# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) # Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. import asyncio import inspect import os import re try: from datetime import UTC, datetime except ImportError: from datetime import datetime UTC = UTC from pathlib import Path from platform import system import shutil from shutil import which import sys import threading import time from types import NoneType from typing import Any, TypeVar, overload import uuid from adbutils import AdbClient from dotenv import load_dotenv from PIL import Image from pydantic import BaseModel from artemis.agents.flash.runner import FlashRunner from artemis.agents.outputter.outputter import outputter from artemis.clients.screen_client_factory import ( ScreenClient, create_screen_client, describe_backend, helper_version, hierarchy_backend_sentence, ) from artemis.config import ( CheckerConfig, OutputConfig, record_events, run_tuning_for_profile, settings, ) from artemis.constants import RECURSION_LIMIT from artemis.context import ( ArtemisContext, DeviceContext, DevicePlatform, ExecutionSetup, ) from artemis.controllers.controller_factory import get_controller from artemis.controllers.platform_specific_commands_controller import ( get_first_device, ) from artemis.runtime import DeviceExecutionLock, trace_store from artemis.runtime.cancel_requests import watch_for_cancel_request from artemis.runtime.cancellation import CancellationReason, reason_from_code from artemis.data_engine.engine import DataEngine from artemis.data_engine.trace import DataEngineCallbackHandler from artemis.graph.graph import get_graph from artemis.graph.state import State from artemis.sdk.builders.agent_config_builder import get_default_agent_config from artemis.sdk.builders.task_request_builder import TaskRequestBuilder from artemis.sdk.types.agent import AgentConfig from artemis.sdk.types.exceptions import ( AgentError, AgentNotInitializedError, AgentProfileNotFoundError, AgentTaskRequestError, DeviceNotFoundError, ExecutableNotFoundError, ) from artemis.sdk.types.task import ( AgentProfile, Task, TaskRequest, ) from artemis.utils.app_launch_utils import _handle_initial_app_launch from artemis.utils.logger import get_logger from artemis.utils.media import ( create_gif_from_trace_folder, create_steps_json_from_trace_folder, remove_images_from_trace_folder, remove_steps_json_from_trace_folder, ) from artemis.utils.startup_progress import publish_startup_progress logger = get_logger(__name__) TOutput = TypeVar("TOutput", bound=BaseModel | None) load_dotenv() def run_tuning_summary(config: AgentConfig, profile: str | None) -> dict[str, str] | None: """Per-run Pro tuning (verification level, explorer mode) for an SDK config. The SDK config carries the Checker switches as flat flags; fold them back into a :class:`CheckerConfig` so the ladder classification stays single- sourced in ``artemis.config``. Returns ``None`` for Flash runs. """ return run_tuning_for_profile( profile, checker=CheckerConfig( enabled=not config.disable_checker, midway_checks=not config.disable_midway_checks, final_check=not config.disable_final_check, assert_failure_policy=config.assert_failure_policy, ), explorer=config.explorer, explorer_versions=dict(config.explorer_versions or {}), ) class Agent: _config: AgentConfig _tasks: list[Task] = [] _tmp_traces_dir: Path _initialized: bool = False _device_context: DeviceContext _adb_client: AdbClient | None _ui_adb_client: ScreenClient | Any | None _current_task: asyncio.Task | None = None _task_lock: asyncio.Lock _cloud_mobile_id: str | None = None def __init__( self, *, config: AgentConfig | None = None, device_id: str | None = None, device_serial: str | None = None, concurrency_mode: str | None = None, max_concurrency: int | None = None, session_id: str | None = None, ): raw_sid = ( session_id or os.getenv("ARTEMIS_SESSION_ID") or os.getenv("ARTEMIS_CLOUD_SESSION_ID") ) self._session_id: str | None = str(raw_sid).strip() if raw_sid else None target_dev = device_serial or device_id if config is None: from artemis.sdk.builders import Builders builder = Builders.AgentConfig if target_dev: builder.for_device(DevicePlatform.ANDROID, target_dev) if concurrency_mode: builder.with_concurrency_mode(concurrency_mode) if max_concurrency is not None: builder.with_max_concurrency(max_concurrency) self._config = builder.build() else: self._config = config updates = {} if target_dev: updates["device_id"] = target_dev updates["device_platform"] = DevicePlatform.ANDROID if concurrency_mode: updates["concurrency_mode"] = str(concurrency_mode).strip().lower() if max_concurrency is not None: updates["max_concurrency"] = max_concurrency if updates: self._config = self._config.model_copy(update=updates) self._tasks = [] self._tmp_traces_dir = Path(settings.TRACES_PATH) self._initialized = False self._task_lock = asyncio.Lock() async def _prewarm_llm_connections(self, api_key: str | None = None): from artemis.llm.router import ModelEndpoint, ModelFactory try: ModelFactory.get_model(ModelEndpoint(api_key=api_key)) message = "模型连接已就绪" except Exception: message = "模型连接将在首次调用时初始化" publish_startup_progress("model_ready", message, session_id=self._session_id) async def init( self, api_key: str | None = None, device_id: str | None = None, device_serial: str | None = None, retry_count: int = 5, retry_wait_seconds: int = 5, execution_target: str = "android", ): if execution_target not in {"android", "computer", "mixed"}: raise ValueError("任务目标必须是 android、computer 或 mixed") self._execution_target = execution_target if execution_target == "computer": from artemis.platform import platform as pal_platform self._device_context = DeviceContext( host_platform=pal_platform.os_type.name, device_id="computer:local", mobile_platform=DevicePlatform.COMPUTER, device_width=0, device_height=0, ) self._adb_client = None self._ui_adb_client = None self._initialized = True return True target_dev = device_serial or device_id if target_dev: self._config = self._config.model_copy( update={"device_id": target_dev, "device_platform": DevicePlatform.ANDROID} ) return await self._init_internal( api_key=api_key, retry_count=retry_count, retry_wait_seconds=retry_wait_seconds, ) async def _init_internal( self, api_key: str | None = None, retry_count: int = 5, retry_wait_seconds: int = 5, ): if os.environ.get("ARTEMIS_CLOUD_MODE") != "1" and not which("adb"): raise ExecutableNotFoundError("adb") if self._initialized: logger.warning("Agent is already initialized. Skipping...") return True # Get first available device ID if os.environ.get("ARTEMIS_CLOUD_MODE") == "1": device_id = os.environ.get("ADB_DEVICE_SERIAL", "cloud_device") platform = DevicePlatform.ANDROID elif not self._config.device_id or not self._config.device_platform: device_id, platform, _ = get_first_device(logger=logger) else: device_id, platform = ( self._config.device_id, self._config.device_platform, ) if not device_id or not platform: error_msg = "No device found. Exiting." logger.error(error_msg) raise DeviceNotFoundError(error_msg) # Initialize clients publish_startup_progress( "device_check", "Checking the Android device", session_id=self._session_id ) if os.environ.get("ARTEMIS_CLOUD_MODE") != "1": self._init_clients( device_id=device_id, platform=platform, ) else: # Cloud mode is pinned to UIAutomator2 through the gateway; the # hierarchy backend switch only applies to locally attached devices. from cloud_service.virtualization import RemoteAdbClient, RemoteUIAutomatorClient self._adb_client = RemoteAdbClient() self._ui_adb_client = RemoteUIAutomatorClient( device_id=device_id, adb_client=self._adb_client, ) self._device_context = await self._get_device_context( device_id=device_id, platform=platform ) logger.info(self._device_context.to_str()) publish_startup_progress( "device_ready", "Android device connected", session_id=self._session_id ) # Asynchronously pre-warm LLM connection pools in the background asyncio.create_task(self._prewarm_llm_connections(api_key)) logger.info("✅ Artemis agent initialized.") self._initialized = True return True async def install_apk(self, apk_path: str | Path) -> None: """Install an APK on the connected device. For cloud mobiles, the APK must be x86_64 compatible. Args: apk_path: Path to the local APK file to install Raises: AgentNotInitializedError: If the agent is not initialized AgentError: If attempting to install on non-Android device or ADB operations fail FileNotFoundError: If the APK file doesn't exist CloudMobileServiceUninitializedError: If cloud service is unavailable """ await self._install_apk_internal(apk_path) async def _install_apk_internal(self, apk_path: str | Path) -> None: if isinstance(apk_path, str): apk_path = Path(apk_path) if not apk_path.exists(): raise FileNotFoundError(f"APK file not found: {apk_path}") if not self._initialized: raise AgentNotInitializedError() device_id = self._device_context.device_id logger.info(f"Installing APK on Android device '{device_id}'") if not self._adb_client: raise AgentError("ADB client not initialized") device = self._adb_client.device(serial=device_id) await asyncio.to_thread(device.install, apk_path) logger.info(f"APK installed successfully on Android device '{device_id}'") async def install_app(self, app_path: str | Path) -> str | None: """Install an app on the connected device. For Android: Installs an APK file using ADB. Args: app_path: Path to the app to install (Android APK file). Returns: None. Raises: AgentNotInitializedError: If the agent is not initialized AgentError: If installation fails or platform is unsupported FileNotFoundError: If the app file/folder doesn't exist """ return await self._install_app_internal(app_path) async def _install_app_internal(self, app_path: str | Path) -> str | None: if isinstance(app_path, str): app_path = Path(app_path) if not app_path.exists(): raise FileNotFoundError(f"App not found: {app_path}") if not self._initialized: raise AgentNotInitializedError() await self._install_apk_internal(app_path) return None def new_task(self, goal: str): """Create a new task request builder. Args: goal: Natural language description of what to accomplish Returns: TaskRequestBuilder that can be configured with: - .with_output_format() for structured output - .with_output_description() for output description - .with_locked_app_package() to restrict execution to a specific app - .using_profile() to specify an LLM profile - .with_max_steps() to set maximum execution steps - .with_trace_recording() to enable trace recording - .with_name() to set a custom task name """ return ( TaskRequestBuilder[None] .from_common( goal=goal, common=self._config.task_request_defaults, ) .for_target(getattr(self, "_execution_target", "android")) ) @overload async def run_task( self, *, goal: str, output: type[TOutput], profile: str | AgentProfile | None = None, name: str | None = None, locked_app_package: str | None = None, app_path: str | Path | None = None, ) -> TOutput | None: ... @overload async def run_task( self, *, goal: str, output: str, profile: str | AgentProfile | None = None, name: str | None = None, locked_app_package: str | None = None, app_path: str | Path | None = None, ) -> str | dict | None: ... @overload async def run_task( self, *, goal: str, output=None, profile: str | AgentProfile | None = None, name: str | None = None, locked_app_package: str | None = None, app_path: str | Path | None = None, ) -> str | None: ... @overload async def run_task( self, *, request: TaskRequest[None], locked_app_package: str | None = None, app_path: str | Path | None = None, ) -> str | dict | None: ... @overload async def run_task( self, *, request: TaskRequest[TOutput], locked_app_package: str | None = None, app_path: str | Path | None = None, ) -> TOutput | None: ... async def run_task( self, *, goal: str | None = None, output: type[TOutput] | str | None = None, profile: str | AgentProfile | None = None, locked_app_package: str | None = None, name: str | None = None, app_path: str | Path | None = None, request: TaskRequest[TOutput] | None = None, ) -> str | dict | TOutput | None: # Normal local execution path if request is not None: if locked_app_package is not None: if request.locked_app_package: logger.warning( "Locked app package specified both in the request and" " as a parameter. Using the parameter value." ) request.locked_app_package = locked_app_package # Handle app_path parameter override if app_path is not None: if request.app_path: logger.warning( "App path specified both in the request and as a" " parameter. Using the parameter value." ) request.app_path = Path(app_path) if isinstance(app_path, str) else app_path return await self._run_task(request=request) if goal is None: raise AgentTaskRequestError("Goal is required") task_request = self.new_task(goal=goal) if output is not None: if isinstance(output, str): task_request.with_output_description(description=output) elif output is not NoneType: task_request.with_output_format(output_format=output) if profile is not None: task_request.using_profile(profile=profile) if name is not None: task_request.with_name(name=name) if locked_app_package is not None: task_request.with_locked_app_package(package_name=locked_app_package) if app_path is not None: task_request.with_app_path(app_path=app_path) return await self._run_task(task_request.build()) async def _run_task( self, request: TaskRequest[TOutput], ) -> str | dict | TOutput | None: if not self._initialized: raise AgentNotInitializedError() if request.execution_target != getattr(self, "_execution_target", "android"): raise AgentTaskRequestError("任务目标与 Agent 初始化目标不一致") if request.execution_target == "computer" and ( request.app_path or request.locked_app_package ): raise AgentTaskRequestError("纯电脑任务不能使用 Android 安装包或应用锁定") if request.profile: agent_profile = self._config.agent_profiles.get(request.profile) if agent_profile is None: if request.profile.lower() in ( "flash", "pro", "ultra", "default", ): agent_profile = self._config.default_profile else: raise AgentProfileNotFoundError(request.profile) else: agent_profile = self._config.default_profile logger.info(str(agent_profile)) on_status_changed = None task_id = str( self._session_id or os.getenv("ARTEMIS_SESSION_ID") or os.getenv("ARTEMIS_CLOUD_SESSION_ID") or uuid.uuid4() ) task = Task( id=task_id, device=self._device_context, status="pending", request=request, created_at=datetime.now(), on_status_changed=on_status_changed, ) self._tasks.append(task) task_name = task.get_name() context = ArtemisContext( device=self._device_context, adb_client=self._adb_client, ui_adb_client=self._ui_adb_client, llm_config=agent_profile.llm_config, agent_config=self._config, execution_target=request.execution_target, ) from artemis.exploration.runtime import configure_context configure_context(context, task_id) from artemis.plugins.runtime import PluginRuntime output_config = None if request.output_description or request.output_format: output_config = OutputConfig( output_description=request.output_description, structured_output=request.output_format, # type: ignore ) logger.info(str(output_config)) logger.info(f"[{task_name}] Starting graph with goal: `{request.goal}`") state = self._get_graph_state(task=task) graph_input = state.model_dump() datetime.now(UTC) async def _execute_task_logic(): last_state: State | None = None last_state_snapshot: dict | None = None output = None task_control = None effective_mode = getattr(self._config, "concurrency_mode", "per_device") effective_max = getattr(self._config, "max_concurrency", None) sess_id = ( self._session_id or os.getenv("ARTEMIS_SESSION_ID") or os.getenv("ARTEMIS_CLOUD_SESSION_ID") or getattr(task, "id", None) or getattr(getattr(task, "request", None), "task_name", None) ) device_lock = None execution_started = False try: # Construct the runtime only inside the task's cleanup boundary. A # cancelled queue wait or setup failure must not leave version pins. context.plugins = PluginRuntime(request.plugins, ctx=context) active_owner = DeviceExecutionLock.get_active_owner(self._device_context.device_id) already_held = ( active_owner is not None and active_owner.pid == os.getpid() and ( (sess_id and str(active_owner.session_id) == str(sess_id)) or active_owner.token == self._device_context.device_id ) ) if not already_held or context.plugins.resources: device_lock = DeviceExecutionLock( self._device_context.device_id, description=f"{request.goal[:120]}", concurrency_mode=effective_mode, max_concurrency=effective_max, session_id=str(sess_id) if sess_id else None, ingress=os.getenv("ARTEMIS_TASK_INGRESS") or "agent", resource_ids=context.plugins.resources, ) context.plugins.ensure_resources_ready() if context.plugins.packages: from artemis.plugins.models import PluginSelection context.plugins.store.hold( str(sess_id), [ PluginSelection(id=m.id, version=m.version, profile=p.name) for m, p, _ in context.plugins.packages.values() ], ) if device_lock is not None and os.environ.get("ARTEMIS_CLOUD_MODE") != "1": from artemis.runtime.task_control import write_control_state if sess_id and os.environ.get("ARTEMIS_TASK_WORKER") == "1": write_control_state(str(sess_id), "queued", reset=True) queue_cancel_event = threading.Event() acquire_task = asyncio.create_task( asyncio.to_thread( device_lock.acquire, cancel_event=queue_cancel_event, ) ) try: await asyncio.shield(acquire_task) except asyncio.CancelledError: queue_cancel_event.set() try: await asyncio.shield(acquire_task) except Exception as exc: # pylint: disable=broad-exception-caught # Draining the lock acquisition after cancellation is best effort. logger.debug( f"Device lock acquisition drain after cancel failed: {exc}", exc_info=True, ) raise # All device mutation and UI initialization happens only after # this task reaches the head of the one global FIFO queue. # Session creation must be inside the same boundary as well: # DataEngine uses a shared database, so a queued task must not # publish a new active session while the current task is still # finishing. self._prepare_tracing(task=task, context=context) from artemis.runtime.diagnostics import capture_runtime_diagnostics await capture_runtime_diagnostics( context.data_engine.base_dir, "start", android=request.execution_target != "computer", ) if device_lock is not None and os.environ.get("ARTEMIS_TASK_WORKER") == "1": from artemis.runtime.task_control import TaskControl if getattr(context, "exploration_id", None): from artemis.exploration.runtime import ExplorationControl task_control = ExplorationControl(str(sess_id), device_lock) task_control.context = context else: task_control = TaskControl(str(sess_id), device_lock) context.task_control = task_control task_control._state("running") self._prepare_output_files(task=task) if ( os.environ.get("ARTEMIS_CLOUD_MODE") != "1" and request.execution_target != "computer" ): if self._ui_adb_client is not None: await self._connect_screen_client(context, str(sess_id)) await self._ensure_device_unlocked() publish_startup_progress( "environment", "Preparing the device environment", session_id=str(sess_id) ) if request.execution_target != "computer": await self._prepare_app_installation(task=task) await self._prepare_device_environment(context=context) await self._prepare_app_lock(task=task, context=context) elif context.execution_setup: context.execution_setup.video_recording_tools_enabled = False async with context: recording_started = False recording_parts = 1 controller = get_controller(context) output_dir = None if context.execution_setup and context.execution_setup.traces_path: output_dir = ( self._tmp_traces_dir / context.execution_setup.trace_name ).resolve() async def pause_capture(): nonlocal recording_started await context.plugins.aclose(release=False) if recording_started: try: await controller.stop_video_recording() finally: recording_started = False from artemis.tools.command_tool import shutdown_adb_background_tasks await shutdown_adb_background_tasks(context) async def resume_capture(): nonlocal recording_started, recording_parts if recording_started: return if request.execution_target == "computer": return if not ( self._config.video_recording_tools_enabled or self._config.record_video_enabled ): return recording_parts += 1 target = ( output_dir / f"resume_{recording_parts:04d}" if output_dir else None ) result = await controller.start_video_recording(output_dir=target) recording_started = bool(result and result.success) context.on_task_pause = pause_capture context.on_task_resume = resume_capture async def review_capture(): nonlocal recording_started await context.plugins.suspend_capture() if recording_started: try: await controller.stop_video_recording() finally: recording_started = False async def continue_capture(): context.plugins.resume_capture() await resume_capture() context.on_execution_review = review_capture context.on_execution_resume = continue_capture if getattr(context, "exploration_id", None) and task_control is not None: task_control.start_budget() if request.execution_target != "computer" and ( self._config.video_recording_tools_enabled or self._config.record_video_enabled ): try: logger.info(f"[{task_name}] Starting automated screen recording...") start_res = await controller.start_video_recording( output_dir=output_dir ) if start_res and start_res.success: recording_started = True else: logger.warning( f"[{task_name}] Failed to start screen" f" recording: {start_res.message if start_res else 'unknown'}" ) except Exception as e: logger.error(f"[{task_name}] Failed to start screen recording: {e}") publish_startup_progress( "environment_ready", "Device environment is ready", session_id=str(sess_id) ) execution_started = True try: if request.profile and request.profile.lower() == "flash": logger.info(f"[{task_name}] Invoking FlashRunner reactive loop...") await task.set_status( status="running", message="Invoking FlashRunner...", ) # An explicit max_steps caps the reactive loop; the # default leaves the cap to agent.flash.max_turns # (0 = unlimited). max_turns = ( request.max_steps if request.max_steps != RECURSION_LIMIT else None ) runner = FlashRunner(context, goal=request.goal, max_turns=max_turns) flash_result = await runner.run(state) from artemis.agents.checker.flash import settle_flash_outcome run_outcome = await settle_flash_outcome( context, request.goal, flash_result, final_report_received=runner.final_report_received, ) output = attach_test_summary(flash_result, run_outcome) state.run_outcome = run_outcome last_state_snapshot = state.model_dump() status = flash_result.get("status") if status == "completed": logger.info(f"Automation '{task_name}' execution completed.") await task.finalize(content=output, state=last_state_snapshot) if context.data_engine: context.data_engine.end_session("completed") else: err = ( f"[{task_name}] FlashRunner failed:" f" {flash_result.get('explanation')}" ) logger.warning(err) await task.finalize( content=output, state=last_state_snapshot, error=err, ) from artemis.runtime.task_result import save_failure_result if sess_id: save_failure_result( str(sess_id), err, code=flash_result.get("code", "execution_failed"), ) if context.data_engine: context.data_engine.end_session("failed") return output else: logger.info(f"[{task_name}] Invoking graph with input: {graph_input}") await task.set_status(status="running", message="Invoking graph...") async for chunk in (await get_graph(context)).astream( input=graph_input, config={ "recursion_limit": task.request.max_steps, "callbacks": ( ( list(self._config.graph_config_callbacks) if self._config.graph_config_callbacks is not None else [] ) + [DataEngineCallbackHandler(context)] if context.data_engine else self._config.graph_config_callbacks ), }, stream_mode=[ "messages", "custom", "updates", "values", ], ): stream_mode, payload = chunk if stream_mode == "values": last_state_snapshot = payload # type: ignore last_state = State(**last_state_snapshot) # type: ignore if not last_state: err = f"[{task_name}] No result received from graph" logger.warning(err) await task.finalize( content=output, state=last_state_snapshot, error=err, ) if context.data_engine: context.data_engine.end_session("failed") return None output = await self._extract_output( task_name=task_name, ctx=context, request=request, output_config=output_config, state=last_state, ) # Run outcome (goal axis x test axis) drives the # wrap-up instead of an unconditional success path. run_outcome = getattr(last_state, "run_outcome", None) context.run_outcome = run_outcome if getattr(context, "exploration_id", None): from artemis.exploration.synthesis import synthesize_completed_run await synthesize_completed_run(context, run_outcome or {}) output = attach_test_summary(output, run_outcome) task_status_axis = ( run_outcome.get("task_status") if run_outcome else None ) tests_failed = ( int((run_outcome.get("tests") or {}).get("failed", 0)) if run_outcome else 0 ) if task_status_axis == "blocked": err = ( f"[{task_name}] Task ended blocked: verify" " criteria unmet after exhausting the final" " check budget." ) logger.warning(err) await task.finalize( content=output, state=last_state_snapshot, error=err, ) if context.data_engine: context.data_engine.end_session("failed") return output if tests_failed > 0: logger.warning( f"[{task_name}] Task completed, but" f" {tests_failed} assertion(s) failed —" " see the test summary." ) else: logger.info(f"Automation '{task_name}' execution completed.") await task.finalize(content=output, state=last_state_snapshot) if context.data_engine: context.data_engine.end_session("completed") return output finally: if recording_started: async def _safe_stop_recording(): try: logger.info( f"[{task_name}] Stopping automated screen recording..." ) controller = get_controller(context) return await controller.stop_video_recording() except Exception as e: logger.error( f"[{task_name}] Error stopping screen recording: {e}" ) return None stop_task = asyncio.create_task(_safe_stop_recording()) try: stop_result = await asyncio.shield(stop_task) except (asyncio.CancelledError, BaseException): try: stop_result = await asyncio.wait_for(stop_task, timeout=15.0) except Exception: stop_result = None if stop_result and stop_result.success and stop_result.video_path: logger.info( f"[{task_name}] Screen recording saved" f" to: {stop_result.video_path}" ) elif stop_result: logger.warning( f"[{task_name}] Failed to save screen" f" recording: {stop_result.message}" ) except asyncio.CancelledError: err = f"[{task_name}] Task cancelled" logger.warning(err) await task.finalize( content=output, state=last_state_snapshot, error=err, cancelled=True, ) if context.data_engine: context.data_engine.end_session("cancelled") raise except Exception as e: err = f"[{task_name}] Error running automation: {e}" logger.error(err) await task.finalize( content=output, state=last_state_snapshot, error=err, ) from artemis.runtime.task_result import save_failure_result if sess_id: save_failure_result( str(sess_id), err, code="execution_failed" if execution_started else "initialization_failed", ) if context.data_engine: context.data_engine.end_session("failed") raise finally: # Close plugins before releasing the whole task's device/resource lease. plugins_stopped = False try: if context.plugins: await context.plugins.aclose() context.plugins.store.release(str(sess_id)) plugins_stopped = not getattr(context, "exploration_capture_unstopped", False) except Exception: logger.error("插件未确认停止，执行资源将保留至工作进程退出。") try: # Background ADB processes (logcat, screenrecord, ...) started # by the Operator must not outlive the automation task. from artemis.tools.command_tool import shutdown_adb_background_tasks await asyncio.wait_for(shutdown_adb_background_tasks(context), timeout=15.0) except Exception as e: logger.warning(f"[{task_name}] Failed to stop background ADB tasks: {e}") try: if ( self._config.video_recording_tools_enabled and not self._config.record_video_enabled ): # Video analysis used a temporary capture during execution. # Keep no recording after the final model/report step. trace_dir = (self._tmp_traces_dir / task_name).resolve() root = self._tmp_traces_dir.resolve() if trace_dir.is_relative_to(root) and trace_dir.is_dir(): for recording in trace_dir.rglob("*"): if recording.is_relative_to( trace_dir / "evidence" / "demonstrations" ): continue if recording.is_file() and ( recording.suffix.lower() in {".mkv", ".mp4", ".webm"} or recording.name == "recording.json" ): recording.unlink(missing_ok=True) if context.data_engine and context.data_engine.current_session_id: context.data_engine.storage.discard_session_recordings( str(context.data_engine.current_session_id), preserve_directory=trace_dir / "evidence" / "demonstrations", ) await self._finalize_tracing_safely(task=task, context=context) finally: if os.environ.get("ARTEMIS_CLOUD_MODE") != "1": try: if self._ui_adb_client is not None: await asyncio.to_thread(self._ui_adb_client.disconnect) finally: if task_control is not None and plugins_stopped: await asyncio.to_thread(task_control.release) elif device_lock is not None and plugins_stopped: await asyncio.to_thread(device_lock.release) async with self._task_lock: if self._current_task and not self._current_task.done(): logger.warning( "Another automation task is already running. " "Stopping it before starting the new one." ) self.stop_current_task(reason=CancellationReason.TASK_REPLACED) try: await self._current_task except asyncio.CancelledError: pass cancel_watcher: asyncio.Task | None = None try: self._current_task = asyncio.create_task(_execute_task_logic()) cancel_watcher = asyncio.create_task(self._watch_external_cancel(task_name)) return await self._current_task finally: self._current_task = None if cancel_watcher is not None and not cancel_watcher.done(): cancel_watcher.cancel() try: await cancel_watcher except asyncio.CancelledError: pass async def _watch_external_cancel(self, task_name: str) -> None: """Cancel the running task when another process drops a cancel marker. The admin console (and other ingresses) cannot deliver a signal to a worker portably, so they write a marker keyed by session id / pid. Reacting here routes the request through the same cancellation path as Ctrl+C: the recording is stopped and remuxed, the trace folder is compiled and renamed, and the device lease is released. """ def _on_cancel(reason: CancellationReason) -> None: logger.warning( f"[{task_name}] External cancel request received; stopping the task gracefully." ) self.stop_current_task(reason=reason) try: await watch_for_cancel_request( _on_cancel, session_id=self._session_id, pass_reason=True ) except asyncio.CancelledError: raise except Exception as exc: logger.debug(f"[{task_name}] Cancel watcher stopped: {exc}") def stop_current_task(self, *, reason: CancellationReason = CancellationReason.USER_STOP): """Requests cancellation of the currently running automation task.""" if self._current_task and not self._current_task.done(): logger.info("Requesting to stop the current automation task...") was_cancelled = self._current_task.cancel(reason_from_code(reason)) if was_cancelled: logger.success("Cancellation request for the current task was sent.") else: logger.warning( "Could not send cancellation request for the current task " "(it may already be completing)." ) else: logger.info("No active automation task to stop.") async def get_screenshot(self) -> Image.Image: """Capture a screenshot from the mobile device. Returns: Screenshot as PIL Image Raises: AgentNotInitializedError: If the agent is not initialized Exception: If screenshot capture fails """ if not self._initialized: raise AgentNotInitializedError() # Use ADB to capture screenshot logger.info("Capturing screenshot from local Android device") if not self._adb_client: raise Exception("ADB client not initialized") device = self._adb_client.device(serial=self._device_context.device_id) screenshot = await asyncio.to_thread(device.screenshot) logger.info("Screenshot captured from local Android device") return screenshot async def clean(self, force: bool = False): if not self._initialized and not force: return if self._ui_adb_client is not None: await asyncio.to_thread(self._ui_adb_client.disconnect) self._initialized = False logger.info("✅ Artemis agent stopped.") async def _ensure_device_unlocked(self) -> None: """Reject secure keyguard instead of allowing an agent to guess credentials.""" if self._adb_client is None: raise AgentError("ADB client is not initialized.") device = self._adb_client.device(serial=self._device_context.device_id) try: trust_state = str(await asyncio.to_thread(device.shell, "dumpsys trust")) except Exception as exc: logger.warning(f"Could not inspect Android keyguard state: {exc}") return # The first deviceLocked value belongs to the current Android user; # later entries may describe a separately locked work profile. match = re.search(r"\bdeviceLocked=(?:true|1|false|0)\b", trust_state, re.IGNORECASE) if match is None: return value = match.group(0).split("=", 1)[1].lower() if value in {"true", "1"}: raise AgentError( "Android secure keyguard is locked. Unlock the device manually before " "running Artemis; automation will not guess a PIN, password, or pattern." ) async def _prepare_app_installation(self, task: Task) -> str | None: """Install app if app_path is specified in the task request. Returns: None. """ if not task.request.app_path: return None task_name = task.get_name() logger.info(f"[{task_name}] Installing app from: {task.request.app_path}") await self.install_app(task.request.app_path) return None async def _prepare_device_environment(self, context: ArtemisContext): """Prepare device environment flags (like forcing Web Accessibility) before the task runs.""" if not self._config.force_web_accessibility: logger.info( "Forcing web accessibility is disabled in AgentConfig. Skipping" " device environment prep..." ) return if not self._adb_client or not self._device_context: logger.warning( "ADB client or device context not available. Skipping device environment prep..." ) return device_id = self._device_context.device_id try: device = self._adb_client.device(serial=device_id) logger.info(f"[{device_id}] ⚙️ Configuring Chrome/WebView Web Accessibility flags...") # 1. Write the force accessibility command flag to Chrome command line await asyncio.to_thread( device.shell, "echo 'chrome --force-renderer-accessibility' >" " /data/local/tmp/chrome-command-line", ) await asyncio.to_thread(device.shell, "chmod 555 /data/local/tmp/chrome-command-line") # 2. Write the same command line flag to System WebView config file await asyncio.to_thread( device.shell, "echo 'chrome --force-renderer-accessibility' >" " /data/local/tmp/webview-command-line", ) await asyncio.to_thread(device.shell, "chmod 555 /data/local/tmp/webview-command-line") # 3. Force stop Chrome so the flag takes effect next time it launches await asyncio.to_thread(device.shell, "am force-stop com.android.chrome") logger.success( f"[{device_id}] ✅ Chrome Web Accessibility command line flags" " configured successfully." ) except Exception as e: logger.warning( "Failed to configure Chrome Web Accessibility flags (needs" f" root/userdebug device): {e}" ) async def _prepare_app_lock(self, task: Task, context: ArtemisContext): """Prepare app lock by launching the locked app if specified.""" if not task.request.locked_app_package: return task_name = task.get_name() logger.info(f"[{task_name}] Preparing app lock for: {task.request.locked_app_package}") app_lock_status = await _handle_initial_app_launch( ctx=context, locked_app_package=task.request.locked_app_package ) if context.execution_setup is None: context.execution_setup = ExecutionSetup(app_lock_status=app_lock_status) else: context.execution_setup.app_lock_status = app_lock_status if app_lock_status.locked_app_initial_launch_success is False: error = app_lock_status.locked_app_initial_launch_error logger.warning(f"[{task_name}] Failed to launch locked app: {error}") def _prepare_tracing(self, task: Task, context: ArtemisContext): """Prepare tracing and data engine setup.""" task_name = task.get_name() temp_trace_path = Path(self._tmp_traces_dir / task_name).resolve() temp_trace_path.mkdir(parents=True, exist_ok=True) if task.request.record_trace: traces_output_path = Path(task.request.trace_path).resolve() logger.info(f"[{task_name}] 📂 Traces output path: {traces_output_path}") logger.info(f"[{task_name}] 📄📂 Traces temp path: {temp_trace_path}") traces_output_path.mkdir(parents=True, exist_ok=True) context.execution_setup = ExecutionSetup( traces_path=self._tmp_traces_dir, trace_name=task_name, enable_remote_tracing=task.request.enable_remote_tracing if task.request.record_trace else False, video_recording_tools_enabled=self._config.video_recording_tools_enabled, record_video_enabled=self._config.record_video_enabled, disable_checker=self._config.disable_checker, disable_midway_checks=self._config.disable_midway_checks, disable_final_check=self._config.disable_final_check or ( (task.request.profile or "").lower() == "flash" and not self._config.flash.final_check_enabled ), checker_max_iterations=self._config.checker_max_iterations, final_check_max_attempts=self._config.final_check_max_attempts, checkpoint_max_repairs=self._config.checkpoint_max_repairs, max_concurrent_checkpoints=self._config.max_concurrent_checkpoints, checkpoint_timeout=self._config.checkpoint_timeout, settlement_timeout=self._config.settlement_timeout, assert_failure_policy=self._config.assert_failure_policy, disable_device_probes=self._config.disable_device_probes, disable_planner_validation=self._config.disable_planner_validation, enable_committee=self._config.enable_committee, committee_debate_rounds=self._config.committee_debate_rounds, disable_outputter=self._config.disable_outputter, outputter=self._config.outputter, explorer=self._config.explorer, explorer_versions=self._config.explorer_versions, ) context.data_engine = DataEngine(ctx=context) device_data = context.device.model_dump() if context.device else {} if task.request.profile: device_data["profile"] = task.request.profile run_tuning = run_tuning_summary(self._config, task.request.profile) if run_tuning: device_data["run_tuning"] = run_tuning target_sid = ( self._session_id or os.getenv("ARTEMIS_SESSION_ID") or os.getenv("ARTEMIS_CLOUD_SESSION_ID") or getattr(task, "id", None) or getattr(getattr(task, "request", None), "task_name", None) ) sess_uuid = None if target_sid: try: sess_uuid = uuid.UUID(str(target_sid)) except Exception: sess_uuid = str(target_sid) context.data_engine.start_session( goal=task.request.goal, device_info=device_data, session_id=sess_uuid, ) async def _finalize_tracing_safely(self, task: Task, context: ArtemisContext): """Finalize optional trace artifacts without changing task semantics.""" try: await self._finalize_tracing(task=task, context=context) except Exception as exc: logger.error( f"[{task.get_name()}] Trace artifact finalization failed after task status " f"was resolved as '{task.status}': {exc}", exc_info=True, ) async def _finalize_tracing(self, task: Task, context: ArtemisContext): if context.data_engine: if task.status == "failed": from artemis.runtime.diagnostics import capture_runtime_diagnostics await capture_runtime_diagnostics( context.data_engine.base_dir, "failure", android=context.execution_target != "computer", ) session_status = "completed" if task.status == "failed": session_status = "failed" elif task.status == "cancelled": session_status = "cancelled" context.data_engine.end_session(status=session_status) exec_setup_ctx = context.execution_setup if not exec_setup_ctx: return if exec_setup_ctx.traces_path is None or exec_setup_ctx.trace_name is None: return task_name = task.get_name() temp_trace_path = (self._tmp_traces_dir / exec_setup_ctx.trace_name).resolve() if not task.request.record_trace: try: if temp_trace_path.exists(): shutil.rmtree(temp_trace_path) logger.info(f"[{task_name}] Cleaned up temporary trace folder.") except Exception as e: logger.error(f"[{task_name}] Failed to clean up temp trace folder: {e}") return status = resolve_trace_suffix(task.status, getattr(context, "run_outcome", None)) ts = task.created_at.strftime("%Y-%m-%dT%H-%M-%S") new_name = f"{exec_setup_ctx.trace_name}{status}_{ts}" traces_output_path = Path(task.request.trace_path).resolve() logger.info(f"[{task_name}] Compiling trace FROM FOLDER: " + str(temp_trace_path)) create_gif_from_trace_folder(temp_trace_path) create_steps_json_from_trace_folder(temp_trace_path) logger.info(f"[{task_name}] Video created, removing dust...") remove_images_from_trace_folder(temp_trace_path) remove_steps_json_from_trace_folder(temp_trace_path) logger.info(f"[{task_name}] 📽️ Trace compiled, moving to output path 📽️") output_folder_path = temp_trace_path.rename(traces_output_path / new_name).resolve() logger.info(f"[{task_name}] 📂✅ Traces located in: {output_folder_path}") new_video_path = output_folder_path / "recording.mp4" if new_video_path.exists() and context.data_engine: context.data_engine.update_video_path(new_video_path) def _prepare_output_files(self, task: Task): if task.request.llm_output_path: _validate_and_prepare_file(file_path=task.request.llm_output_path) async def _extract_output( self, task_name: str, ctx: ArtemisContext, request: TaskRequest[TOutput], output_config: OutputConfig | None, state: State, ) -> str | dict | TOutput | None: exec_setup = getattr(ctx, "execution_setup", None) outputter_cfg = getattr(exec_setup, "outputter", None) or ( getattr(ctx.agent_config, "outputter", None) if ctx and ctx.agent_config else None ) outputter_enabled = getattr(outputter_cfg, "enabled", True) if outputter_cfg else True force_synthesis = ( getattr(outputter_cfg, "force_synthesis", False) if outputter_cfg else False ) if exec_setup and getattr(exec_setup, "disable_outputter", False): outputter_enabled = False should_run = outputter_enabled and ( ( output_config and output_config.needs_structured_format(default_enabled=outputter_enabled) ) or force_synthesis or (output_config and output_config.enable_outputter is True) ) if output_config and output_config.enable_outputter is False: should_run = False execution_report = getattr(state, "execution_report", None) if isinstance(execution_report, str) and execution_report.strip() and not should_run: record_events(output_path=request.llm_output_path, events={"result": execution_report}) return execution_report if should_run: logger.info(f"[{task_name}] Generating structured output via Outputter...") try: structured_output = await outputter( ctx=ctx, output_config=output_config or OutputConfig(), graph_output=state, ) logger.info(f"[{task_name}] Structured output: {structured_output}") record_events( output_path=request.llm_output_path, events=structured_output, ) if request.output_format is not None and request.output_format is not NoneType: return request.output_format.model_validate(structured_output) return structured_output except Exception as e: logger.error(f"[{task_name}] Failed to generate structured output: {e}") return None return None def _get_graph_state(self, task: Task): return State.initial(task.request.goal) # ------------------------------------------------------------------ # # UI hierarchy backend: connect, and keep the user told which one serves # ------------------------------------------------------------------ # async def _connect_screen_client(self, context: ArtemisContext, session_id: str) -> None: """Connect the screen client with visible progress for slow first-time steps.""" client = self._ui_adb_client previous_listener = getattr(self, "_hierarchy_backend_listener", None) if previous_listener is not None: previous_client, listener = previous_listener previous_client.remove_backend_listener(listener) self._hierarchy_backend_listener = None publish_startup_progress( "uiautomator", "Connecting to the UI hierarchy service", session_id=session_id ) def on_provision(event: str, details: dict) -> None: version = details.get("version_name") or details.get("to_version") if event == "installing": publish_startup_progress( "helper_install", "Installing the Artemis accessibility helper on this device for the " f"first time (v{version}, about 3 seconds)", session_id=session_id, **details, ) elif event == "upgrading": publish_startup_progress( "helper_upgrade", "Upgrading the Artemis accessibility helper " f"(v{details.get('from_version')} -> v{details.get('to_version')})", session_id=session_id, **details, ) # Clients without provisioning (UIAutomator2, cloud) take no callback. try: accepts_events = "on_event" in inspect.signature(client.connect).parameters except (TypeError, ValueError): accepts_events = False if accepts_events: await asyncio.to_thread(client.connect, on_event=on_provision) else: await asyncio.to_thread(client.connect) backend = describe_backend(client) or "uiautomator" publish_startup_progress( "uiautomator_ready", f"UI hierarchy service is ready ({backend})", session_id=session_id, ) self._announce_hierarchy_backend(context, session_id, None, backend, None) add_listener = getattr(client, "add_backend_listener", None) if callable(add_listener): def listener(previous, new, reason): self._announce_hierarchy_backend(context, session_id, previous, new, reason) add_listener(listener) self._hierarchy_backend_listener = (client, listener) def _announce_hierarchy_backend( self, context: ArtemisContext, session_id: str, previous: str | None, backend: str, reason: str | None, ) -> None: """One line everywhere a person or an agent looks for it. Startup-progress event (UI timeline), a named log trace, the session's device_info, and status.json for ``mobile_manage_task``. Runs from worker threads too, so every sink is best effort. """ label = {"helper": "Artemis accessibility helper", "uiautomator": "UIAutomator2"} version = helper_version(self._ui_adb_client) if previous is None: message = f"UI hierarchy source: {label.get(backend, backend)}" if backend == "helper" and version: message += f" v{version}" stage = "hierarchy_backend" else: message = ( f"UI hierarchy source switched from {label.get(previous, previous)} to " f"{label.get(backend, backend)}" ) if reason: message += f" because {reason}" stage = "hierarchy_backend_changed" logger.info(message) try: publish_startup_progress( stage, message, session_id=session_id, backend=backend, previous_backend=previous, reason=reason, helper_version=version, ) except (OSError, ValueError, RuntimeError) as exc: logger.debug(f"Could not publish hierarchy backend progress: {exc}") engine = getattr(context, "data_engine", None) if engine is not None and getattr(engine, "current_session_id", None): try: engine.record_trace( type="log", name="hierarchy_backend", payload={ "message": message, "backend": backend, "previous_backend": previous, "reason": reason, "helper_version": version, "level": "WARNING" if previous is not None else "INFO", }, ) engine.update_session_device_info( hierarchy_backend=backend, hierarchy_backend_note=hierarchy_backend_sentence( self._ui_adb_client, relative_time=engine.get_relative_time ), ) except (OSError, ValueError, RuntimeError) as exc: logger.debug(f"Could not record hierarchy backend in the data engine: {exc}") try: trace_store.update_trace_fields( session_id, hierarchy_backend=backend, hierarchy_backend_note=hierarchy_backend_sentence(self._ui_adb_client), ) except (OSError, ValueError, RuntimeError) as exc: logger.debug(f"Could not record hierarchy backend in status.json: {exc}") def _init_clients( self, device_id: str, platform: DevicePlatform, ): self._adb_client = AdbClient( host=self._config.servers.adb_host, port=self._config.servers.adb_port, ) self._ui_adb_client = create_screen_client(device_id) async def _get_device_context( self, device_id: str, platform: DevicePlatform, ) -> DeviceContext: host_platform = system() if os.environ.get("ARTEMIS_CLOUD_MODE") == "1": width, height = (1080, 2400) if self._adb_client: try: width, height = self._adb_client.device(device_id).window_size() logger.info( f"Retrieved remote device screen dimensions dynamically: {width}x{height}" ) except Exception as e: logger.debug( f"Failed to query remote window size: {e}. Defaulting to 1080x2400." ) return DeviceContext( host_platform="LINUX", mobile_platform=platform, device_id=device_id, device_width=width, device_height=height, ) # Query dimensions without starting UIAutomator or acquiring an awake # strategy. Those belong inside the global execution queue lease. if self._adb_client: try: device_width, device_height = self._adb_client.device(device_id).window_size() logger.info(f"Retrieved Android screen dimensions: {device_width}x{device_height}") except Exception as e: logger.warning(f"Failed to get Android window size: {e}, using defaults") device_width, device_height = 1080, 2340 else: logger.warning("ADB client not available, using default dimensions") device_width, device_height = 1080, 2340 from artemis.platform import platform as pal_platform return DeviceContext( host_platform=pal_platform.os_type.name, mobile_platform=platform, device_id=device_id, device_width=device_width, device_height=device_height, ) def _validate_and_prepare_file(file_path: Path): path_obj = Path(file_path) if path_obj.exists() and path_obj.is_dir(): raise AgentTaskRequestError(f"Error: Path '{file_path}' is a directory, not a file.") try: path_obj.parent.mkdir(parents=True, exist_ok=True) path_obj.touch(exist_ok=True) except OSError as e: raise AgentTaskRequestError(f"Error creating file '{file_path}': {e}") def attach_test_summary(output, run_outcome: dict | None): """Surfaces the machine-readable test summary in the task's return value. Only applies when the run actually had check items (the summary carries at least one counted item) — otherwise the historical return shape is kept untouched. String outputs are wrapped into a dict so that callers such as ``mobile_run_task`` receive the summary without parsing report prose. """ if not run_outcome: return output tests = run_outcome.get("tests") or {} total = ( int(tests.get("passed", 0)) + int(tests.get("failed", 0)) + int(tests.get("inconclusive", 0)) + int(tests.get("unchecked", 0)) ) test_result = run_outcome.get("test_result") if total <= 0 and not test_result: return output summary = {"task_status": run_outcome.get("task_status"), **tests} additions = {"test_summary": summary} if total > 0 else {} if test_result: additions["test_result"] = test_result if isinstance(output, dict): merged = dict(output) merged.update(additions) return merged if output is None: return additions if isinstance(output, str): return {"result": output, **additions} # Typed/structured outputs keep their shape; the summary stays available in # run_outcome.json and the session state. return output def resolve_trace_suffix(task_status: str, run_outcome: dict | None) -> str: """Keep execution and the independently verified test verdict distinct.""" if task_status != "completed": return "_FAIL" tests = (run_outcome or {}).get("tests") or {} verdict = ((run_outcome or {}).get("test_result") or {}).get("status") if verdict == "blocked": return "_BLOCKED" if verdict == "inconclusive": return "_UNVERIFIED" if verdict == "failed" or int(tests.get("failed", 0)) > 0: return "_TESTFAIL" return "_PASS" if verdict == "passed" else "_UNVERIFIED"