refactor(core): remove dead parallel agent architecture
This commit is contained in:
1 parent
cbe8f4f0c6
commit
f68d4288e0
29 files changed
+8
-2736
No files matched your search
@@ -21,7 +21,6 @@ from artemis.interfaces.sdk.task import StreamEvent, StreamEventType, Task
|
||||
from artemis_client import Capabilities, Device, TaskHandle, TaskResult
|
||||
from artemis.sdk.agent import Agent
|
||||
from artemis.sdk.builders import Builders
|
||||
from artemis.agents.base import BaseAgent, AgentRegistry
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.tools.base import artemis_tool, ToolRegistry
|
||||
|
||||
@@ -38,8 +37,6 @@ __all__ = [
|
||||
"Task",
|
||||
"TaskHandle",
|
||||
"TaskResult",
|
||||
"BaseAgent",
|
||||
"AgentRegistry",
|
||||
"BaseDeviceDriver",
|
||||
"artemis_tool",
|
||||
"ToolRegistry",
|
||||
|
||||
@@ -12,26 +12,6 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Modular Agent Framework for ARTEMIS."""
|
||||
"""Agent implementations for ARTEMIS."""
|
||||
|
||||
from artemis.agents.base import (
|
||||
AgentConfig,
|
||||
AgentRegistry,
|
||||
AgentResponse,
|
||||
BaseAgent,
|
||||
)
|
||||
from artemis.agents.operator.operator_agent import OperatorAgent
|
||||
from artemis.agents.planner.planner_agent import PlannerAgent
|
||||
from artemis.agents.summarizer.summarizer_agent import SummarizerAgent
|
||||
from artemis.agents.validator.failure_analyzer_agent import FailureAnalyzerAgent
|
||||
|
||||
__all__ = [
|
||||
"BaseAgent",
|
||||
"AgentConfig",
|
||||
"AgentResponse",
|
||||
"AgentRegistry",
|
||||
"PlannerAgent",
|
||||
"OperatorAgent",
|
||||
"SummarizerAgent",
|
||||
"FailureAnalyzerAgent",
|
||||
]
|
||||
__all__: list[str] = []
|
||||
@@ -1,146 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Base Agent Protocol & Lifecycle Framework for ARTEMIS.
|
||||
|
||||
Provides a unified interface and lifecycle management for all specialized
|
||||
multimodal agents (Planner, Operator, FailureAnalyzer, Explorer, Summarizer, etc.).
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from artemis.drivers.factory import get_driver
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
pass
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class AgentConfig(BaseModel):
|
||||
"""Configuration options for an individual agent."""
|
||||
|
||||
model_name: str | None = Field(default=None, description="LLM/VLM model override")
|
||||
temperature: float = Field(default=0.0, description="Sampling temperature")
|
||||
max_tokens: int = Field(default=4096, description="Maximum output tokens")
|
||||
system_prompt_override: str | None = Field(
|
||||
default=None, description="Custom system prompt override"
|
||||
)
|
||||
enabled_tools: list[str] = Field(
|
||||
default_factory=list, description="Tool names enabled for this agent"
|
||||
)
|
||||
|
||||
|
||||
class AgentResponse(BaseModel):
|
||||
"""Standardized structured response from an agent invocation."""
|
||||
|
||||
status: str = Field(
|
||||
default="success", description="Execution status ('success', 'retry', 'fail', 'done')"
|
||||
)
|
||||
thought: str | None = Field(default=None, description="Reasoning and internal monologue")
|
||||
action_taken: str | None = Field(default=None, description="Action or tool called")
|
||||
output: Any = Field(default=None, description="Output payload or state updates")
|
||||
metadata: dict[str, Any] = Field(
|
||||
default_factory=dict, description="Diagnostic and profiling metadata"
|
||||
)
|
||||
|
||||
|
||||
class BaseAgent(ABC):
|
||||
"""Abstract Base Class defining the standard lifecycle for all ARTEMIS agents."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
role: str,
|
||||
description: str,
|
||||
config: AgentConfig | None = None,
|
||||
ctx: Any | None = None,
|
||||
):
|
||||
self.name = name
|
||||
self.role = role
|
||||
self.description = description
|
||||
self.config = config or AgentConfig()
|
||||
self.ctx: Any | None = ctx
|
||||
self._is_initialized = False
|
||||
|
||||
@property
|
||||
def driver(self) -> Any | None:
|
||||
"""Helper to get the active device driver from the context."""
|
||||
if self.ctx:
|
||||
return get_driver(self.ctx)
|
||||
return None
|
||||
|
||||
async def initialize(self, ctx: Any) -> None:
|
||||
"""Lifecycle hook: Called once when the agent is loaded into the workflow."""
|
||||
self.ctx = ctx
|
||||
self._is_initialized = True
|
||||
logger.info(f"Agent '{self.name}' ({self.role}) initialized.")
|
||||
|
||||
@abstractmethod
|
||||
async def process(self, state: Any) -> dict[str, Any]:
|
||||
"""Main execution hook: Receives state, reasons, interacts with tools, and returns state updates."""
|
||||
...
|
||||
|
||||
async def cleanup(self) -> None:
|
||||
"""Lifecycle hook: Called upon session or workflow termination."""
|
||||
self._is_initialized = False
|
||||
logger.debug(f"Agent '{self.name}' cleanup complete.")
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"<{self.__class__.__name__} name={self.name!r} role={self.role!r}>"
|
||||
|
||||
|
||||
class AgentRegistry:
|
||||
"""Central registry and factory for ARTEMIS agents."""
|
||||
|
||||
_registry: dict[str, type[BaseAgent]] = {}
|
||||
|
||||
@classmethod
|
||||
def register(cls, name: str | None = None):
|
||||
"""Decorator to register an agent class with a unique identifier."""
|
||||
|
||||
def decorator(subclass: type[BaseAgent]) -> type[BaseAgent]:
|
||||
reg_name = name or subclass.__name__.lower()
|
||||
cls._registry[reg_name] = subclass
|
||||
return subclass
|
||||
|
||||
return decorator
|
||||
|
||||
@classmethod
|
||||
def get_agent_class(cls, name: str) -> type[BaseAgent] | None:
|
||||
return cls._registry.get(name.lower())
|
||||
|
||||
@classmethod
|
||||
def create_agent(
|
||||
cls,
|
||||
name: str,
|
||||
ctx: Any | None = None,
|
||||
config: AgentConfig | None = None,
|
||||
**kwargs: Any,
|
||||
) -> BaseAgent:
|
||||
"""Instantiates an agent by registered name."""
|
||||
agent_cls = cls.get_agent_class(name)
|
||||
if not agent_cls:
|
||||
raise KeyError(
|
||||
f"Agent '{name}' is not registered. Available: {list(cls._registry.keys())}"
|
||||
)
|
||||
return agent_cls(ctx=ctx, config=config, **kwargs)
|
||||
|
||||
@classmethod
|
||||
def list_agents(cls) -> list[str]:
|
||||
return list(cls._registry.keys())
|
||||
@@ -1,42 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Loop, deadlock, and action repetition detector for Operator Agent."""
|
||||
|
||||
from typing import Any
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class LoopDetector:
|
||||
"""Monitors recent actions to detect and break infinite interaction loops."""
|
||||
|
||||
def __init__(self, repetition_threshold: int = 3):
|
||||
self.repetition_threshold = repetition_threshold
|
||||
self._action_signatures: list[str] = []
|
||||
|
||||
def record_action(self, action_name: str, params: dict[str, Any]) -> None:
|
||||
sig = f"{action_name}:{sorted(params.items())}"
|
||||
self._action_signatures.append(sig)
|
||||
|
||||
def is_loop_detected(self) -> bool:
|
||||
"""Returns True if the identical action has been repeated >= threshold times."""
|
||||
if len(self._action_signatures) < self.repetition_threshold:
|
||||
return False
|
||||
last_sig = self._action_signatures[-1]
|
||||
return all(s == last_sig for s in self._action_signatures[-self.repetition_threshold :])
|
||||
|
||||
def reset(self) -> None:
|
||||
self._action_signatures.clear()
|
||||
@@ -1,72 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Modular Operator Agent implementation adhering to BaseAgent lifecycle."""
|
||||
|
||||
from typing import Any
|
||||
from artemis.agents.base import AgentConfig, AgentRegistry, BaseAgent
|
||||
from artemis.agents.operator.loop_detector import LoopDetector
|
||||
from artemis.agents.operator.prompt_builder import OperatorPromptBuilder
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
@AgentRegistry.register("operator")
|
||||
class OperatorAgent(BaseAgent):
|
||||
"""Specialized agent responsible for real-time screen analysis and UI interaction decisions."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: AgentConfig | None = None,
|
||||
ctx: Any | None = None,
|
||||
**kwargs: Any,
|
||||
):
|
||||
super().__init__(
|
||||
name="operator",
|
||||
role="Mobile UI Interaction Operator",
|
||||
description="Perceives mobile UI and executes touch/key gestures to progress tasks.",
|
||||
config=config,
|
||||
ctx=ctx,
|
||||
)
|
||||
self.loop_detector = LoopDetector(repetition_threshold=3)
|
||||
|
||||
async def process(self, state: Any) -> dict[str, Any]:
|
||||
"""Observes screen state, reasons next action, and dispatches tool call."""
|
||||
if not self.driver:
|
||||
raise RuntimeError("OperatorAgent requires an active device driver in context.")
|
||||
|
||||
# 1. Capture screen data
|
||||
screen_data = await self.driver.get_screen_data()
|
||||
|
||||
# 2. Check for loop deadlock
|
||||
if self.loop_detector.is_loop_detected():
|
||||
logger.warning(
|
||||
"Operator detected repetitive interaction loop! Injecting recovery back-press."
|
||||
)
|
||||
await self.driver.press_key("back")
|
||||
self.loop_detector.reset()
|
||||
return {"action_taken": "press_key:back", "status": "recovering"}
|
||||
|
||||
# 3. Formulate prompt & messages
|
||||
goal = getattr(state, "task_goal", "")
|
||||
human_text = OperatorPromptBuilder.build_human_message(goal=goal)
|
||||
|
||||
return {
|
||||
"thought": "Observed screen and deciding next interaction.",
|
||||
"action_taken": "click",
|
||||
"screen_width": screen_data.width,
|
||||
"screen_height": screen_data.height,
|
||||
"status": "success",
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Planner Agent implementation adhering to BaseAgent lifecycle."""
|
||||
|
||||
from typing import Any
|
||||
from artemis.agents.base import AgentConfig, AgentRegistry, BaseAgent
|
||||
from artemis.agents.planner.planner import PlannerNode
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
@AgentRegistry.register("planner")
|
||||
class PlannerAgent(BaseAgent):
|
||||
"""Specialized agent responsible for high-level goal decomposition and strategic task planning."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: AgentConfig | None = None,
|
||||
ctx: Any | None = None,
|
||||
**kwargs: Any,
|
||||
):
|
||||
super().__init__(
|
||||
name="planner",
|
||||
role="Strategic Task Planner",
|
||||
description="Decomposes user objectives into structured, observable milestone steps.",
|
||||
config=config,
|
||||
ctx=ctx,
|
||||
)
|
||||
self._node = PlannerNode(ctx=ctx, **kwargs) if ctx else None
|
||||
|
||||
async def initialize(self, ctx: Any) -> None:
|
||||
await super().initialize(ctx)
|
||||
self._node = PlannerNode(ctx=ctx)
|
||||
|
||||
async def process(self, state: Any) -> dict[str, Any]:
|
||||
if not self._node and self.ctx:
|
||||
self._node = PlannerNode(ctx=self.ctx)
|
||||
if not self._node:
|
||||
raise RuntimeError("PlannerAgent has not been initialized with a ArtemisContext.")
|
||||
return await self._node(state)
|
||||
@@ -1,85 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from typing import Annotated
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class _CyFunctionDetectorMeta(type):
|
||||
def __instancecheck__(self, instance):
|
||||
name = type(instance).__name__
|
||||
return (
|
||||
name
|
||||
in (
|
||||
"cyfunction",
|
||||
"cython_function_or_method",
|
||||
"builtin_function_or_method",
|
||||
)
|
||||
or "cyfunction" in name.lower()
|
||||
)
|
||||
|
||||
|
||||
class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta):
|
||||
pass
|
||||
|
||||
|
||||
class PlannerSubgoalOutput(BaseModel):
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
description: str
|
||||
|
||||
|
||||
class PlannerOutput(BaseModel):
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
subgoals: list[PlannerSubgoalOutput]
|
||||
|
||||
|
||||
class SubgoalStatus(Enum):
|
||||
NOT_STARTED = "NOT_STARTED"
|
||||
PENDING = "PENDING"
|
||||
SUCCESS = "SUCCESS"
|
||||
FAILURE = "FAILURE"
|
||||
|
||||
|
||||
class Subgoal(BaseModel):
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
id: Annotated[str, "Unique identifier of the subgoal"]
|
||||
description: Annotated[str, "Description of the subgoal"]
|
||||
completion_reason: Annotated[
|
||||
str | None, "Reason why the subgoal was completed (failure or success)"
|
||||
] = None
|
||||
status: SubgoalStatus
|
||||
started_at: Annotated[datetime | None, "When the subgoal started"] = None
|
||||
ended_at: Annotated[datetime | None, "When the subgoal ended"] = None
|
||||
|
||||
def __str__(self):
|
||||
status_emoji = "❓"
|
||||
if self.status == SubgoalStatus.SUCCESS:
|
||||
status_emoji = "✅"
|
||||
elif self.status == SubgoalStatus.FAILURE:
|
||||
status_emoji = "❌"
|
||||
elif self.status == SubgoalStatus.PENDING:
|
||||
status_emoji = "⏳"
|
||||
elif self.status == SubgoalStatus.NOT_STARTED:
|
||||
status_emoji = "(not started yet)"
|
||||
|
||||
output = f"- [ID:{self.id}]: {self.description} : {status_emoji}."
|
||||
if self.completion_reason:
|
||||
output += f" Completion reason: {self.completion_reason}"
|
||||
return output
|
||||
|
||||
def __repr__(self):
|
||||
return str(self)
|
||||
@@ -1,89 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
try:
|
||||
from datetime import UTC, datetime
|
||||
except ImportError:
|
||||
from datetime import datetime
|
||||
|
||||
UTC = UTC
|
||||
import random
|
||||
import string
|
||||
|
||||
from artemis.agents.planner.types import Subgoal, SubgoalStatus
|
||||
|
||||
|
||||
def get_current_subgoal(subgoals: list[Subgoal]) -> Subgoal | None:
|
||||
return next((s for s in subgoals if s.status == SubgoalStatus.PENDING), None)
|
||||
|
||||
|
||||
def get_subgoals_by_ids(subgoals: list[Subgoal], ids: list[str]) -> list[Subgoal]:
|
||||
return [s for s in subgoals if s.id in ids]
|
||||
|
||||
|
||||
def get_next_subgoal(subgoals: list[Subgoal]) -> Subgoal | None:
|
||||
return next((s for s in subgoals if s.status == SubgoalStatus.NOT_STARTED), None)
|
||||
|
||||
|
||||
def nothing_started(subgoals: list[Subgoal]) -> bool:
|
||||
return all(s.status == SubgoalStatus.NOT_STARTED for s in subgoals)
|
||||
|
||||
|
||||
def complete_current_subgoal(subgoals: list[Subgoal]) -> list[Subgoal]:
|
||||
current_subgoal = get_current_subgoal(subgoals)
|
||||
if not current_subgoal:
|
||||
return subgoals
|
||||
current_subgoal.status = SubgoalStatus.SUCCESS
|
||||
current_subgoal.ended_at = datetime.now(UTC)
|
||||
return subgoals
|
||||
|
||||
|
||||
def complete_subgoals_by_ids(subgoals: list[Subgoal], ids: list[str]) -> list[Subgoal]:
|
||||
for subgoal in subgoals:
|
||||
if subgoal.id in ids:
|
||||
subgoal.status = SubgoalStatus.SUCCESS
|
||||
subgoal.ended_at = datetime.now(UTC)
|
||||
return subgoals
|
||||
|
||||
|
||||
def fail_current_subgoal(subgoals: list[Subgoal]) -> list[Subgoal]:
|
||||
current_subgoal = get_current_subgoal(subgoals)
|
||||
if not current_subgoal:
|
||||
return subgoals
|
||||
current_subgoal.status = SubgoalStatus.FAILURE
|
||||
current_subgoal.ended_at = datetime.now(UTC)
|
||||
return subgoals
|
||||
|
||||
|
||||
def all_completed(subgoals: list[Subgoal]) -> bool:
|
||||
return all(s.status == SubgoalStatus.SUCCESS for s in subgoals)
|
||||
|
||||
|
||||
def one_of_them_is_failure(subgoals: list[Subgoal]) -> bool:
|
||||
return any(s.status == SubgoalStatus.FAILURE for s in subgoals)
|
||||
|
||||
|
||||
def start_next_subgoal(subgoals: list[Subgoal]) -> list[Subgoal]:
|
||||
next_subgoal = get_next_subgoal(subgoals)
|
||||
if not next_subgoal:
|
||||
return subgoals
|
||||
next_subgoal.status = SubgoalStatus.PENDING
|
||||
next_subgoal.started_at = datetime.now(UTC)
|
||||
return subgoals
|
||||
|
||||
|
||||
def generate_id(length: int = 6) -> str:
|
||||
"""Generates a small and distinct random string ID."""
|
||||
chars = string.ascii_lowercase + string.digits
|
||||
return "".join(random.choice(chars) for _ in range(length))
|
||||
@@ -1,53 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Summarizer Agent implementation adhering to BaseAgent lifecycle."""
|
||||
|
||||
from typing import Any
|
||||
from artemis.agents.base import AgentConfig, AgentRegistry, BaseAgent
|
||||
from artemis.agents.summarizer.summarizer import SummarizerNode
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
@AgentRegistry.register("summarizer")
|
||||
class SummarizerAgent(BaseAgent):
|
||||
"""Specialized agent responsible for step execution summarization and trajectory logging."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: AgentConfig | None = None,
|
||||
ctx: Any | None = None,
|
||||
**kwargs: Any,
|
||||
):
|
||||
super().__init__(
|
||||
name="summarizer",
|
||||
role="Execution History Summarizer",
|
||||
description="Synthesizes actions and observations into concise milestone summaries.",
|
||||
config=config,
|
||||
ctx=ctx,
|
||||
)
|
||||
self._node = SummarizerNode(ctx=ctx) if ctx else None
|
||||
|
||||
async def initialize(self, ctx: Any) -> None:
|
||||
await super().initialize(ctx)
|
||||
self._node = SummarizerNode(ctx=ctx)
|
||||
|
||||
async def process(self, state: Any) -> dict[str, Any]:
|
||||
if not self._node and self.ctx:
|
||||
self._node = SummarizerNode(ctx=self.ctx)
|
||||
if not self._node:
|
||||
raise RuntimeError("SummarizerAgent has not been initialized with a ArtemisContext.")
|
||||
return await self._node(state)
|
||||
@@ -1,69 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Modular Failure Analyzer Agent adhering to BaseAgent lifecycle."""
|
||||
|
||||
from typing import Any
|
||||
from artemis.agents.base import AgentConfig, AgentRegistry, BaseAgent
|
||||
from artemis.agents.validator.recovery_strategies import RecoveryStrategyRegistry
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
@AgentRegistry.register("failure_analyzer")
|
||||
class FailureAnalyzerAgent(BaseAgent):
|
||||
"""Specialized agent responsible for diagnosing step failures and triggering autonomous recovery."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: AgentConfig | None = None,
|
||||
ctx: Any | None = None,
|
||||
**kwargs: Any,
|
||||
):
|
||||
super().__init__(
|
||||
name="failure_analyzer",
|
||||
role="Autonomous Self-Healing Failure Analyzer",
|
||||
description="Diagnoses execution exceptions, analyzes screen discrepancies, and runs self-healing sequences.",
|
||||
config=config,
|
||||
ctx=ctx,
|
||||
)
|
||||
|
||||
async def process(self, state: Any) -> dict[str, Any]:
|
||||
"""Analyzes failure context and executes appropriate recovery strategy."""
|
||||
if not self.driver:
|
||||
raise RuntimeError("FailureAnalyzerAgent requires an active device driver in context.")
|
||||
|
||||
failure_context = getattr(state, "failure_context", {}) or {}
|
||||
if isinstance(state, dict):
|
||||
failure_context = state.get("failure_context", {})
|
||||
|
||||
logger.info(f"FailureAnalyzer diagnosing failure context: {failure_context}")
|
||||
|
||||
recovery_result = await RecoveryStrategyRegistry.attempt_recovery(
|
||||
driver=self.driver,
|
||||
failure_context=failure_context,
|
||||
)
|
||||
|
||||
if recovery_result and recovery_result.success:
|
||||
return {
|
||||
"status": "recovered",
|
||||
"recovery_action": recovery_result.action_type.value,
|
||||
"reason": recovery_result.reason,
|
||||
}
|
||||
|
||||
return {
|
||||
"status": "unrecoverable",
|
||||
"reason": "No applicable recovery strategy succeeded.",
|
||||
}
|
||||
@@ -1,177 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Modular Recovery Strategies for the Validation and Self-Healing Engine."""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.utils.logger import get_logger
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class RecoveryActionType(str, Enum):
|
||||
RETRY_DYNAMIC_LOCATOR = "retry_dynamic_locator"
|
||||
DISMISS_DIALOG = "dismiss_dialog"
|
||||
NAVIGATE_BACK = "navigate_back"
|
||||
SCROLL_AND_RETRY = "scroll_and_retry"
|
||||
RESTART_APP = "restart_app"
|
||||
FALLBACK_COORDINATES = "fallback_coordinates"
|
||||
|
||||
|
||||
class RecoveryResult(BaseModel):
|
||||
"""Result of executing an autonomous self-healing recovery strategy."""
|
||||
|
||||
success: bool
|
||||
action_type: RecoveryActionType
|
||||
reason: str
|
||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class BaseRecoveryStrategy(ABC):
|
||||
"""Abstract interface for modular self-healing failure recovery tactics."""
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def name(self) -> str:
|
||||
"""Strategy identifier."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
async def can_handle(self, failure_context: dict[str, Any]) -> bool:
|
||||
"""Determines if this strategy applies to the observed failure mode."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver,
|
||||
failure_context: dict[str, Any],
|
||||
) -> RecoveryResult:
|
||||
"""Executes the self-healing corrective actions."""
|
||||
...
|
||||
|
||||
|
||||
class DismissDialogRecoveryStrategy(BaseRecoveryStrategy):
|
||||
"""Strategy to detect and dismiss unexpected modal dialogs, popups, or permission prompts."""
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "dismiss_dialog"
|
||||
|
||||
async def can_handle(self, failure_context: dict[str, Any]) -> bool:
|
||||
category = failure_context.get("category", "")
|
||||
error_msg = failure_context.get("error_message", "").lower()
|
||||
return (
|
||||
category == "ui_obstruction"
|
||||
or "permission" in error_msg
|
||||
or "dialog" in error_msg
|
||||
or "popup" in error_msg
|
||||
)
|
||||
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver,
|
||||
failure_context: dict[str, Any],
|
||||
) -> RecoveryResult:
|
||||
logger.info(
|
||||
"[Recovery] Attempting to dismiss unexpected dialog or overlay via Back button..."
|
||||
)
|
||||
success = await driver.press_key("back")
|
||||
return RecoveryResult(
|
||||
success=success,
|
||||
action_type=RecoveryActionType.DISMISS_DIALOG,
|
||||
reason="Sent BACK keyevent to dismiss modal overlay",
|
||||
)
|
||||
|
||||
|
||||
class NavigateBackRecoveryStrategy(BaseRecoveryStrategy):
|
||||
"""Strategy to navigate back when agent navigates to a wrong screen."""
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "navigate_back"
|
||||
|
||||
async def can_handle(self, failure_context: dict[str, Any]) -> bool:
|
||||
category = failure_context.get("category", "")
|
||||
return category in ("wrong_screen", "dead_end", "navigation_error")
|
||||
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver,
|
||||
failure_context: dict[str, Any],
|
||||
) -> RecoveryResult:
|
||||
logger.info("[Recovery] Wrong screen detected, rolling back navigation...")
|
||||
success = await driver.press_key("back")
|
||||
return RecoveryResult(
|
||||
success=success,
|
||||
action_type=RecoveryActionType.NAVIGATE_BACK,
|
||||
reason="Navigated back from incorrect screen",
|
||||
)
|
||||
|
||||
|
||||
class ScrollAndSearchRecoveryStrategy(BaseRecoveryStrategy):
|
||||
"""Strategy to scroll viewport when target element is off-screen."""
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "scroll_and_search"
|
||||
|
||||
async def can_handle(self, failure_context: dict[str, Any]) -> bool:
|
||||
category = failure_context.get("category", "")
|
||||
return category in ("element_not_found", "off_screen", "target_missing")
|
||||
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver,
|
||||
failure_context: dict[str, Any],
|
||||
) -> RecoveryResult:
|
||||
direction = failure_context.get("scroll_direction", "down")
|
||||
logger.info(f"[Recovery] Target missing, scrolling {direction} to reveal element...")
|
||||
success = await driver.swipe_direction(direction=direction)
|
||||
return RecoveryResult(
|
||||
success=success,
|
||||
action_type=RecoveryActionType.SCROLL_AND_RETRY,
|
||||
reason=f"Scrolled {direction} to discover offscreen element",
|
||||
)
|
||||
|
||||
|
||||
class RecoveryStrategyRegistry:
|
||||
"""Central registry and chain-of-responsibility for self-healing recovery strategies."""
|
||||
|
||||
_strategies: list[BaseRecoveryStrategy] = [
|
||||
DismissDialogRecoveryStrategy(),
|
||||
NavigateBackRecoveryStrategy(),
|
||||
ScrollAndSearchRecoveryStrategy(),
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def register(cls, strategy: BaseRecoveryStrategy) -> None:
|
||||
cls._strategies.append(strategy)
|
||||
|
||||
@classmethod
|
||||
async def attempt_recovery(
|
||||
cls,
|
||||
driver: BaseDeviceDriver,
|
||||
failure_context: dict[str, Any],
|
||||
) -> RecoveryResult | None:
|
||||
"""Finds the first applicable strategy and executes recovery."""
|
||||
for strategy in cls._strategies:
|
||||
if await strategy.can_handle(failure_context):
|
||||
logger.info(f"Applying recovery strategy: {strategy.name}")
|
||||
return await strategy.execute(driver=driver, failure_context=failure_context)
|
||||
return None
|
||||
@@ -1,48 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Safety net and system dialog auto-recovery."""
|
||||
|
||||
from typing import Any
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class SafetyNet:
|
||||
"""Detects and resolves unwanted popups, system ANRs, and permission prompts."""
|
||||
|
||||
SYSTEM_DIALOG_PATTERNS = [
|
||||
"is not responding",
|
||||
"Wait",
|
||||
"Close app",
|
||||
"Allow only while using the app",
|
||||
"While using the app",
|
||||
"Don't allow",
|
||||
]
|
||||
|
||||
@classmethod
|
||||
async def check_and_recover(cls, screen_data: Any, driver: BaseDeviceDriver) -> bool:
|
||||
"""Inspects screen elements for crash/permission dialogs and dismisses them."""
|
||||
for element in screen_data.ui_elements:
|
||||
text = element.get("text", "")
|
||||
if any(pattern.lower() in text.lower() for pattern in cls.SYSTEM_DIALOG_PATTERNS):
|
||||
logger.info(
|
||||
f"SafetyNet detected system dialog '{text}', attempting auto-dismiss..."
|
||||
)
|
||||
center = element.get("center", [500, 500])
|
||||
await driver.tap(center[0], center[1])
|
||||
return True
|
||||
return False
|
||||
@@ -1,925 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
import re
|
||||
import tempfile
|
||||
import time
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from adbutils import AdbClient, AdbDevice
|
||||
from PIL import Image
|
||||
|
||||
from artemis.clients.ui_automator_client import (
|
||||
UIAutomatorClient,
|
||||
_parse_hierarchy_xml_to_elements,
|
||||
)
|
||||
from artemis.config import get_temp_dir
|
||||
from artemis.controllers.device_controller import (
|
||||
MobileDeviceController,
|
||||
ScreenDataResponse,
|
||||
)
|
||||
from artemis.controllers.types import (
|
||||
Bounds,
|
||||
CoordinatesSelectorRequest,
|
||||
TapOutput,
|
||||
)
|
||||
from artemis.data_engine.models import VideoRecordingRecord
|
||||
from artemis.utils.logger import get_logger
|
||||
from artemis.utils.video import (
|
||||
DEFAULT_MAX_DURATION_SECONDS,
|
||||
RecordingSession,
|
||||
VideoRecordingResult,
|
||||
build_scrcpy_record_command,
|
||||
cleanup_video_segments,
|
||||
concatenate_videos,
|
||||
get_active_session,
|
||||
has_active_session,
|
||||
normalize_recording_to_mp4,
|
||||
remove_active_session,
|
||||
set_active_session,
|
||||
trim_video,
|
||||
)
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class AndroidDeviceController(MobileDeviceController):
|
||||
def __init__(
|
||||
self,
|
||||
device_id: str,
|
||||
adb_client: AdbClient,
|
||||
ui_adb_client: UIAutomatorClient,
|
||||
device_width: int,
|
||||
device_height: int,
|
||||
data_engine_start_time: float | None = None,
|
||||
data_engine: Any | None = None,
|
||||
):
|
||||
self.device_id = device_id
|
||||
self.adb_client = adb_client
|
||||
self.ui_adb_client = ui_adb_client
|
||||
self.device_width = device_width
|
||||
self.device_height = device_height
|
||||
self.data_engine_start_time = data_engine_start_time
|
||||
self.data_engine = data_engine
|
||||
self._device: AdbDevice | None = None
|
||||
self._segment_cache: dict[
|
||||
tuple[str, int, float, float], VideoRecordingResult
|
||||
] = {}
|
||||
|
||||
@property
|
||||
def device(self) -> AdbDevice:
|
||||
if self._device is None:
|
||||
self._device = self.adb_client.device(serial=self.device_id)
|
||||
return self._device
|
||||
|
||||
async def tap(
|
||||
self,
|
||||
coords: CoordinatesSelectorRequest,
|
||||
long_press: bool = False,
|
||||
long_press_duration: int = 1000,
|
||||
times: int = 1,
|
||||
delay_ms: int = 100,
|
||||
) -> TapOutput:
|
||||
try:
|
||||
if long_press:
|
||||
cmd = (
|
||||
f"input swipe {coords.x} {coords.y} {coords.x} {coords.y} {long_press_duration}"
|
||||
)
|
||||
else:
|
||||
if times <= 1:
|
||||
cmd = f"input tap {coords.x} {coords.y}"
|
||||
else:
|
||||
# Chain multiple taps in a single shell command to eliminate shell spawning overhead
|
||||
# between rapid consecutive clicks (e.g. 7 taps to enter developer mode)
|
||||
taps = [f"input tap {coords.x} {coords.y}"] * times
|
||||
cmd = f" && sleep {delay_ms / 1000.0:.3f} && ".join(taps)
|
||||
|
||||
logger.info(f"Executing ADB shell: {cmd}")
|
||||
self.device.shell(cmd)
|
||||
return TapOutput(error=None)
|
||||
except Exception as e:
|
||||
return TapOutput(error=f"ADB tap failed: {str(e)}")
|
||||
|
||||
async def swipe(
|
||||
self,
|
||||
start: CoordinatesSelectorRequest,
|
||||
end: CoordinatesSelectorRequest,
|
||||
duration: int = 400,
|
||||
) -> str | None:
|
||||
try:
|
||||
cmd = f"input touchscreen swipe {start.x} {start.y} {end.x} {end.y} {duration}"
|
||||
logger.info(f"Executing ADB command: {cmd}")
|
||||
self.device.shell(cmd)
|
||||
return None
|
||||
except Exception as e:
|
||||
return f"ADB swipe failed: {str(e)}"
|
||||
|
||||
async def get_screen_data(self) -> ScreenDataResponse:
|
||||
"""Get screen data using the UIAutomator2 client"""
|
||||
try:
|
||||
logger.info("Using UIAutomator2 for screen data retrieval")
|
||||
ui_data = self.ui_adb_client.get_screen_data()
|
||||
return ScreenDataResponse(
|
||||
base64=ui_data.base64,
|
||||
elements=ui_data.elements,
|
||||
width=ui_data.width,
|
||||
height=ui_data.height,
|
||||
platform="android",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get screen data: {e}")
|
||||
raise
|
||||
|
||||
async def screenshot(self) -> str:
|
||||
try:
|
||||
b64 = self.ui_adb_client.get_screenshot_base64()
|
||||
if b64 is None:
|
||||
raise RuntimeError("Failed to capture screenshot")
|
||||
return b64
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to take screenshot: {e}")
|
||||
raise
|
||||
|
||||
async def input_text(self, text: str) -> bool:
|
||||
try:
|
||||
norm_text = (
|
||||
text.replace(r"\r\n", "\n")
|
||||
.replace(r"\n", "\n")
|
||||
.replace(r"\r", "\n")
|
||||
)
|
||||
|
||||
# 1. Tier 1: Try clipboard injection + KEYCODE_PASTE (Zero IME interference, preserves multiline, works for all charsets)
|
||||
if self.ui_adb_client:
|
||||
try:
|
||||
set_clip_ok = False
|
||||
if hasattr(self.ui_adb_client, "set_clipboard"):
|
||||
set_clip_ok = self.ui_adb_client.set_clipboard(norm_text)
|
||||
elif hasattr(self.ui_adb_client, "_device") and self.ui_adb_client._device:
|
||||
self.ui_adb_client._device.set_clipboard(norm_text)
|
||||
set_clip_ok = True
|
||||
elif hasattr(self.ui_adb_client, "_ensure_connected"):
|
||||
dev = self.ui_adb_client._ensure_connected()
|
||||
dev.set_clipboard(norm_text)
|
||||
set_clip_ok = True
|
||||
|
||||
if set_clip_ok:
|
||||
self.device.shell("input keyevent 279")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.debug(f"Clipboard paste fallback: {e}")
|
||||
|
||||
return self._input_text_adb_fallback(norm_text)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to input text: {e}")
|
||||
return False
|
||||
|
||||
def _input_text_adb_fallback(self, text: str) -> bool:
|
||||
"""Fallback method using ADB shell input text command."""
|
||||
try:
|
||||
norm_text = (
|
||||
text.replace(r"\r\n", "\n")
|
||||
.replace(r"\n", "\n")
|
||||
.replace(r"\r", "\n")
|
||||
)
|
||||
lines = norm_text.split("\n")
|
||||
for line_idx, line in enumerate(lines):
|
||||
if line_idx > 0:
|
||||
self.device.shell("input keyevent 66")
|
||||
if not line:
|
||||
continue
|
||||
parts = line.split("%s")
|
||||
for i, part in enumerate(parts):
|
||||
to_write = ""
|
||||
for char in part:
|
||||
if char == " ":
|
||||
to_write += "%s"
|
||||
elif char in [
|
||||
"&",
|
||||
"<",
|
||||
">",
|
||||
"|",
|
||||
";",
|
||||
"(",
|
||||
")",
|
||||
"$",
|
||||
"`",
|
||||
"\\",
|
||||
'"',
|
||||
"'",
|
||||
]:
|
||||
to_write += f"\\{char}"
|
||||
else:
|
||||
to_write += char
|
||||
|
||||
if to_write:
|
||||
self.device.shell(f"input text '{to_write}'")
|
||||
|
||||
if i < len(parts) - 1:
|
||||
self.device.shell("input keyevent 62")
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to input text via ADB fallback: {e}")
|
||||
return False
|
||||
|
||||
async def launch_app(self, package_or_bundle_id: str) -> bool:
|
||||
try:
|
||||
# Clear foreground obstructions before launching
|
||||
logger.info(
|
||||
"Clearing foreground obstructions (status bar, system dialogs,"
|
||||
" keyboard) before launching app..."
|
||||
)
|
||||
try:
|
||||
# 1. Collapse status bar / notification shade
|
||||
self.device.shell("cmd statusbar collapse")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to collapse status bar: {e}")
|
||||
|
||||
try:
|
||||
# 2. Close system dialogs
|
||||
self.device.shell("am broadcast -a android.intent.action.CLOSE_SYSTEM_DIALOGS")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to close system dialogs: {e}")
|
||||
|
||||
# 3. Dismiss keyboard if visible
|
||||
if self.is_keyboard_visible():
|
||||
await self.dismiss_keyboard()
|
||||
|
||||
self.device.app_start(package_or_bundle_id)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to launch app {package_or_bundle_id}: {e}")
|
||||
return False
|
||||
|
||||
async def terminate_app(self, package_or_bundle_id: str | None) -> bool:
|
||||
try:
|
||||
if package_or_bundle_id is None:
|
||||
current_app = self._get_current_foreground_package()
|
||||
if current_app:
|
||||
logger.info(f"Stopping currently running app: {current_app}")
|
||||
self.device.app_stop(current_app)
|
||||
else:
|
||||
logger.warning("No foreground app detected")
|
||||
return False
|
||||
else:
|
||||
self.device.app_stop(package_or_bundle_id)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to terminate app {package_or_bundle_id}: {e}")
|
||||
return False
|
||||
|
||||
async def open_url(self, url: str) -> bool:
|
||||
try:
|
||||
cmd = f"am start -a android.intent.action.VIEW -d {url}"
|
||||
logger.info(f"Executing ADB shell: {cmd}")
|
||||
self.device.shell(cmd)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to open URL {url}: {e}")
|
||||
return False
|
||||
|
||||
def is_keyboard_visible(self) -> bool:
|
||||
try:
|
||||
res = self.device.shell("dumpsys input_method")
|
||||
if isinstance(res, bytes):
|
||||
res = res.decode("utf-8")
|
||||
|
||||
# Common patterns for open keyboard in dumpsys
|
||||
patterns = [
|
||||
"mInputShown=true",
|
||||
"mInputViewShowing=true",
|
||||
"mInputViewShowing=VISIBLE",
|
||||
]
|
||||
for p in patterns:
|
||||
if p in res:
|
||||
return True
|
||||
|
||||
# Alternate check using dumpsys window
|
||||
res_win = self.device.shell(
|
||||
"dumpsys window | grep -E 'mInputMethodWindow|mShowingInputMethod'"
|
||||
)
|
||||
if isinstance(res_win, bytes):
|
||||
res_win = res_win.decode("utf-8")
|
||||
if "mInputMethodWindow" in res_win and "visible=true" in res_win.lower():
|
||||
return True
|
||||
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to check keyboard visibility: {e}")
|
||||
return False
|
||||
|
||||
async def dismiss_keyboard(self) -> bool:
|
||||
try:
|
||||
if not self.is_keyboard_visible():
|
||||
return True
|
||||
|
||||
logger.info("Keyboard is visible, attempting to dismiss via Keyevent 111 (Escape)")
|
||||
self.device.shell("input keyevent 111")
|
||||
await asyncio.sleep(0.3)
|
||||
|
||||
if not self.is_keyboard_visible():
|
||||
logger.info("Keyboard successfully dismissed via Keyevent 111")
|
||||
return True
|
||||
|
||||
logger.info("Keyboard still visible, trying Keyevent 4 (BACK) as fallback to dismiss")
|
||||
self.device.shell("input keyevent 4")
|
||||
await asyncio.sleep(0.3)
|
||||
|
||||
if not self.is_keyboard_visible():
|
||||
logger.info("Keyboard successfully dismissed via Keyevent 4")
|
||||
return True
|
||||
|
||||
logger.warning("Failed to dismiss keyboard after Escape and BACK")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"Error in dismiss_keyboard: {e}")
|
||||
return False
|
||||
|
||||
async def press_back(self) -> bool:
|
||||
try:
|
||||
if self.is_keyboard_visible():
|
||||
logger.info(
|
||||
"Intercepted press_back: Keyboard is visible. Attempting to"
|
||||
" dismiss keyboard first."
|
||||
)
|
||||
return await self.dismiss_keyboard()
|
||||
|
||||
logger.info("Executing normal press_back (keyboard not visible)")
|
||||
self.device.shell("input keyevent 4")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to press back: {e}")
|
||||
return False
|
||||
|
||||
async def press_home(self) -> bool:
|
||||
try:
|
||||
self.device.shell("input keyevent 3")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to press home: {e}")
|
||||
return False
|
||||
|
||||
async def press_enter(self) -> bool:
|
||||
try:
|
||||
self.device.shell("input keyevent 66")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to press enter: {e}")
|
||||
return False
|
||||
|
||||
async def press_key(self, keycode: str) -> bool:
|
||||
try:
|
||||
# Intercept BACK keycodes to handle keyboard dismissal first
|
||||
clean_keycode = str(keycode).strip().upper()
|
||||
if clean_keycode in ("4", "KEYCODE_BACK", "BACK"):
|
||||
if self.is_keyboard_visible():
|
||||
logger.info(
|
||||
f"Intercepted press_key({keycode}): Keyboard is"
|
||||
" visible. Dismissing keyboard first."
|
||||
)
|
||||
return await self.dismiss_keyboard()
|
||||
|
||||
self.device.shell(f"input keyevent {keycode}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to press key {keycode}: {e}")
|
||||
return False
|
||||
|
||||
async def get_ui_hierarchy(self) -> list[dict]:
|
||||
|
||||
max_attempts = 4
|
||||
retry_delay = 0.5
|
||||
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
try:
|
||||
hierarchy_xml = self.ui_adb_client.get_hierarchy()
|
||||
elements = _parse_hierarchy_xml_to_elements(hierarchy_xml)
|
||||
|
||||
# Check if hierarchy contains actual UI elements besides the root node
|
||||
if len(elements) > 1:
|
||||
if attempt > 1:
|
||||
logger.info(
|
||||
f"Successfully retrieved non-empty UI hierarchy on attempt {attempt}"
|
||||
)
|
||||
return elements
|
||||
|
||||
logger.warning(
|
||||
"Retrieved empty or root-only UI hierarchy (length"
|
||||
f" {len(elements)}) on attempt {attempt}/{max_attempts}."
|
||||
f" Retrying in {retry_delay}s..."
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Failed to get UI hierarchy on attempt {attempt}/{max_attempts}: {e}"
|
||||
)
|
||||
if attempt == max_attempts:
|
||||
logger.error(f"Failed to get UI hierarchy after {max_attempts} attempts: {e}")
|
||||
return []
|
||||
|
||||
if attempt < max_attempts:
|
||||
await asyncio.sleep(retry_delay)
|
||||
|
||||
return []
|
||||
|
||||
def find_element(
|
||||
self,
|
||||
ui_hierarchy: list[dict],
|
||||
resource_id: str | None = None,
|
||||
text: str | None = None,
|
||||
index: int = 0,
|
||||
) -> tuple[dict | None, Bounds | None, str | None]:
|
||||
if not resource_id and not text:
|
||||
return None, None, "No resource_id or text provided"
|
||||
|
||||
matches = []
|
||||
for element in ui_hierarchy:
|
||||
if resource_id and element.get("resource-id") == resource_id:
|
||||
matches.append(element)
|
||||
elif text and (element.get("text") == text or element.get("accessibilityText") == text):
|
||||
matches.append(element)
|
||||
|
||||
if not matches:
|
||||
criteria = f"resource_id='{resource_id}'" if resource_id else f"text='{text}'"
|
||||
return None, None, f"No element found with {criteria}"
|
||||
|
||||
if index >= len(matches):
|
||||
criteria = f"resource_id='{resource_id}'" if resource_id else f"text='{text}'"
|
||||
return (
|
||||
None,
|
||||
None,
|
||||
(f"Index {index} out of range for {criteria} (found {len(matches)} matches)"),
|
||||
)
|
||||
|
||||
element = matches[index]
|
||||
bounds = self._extract_bounds(element)
|
||||
|
||||
return element, bounds, None
|
||||
|
||||
def _get_current_foreground_package(self) -> str | None:
|
||||
try:
|
||||
app_info = self.device.current_app()
|
||||
if app_info and app_info.package:
|
||||
return app_info.package
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Failed to get current foreground package using adbutils: {e}."
|
||||
" Falling back to dumpsys parsing."
|
||||
)
|
||||
try:
|
||||
result = self.device.shell("dumpsys window | grep mCurrentFocus")
|
||||
|
||||
# Convert to string if bytes
|
||||
if isinstance(result, bytes):
|
||||
result_str = result.decode("utf-8")
|
||||
elif isinstance(result, str):
|
||||
result_str = result
|
||||
else:
|
||||
return None
|
||||
|
||||
if result_str and "=" in result_str:
|
||||
parts = result_str.split("/")
|
||||
if len(parts) > 0:
|
||||
package = parts[0].split()[-1]
|
||||
return package if package else None
|
||||
except Exception as fallback_err:
|
||||
logger.error(f"Fallback dumpsys parsing also failed: {fallback_err}")
|
||||
return None
|
||||
|
||||
def _extract_bounds(self, element: dict) -> Bounds | None:
|
||||
bounds_str = element.get("bounds")
|
||||
if not bounds_str or not isinstance(bounds_str, str):
|
||||
return None
|
||||
|
||||
try:
|
||||
match = re.match(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", bounds_str)
|
||||
if match:
|
||||
return Bounds(
|
||||
x1=int(match.group(1)),
|
||||
y1=int(match.group(2)),
|
||||
x2=int(match.group(3)),
|
||||
y2=int(match.group(4)),
|
||||
)
|
||||
except (ValueError, IndexError):
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
async def erase_text(self, nb_chars: int | None = None) -> bool:
|
||||
# 1. If clearing the whole field, try fast UIAutomator2 clear first
|
||||
if nb_chars is None:
|
||||
logger.info("Attempting fast UIAutomator2 clear_text...")
|
||||
if self.ui_adb_client.clear_text():
|
||||
return True
|
||||
logger.warning("UIAutomator2 clear_text failed, falling back to optimized ADB shell")
|
||||
|
||||
# 2. Optimized ADB fallback: combine multiple KEYCODE_DEL keyevents into a single command
|
||||
try:
|
||||
chars_to_delete = nb_chars if nb_chars is not None else 50
|
||||
if chars_to_delete > 0:
|
||||
# KEYCODE_DEL is 67. Build arguments list "67 67 67 ..."
|
||||
keycode_args = " ".join(["67"] * chars_to_delete)
|
||||
cmd = f"input keyevent {keycode_args}"
|
||||
logger.info(f"Executing combined ADB shell delete: {cmd}")
|
||||
self.device.shell(cmd)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to erase text via ADB fallback: {e}")
|
||||
return False
|
||||
|
||||
async def cleanup(self) -> None:
|
||||
pass
|
||||
|
||||
def get_compressed_b64_screenshot(self, image_base64: str, quality: int = 50) -> str:
|
||||
if image_base64.startswith("data:image"):
|
||||
image_base64 = image_base64.split(",")[1]
|
||||
|
||||
image_data = base64.b64decode(image_base64)
|
||||
image = Image.open(BytesIO(image_data))
|
||||
|
||||
compressed_io = BytesIO()
|
||||
image.save(compressed_io, format="JPEG", quality=quality, optimize=True)
|
||||
|
||||
compressed_base64 = base64.b64encode(compressed_io.getvalue()).decode("utf-8")
|
||||
return compressed_base64
|
||||
|
||||
async def start_video_recording(
|
||||
self,
|
||||
max_duration_seconds: int = DEFAULT_MAX_DURATION_SECONDS,
|
||||
output_dir: Path | None = None,
|
||||
) -> VideoRecordingResult:
|
||||
"""Start screen recording on Android device using scrcpy."""
|
||||
self._segment_cache.clear()
|
||||
if has_active_session(self.device_id):
|
||||
return VideoRecordingResult(
|
||||
success=False,
|
||||
message=(f"Recording already in progress for device {self.device_id}"),
|
||||
)
|
||||
|
||||
try:
|
||||
# Create a temp file for the MKV recording if output_dir not provided
|
||||
if not output_dir:
|
||||
output_dir = Path(
|
||||
tempfile.mkdtemp(prefix="scrcpy_", dir=get_temp_dir("recordings"))
|
||||
)
|
||||
else:
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
local_video_path = output_dir / "recording.mkv"
|
||||
|
||||
video_id = uuid4()
|
||||
session = RecordingSession(
|
||||
video_id=video_id,
|
||||
device_id=self.device_id,
|
||||
start_time=time.time(),
|
||||
data_engine_start_time=self.data_engine_start_time,
|
||||
local_video_path=local_video_path,
|
||||
capture_width=self.device_width,
|
||||
capture_height=self.device_height,
|
||||
)
|
||||
|
||||
# Persist to local database if Data Engine is active
|
||||
if self.data_engine and self.data_engine.storage:
|
||||
try:
|
||||
record = VideoRecordingRecord(
|
||||
video_id=video_id,
|
||||
session_id=self.data_engine.current_session_id,
|
||||
device_id=self.device_id,
|
||||
start_time=session.start_time,
|
||||
local_video_path=str(local_video_path),
|
||||
)
|
||||
self.data_engine.storage.create_video_recording(record)
|
||||
except Exception as db_err:
|
||||
logger.error(f"Failed to persist video recording start to DB: {db_err}")
|
||||
|
||||
# Start scrcpy in background
|
||||
cmd = build_scrcpy_record_command(
|
||||
"scrcpy", self.device_id, local_video_path, lock_capture_orientation=False
|
||||
)
|
||||
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
|
||||
session.process = process
|
||||
set_active_session(self.device_id, session)
|
||||
|
||||
logger.info(
|
||||
f"Started scrcpy recording on {self.device_id}, saving to {local_video_path}"
|
||||
)
|
||||
|
||||
# Wait a bit and check if it's still running.
|
||||
await asyncio.sleep(1.0)
|
||||
if process.returncode is not None:
|
||||
stderr = await process.stderr.read()
|
||||
logger.error(f"scrcpy failed to start: {stderr.decode()}")
|
||||
remove_active_session(self.device_id)
|
||||
return VideoRecordingResult(
|
||||
success=False,
|
||||
message=f"scrcpy failed to start: {stderr.decode()}",
|
||||
)
|
||||
|
||||
return VideoRecordingResult(
|
||||
success=True,
|
||||
message=f"Recording started on {self.device_id}",
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to start scrcpy recording: {e}")
|
||||
remove_active_session(self.device_id)
|
||||
return VideoRecordingResult(
|
||||
success=False,
|
||||
message=f"Failed to start recording: {e}",
|
||||
)
|
||||
|
||||
async def stop_video_recording(self) -> VideoRecordingResult:
|
||||
"""Stop scrcpy recording and return the video file."""
|
||||
self._segment_cache.clear()
|
||||
session = get_active_session(self.device_id)
|
||||
if not session:
|
||||
return VideoRecordingResult(
|
||||
success=False,
|
||||
message=f"No active recording for device {self.device_id}",
|
||||
)
|
||||
|
||||
try:
|
||||
process = session.process
|
||||
if process is not None:
|
||||
process.terminate()
|
||||
await asyncio.wait_for(process.wait(), timeout=5.0)
|
||||
|
||||
output_path = session.local_video_path
|
||||
|
||||
# Fix: Concatenate previous segments if scrcpy crashed and restarted
|
||||
if session.android_video_segments:
|
||||
all_segments = session.android_video_segments + [output_path]
|
||||
concatenated_mkv = output_path.parent / "full_recording.mkv"
|
||||
|
||||
if await concatenate_videos(all_segments, concatenated_mkv):
|
||||
cleanup_video_segments(all_segments)
|
||||
output_path = concatenated_mkv
|
||||
|
||||
# Convert MKV to MP4
|
||||
mp4_path = (
|
||||
output_path.parent / "recording.mp4"
|
||||
if output_path.name == "full_recording.mkv"
|
||||
else output_path.with_suffix(".mp4")
|
||||
)
|
||||
success = await self._convert_mkv_to_mp4(output_path, mp4_path)
|
||||
|
||||
remove_active_session(self.device_id)
|
||||
|
||||
final_video_path = mp4_path if (success and mp4_path.exists()) else output_path
|
||||
|
||||
# Persist update to local database if Data Engine is active
|
||||
if self.data_engine and self.data_engine.storage:
|
||||
try:
|
||||
record = VideoRecordingRecord(
|
||||
video_id=session.video_id,
|
||||
session_id=self.data_engine.current_session_id,
|
||||
device_id=self.device_id,
|
||||
start_time=session.start_time,
|
||||
end_time=time.time(),
|
||||
local_video_path=str(final_video_path),
|
||||
)
|
||||
self.data_engine.storage.update_video_recording(record)
|
||||
except Exception as db_err:
|
||||
logger.error(f"Failed to persist video recording stop to DB: {db_err}")
|
||||
|
||||
if success and mp4_path.exists():
|
||||
try:
|
||||
output_path.unlink()
|
||||
except Exception:
|
||||
pass
|
||||
return VideoRecordingResult(
|
||||
success=True,
|
||||
message=f"Recording stopped, saved to {mp4_path}",
|
||||
video_path=mp4_path,
|
||||
)
|
||||
else:
|
||||
return VideoRecordingResult(
|
||||
success=True,
|
||||
message=(
|
||||
f"Recording stopped, saved as MKV to {output_path} (conversion failed)"
|
||||
),
|
||||
video_path=output_path,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to stop scrcpy recording: {e}")
|
||||
remove_active_session(self.device_id)
|
||||
return VideoRecordingResult(
|
||||
success=False,
|
||||
message=f"Failed to stop recording: {e}",
|
||||
)
|
||||
|
||||
async def extract_segment_metadata(
|
||||
self,
|
||||
start_relative_time: float,
|
||||
end_relative_time: float | None = None,
|
||||
) -> VideoRecordingResult:
|
||||
"""Get a video segment for a specific time range (relative to video start)."""
|
||||
session = get_active_session(self.device_id)
|
||||
if not session:
|
||||
return VideoRecordingResult(
|
||||
success=False,
|
||||
message=f"No active recording for device {self.device_id}",
|
||||
)
|
||||
|
||||
cache_key = None
|
||||
if end_relative_time is not None:
|
||||
cache_key = (
|
||||
str(session.video_id),
|
||||
session.generation,
|
||||
round(start_relative_time, 1),
|
||||
round(end_relative_time, 1),
|
||||
)
|
||||
cached_res = self._segment_cache.get(cache_key)
|
||||
if cached_res and cached_res.success and cached_res.video_path and cached_res.video_path.exists():
|
||||
logger.info(
|
||||
"Reusing generation-scoped trimmed video segment for range"
|
||||
f" {cache_key[2]}s to {cache_key[3]}s"
|
||||
)
|
||||
return cached_res
|
||||
|
||||
try:
|
||||
mkv_path = session.local_video_path
|
||||
if not mkv_path or not mkv_path.exists():
|
||||
return VideoRecordingResult(
|
||||
success=False,
|
||||
message="Recording file not found",
|
||||
)
|
||||
|
||||
# Check if process is still running (best effort for disconnection check)
|
||||
crashed = False
|
||||
restart_message = ""
|
||||
|
||||
if session.process and session.process.returncode is not None:
|
||||
logger.warning(f"scrcpy process for {self.device_id} has terminated unexpectedly")
|
||||
crashed = True
|
||||
|
||||
# Move current file to segments list (save it before overwriting session state)
|
||||
session.android_video_segments.append(mkv_path)
|
||||
|
||||
# Try to restart scrcpy
|
||||
try:
|
||||
session.android_segment_index += 1
|
||||
session.generation = session.android_segment_index
|
||||
output_dir = mkv_path.parent
|
||||
new_video_path = output_dir / f"recording_{session.android_segment_index}.mkv"
|
||||
|
||||
cmd = build_scrcpy_record_command(
|
||||
"scrcpy", self.device_id, new_video_path, lock_capture_orientation=False
|
||||
)
|
||||
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
|
||||
# Wait a bit and check if it's still running.
|
||||
await asyncio.sleep(1.0)
|
||||
if process.returncode is None:
|
||||
session.process = process
|
||||
session.local_video_path = new_video_path
|
||||
restart_message = "Successfully restarted scrcpy."
|
||||
else:
|
||||
stderr = await process.stderr.read()
|
||||
restart_message = f"Failed to restart scrcpy: {stderr.decode()}"
|
||||
except Exception as e:
|
||||
restart_message = f"Failed to restart scrcpy: {e}"
|
||||
|
||||
current_time = time.time()
|
||||
video_duration = current_time - session.start_time
|
||||
|
||||
# Calculate offset
|
||||
offset = 0.0
|
||||
if session.data_engine_start_time is not None:
|
||||
offset = session.start_time - session.data_engine_start_time
|
||||
if offset < 0:
|
||||
offset = 0.0
|
||||
|
||||
video_start_relative_time = start_relative_time - offset
|
||||
video_end_relative_time = (
|
||||
end_relative_time - offset if end_relative_time is not None else None
|
||||
)
|
||||
|
||||
# Safety buffer: avoid reading the very last second being written
|
||||
safe_duration = max(0.0, video_duration - 1.0)
|
||||
|
||||
truncation_warning = None
|
||||
if video_end_relative_time is not None and video_end_relative_time > safe_duration:
|
||||
truncation_warning = (
|
||||
"Video segment truncated. Requested end time"
|
||||
f" {end_relative_time:.1f}s exceeded available safe"
|
||||
f" duration. Truncated to {safe_duration + offset:.1f}s."
|
||||
)
|
||||
|
||||
if video_end_relative_time is None or video_end_relative_time > safe_duration:
|
||||
video_end_relative_time = safe_duration
|
||||
|
||||
if video_start_relative_time < 0:
|
||||
video_start_relative_time = 0.0
|
||||
|
||||
if video_start_relative_time >= video_end_relative_time:
|
||||
return VideoRecordingResult(
|
||||
success=False,
|
||||
message=(
|
||||
"Invalid time range or requested too close to current"
|
||||
f" time. Available duration: {safe_duration:.1f}s"
|
||||
),
|
||||
)
|
||||
|
||||
# Trim directly from MKV to MP4 (using mkv_path which points to the file before we potentially changed it for restart)
|
||||
|
||||
trim_output_dir = tempfile.mkdtemp(
|
||||
prefix="video_trimmed_",
|
||||
dir=get_temp_dir("trimmed_videos"),
|
||||
)
|
||||
trim_output_path = Path(trim_output_dir) / "segment.mp4"
|
||||
|
||||
# Retry mechanism for ffmpeg stability
|
||||
success = False
|
||||
for attempt in range(3):
|
||||
success = await trim_video(
|
||||
mkv_path,
|
||||
video_start_relative_time,
|
||||
video_end_relative_time,
|
||||
trim_output_path,
|
||||
)
|
||||
if success and trim_output_path.exists():
|
||||
break
|
||||
logger.warning(f"ffmpeg trim attempt {attempt + 1} failed, retrying in 0.5s...")
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
if not success or not trim_output_path.exists():
|
||||
return VideoRecordingResult(
|
||||
success=False,
|
||||
message="Failed to trim video segment after retries",
|
||||
)
|
||||
|
||||
actual_start = video_start_relative_time + offset
|
||||
actual_end = video_end_relative_time + offset
|
||||
|
||||
message = f"Video segment retrieved for range {actual_start:.1f}s to {actual_end:.1f}s"
|
||||
if crashed:
|
||||
message += (
|
||||
f". Warning: scrcpy crashed. Data loss may have occurred. {restart_message}"
|
||||
)
|
||||
|
||||
file_size_mb = trim_output_path.stat().st_size / (1024 * 1024)
|
||||
duration = video_end_relative_time - video_start_relative_time
|
||||
|
||||
res = VideoRecordingResult(
|
||||
success=True,
|
||||
message=message,
|
||||
video_path=trim_output_path,
|
||||
file_size_mb=round(file_size_mb, 2),
|
||||
duration_seconds=round(duration, 2),
|
||||
actual_start_relative_time=actual_start,
|
||||
warning=truncation_warning,
|
||||
video_id=session.video_id,
|
||||
generation=session.generation,
|
||||
sealed_until=session.sealed_until,
|
||||
source_revision=(
|
||||
f"{session.video_id}:{session.generation}:{round(actual_end, 3)}"
|
||||
),
|
||||
)
|
||||
if cache_key is not None:
|
||||
self._segment_cache[cache_key] = res
|
||||
return res
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get video segment: {e}")
|
||||
return VideoRecordingResult(
|
||||
success=False,
|
||||
message=f"Failed to get video segment: {e}",
|
||||
)
|
||||
|
||||
async def _convert_mkv_to_mp4(self, mkv_path: Path, mp4_path: Path) -> bool:
|
||||
"""Normalize MKV into a fixed-size, browser-safe MP4."""
|
||||
session = get_active_session(self.device_id)
|
||||
return await normalize_recording_to_mp4(
|
||||
mkv_path,
|
||||
mp4_path,
|
||||
session.capture_width if session else self.device_width,
|
||||
session.capture_height if session else self.device_height,
|
||||
)
|
||||
@@ -12,40 +12,10 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Core framework contracts and lifecycle primitives."""
|
||||
"""Core framework primitives.
|
||||
|
||||
from artemis.core.context import ExecutionContext, DeviceInfoContext
|
||||
from artemis.core.state import ExecutionContextState, ExecutionStatus, StepRecord
|
||||
from artemis.core.events import Event, EventBus, HookType, global_event_bus
|
||||
from artemis.core.exceptions import (
|
||||
ArtemisException,
|
||||
DeviceDriverException,
|
||||
DeviceNotFoundException,
|
||||
AgentExecutionException,
|
||||
ToolExecutionException,
|
||||
LLMProviderException,
|
||||
StateTransitionException,
|
||||
)
|
||||
from artemis.core.registry import AgentRegistry, ToolRegistry, DriverRegistry
|
||||
Live submodules: ``artemis.core.tool_declaration`` and ``artemis.core.diagnostics``
|
||||
(imported directly by module path by their consumers).
|
||||
"""
|
||||
|
||||
__all__ = [
|
||||
"ExecutionContext",
|
||||
"DeviceInfoContext",
|
||||
"ExecutionContextState",
|
||||
"ExecutionStatus",
|
||||
"StepRecord",
|
||||
"Event",
|
||||
"EventBus",
|
||||
"HookType",
|
||||
"global_event_bus",
|
||||
"ArtemisException",
|
||||
"DeviceDriverException",
|
||||
"DeviceNotFoundException",
|
||||
"AgentExecutionException",
|
||||
"ToolExecutionException",
|
||||
"LLMProviderException",
|
||||
"StateTransitionException",
|
||||
"AgentRegistry",
|
||||
"ToolRegistry",
|
||||
"DriverRegistry",
|
||||
]
|
||||
__all__: list[str] = []
|
||||
@@ -1,62 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Clean Execution Context for ARTEMIS."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class DeviceInfoContext(BaseModel):
|
||||
"""Device metadata and configuration."""
|
||||
|
||||
device_id: str = Field(default="default-device", description="ADB serial or device identifier")
|
||||
platform: str = Field(
|
||||
default="android", description="Platform type ('android', 'cloud', 'mock')"
|
||||
)
|
||||
width: int = Field(default=1080, description="Screen width in pixels")
|
||||
height: int = Field(default=2400, description="Screen height in pixels")
|
||||
|
||||
|
||||
class ExecutionContext:
|
||||
"""Central context holding device driver, session identifiers, and telemetry tracer."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
task_goal: str,
|
||||
device_id: str = "default-device",
|
||||
trace_id: str | None = None,
|
||||
traces_path: Path | str | None = None,
|
||||
locked_package: str | None = None,
|
||||
platform: str = "android",
|
||||
):
|
||||
self.task_goal = task_goal
|
||||
self.trace_id = trace_id or str(uuid4())
|
||||
self.traces_path = Path(traces_path) if traces_path else Path("traces") / self.trace_id
|
||||
self.locked_package = locked_package
|
||||
self.device = DeviceInfoContext(device_id=device_id, platform=platform)
|
||||
self.driver: Any | None = None
|
||||
self.telemetry: Any | None = None
|
||||
# Fully-wired ArtemisContext (data_engine, adb clients, execution setup).
|
||||
# Required by the live (non-mock) paths of GraphRunner/ReactiveRunner.
|
||||
self.artemis_context: Any | None = None
|
||||
self.custom_attributes: dict[str, Any] = {}
|
||||
|
||||
def set_attribute(self, key: str, value: Any) -> None:
|
||||
self.custom_attributes[key] = value
|
||||
|
||||
def get_attribute(self, key: str, default: Any = None) -> Any:
|
||||
return self.custom_attributes.get(key, default)
|
||||
@@ -1,77 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Event Bus and Lifecycle Hooks System for ARTEMIS."""
|
||||
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable
|
||||
from enum import Enum
|
||||
import inspect
|
||||
import time
|
||||
from typing import Any
|
||||
from pydantic import BaseModel, Field
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class HookType(str, Enum):
|
||||
BEFORE_TASK = "before_task"
|
||||
AFTER_TASK = "after_task"
|
||||
BEFORE_STEP = "before_step"
|
||||
AFTER_STEP = "after_step"
|
||||
ON_ACTION = "on_action"
|
||||
ON_ERROR = "on_error"
|
||||
ON_SCREEN_CHANGE = "on_screen_change"
|
||||
|
||||
|
||||
class Event(BaseModel):
|
||||
"""Encapsulates an event payload emitted through the event bus."""
|
||||
|
||||
event_type: HookType = Field(..., description="Type of the lifecycle event")
|
||||
timestamp: float = Field(default_factory=time.time, description="Event occurrence timestamp")
|
||||
trace_id: str = Field(..., description="Active session trace identifier")
|
||||
payload: dict[str, Any] = Field(default_factory=dict, description="Event data dictionary")
|
||||
|
||||
|
||||
class EventBus:
|
||||
"""Asynchronous event bus supporting subscription and broadcast of lifecycle events."""
|
||||
|
||||
def __init__(self):
|
||||
self._subscribers: dict[HookType, list[Callable[..., Any]]] = defaultdict(list)
|
||||
|
||||
def subscribe(self, hook: HookType, callback: Callable[..., Any]) -> None:
|
||||
"""Registers a listener callback for a specific lifecycle event."""
|
||||
self._subscribers[hook].append(callback)
|
||||
|
||||
def unsubscribe(self, hook: HookType, callback: Callable[..., Any]) -> None:
|
||||
"""Removes a registered callback."""
|
||||
if callback in self._subscribers[hook]:
|
||||
self._subscribers[hook].remove(callback)
|
||||
|
||||
async def emit(self, event: Event) -> None:
|
||||
"""Asynchronously dispatches an event to all subscribed listeners."""
|
||||
listeners = self._subscribers.get(event.event_type, [])
|
||||
for listener in listeners:
|
||||
try:
|
||||
if inspect.iscoroutinefunction(listener):
|
||||
await listener(event)
|
||||
else:
|
||||
listener(event)
|
||||
except Exception as e:
|
||||
logger.warning(f"Error executing event listener for {event.event_type}: {e}")
|
||||
|
||||
|
||||
# Global singleton event bus instance
|
||||
global_event_bus = EventBus()
|
||||
@@ -1,57 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Standardized exception hierarchy for ARTEMIS."""
|
||||
|
||||
|
||||
class ArtemisException(Exception):
|
||||
"""Base exception for all ARTEMIS framework errors."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class DeviceDriverException(ArtemisException):
|
||||
"""Raised when hardware, emulator, or ADB interactions fail."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class DeviceNotFoundException(DeviceDriverException):
|
||||
"""Raised when the specified target device cannot be located or connected."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class AgentExecutionException(ArtemisException):
|
||||
"""Raised when an autonomous agent fails during reasoning or execution."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class ToolExecutionException(ArtemisException):
|
||||
"""Raised when a tool action fails or receives invalid arguments."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class LLMProviderException(ArtemisException):
|
||||
"""Raised when an upstream LLM/VLM API call fails or times out."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class StateTransitionException(ArtemisException):
|
||||
"""Raised when graph execution encounters an invalid state transition."""
|
||||
|
||||
pass
|
||||
@@ -1,46 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Global Plugin and Component Registry for Agents, Tools, and Drivers."""
|
||||
|
||||
from typing import Any, TypeVar
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
class ComponentRegistry:
|
||||
"""Generic typed component registry supporting runtime dynamic plugin loading."""
|
||||
|
||||
def __init__(self, name: str):
|
||||
self.name = name
|
||||
self._items: dict[str, Any] = {}
|
||||
|
||||
def register(self, key: str, item: Any) -> Any:
|
||||
self._items[key.lower()] = item
|
||||
return item
|
||||
|
||||
def get(self, key: str) -> Any | None:
|
||||
return self._items.get(key.lower())
|
||||
|
||||
def list_all(self) -> dict[str, Any]:
|
||||
return dict(self._items)
|
||||
|
||||
def contains(self, key: str) -> bool:
|
||||
return key.lower() in self._items
|
||||
|
||||
|
||||
# Global Registries
|
||||
AgentRegistry = ComponentRegistry("Agents")
|
||||
ToolRegistry = ComponentRegistry("Tools")
|
||||
DriverRegistry = ComponentRegistry("Drivers")
|
||||
@@ -1,70 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Core Execution State and Lifecycle Data Structures."""
|
||||
|
||||
from enum import Enum
|
||||
import time
|
||||
from typing import Any
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class ExecutionStatus(str, Enum):
|
||||
PENDING = "pending"
|
||||
RUNNING = "running"
|
||||
SUCCESS = "success"
|
||||
FAILED = "failed"
|
||||
PAUSED = "paused"
|
||||
INTERRUPTED = "interrupted"
|
||||
|
||||
|
||||
class StepRecord(BaseModel):
|
||||
"""Encapsulates a single step's observation, thought, and action."""
|
||||
|
||||
step_number: int = Field(..., description="1-indexed step turn counter")
|
||||
thought: str | None = Field(default=None, description="Reasoning text")
|
||||
action_name: str | None = Field(default=None, description="Triggered action name")
|
||||
action_params: dict[str, Any] = Field(default_factory=dict, description="Action arguments")
|
||||
result: str | None = Field(default=None, description="Action execution result output")
|
||||
screenshot_path: str | None = Field(default=None, description="Saved step screenshot file URI")
|
||||
timestamp: float = Field(default_factory=time.time, description="Timestamp of step start")
|
||||
duration_seconds: float = Field(default=0.0, description="Step processing duration in seconds")
|
||||
|
||||
|
||||
class ExecutionContextState(BaseModel):
|
||||
"""Full execution state passed across agents and runners."""
|
||||
|
||||
task_goal: str = Field(..., description="Initial user task objective")
|
||||
trace_id: str = Field(..., description="Unique task trace identifier")
|
||||
status: ExecutionStatus = Field(
|
||||
default=ExecutionStatus.RUNNING, description="Current execution lifecycle status"
|
||||
)
|
||||
current_turn: int = Field(default=0, description="Current step turn count")
|
||||
max_turns: int = Field(default=30, description="Maximum allowed step turns before timeout")
|
||||
steps: list[StepRecord] = Field(
|
||||
default_factory=list, description="Chronological execution step history"
|
||||
)
|
||||
active_plan: list[str] = Field(
|
||||
default_factory=list, description="Decomposed milestone plan items"
|
||||
)
|
||||
current_step_index: int = Field(default=0, description="Current index within active_plan")
|
||||
metadata: dict[str, Any] = Field(
|
||||
default_factory=dict, description="Custom metadata and state parameters"
|
||||
)
|
||||
error_message: str | None = Field(
|
||||
default=None, description="Failure reason if status is FAILED"
|
||||
)
|
||||
final_output: Any | None = Field(
|
||||
default=None, description="Final synthesized structured output from Outputter or runner"
|
||||
)
|
||||
@@ -1,27 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Execution Engine and Pipeline Orchestration layer."""
|
||||
|
||||
from artemis.engine.base_runner import BaseRunner
|
||||
from artemis.engine.graph_runner import GraphRunner
|
||||
from artemis.engine.pipeline import Pipeline
|
||||
from artemis.engine.reactive_runner import ReactiveRunner
|
||||
|
||||
__all__ = [
|
||||
"BaseRunner",
|
||||
"ReactiveRunner",
|
||||
"GraphRunner",
|
||||
"Pipeline",
|
||||
]
|
||||
@@ -1,36 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Base Execution Runner protocol and lifecycle manager."""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from artemis.core.context import ExecutionContext
|
||||
from artemis.core.state import ExecutionContextState
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class BaseRunner(ABC):
|
||||
"""Abstract Base Runner for orchestrating mobile agent workflows."""
|
||||
|
||||
def __init__(self, ctx: ExecutionContext, driver: BaseDeviceDriver):
|
||||
self.ctx = ctx
|
||||
self.driver = driver
|
||||
|
||||
@abstractmethod
|
||||
async def run(self, max_turns: int = 30) -> ExecutionContextState:
|
||||
"""Executes the autonomous automation workflow until completion or limit."""
|
||||
...
|
||||
@@ -1,89 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Pro Graph Runner orchestrating multi-agent state machines with self-healing."""
|
||||
|
||||
from artemis.core.state import ExecutionContextState, ExecutionStatus, StepRecord
|
||||
from artemis.drivers.mock.mock_driver import MockDeviceDriver
|
||||
from artemis.engine.base_runner import BaseRunner
|
||||
from artemis.graph.graph import get_graph
|
||||
from artemis.graph.state import State
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class GraphRunner(BaseRunner):
|
||||
"""Deep reasoning closed-loop runner coordinating Planner, Operator, Validator, and Summarizer."""
|
||||
|
||||
async def run(self, max_turns: int = 50) -> ExecutionContextState:
|
||||
state = ExecutionContextState(
|
||||
task_goal=self.ctx.task_goal,
|
||||
trace_id=self.ctx.trace_id,
|
||||
max_turns=max_turns,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"Starting GraphRunner for deep closed-loop goal: '{self.ctx.task_goal}' on device: '{self.driver.device_id}'"
|
||||
)
|
||||
|
||||
is_mock = isinstance(self.driver, MockDeviceDriver)
|
||||
|
||||
if not is_mock:
|
||||
try:
|
||||
artemis_ctx = self.ctx.artemis_context
|
||||
if artemis_ctx is None:
|
||||
# Fail fast with a clear contract violation instead of
|
||||
# silently building an empty ArtemisContext that drops
|
||||
# data_engine/llm_config and dies later in perception.
|
||||
raise RuntimeError(
|
||||
"GraphRunner requires ExecutionContext.artemis_context"
|
||||
" to be wired (data_engine, adb clients, execution"
|
||||
" setup) before running the live graph path."
|
||||
)
|
||||
graph = await get_graph(artemis_ctx)
|
||||
graph_input = State.initial(self.ctx.task_goal)
|
||||
async for _ in graph.astream(
|
||||
input=graph_input, config={"recursion_limit": max_turns}
|
||||
):
|
||||
pass
|
||||
state.status = ExecutionStatus.SUCCESS
|
||||
return state
|
||||
except Exception as e:
|
||||
logger.error(f"Live GraphRunner execution failed: {e}", exc_info=True)
|
||||
state.status = ExecutionStatus.FAILED
|
||||
state.error_message = str(e)
|
||||
return state
|
||||
|
||||
# Plan-Execute-Validate closed loop on driver
|
||||
state.active_plan = [
|
||||
f"Step 1: Inspect screen for goal '{self.ctx.task_goal}'",
|
||||
"Step 2: Perform required UI interactions",
|
||||
"Step 3: Verify target screen state",
|
||||
]
|
||||
|
||||
screen_data = await self.driver.get_screen_data()
|
||||
state.steps.append(
|
||||
StepRecord(
|
||||
step_number=1,
|
||||
thought="Decomposed goal into structured milestones; inspecting initial UI state.",
|
||||
action_name="get_screen_data",
|
||||
action_params={},
|
||||
result="Screen captured successfully",
|
||||
duration_seconds=0.05,
|
||||
)
|
||||
)
|
||||
state.current_turn = 1
|
||||
state.status = ExecutionStatus.SUCCESS
|
||||
return state
|
||||
@@ -1,95 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
from typing import Any, Literal
|
||||
|
||||
from artemis.config import OutputConfig, load_agent_config
|
||||
from artemis.core.context import ExecutionContext
|
||||
from artemis.core.state import ExecutionContextState
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.drivers.mock.mock_driver import MockDeviceDriver
|
||||
from artemis.engine.base_runner import BaseRunner
|
||||
from artemis.engine.graph_runner import GraphRunner
|
||||
from artemis.engine.reactive_runner import ReactiveRunner
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class Pipeline:
|
||||
"""Orchestrates runner selection and execution flow."""
|
||||
|
||||
@classmethod
|
||||
def create_runner(
|
||||
cls,
|
||||
profile: Literal["flash", "pro"],
|
||||
ctx: ExecutionContext,
|
||||
driver: BaseDeviceDriver,
|
||||
) -> BaseRunner:
|
||||
if profile.lower() == "flash":
|
||||
return ReactiveRunner(ctx=ctx, driver=driver)
|
||||
return GraphRunner(ctx=ctx, driver=driver)
|
||||
|
||||
@classmethod
|
||||
async def execute(
|
||||
cls,
|
||||
goal: str,
|
||||
profile: Literal["flash", "pro"] = "pro",
|
||||
driver: BaseDeviceDriver | None = None,
|
||||
device_id: str = "default-device",
|
||||
output_config: OutputConfig | None = None,
|
||||
enable_outputter: bool | None = None,
|
||||
locked_package: str | None = None,
|
||||
**kwargs: Any,
|
||||
) -> ExecutionContextState:
|
||||
ctx = ExecutionContext(task_goal=goal, device_id=device_id)
|
||||
if not driver:
|
||||
driver = MockDeviceDriver(device_id=device_id)
|
||||
|
||||
max_turns = kwargs.get("max_turns", 5)
|
||||
runner = cls.create_runner(profile=profile, ctx=ctx, driver=driver)
|
||||
state = await runner.run(max_turns=max_turns)
|
||||
|
||||
# Check Outputter configuration
|
||||
try:
|
||||
agent_cfg = load_agent_config()
|
||||
outputter_cfg = getattr(agent_cfg, "outputter", None)
|
||||
is_enabled = (
|
||||
enable_outputter
|
||||
if enable_outputter is not None
|
||||
else (outputter_cfg.enabled if outputter_cfg else True)
|
||||
)
|
||||
force_synth = outputter_cfg.force_synthesis if outputter_cfg else False
|
||||
|
||||
should_synthesize = is_enabled and (
|
||||
(
|
||||
output_config
|
||||
and output_config.needs_structured_format(default_enabled=is_enabled)
|
||||
)
|
||||
or force_synth
|
||||
)
|
||||
|
||||
if should_synthesize:
|
||||
logger.info("Executing post-execution Outputter synthesis in Pipeline...")
|
||||
summary = "\n".join(
|
||||
f"- Step {s.step_number}: {s.action_name}({s.action_params}) -> {s.result or s.thought or 'completed'}"
|
||||
for s in state.steps
|
||||
)
|
||||
state.metadata["structured_output"] = summary
|
||||
state.final_output = summary
|
||||
except Exception as e:
|
||||
logger.warning(f"Pipeline outputter post-processing skipped: {e}")
|
||||
|
||||
return state
|
||||
@@ -1,112 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Flash Reactive Runner executing rapid Observe-Think-Act cycles."""
|
||||
|
||||
import time
|
||||
|
||||
from artemis.agents.flash.runner import FlashRunner
|
||||
from artemis.config import load_agent_config
|
||||
from artemis.core.state import ExecutionContextState, ExecutionStatus, StepRecord
|
||||
from artemis.drivers.mock.mock_driver import MockDeviceDriver
|
||||
from artemis.engine.base_runner import BaseRunner
|
||||
from artemis.graph.state import State
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class ReactiveRunner(BaseRunner):
|
||||
"""Fast, single-loop reactive runner for short-to-medium workflows (< 30 steps)."""
|
||||
|
||||
async def run(self, max_turns: int | None = None) -> ExecutionContextState:
|
||||
if max_turns is None:
|
||||
try:
|
||||
cfg = load_agent_config()
|
||||
max_turns = cfg.flash.max_turns
|
||||
except Exception:
|
||||
max_turns = 30
|
||||
|
||||
state = ExecutionContextState(
|
||||
task_goal=self.ctx.task_goal,
|
||||
trace_id=self.ctx.trace_id,
|
||||
max_turns=max_turns,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"Starting ReactiveRunner for goal: '{self.ctx.task_goal}' on device: '{self.driver.device_id}'"
|
||||
)
|
||||
|
||||
is_mock = isinstance(self.driver, MockDeviceDriver)
|
||||
|
||||
if not is_mock:
|
||||
try:
|
||||
artemis_ctx = self.ctx.artemis_context
|
||||
if artemis_ctx is None:
|
||||
# Fail fast with a clear contract violation instead of
|
||||
# silently building an empty ArtemisContext that drops
|
||||
# data_engine/llm_config and dies later mid-run.
|
||||
raise RuntimeError(
|
||||
"ReactiveRunner requires ExecutionContext.artemis_context"
|
||||
" to be wired (data_engine, adb clients, execution"
|
||||
" setup) before running the live Flash path."
|
||||
)
|
||||
flash_runner = FlashRunner(
|
||||
artemis_ctx, goal=self.ctx.task_goal, max_turns=max_turns
|
||||
)
|
||||
legacy_state = State.initial(self.ctx.task_goal)
|
||||
result = await flash_runner.run(legacy_state)
|
||||
|
||||
state.current_turn = 1
|
||||
if result.get("status") == "completed":
|
||||
state.status = ExecutionStatus.SUCCESS
|
||||
else:
|
||||
state.status = ExecutionStatus.FAILED
|
||||
state.error_message = result.get("explanation") or "Task failed"
|
||||
return state
|
||||
except Exception as e:
|
||||
logger.error(f"Live FlashRunner execution failed: {e}", exc_info=True)
|
||||
state.status = ExecutionStatus.FAILED
|
||||
state.error_message = str(e)
|
||||
return state
|
||||
|
||||
# Driver-based execution loop (for MockDriver and direct driver automation)
|
||||
for turn in range(1, max_turns + 1):
|
||||
state.current_turn = turn
|
||||
step_start = time.time()
|
||||
|
||||
# 1. Observe screen
|
||||
screen_data = await self.driver.get_screen_data()
|
||||
|
||||
# 2. Execute action step
|
||||
action_name = "wait_for_delay"
|
||||
action_params = {"seconds": 0.1}
|
||||
await self.driver.wait_for_delay(0.1)
|
||||
|
||||
# 3. Record step
|
||||
step_record = StepRecord(
|
||||
step_number=turn,
|
||||
thought=f"Turn {turn}: Observing screen state and progressing toward goal: '{self.ctx.task_goal}'.",
|
||||
action_name=action_name,
|
||||
action_params=action_params,
|
||||
result="Executed successfully",
|
||||
duration_seconds=time.time() - step_start,
|
||||
)
|
||||
state.steps.append(step_record)
|
||||
|
||||
if turn >= 1:
|
||||
state.status = ExecutionStatus.SUCCESS
|
||||
break
|
||||
|
||||
return state
|
||||
@@ -1,25 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""LLM Provider Client Factory."""
|
||||
|
||||
from langchain_core.language_models.chat_models import BaseChatModel
|
||||
from artemis.llm.router import ModelEndpoint, default_router
|
||||
|
||||
|
||||
def get_chat_model(role: str = "operator", endpoint: ModelEndpoint | None = None) -> BaseChatModel:
|
||||
"""Instantiates a configured chat model for the requested role or endpoint."""
|
||||
if endpoint:
|
||||
return default_router.instantiate_model(endpoint)
|
||||
return default_router.create_chat_model(role)
|
||||
@@ -15,7 +15,6 @@
|
||||
"""Global pytest fixtures for ARTEMIS test suite."""
|
||||
|
||||
import pytest
|
||||
from artemis.core.context import ExecutionContext
|
||||
from artemis.drivers.mock.mock_driver import MockDeviceDriver
|
||||
|
||||
|
||||
@@ -23,9 +22,3 @@ from artemis.drivers.mock.mock_driver import MockDeviceDriver
|
||||
def mock_driver():
|
||||
"""Provides an isolated MockDeviceDriver instance."""
|
||||
return MockDeviceDriver(device_id="fixture-mock-device", width=1080, height=2400)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def test_context():
|
||||
"""Provides a fresh ExecutionContext."""
|
||||
return ExecutionContext(task_goal="Test sample task objective", device_id="fixture-mock-device")
|
||||
@@ -1,45 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Integration tests for ReactiveRunner and Pipeline."""
|
||||
|
||||
import pytest
|
||||
from artemis.core.context import ExecutionContext
|
||||
from artemis.core.state import ExecutionStatus
|
||||
from artemis.drivers.mock.mock_driver import MockDeviceDriver
|
||||
from artemis.engine.pipeline import Pipeline
|
||||
from artemis.engine.reactive_runner import ReactiveRunner
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reactive_runner_execution():
|
||||
"""Verify ReactiveRunner executes workflow cycles on mock driver."""
|
||||
ctx = ExecutionContext(task_goal="Turn on Bluetooth in Settings", device_id="test-emulator")
|
||||
driver = MockDeviceDriver(device_id="test-emulator")
|
||||
|
||||
runner = ReactiveRunner(ctx=ctx, driver=driver)
|
||||
state = await runner.run(max_turns=5)
|
||||
|
||||
assert state.status == ExecutionStatus.SUCCESS
|
||||
assert state.current_turn >= 1
|
||||
assert len(state.steps) >= 1
|
||||
assert state.steps[0].action_name == "wait_for_delay"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pipeline_dispatch():
|
||||
"""Verify Pipeline executes with default mock driver."""
|
||||
state = await Pipeline.execute(goal="Open YouTube and play video", profile="flash")
|
||||
assert state.status == ExecutionStatus.SUCCESS
|
||||
assert state.task_goal == "Open YouTube and play video"
|
||||
@@ -1,146 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for Base Agent Protocol & Lifecycle Registry."""
|
||||
|
||||
import pytest
|
||||
from typing import Any
|
||||
from artemis.agents.base import AgentConfig, AgentRegistry, BaseAgent
|
||||
from artemis.agents.validator.failure_analyzer_agent import FailureAnalyzerAgent
|
||||
from artemis.agents.validator.recovery_strategies import (
|
||||
RecoveryStrategyRegistry,
|
||||
)
|
||||
from artemis.drivers.mock.mock_driver import MockDeviceDriver
|
||||
|
||||
|
||||
class DummyTestAgent(BaseAgent):
|
||||
"""Simple test agent implementing BaseAgent protocol."""
|
||||
|
||||
def __init__(self, config: AgentConfig | None = None, ctx: Any | None = None, **kwargs: Any):
|
||||
super().__init__(
|
||||
name="dummy_test_agent",
|
||||
role="Test Worker",
|
||||
description="Agent used for testing lifecycle transitions.",
|
||||
config=config,
|
||||
ctx=ctx,
|
||||
)
|
||||
|
||||
async def process(self, state: Any) -> dict[str, Any]:
|
||||
return {"status": "success", "processed_state": True}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_base_agent_lifecycle():
|
||||
"""Verify BaseAgent initialization, execution, and cleanup lifecycle."""
|
||||
agent = DummyTestAgent()
|
||||
assert agent.name == "dummy_test_agent"
|
||||
assert not agent._is_initialized
|
||||
|
||||
# 1. Initialize
|
||||
mock_ctx = object()
|
||||
await agent.initialize(mock_ctx)
|
||||
assert agent._is_initialized
|
||||
assert agent.ctx is mock_ctx
|
||||
|
||||
# 2. Process
|
||||
result = await agent.process({})
|
||||
assert result.get("status") == "success"
|
||||
assert result.get("processed_state") is True
|
||||
|
||||
# 3. Cleanup
|
||||
await agent.cleanup()
|
||||
assert not agent._is_initialized
|
||||
|
||||
|
||||
def test_agent_registry():
|
||||
"""Verify AgentRegistry registers and instantiates agents correctly."""
|
||||
registered = AgentRegistry.list_agents()
|
||||
assert "planner" in registered
|
||||
assert "summarizer" in registered
|
||||
assert "operator" in registered
|
||||
assert "failure_analyzer" in registered
|
||||
|
||||
# Instantiate PlannerAgent via registry
|
||||
planner = AgentRegistry.create_agent("planner", ctx=None)
|
||||
assert planner.name == "planner"
|
||||
assert planner.role == "Strategic Task Planner"
|
||||
|
||||
# Instantiate SummarizerAgent via registry
|
||||
summarizer = AgentRegistry.create_agent("summarizer", ctx=None)
|
||||
assert summarizer.name == "summarizer"
|
||||
assert summarizer.role == "Execution History Summarizer"
|
||||
|
||||
# Instantiate OperatorAgent via registry
|
||||
operator = AgentRegistry.create_agent("operator", ctx=None)
|
||||
assert operator.name == "operator"
|
||||
assert operator.role == "Mobile UI Interaction Operator"
|
||||
|
||||
# Instantiate FailureAnalyzerAgent via registry
|
||||
fa = AgentRegistry.create_agent("failure_analyzer", ctx=None)
|
||||
assert fa.name == "failure_analyzer"
|
||||
assert fa.role == "Autonomous Self-Healing Failure Analyzer"
|
||||
|
||||
|
||||
def test_agent_config():
|
||||
"""Verify AgentConfig parameters can be customized."""
|
||||
config = AgentConfig(
|
||||
model_name="gemini-2.5-pro",
|
||||
temperature=0.7,
|
||||
max_tokens=2048,
|
||||
enabled_tools=["click", "swipe"],
|
||||
)
|
||||
agent = DummyTestAgent(config=config)
|
||||
assert agent.config.model_name == "gemini-2.5-pro"
|
||||
assert agent.config.temperature == 0.7
|
||||
assert agent.config.enabled_tools == ["click", "swipe"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recovery_strategies_and_failure_analyzer():
|
||||
"""Verify self-healing recovery strategies and FailureAnalyzerAgent execution."""
|
||||
driver = MockDeviceDriver(device_id="mock-rec-1")
|
||||
|
||||
# 1. Test dialog dismissal strategy
|
||||
dialog_ctx = {"category": "ui_obstruction", "error_message": "Unexpected permission popup"}
|
||||
dialog_res = await RecoveryStrategyRegistry.attempt_recovery(driver, dialog_ctx)
|
||||
assert dialog_res is not None
|
||||
assert dialog_res.success
|
||||
assert dialog_res.action_type.value == "dismiss_dialog"
|
||||
assert driver.action_history[-1].get("key") == "back"
|
||||
|
||||
# 2. Test navigate back strategy
|
||||
nav_ctx = {"category": "wrong_screen"}
|
||||
nav_res = await RecoveryStrategyRegistry.attempt_recovery(driver, nav_ctx)
|
||||
assert nav_res is not None
|
||||
assert nav_res.success
|
||||
assert nav_res.action_type.value == "navigate_back"
|
||||
|
||||
# 3. Test scroll and search strategy
|
||||
scroll_ctx = {"category": "element_not_found", "scroll_direction": "down"}
|
||||
scroll_res = await RecoveryStrategyRegistry.attempt_recovery(driver, scroll_ctx)
|
||||
assert scroll_res is not None
|
||||
assert scroll_res.success
|
||||
assert scroll_res.action_type.value == "scroll_and_retry"
|
||||
|
||||
# 4. Test FailureAnalyzerAgent process
|
||||
class MockContext:
|
||||
pass
|
||||
|
||||
ctx = MockContext()
|
||||
setattr(ctx, "_active_driver", driver)
|
||||
|
||||
fa_agent = FailureAnalyzerAgent(ctx=ctx)
|
||||
fa_result = await fa_agent.process({"failure_context": {"category": "wrong_screen"}})
|
||||
assert fa_result["status"] == "recovered"
|
||||
assert fa_result["recovery_action"] == "navigate_back"
|
||||
@@ -12,9 +12,8 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for OperatorPromptBuilder and LoopDetector."""
|
||||
"""Unit tests for OperatorPromptBuilder."""
|
||||
|
||||
from artemis.agents.operator.loop_detector import LoopDetector
|
||||
from artemis.agents.operator.prompt_builder import OperatorPromptBuilder
|
||||
|
||||
|
||||
@@ -33,24 +32,3 @@ def test_operator_prompt_builder():
|
||||
assert "Open Settings" in human_msg
|
||||
assert "Tap Network & internet" in human_msg
|
||||
assert "Current Turn: 2" in human_msg
|
||||
|
||||
|
||||
def test_loop_detector():
|
||||
"""Verify loop detector flags repeated action sequences."""
|
||||
detector = LoopDetector(repetition_threshold=3)
|
||||
assert not detector.is_loop_detected()
|
||||
|
||||
# Record distinct actions
|
||||
detector.record_action("click", {"target": [100, 200]})
|
||||
detector.record_action("swipe", {"action": "up"})
|
||||
assert not detector.is_loop_detected()
|
||||
|
||||
# Record 3 identical actions
|
||||
detector.record_action("click", {"target": [500, 500]})
|
||||
detector.record_action("click", {"target": [500, 500]})
|
||||
detector.record_action("click", {"target": [500, 500]})
|
||||
assert detector.is_loop_detected()
|
||||
|
||||
# Reset
|
||||
detector.reset()
|
||||
assert not detector.is_loop_detected()
|
||||
Reference in new issue
Block a user