Files
somew1nd 206a9ce921 feat: initial release of ARTEMIS Android Agent & Automation Platform
ARTEMIS is an autonomous multimodal Android agent and mobile UI automation
platform engineered for physical devices and emulators. It combines state-of-the-art
multimodal Vision-Language Models (VLMs) with hierarchical closed-loop planning,
resilient self-healing, and deep Android subsystem integration.

Key Features & Architectural Highlights:
- Dual-Engine Orchestration:
  * Pro Engine: Multi-agent closed-loop (Planner -> Operator -> Validator/Failure Analyzer -> Summarizer) powered by LangGraph.
  * Flash Engine: Ultra-fast reactive loop for deterministic, low-latency UI execution (<35 steps).
- Multimodal Perception & Grounding:
  * Hybrid visual screenshot grounding, real-time OCR extraction, and XML accessibility hierarchy parsing.
  * Dynamic-First, Coordinate-Fallback locating strategy for maximum resilience against UI drift.
- Autonomous Self-Healing & Diagnostics:
  * Failure analyzer and safety net for popup dismissal, crash recovery, and auto-retry.
  * Diagnostic sub-agents for log analysis, video recording inspection, and object detection.
- Unified CLI & Python SDK:
  * Rich CLI with commands: init, doctor, run, trace, batch, bench, server, and mcp.
  * Fluent Python SDK with Pydantic structured output models and package locking.
- Multi-Provider LLM Architecture:
  * Support for Google Gemini, OpenAI, Anthropic Claude, Vertex AI, OpenRouter, xAI Grok, and local endpoints (Ollama/vLLM).
  * Centralized configuration in config/artemis.jsonc.
- Native MCP & Observability:
  * Model Context Protocol (MCP) server endpoints for AI IDEs.
  * Full trace recording with before/after screenshots, action overlays, and execution replays.

Change-Id: I430b93ee2b76b54cc5a3c6eeccd15d2cf29830a1
2026-08-13 11:46:47 -07:00

138 lines
4.6 KiB
Python

# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
logger = logging.getLogger(__name__)
class ConflictResolutionService:
@staticmethod
def calculate_1d_iou(a, b):
start = max(a[0], b[0])
end = min(a[1], b[1])
intersection = max(0.0, end - start)
union = (a[1] - a[0]) + (b[1] - b[0]) - intersection
return intersection / (union + 1e-6)
@staticmethod
def calculate_1d_iomin(a, b):
start = max(a[0], b[0])
end = min(a[1], b[1])
intersection = max(0.0, end - start)
min_len = min(a[1] - a[0], b[1] - b[0])
if min_len <= 0:
return 0.0
return intersection / min_len
@staticmethod
def sanitize_interval(interval):
start = interval.get("start", 0.0)
end = interval.get("end", "unknown")
if isinstance(start, (int, float)):
start = max(0.0, float(start))
if end != "unknown" and isinstance(end, (int, float)):
end = max(0.0, float(end))
if start > end:
start, end = end, start
interval["start"] = start
interval["end"] = end
if "confidence_score" not in interval or interval["confidence_score"] is None:
interval["confidence_score"] = 0.0
if end != "unknown":
interval["duration"] = float(end) - float(start)
else:
interval["duration"] = 0.0
return interval
@staticmethod
def group_by_semantic_class(proposals):
groups = {}
for p in proposals:
target = p.get("target", "unknown")
if target not in groups:
groups[target] = []
groups[target].append(p)
return groups
@staticmethod
def apply_1d_nms(group_proposals):
if len(group_proposals) <= 1:
return group_proposals
unknown_end_proposals = [p for p in group_proposals if p.get("end") == "unknown"]
valid_proposals = [p for p in group_proposals if p.get("end") != "unknown"]
valid_proposals.sort(
key=lambda x: (
x.get("confidence_score", 0.0),
-x.get("duration", 0.0),
),
reverse=True,
)
n = len(valid_proposals)
suppressed_mask = [False] * n
for i in range(n):
if suppressed_mask[i]:
continue
a = (valid_proposals[i]["start"], valid_proposals[i]["end"])
for j in range(i + 1, n):
if suppressed_mask[j]:
continue
b = (valid_proposals[j]["start"], valid_proposals[j]["end"])
iou = ConflictResolutionService.calculate_1d_iou(a, b)
iomin = ConflictResolutionService.calculate_1d_iomin(a, b)
if iou > 0.5 or iomin > 0.8:
suppressed_mask[j] = True
kept_proposals = [valid_proposals[i] for i in range(n) if not suppressed_mask[i]]
return kept_proposals + unknown_end_proposals
@classmethod
def clean(cls, blackboard_entries):
try:
sanitized = [cls.sanitize_interval(dict(e)) for e in blackboard_entries]
groups = cls.group_by_semantic_class(sanitized)
cleaned_entries = []
for target, proposals in groups.items():
if len(proposals) <= 1:
cleaned_entries.extend(proposals)
else:
cleaned_entries.extend(cls.apply_1d_nms(proposals))
suppressed_count = len(blackboard_entries) - len(cleaned_entries)
if suppressed_count > 0:
logger.info(
"NMS filtering telemetry: Successfully suppressed"
f" {suppressed_count} redundant proposals."
)
# Sort chronologically by start time before returning so downstream systems read it in order
cleaned_entries.sort(key=lambda x: float(x.get("start", 0.0)))
return cleaned_entries
except Exception as e:
logger.error(f"NMS filter crashed: {e}. Falling back to raw ledger state.")
return blackboard_entries