feat: introduce MCP server, cross-platform runtime, and Showcase UI reorganization
- Showcase UI Reorganization: * Move web UI dashboard from root directory into apps/showcase_ui/ for cleaner project structure * Add 'artemis ui' CLI command (artemis/interfaces/cli/commands/ui.py) and launcher scripts for the interactive showcase console - Standalone MCP Server & Notifiers (mcp_server/): * Implement dedicated MCP server package with mobile automation tools (device_state, inspect_trace, task_manager, task_runner) * Add multi-channel notifiers (AgentAPI, Desktop notifications, Webhooks, File output, and Composite routing) * Introduce background task runner and trace store for autonomous mobile execution monitoring - Cross-Platform Runtime & Toolchain Support: * Add artemis/platform/ with OS-specific implementations for Linux, macOS (Darwin), and Windows * Add artemis/runtime/supervisor.py for resilient process and agent runtime supervision * Add artemis/toolchain/ with dependency resolution and SDK descriptors * Add cross-platform launcher scripts (start.sh, start.bat) and installer scripts (scripts/install_deps.*) - Core Enhancements & Diagnostics: * Enhance coordinate normalization, OCR API fallback, and perception pipeline * Add credential validation (credentials_validator.py) and system diagnostics * Update bilingual documentation (README.md, README_CN.md) and architecture diagrams - Test Suite Expansion: * Add integration tests for cloud gateway, remote ADB, and Artemis login * Add unit tests covering core diagnostics, platform detection, toolchain, credentials validator, optional OCR, and MCP tools
This commit is contained in:
1 parent
78e478f502
commit
9a5aaf99b7
249 files changed
+37226
-9339
No files matched your search
+89
-58
@@ -358,6 +358,61 @@ class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta):
|
||||
pass
|
||||
|
||||
|
||||
def check_plan_mutation_rejections(
|
||||
content_before: str, content_after: str, state: State | None = None
|
||||
) -> str | None:
|
||||
"""Checks if a task plan modification violates system safety or loop integrity rules."""
|
||||
lines_after = content_after.splitlines()
|
||||
top_level_after = [line for line in lines_after if line.startswith("- [")]
|
||||
|
||||
# 1. Incomplete nested subgoals check
|
||||
if top_level_after and all(line.startswith("- [x]") for line in top_level_after):
|
||||
has_incomplete = any(re.match(r"^\s*-\s*\[[ /]\]", line) for line in lines_after)
|
||||
if has_incomplete:
|
||||
return (
|
||||
"Your changes to the task plan were not applied. Your changes would cause the"
|
||||
" task to end completely, but there seem to be goals in the task plan"
|
||||
" that have not been marked as completed. If you believe these goals"
|
||||
" are already completed, please update their statuses."
|
||||
)
|
||||
|
||||
# 2. Continuous monitoring / Loop milestone protection check
|
||||
lines_before = content_before.splitlines()
|
||||
top_level_before = [line for line in lines_before if line.startswith("- [")]
|
||||
|
||||
continuous_pattern = re.compile(r"\[Loop\].*?(?:continuous|until\s*manual)", re.IGNORECASE)
|
||||
has_continuous_before = any(continuous_pattern.search(line) for line in top_level_before)
|
||||
|
||||
injected = getattr(state, "injected_instruction", None) if state else None
|
||||
user_stopped = bool(
|
||||
injected
|
||||
and any(
|
||||
w in str(injected).lower()
|
||||
for w in ["stop", "finish", "end", "停止", "结束", "退出", "quit"]
|
||||
)
|
||||
)
|
||||
|
||||
if has_continuous_before and not user_stopped:
|
||||
has_continuous_after = any(continuous_pattern.search(line) for line in top_level_after)
|
||||
if not has_continuous_after:
|
||||
return (
|
||||
"Your changes to the task plan were rejected. You cannot delete an active [Loop] "
|
||||
"continuous monitoring milestone or remove its [Loop] tag. Please keep the [Loop] "
|
||||
"milestone active (in progress [/]) and record each check cycle as an indented "
|
||||
"subtask."
|
||||
)
|
||||
for line in top_level_after:
|
||||
if continuous_pattern.search(line) and line.startswith("- [x]"):
|
||||
return (
|
||||
"Your changes to the task plan were rejected. Ongoing continuous monitoring "
|
||||
"milestones (exit condition: 'until manually stopped') must remain in progress "
|
||||
"([/]) and cannot be unilaterally marked as completed [x]. Please keep it "
|
||||
"active ([/]) and record each polling check as an indented subtask."
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
class NoteArgs(BaseModel):
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
key: str = Field(..., description=SAVE_NOTE_ARG_KEY_DESC)
|
||||
@@ -392,36 +447,15 @@ def wrap_note_tool(ctx: ArtemisContext, original_tool):
|
||||
if task_plan_path.exists():
|
||||
content_after = task_plan_path.read_text(encoding="utf-8")
|
||||
|
||||
lines = content_after.splitlines()
|
||||
top_level_lines = [line for line in lines if line.startswith("- [")]
|
||||
if top_level_lines and all(line.startswith("- [x]") for line in top_level_lines):
|
||||
has_incomplete = any(re.match(r"^\s*-\s*\[[ /]\]", line) for line in lines)
|
||||
if has_incomplete:
|
||||
logger.info(
|
||||
"Rejected: Top-level subgoals are complete but"
|
||||
" incomplete nested subgoals remain."
|
||||
)
|
||||
if task_plan_path.exists():
|
||||
task_plan_path.write_text(content_before, encoding="utf-8")
|
||||
rejection_message = check_plan_mutation_rejections(content_before, content_after, state)
|
||||
if rejection_message:
|
||||
logger.info(f"Rejected task plan update: {rejection_message}")
|
||||
if task_plan_path.exists():
|
||||
task_plan_path.write_text(content_before, encoding="utf-8")
|
||||
|
||||
return Command(
|
||||
update={
|
||||
VALIDATOR_MESSAGES_KEY: [
|
||||
SystemMessage(
|
||||
content=(
|
||||
"Your changes to the task plan were not"
|
||||
" applied. Your changes would cause the"
|
||||
" task to end completely, but there"
|
||||
" seem to be goals in the task plan"
|
||||
" that have not been marked as"
|
||||
" completed. If you believe these goals"
|
||||
" are already completed, please update"
|
||||
" their statuses."
|
||||
)
|
||||
)
|
||||
]
|
||||
}
|
||||
)
|
||||
return Command(
|
||||
update={VALIDATOR_MESSAGES_KEY: [SystemMessage(content=rejection_message)]}
|
||||
)
|
||||
|
||||
threshold = (
|
||||
ctx.execution_setup.planner_validation_threshold
|
||||
@@ -558,36 +592,15 @@ def wrap_update_note_tool(ctx: ArtemisContext, original_tool):
|
||||
if task_plan_path.exists():
|
||||
content_after = task_plan_path.read_text(encoding="utf-8")
|
||||
|
||||
lines = content_after.splitlines()
|
||||
top_level_lines = [line for line in lines if line.startswith("- [")]
|
||||
if top_level_lines and all(line.startswith("- [x]") for line in top_level_lines):
|
||||
has_incomplete = any(re.match(r"^\s*-\s*\[[ /]\]", line) for line in lines)
|
||||
if has_incomplete:
|
||||
logger.info(
|
||||
"Rejected: Top-level subgoals are complete but"
|
||||
" incomplete nested subgoals remain."
|
||||
)
|
||||
if task_plan_path.exists():
|
||||
task_plan_path.write_text(content_before, encoding="utf-8")
|
||||
rejection_message = check_plan_mutation_rejections(content_before, content_after, state)
|
||||
if rejection_message:
|
||||
logger.info(f"Rejected task plan update via update_note: {rejection_message}")
|
||||
if task_plan_path.exists():
|
||||
task_plan_path.write_text(content_before, encoding="utf-8")
|
||||
|
||||
return Command(
|
||||
update={
|
||||
VALIDATOR_MESSAGES_KEY: [
|
||||
SystemMessage(
|
||||
content=(
|
||||
"Your changes to the task plan were not"
|
||||
" applied. Your changes would cause the"
|
||||
" task to end completely, but there"
|
||||
" seem to be goals in the task plan"
|
||||
" that have not been marked as"
|
||||
" completed. If you believe these goals"
|
||||
" are already completed, please update"
|
||||
" their statuses."
|
||||
)
|
||||
)
|
||||
]
|
||||
}
|
||||
)
|
||||
return Command(
|
||||
update={VALIDATOR_MESSAGES_KEY: [SystemMessage(content=rejection_message)]}
|
||||
)
|
||||
|
||||
threshold = (
|
||||
ctx.execution_setup.planner_validation_threshold
|
||||
@@ -710,6 +723,18 @@ async def get_graph(ctx: ArtemisContext) -> CompiledStateGraph:
|
||||
|
||||
lines = task_plan_content.splitlines()
|
||||
|
||||
# Check for continuous monitoring loop that should not terminate prematurely
|
||||
continuous_pattern = re.compile(r"\[Loop\].*?(?:continuous|until\s*manual)", re.IGNORECASE)
|
||||
has_continuous_monitoring = any(continuous_pattern.search(line) for line in lines)
|
||||
injected = getattr(state, "injected_instruction", None) if state else None
|
||||
user_stopped = bool(
|
||||
injected
|
||||
and any(
|
||||
w in str(injected).lower()
|
||||
for w in ["stop", "finish", "end", "停止", "结束", "退出", "quit"]
|
||||
)
|
||||
)
|
||||
|
||||
# Check for all completed
|
||||
top_level_lines = [line for line in lines if line.startswith("- [")]
|
||||
|
||||
@@ -719,6 +744,12 @@ async def get_graph(ctx: ArtemisContext) -> CompiledStateGraph:
|
||||
|
||||
all_done = all(line.startswith("- [x]") for line in top_level_lines)
|
||||
if all_done:
|
||||
if has_continuous_monitoring and not user_stopped:
|
||||
logger.warning(
|
||||
"All subgoals marked [x] but task plan has active continuous monitoring"
|
||||
" without user stop signal. Continuing."
|
||||
)
|
||||
return "continue"
|
||||
logger.info("All subgoals are completed, ending the goal")
|
||||
return "end"
|
||||
|
||||
|
||||
+28
-18
@@ -23,7 +23,7 @@ from artemis.controllers.unified_controller import UnifiedMobileController
|
||||
from artemis.data_engine.trace import trace
|
||||
from artemis.graph.state import State
|
||||
from artemis.utils.logger import get_logger
|
||||
from artemis.utils.ocr_api import perform_ocr
|
||||
from artemis.utils.ocr_api import is_ocr_configured, perform_ocr
|
||||
from artemis.utils.ocr_xml_fusion import (
|
||||
_crop_image_remove_status_bar,
|
||||
_detect_status_bar_height,
|
||||
@@ -124,25 +124,35 @@ async def perception_node(state: State, ctx: ArtemisContext) -> dict:
|
||||
ctx.device.device_height = device_data.height
|
||||
logger.info(f"Updated context device dimensions: {device_data.width}x{device_data.height}")
|
||||
|
||||
# 3. Perform OCR
|
||||
screen_height = device_data.height
|
||||
status_bar_height = _detect_status_bar_height(xml_hierarchy, screen_height)
|
||||
if status_bar_height > 0:
|
||||
logger.info(
|
||||
f"Detected status bar height: {status_bar_height}. Cropping screenshot for OCR."
|
||||
)
|
||||
cropped_b64, _, _ = await asyncio.to_thread(
|
||||
_crop_image_remove_status_bar,
|
||||
latest_screenshot_b64,
|
||||
status_bar_height,
|
||||
)
|
||||
raw_ocr_results = await perform_ocr(cropped_b64)
|
||||
ocr_results = _map_coordinates_back(raw_ocr_results, status_bar_height)
|
||||
# 3. Perform OCR (if configured, else fallback to pure XML layout)
|
||||
ocr_results = []
|
||||
if is_ocr_configured():
|
||||
screen_height = device_data.height
|
||||
status_bar_height = _detect_status_bar_height(xml_hierarchy, screen_height)
|
||||
try:
|
||||
if status_bar_height > 0:
|
||||
logger.info(
|
||||
f"Detected status bar height: {status_bar_height}. Cropping screenshot for OCR."
|
||||
)
|
||||
cropped_b64, _, _ = await asyncio.to_thread(
|
||||
_crop_image_remove_status_bar,
|
||||
latest_screenshot_b64,
|
||||
status_bar_height,
|
||||
)
|
||||
raw_ocr_results = await perform_ocr(cropped_b64)
|
||||
ocr_results = _map_coordinates_back(raw_ocr_results, status_bar_height)
|
||||
else:
|
||||
logger.info("No status bar detected. Performing OCR on full screenshot.")
|
||||
ocr_results = await perform_ocr(latest_screenshot_b64)
|
||||
except Exception as ocr_err:
|
||||
logger.warning(
|
||||
f"Perception OCR execution encountered error: {ocr_err}. Proceeding with XML-only layout."
|
||||
)
|
||||
ocr_results = []
|
||||
else:
|
||||
logger.info("No status bar detected. Performing OCR on full screenshot.")
|
||||
ocr_results = await perform_ocr(latest_screenshot_b64)
|
||||
logger.debug("OCR is not configured. Proceeding with XML-only layout.")
|
||||
|
||||
# 4. Fuse OCR with XML
|
||||
# 4. Fuse OCR with XML (if ocr_results is empty, returns original xml_hierarchy intact)
|
||||
fused_xml = fuse_ocr_with_xml(xml_hierarchy, ocr_results)
|
||||
|
||||
# 5. Save to Data Engine in background thread
|
||||
|
||||
Reference in new issue
Block a user