feat: add plugin runtime and cross-device task orchestration

This commit is contained in:
chenmingxuan committed 2026-10-14 20:21:44 +08:00
1 parent 3ee4a5f7a8
commit 6663233ab8
82 files changed
+4118 -95

No files matched your search

+10
View File
@@ -75,3 +75,13 @@ ARTEMIS_DEFAULT_PROFILE=flash
ARTEMIS_IMPORT_MAX_ROWS=1000 ARTEMIS_IMPORT_MAX_ROWS=1000
ARTEMIS_BATCH_MAX_EXECUTIONS=10000 ARTEMIS_BATCH_MAX_EXECUTIONS=10000
ARTEMIS_IMPORT_MAX_BYTES=10485760 ARTEMIS_IMPORT_MAX_BYTES=10485760
# 插件连接与安装预算(修改后重启服务)
ARTEMIS_PLUGIN_MARKETPLACE_URL=https://git.playones.com/chenmingxuan/Artemis-Plugins/raw/branch/main/catalog.json
ARTEMIS_PLUGIN_START_TIMEOUT_SECONDS=30
ARTEMIS_PLUGIN_TOOL_TIMEOUT_SECONDS=60
ARTEMIS_PLUGIN_SHUTDOWN_TIMEOUT_SECONDS=15
ARTEMIS_PLUGIN_DOWNLOAD_TIMEOUT_SECONDS=300
ARTEMIS_PLUGIN_DOWNLOAD_MAX_BYTES=134217728
ARTEMIS_PLUGIN_EXTRACT_MAX_BYTES=536870912
ARTEMIS_PLUGIN_RESULT_MAX_BYTES=16777216
+2
View File
@@ -78,6 +78,8 @@ uv run artemis mcp --install codex
1.1.1 及更新版本的发布 ZIP 含浅层 Git 工作树;包含独立升级入口的新版源码包可直接用上述脚本升级。1.1.0 及更早 ZIP 不含 Git 信息,需改用 Git 克隆或新版源码包。遇到启动或设备问题,在项目目录运行 `uv run artemis doctor`。 1.1.1 及更新版本的发布 ZIP 含浅层 Git 工作树;包含独立升级入口的新版源码包可直接用上述脚本升级。1.1.0 及更早 ZIP 不含 Git 信息,需改用 Git 克隆或新版源码包。遇到启动或设备问题,在项目目录运行 `uv run artemis doctor`。
2.0.0 开发分支增加了插件页面、电脑/混合执行目标、命名连接配置及共享资源保护。首发电脑插件仍在实测,市场索引暂不收录未完成验证的包。配置和开发约定见 [插件与跨端任务](docs/plugins.md)。
更多说明:[批量任务与报告](docs/automation-workspace.md) · [AI 接入](docs/ai-automation.md) · [开发协作指南](AGENTS.md) 更多说明:[批量任务与报告](docs/automation-workspace.md) · [AI 接入](docs/ai-automation.md) · [开发协作指南](AGENTS.md)
基于上游 ARTEMIS 开发,遵循 [Apache-2.0 许可证](LICENSE)。 基于上游 ARTEMIS 开发,遵循 [Apache-2.0 许可证](LICENSE)。
@@ -183,6 +183,13 @@ class StepRepository:
if not isinstance(payload_obj, dict): if not isinstance(payload_obj, dict):
return None return None
action = payload_obj.get("action")
if payload_obj.get("plugin_id") or (isinstance(action, dict) and action.get("plugin_id")):
# Plugin payloads contain already-redacted evidence and explicit image
# references. The legacy compact-tool formatter would discard their
# provenance and make a desktop image inherit the phone's step source.
return self._clean_value(payload_obj)
if trace_type == "llm_call": if trace_type == "llm_call":
allowed_keys = ( allowed_keys = (
"error", "error",
+1 -1
View File
@@ -34,7 +34,7 @@ class BatchRequest(ReportPreferences):
request_id: str | None = Field(default=None, pattern=r"^[A-Za-z0-9_-]{1,100}$") request_id: str | None = Field(default=None, pattern=r"^[A-Za-z0-9_-]{1,100}$")
name: str = Field(min_length=1, max_length=200) name: str = Field(min_length=1, max_length=200)
cases: list[task_import.TaskCase] = Field(min_length=1, max_length=task_import.MAX_ROWS) cases: list[task_import.TaskCase] = Field(min_length=1, max_length=task_import.MAX_ROWS)
device_serial: str = Field(min_length=1, max_length=200) device_serial: str | None = Field(default=None, min_length=1, max_length=200)
profile: Literal["flash", "pro"] = "flash" profile: Literal["flash", "pro"] = "flash"
stop_on_failure: bool = False stop_on_failure: bool = False
repeat_count: int = Field(default=1, ge=1, strict=True) repeat_count: int = Field(default=1, ge=1, strict=True)
+245
View File
@@ -0,0 +1,245 @@
"""Plugin management for the local console; credentials never leave this API."""
from concurrent.futures import ThreadPoolExecutor
import io
from pathlib import Path
from threading import Lock
from typing import Literal
import uuid
import zipfile
from fastapi import APIRouter, HTTPException
from fastapi.exceptions import RequestValidationError
from fastapi.routing import APIRoute
from fastapi.responses import JSONResponse
from pydantic import BaseModel, Field, ValidationError
from artemis.plugins.installer import download, install_archive, install_local, load_catalog
from artemis.plugins.models import PluginManifest, PluginProfile, PluginSelection
from artemis.plugins.runtime import PluginRuntime
from artemis.plugins.store import PluginStore
class PluginRoute(APIRoute):
def get_route_handler(self):
handler = super().get_route_handler()
async def sanitized(request):
try:
return await handler(request)
except RequestValidationError:
return JSONResponse(
status_code=422,
content={
"detail": "插件配置格式不正确,请检查字段类型和清单;输入内容不会回显。"
},
)
return sanitized
router = APIRouter(prefix="/api/plugins", tags=["plugins"], route_class=PluginRoute)
_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="plugin-installer")
_jobs: dict[str, dict] = {}
_lock = Lock()
def get_store() -> PluginStore:
return PluginStore()
class InstallRequest(BaseModel):
plugin_id: str | None = None
version: str | None = None
local_path: str | None = None
class EnableRequest(BaseModel):
enabled: bool
class DiagnoseRequest(BaseModel):
profile: str = "default"
class CustomRequest(BaseModel):
manifest: PluginManifest
skill_content: str = Field(default="", max_length=262144)
class ResourceConfirmation(BaseModel):
resource: str = Field(min_length=1, max_length=256)
confirmed_stopped: Literal[True]
def public_error(exc: Exception) -> str:
return "插件清单或配置格式不正确" if isinstance(exc, ValidationError) else str(exc)
@router.get("")
def list_plugins():
store = get_store()
return {"plugins": store.list(), "blocked_resources": store.blocked_resources()}
@router.post("/resources/confirm-stopped")
def confirm_resource_stopped(request: ResourceConfirmation):
from artemis.runtime.device_lock import DeviceExecutionLock
if any(
request.resource in owner.resource_ids
for owner in DeviceExecutionLock.get_active_owners().values()
):
raise HTTPException(409, "此资源仍被任务占用,请先停止相关任务并确认插件动作已停止")
get_store().confirm_resource_stopped(request.resource)
return {"ok": True}
@router.get("/catalog")
def marketplace():
try:
return load_catalog().model_dump()
except Exception:
raise HTTPException(502, "暂时无法读取插件市场,请稍后重试;已安装插件不受影响") from None
@router.post("/install", status_code=202)
def install_plugin(request: InstallRequest):
if bool(request.local_path) == bool(request.plugin_id):
raise HTTPException(400, "请选择一个市场插件或本地安装包")
with _lock:
if any(j["status"] in {"queued", "running"} for j in _jobs.values()):
raise HTTPException(409, "已有插件正在安装,请等待安装结束")
for key in list(_jobs)[:-30]:
del _jobs[key]
job_id = str(uuid.uuid4())
_jobs[job_id] = {"id": job_id, "status": "queued", "stage": "等待安装", "progress": 0}
def progress(stage, percent):
with _lock:
_jobs[job_id].update(status="running", stage=stage, progress=percent)
def run():
try:
store = get_store()
if request.local_path:
installed = install_local(store, Path(request.local_path), progress)
else:
progress("获取市场索引", 0)
candidates = [
p
for p in load_catalog().plugins
if p.manifest.id == request.plugin_id
and (request.version is None or p.manifest.version == request.version)
]
from packaging.version import Version
compatible = []
for entry in candidates:
try:
entry.manifest.check_compatible()
compatible.append(entry)
except ValueError:
continue
if not compatible:
raise ValueError("没有已验证且兼容的插件版本")
entry = max(compatible, key=lambda p: Version(p.manifest.version))
installed = install_archive(
store, download(entry.archive_url, progress), entry=entry, progress=progress
)
with _lock:
_jobs[job_id].update(
status="completed",
stage="安装完成,请配置并启用",
progress=100,
plugin_id=installed["manifest"]["id"],
)
except Exception as exc:
with _lock:
# HTTP exceptions may contain signed URLs; expose only our validation messages.
message = (
public_error(exc)
if isinstance(exc, (ValueError, FileNotFoundError))
else "下载或安装失败,请检查网络及安装包"
)
_jobs[job_id].update(status="failed", stage=message)
_executor.submit(run)
return {"job_id": job_id}
@router.get("/install/{job_id}")
def install_progress(job_id: str):
with _lock:
if job_id not in _jobs:
raise HTTPException(404, "安装记录不存在,服务可能已重启;请刷新插件列表")
return dict(_jobs[job_id])
@router.post("/custom")
def register_custom(request: CustomRequest):
buffer = io.BytesIO()
with zipfile.ZipFile(buffer, "w") as archive:
archive.writestr("plugin.json", request.manifest.model_dump_json())
if request.manifest.skill:
archive.writestr(request.manifest.skill, request.skill_content)
try:
return install_archive(get_store(), buffer.getvalue())
except (ValueError, OSError) as exc:
raise HTTPException(400, public_error(exc)) from None
@router.put("/{plugin_id}/profiles")
def save_profile(plugin_id: str, profile: PluginProfile):
try:
get_store().save_profile(plugin_id, profile)
return get_store().get(plugin_id)
except ValueError as exc:
raise HTTPException(400, public_error(exc)) from None
@router.put("/{plugin_id}/enabled")
def enable_plugin(plugin_id: str, request: EnableRequest):
try:
get_store().set_enabled(plugin_id, request.enabled)
return get_store().get(plugin_id)
except ValueError as exc:
raise HTTPException(400, public_error(exc)) from None
@router.post("/{plugin_id}/diagnose")
async def diagnose_plugin(plugin_id: str, request: DiagnoseRequest):
runtime = None
result = {"ok": False, "message": "插件连接失败,请检查启用状态、授权、命令和配置"}
try:
store = get_store()
manifest = PluginManifest.model_validate(store.get(plugin_id)["manifest"])
runtime = PluginRuntime(
[PluginSelection(id=plugin_id, profile=request.profile)],
store=store,
target=manifest.targets[0],
)
loaded = await runtime.load(plugin_id)
result = {
"ok": loaded.status == "success",
"tools": loaded.structured_content or {},
"message": "插件连接正常",
}
except Exception:
result = {"ok": False, "message": "插件连接失败,请检查启用状态、授权、命令和配置"}
finally:
if runtime:
try:
await runtime.aclose()
except RuntimeError:
result = {"ok": False, "message": "插件未确认停止,请检查受保护的资源"}
return result
@router.delete("/{plugin_id}")
def uninstall_plugin(plugin_id: str):
try:
get_store().uninstall(plugin_id)
return {"ok": True}
except (ValueError, OSError) as exc:
raise HTTPException(409, public_error(exc)) from None
+11 -2
View File
@@ -183,7 +183,10 @@ async def run_task(request: RunRequest):
# never be selected. Only a successful, non-empty enumeration may reject: # never be selected. Only a successful, non-empty enumeration may reject:
# an indeterminate one (adb blip, startup) lets the submission queue and # an indeterminate one (adb blip, startup) lets the submission queue and
# fail downstream with a clear error instead. # fail downstream with a clear error instead.
if request.device_serial: execution_target = (
request.runtime_overrides.execution_target if request.runtime_overrides else "android"
)
if request.device_serial and execution_target != "computer":
try: try:
rejection = await device_pool.validate_explicit_serial_async(request.device_serial) rejection = await device_pool.validate_explicit_serial_async(request.device_serial)
except Exception: except Exception:
@@ -207,7 +210,11 @@ async def run_task(request: RunRequest):
target_serial = ( target_serial = (
request.runtime_overrides.device_serial if request.runtime_overrides else None request.runtime_overrides.device_serial if request.runtime_overrides else None
) or request.device_serial ) or request.device_serial
device_probe = await readiness_engine.run_device_submission_probe(target_serial=target_serial) device_probe = (
await readiness_engine.run_device_submission_probe(target_serial=target_serial)
if execution_target != "computer"
else None
)
if device_probe and device_probe.summary in {"Device Locked", "Lock State Unknown"}: if device_probe and device_probe.summary in {"Device Locked", "Lock State Unknown"}:
locked_serial = ( locked_serial = (
device_probe.metadata.get("active_device", {}).get("serial") or target_serial or "" device_probe.metadata.get("active_device", {}).get("serial") or target_serial or ""
@@ -251,6 +258,8 @@ async def run_task(request: RunRequest):
from apps.admin_console.services.automation_service import register_submission from apps.admin_console.services.automation_service import register_submission
if response.get("status") == "error":
raise HTTPException(400, response.get("message", "任务参数无效"))
for item in response.get("tasks", []): for item in response.get("tasks", []):
register_submission(item) register_submission(item)
return response return response
+4
View File
@@ -242,6 +242,10 @@ from apps.admin_console.routers.automation import router as automation_router
app.include_router(automation_router) app.include_router(automation_router)
from apps.admin_console.routers.plugins import router as plugins_router
app.include_router(plugins_router)
# Mount cloud gateway router for Frappe / Cloud integration if present # Mount cloud gateway router for Frappe / Cloud integration if present
try: try:
from cloud_service.gateway import cloud_router from cloud_service.gateway import cloud_router
@@ -270,10 +270,12 @@ async def tick_batches():
job = pending[0] job = pending[0]
options = job["case"].get("task_config") or {} options = job["case"].get("task_config") or {}
serial = options.get("device_serial") or batch.get("device_serial") serial = options.get("device_serial") or batch.get("device_serial")
devices = await device_pool.try_list_devices_async() computer = options.get("execution_target") == "computer"
devices = await device_pool.try_list_devices_async() if not computer else []
rejection = ( rejection = (
None None
if serial and any(d.serial == serial and d.state == "device" for d in devices or []) if computer
or (serial and any(d.serial == serial and d.state == "device" for d in devices or []))
else "无法确认所选设备在线且已授权,请恢复设备连接后继续批次" else "无法确认所选设备在线且已授权,请恢复设备连接后继续批次"
) )
if rejection: if rejection:
@@ -450,6 +450,47 @@ def add_markdown_content(doc, content: str):
) )
def plugin_operations(steps: list[dict]) -> list[dict]:
operations = []
for step in steps:
for trace in step.get("generic_tools", []):
payload = trace.get("payload") or {}
if isinstance(payload, str):
try:
payload = json.loads(payload)
except ValueError:
continue
if not isinstance(payload, dict) or not payload.get("plugin_id"):
continue
result = payload.get("result") or {}
if isinstance(result, str):
try:
result = json.loads(result)
except ValueError:
result = {}
if not isinstance(result, dict):
result = {}
content = "\n".join(
str(block.get("text", ""))
for block in result.get("content", [])
if isinstance(block, dict) and block.get("type") == "text"
)
operations.append(
{
"step": step.get("step_number"),
"plugin": payload["plugin_id"],
"version": payload.get("plugin_version", ""),
"tool": result.get("tool") or trace.get("name"),
"resources": ", ".join(payload.get("resource_ids") or []),
"status": result.get("status") or trace.get("status", "unknown"),
"content": (
content or payload.get("error") or result.get("message") or "未返回文字证据"
)[:6000],
}
)
return operations
def word_report(report: dict, path: Path): def word_report(report: dict, path: Path):
doc = word_styles(Document()) doc = word_styles(Document())
doc.add_paragraph(report["name"], style="Title") doc.add_paragraph(report["name"], style="Title")
@@ -490,6 +531,15 @@ def word_report(report: dict, path: Path):
f"步骤 {step['step_number']} · {time_text(step.get('timestamp'))}\n" f"步骤 {step['step_number']} · {time_text(step.get('timestamp'))}\n"
).bold = True ).bold = True
paragraph.add_run(step.get("summary") or "已记录操作") paragraph.add_run(step.get("summary") or "已记录操作")
operations = plugin_operations(report["steps"])
if operations:
doc.add_heading("扩展插件操作", 2)
for operation in operations:
doc.add_paragraph(
f"步骤 {operation['step']} · {operation['plugin']} v{operation['version']} · {operation['tool']} · {operation['status']}"
)
doc.add_paragraph("资源:" + operation["resources"])
doc.add_paragraph(operation["content"])
evidence = report["steps"][:1] + report["steps"][-1:] evidence = report["steps"][:1] + report["steps"][-1:]
seen: set[str] = set() seen: set[str] = set()
images: list[tuple[Path, int]] = [] images: list[tuple[Path, int]] = []
@@ -630,6 +680,27 @@ def excel_report(report: dict, path: Path):
], ],
(10, 22, 68, 26, 24), (10, 22, 68, 26, 24),
) )
operations = plugin_operations(report["steps"])
if operations:
sheet_rows(
wb.create_sheet("插件操作"),
[
("步骤", "插件", "版本", "工具", "资源", "调用状态", "观察与结果"),
*[
(
op["step"],
op["plugin"],
op["version"],
op["tool"],
op["resources"],
op["status"],
op["content"],
)
for op in operations
],
],
(10, 24, 14, 26, 24, 16, 80),
)
wb.save(path) wb.save(path)
@@ -663,6 +734,30 @@ def session_screenshots(report: dict) -> list[tuple[Path, str]]:
f"{index:04d}_步骤{int(step.get('step_number') or index):04d}_{phase}{source.suffix.lower()}", f"{index:04d}_步骤{int(step.get('step_number') or index):04d}_{phase}{source.suffix.lower()}",
) )
) )
# Plugin observations are attached to their own tool traces rather than
# replacing a phone's step image in a mixed task.
seen: set[tuple[str, str]] = set()
for tool_index, trace in enumerate(step.get("generic_tools", []), 1):
payload = trace.get("payload") or {}
if isinstance(payload, str):
try:
payload = json.loads(payload)
except ValueError:
continue
if not isinstance(payload, dict) or not payload.get("plugin_id"):
continue
plugin_id = re.sub(r"[^a-zA-Z0-9_-]", "_", str(payload["plugin_id"]))[:48]
names = payload.get("image_names") or [payload.get("post_image_name")]
for image_index, name in enumerate(names, 1):
source = image_path(name)
if source and (plugin_id, str(name)) not in seen:
seen.add((plugin_id, str(name)))
screenshots.append(
(
source,
f"{index:04d}_插件_{plugin_id}_{tool_index:03d}_{image_index:02d}{source.suffix.lower()}",
)
)
return screenshots return screenshots
@@ -9,6 +9,7 @@ from openpyxl import Workbook, load_workbook
from openpyxl.styles import Alignment, Font, PatternFill from openpyxl.styles import Alignment, Font, PatternFill
from openpyxl.utils import get_column_letter from openpyxl.utils import get_column_letter
from pydantic import BaseModel, ConfigDict, Field from pydantic import BaseModel, ConfigDict, Field
from artemis.plugins.models import ExecutionTarget, PluginSelection
HEADERS = ( HEADERS = (
"用例编号", "用例编号",
@@ -47,6 +48,8 @@ class TaskOptions(BaseModel):
device_command_timeout_seconds: float | None = Field(default=None, gt=0, le=60) device_command_timeout_seconds: float | None = Field(default=None, gt=0, le=60)
action_failure_attempts: int | None = Field(default=None, ge=1, le=10) action_failure_attempts: int | None = Field(default=None, ge=1, le=10)
worker_cleanup_timeout_seconds: float | None = Field(default=None, gt=0, le=300) worker_cleanup_timeout_seconds: float | None = Field(default=None, gt=0, le=300)
plugins: list[PluginSelection] | None = None
execution_target: ExecutionTarget | None = None
class TaskCase(BaseModel): class TaskCase(BaseModel):
@@ -13,6 +13,7 @@
# limitations under the License. # limitations under the License.
import asyncio import asyncio
import json
from datetime import datetime from datetime import datetime
import logging import logging
import os import os
@@ -275,6 +276,11 @@ class TaskQueueService:
] ]
for item in removed_items: for item in removed_items:
DeviceExecutionLock.cancel_reservation(item.get("queue_ticket")) DeviceExecutionLock.cancel_reservation(item.get("queue_ticket"))
from artemis.plugins.store import PluginStore
# A stop request may already have removed this row. The worker finalizer
# must still release the server-owned version reservation after exit.
PluginStore().release(str(session_id))
state.queue_items = [ state.queue_items = [
t t
for t in state.queue_items for t in state.queue_items
@@ -417,6 +423,9 @@ class TaskQueueService:
busy_devices = state.busy_device_ids | { busy_devices = state.busy_device_ids | {
cls._task_target(i).lock_key for i in in_flight if i.get("device_serial") cls._task_target(i).lock_key for i in in_flight if i.get("device_serial")
} }
busy_resources = {r for item in in_flight for r in item.get("plugin_resources", [])}
for owner in DeviceExecutionLock.get_active_owners().values():
busy_resources.update(owner.resource_ids)
dispatched_any = False dispatched_any = False
loop = asyncio.get_running_loop() loop = asyncio.get_running_loop()
for item in list(state.queue_items): for item in list(state.queue_items):
@@ -429,6 +438,9 @@ class TaskQueueService:
device = item.get("device_serial") device = item.get("device_serial")
target = cls._task_target(item) target = cls._task_target(item)
resources = set(item.get("plugin_resources", []))
if resources & busy_resources:
continue
if limit == 0: if limit == 0:
# A task without a resolved device may bind to any serial, so it # A task without a resolved device may bind to any serial, so it
# only launches on an otherwise idle scheduler; the device lock # only launches on an otherwise idle scheduler; the device lock
@@ -446,6 +458,7 @@ class TaskQueueService:
continue continue
item["status"] = "running" item["status"] = "running"
busy_resources.update(resources)
dispatched_any = True dispatched_any = True
if device is not None: if device is not None:
busy_devices.add(target.lock_key) busy_devices.add(target.lock_key)
@@ -593,6 +606,11 @@ class TaskQueueService:
else str(overrides[option]) else str(overrides[option])
) )
target.endpoint.apply_to_environment(env) target.endpoint.apply_to_environment(env)
if "plugins" in overrides:
env["ARTEMIS_TASK_PLUGINS"] = json.dumps(overrides["plugins"])
else:
env.pop("ARTEMIS_TASK_PLUGINS", None)
env["ARTEMIS_EXECUTION_TARGET"] = overrides.get("execution_target") or "android"
env[DeviceExecutionLock.LOCK_SCOPE_ENV] = target.lock_scope env[DeviceExecutionLock.LOCK_SCOPE_ENV] = target.lock_scope
queue_ticket = task_item.get("queue_ticket") queue_ticket = task_item.get("queue_ticket")
if queue_ticket: if queue_ticket:
@@ -623,7 +641,7 @@ class TaskQueueService:
if app_path: if app_path:
cmd.extend(["--app-path", str(app_path)]) cmd.extend(["--app-path", str(app_path)])
device_serial = task_item.get("device_serial") device_serial = task_item.get("device_serial")
if device_serial: if device_serial and env["ARTEMIS_EXECUTION_TARGET"] != "computer":
cmd.extend(["--device-serial", str(device_serial)]) cmd.extend(["--device-serial", str(device_serial)])
env["ADB_DEVICE_SERIAL"] = str(device_serial) env["ADB_DEVICE_SERIAL"] = str(device_serial)
from apps.admin_console.services.automation_store import store from apps.admin_console.services.automation_store import store
@@ -1237,11 +1255,40 @@ class TaskQueueService:
if duplicate_response is not None: if duplicate_response is not None:
return duplicate_response return duplicate_response
rejection_response = await cls._reject_unavailable_device(device_serial) execution_target = (runtime_overrides or {}).get("execution_target") or "android"
from pydantic import TypeAdapter
from artemis.plugins.models import PluginSelection
from artemis.plugins.store import PluginStore
choices = (runtime_overrides or {}).get("plugins")
try:
pinned = PluginStore().pin(
TypeAdapter(list[PluginSelection]).validate_python(choices)
if choices is not None
else None,
execution_target,
)
except ValueError:
return {
"status": "error",
"message": "插件未就绪、未授权或不支持所选任务目标,请检查插件配置",
}
runtime_overrides = {
**(runtime_overrides or {}),
"execution_target": execution_target,
"plugins": [p.model_dump() for p in pinned],
}
rejection_response = (
await cls._reject_unavailable_device(device_serial)
if execution_target != "computer"
else None
)
if rejection_response is not None: if rejection_response is not None:
return rejection_response return rejection_response
single_session_id = session_id if (session_id and len(goals) == 1) else None single_session_id = session_id if (session_id and len(goals) == 1) else None
if execution_target == "computer":
device_serial = "computer:local"
if not device_serial: if not device_serial:
# Device enumeration may block on ADB. # Device enumeration may block on ADB.
from artemis.runtime import device_pool from artemis.runtime import device_pool
@@ -1270,6 +1317,19 @@ class TaskQueueService:
) )
if runtime_overrides: if runtime_overrides:
task_item["runtime_overrides"] = runtime_overrides task_item["runtime_overrides"] = runtime_overrides
if pinned:
try:
task_item["plugin_resources"] = PluginStore().reserve(
task_item["session_id"], pinned, execution_target
)
except (ValueError, OSError):
DeviceExecutionLock.cancel_reservation(task_item.get("queue_ticket"))
for previous in enqueued_tasks:
cls._remove_task(previous["session_id"])
return {
"status": "error",
"message": "插件配置在入队前发生变化,请检查后重新提交",
}
if display_name: if display_name:
task_item["display_name"] = display_name task_item["display_name"] = display_name
state.queue_items.append(task_item) state.queue_items.append(task_item)
@@ -1380,6 +1440,7 @@ class TaskQueueService:
"""Terminate all active device owners and clear pending queue submissions.""" """Terminate all active device owners and clear pending queue submissions."""
from apps.admin_console.services.automation_store import store from apps.admin_console.services.automation_store import store
from apps.admin_console.services.automation_service import on_terminal, spawn from apps.admin_console.services.automation_service import on_terminal, spawn
from artemis.plugins.store import PluginStore
for batch in store.records("batch"): for batch in store.records("batch"):
if batch["status"] == "running": if batch["status"] == "running":
@@ -1389,6 +1450,7 @@ class TaskQueueService:
for item in state.queue_items: for item in state.queue_items:
if isinstance(item, dict) and item.get("status") != "running": if isinstance(item, dict) and item.get("status") != "running":
DeviceExecutionLock.cancel_reservation(item.get("queue_ticket")) DeviceExecutionLock.cancel_reservation(item.get("queue_ticket"))
PluginStore().release(str(item.get("session_id")))
if store.job(str(item.get("session_id"))): if store.job(str(item.get("session_id"))):
spawn(on_terminal(item, "cancelled")) spawn(on_terminal(item, "cancelled"))
state.clear_queue() state.clear_queue()
@@ -1712,7 +1774,9 @@ class TaskQueueService:
if stopped_item and stopped_item.get("status") != "running": if stopped_item and stopped_item.get("status") != "running":
from apps.admin_console.services.automation_store import store from apps.admin_console.services.automation_store import store
from apps.admin_console.services.automation_service import on_terminal, spawn from apps.admin_console.services.automation_service import on_terminal, spawn
from artemis.plugins.store import PluginStore
PluginStore().release(str(stopped_session_id))
if store.job(str(stopped_session_id)): if store.job(str(stopped_session_id)):
spawn(on_terminal(stopped_item, "cancelled")) spawn(on_terminal(stopped_item, "cancelled"))
if stopped_item and not reservation_cancelled: if stopped_item and not reservation_cancelled:
+1
View File
@@ -22,6 +22,7 @@ import { LegacyWorkspaceComponent } from './pages/legacy-workspace/legacy-worksp
export const routes: Routes = [ export const routes: Routes = [
{ path: '', component: HomeComponent }, { path: '', component: HomeComponent },
{ path: 'workspace', component: WorkspaceComponent }, { path: 'workspace', component: WorkspaceComponent },
{ path: 'plugins', loadComponent: () => import('./pages/plugins/plugins.component').then(m => m.PluginsComponent) },
{ path: 'check', component: LegacyWorkspaceComponent }, { path: 'check', component: LegacyWorkspaceComponent },
{ path: '**', redirectTo: '' } { path: '**', redirectTo: '' }
]; ];
@@ -43,7 +43,7 @@
</section> </section>
} }
@if (tab() !== 'reports') { @if (tab() !== 'reports') {
<section class="device"><label>执行设备<select [(ngModel)]="device"><option value="">请选择设备</option>@for(d of system.connectedDevices(); track d.serial){<option [value]="d.serial">{{d.serial}}</option>}</select></label></section> <app-task-target (changed)="taskTarget = $event" /><section class="device"><label>执行设备<select [(ngModel)]="device"><option value="">请选择设备</option>@for(d of system.connectedDevices(); track d.serial){<option [value]="d.serial">{{d.serial}}</option>}</select></label></section>
} }
@if (tab() === 'batches') { @if (tab() === 'batches') {
<section><h3>导入任务</h3><p>每行一个用例,仅「Artemis提示词」作为任务正文;「任务配置(JSON)」可设置每项模型、模式、超时和录屏。「前置约束提取」仅供查看与搜索。示例在模板的「填写说明」页。</p> <section><h3>导入任务</h3><p>每行一个用例,仅「Artemis提示词」作为任务正文;「任务配置(JSON)」可设置每项模型、模式、超时和录屏。「前置约束提取」仅供查看与搜索。示例在模板的「填写说明」页。</p>
@@ -70,7 +70,7 @@
@if(!validRepetitions()){<p class="error">循环次数须为正整数,执行总数不超过 {{maxExecutions}}。</p>} @if(!validRepetitions()){<p class="error">循环次数须为正整数,执行总数不超过 {{maxExecutions}}。</p>}
<label><input type="checkbox" [(ngModel)]="stopOnFailure"> 失败即停止后续任务(默认继续)</label> <label><input type="checkbox" [(ngModel)]="stopOnFailure"> 失败即停止后续任务(默认继续)</label>
<label><input type="checkbox" [(ngModel)]="word"> Word</label><label><input type="checkbox" [(ngModel)]="excel"> Excel</label> <label><input type="checkbox" [(ngModel)]="word"> Word</label><label><input type="checkbox" [(ngModel)]="excel"> Excel</label>
<button class="primary" [disabled]="busy() || !p.valid || !selectedCount() || !validRepetitions() || !device || !name.trim() || !formats().length" (click)="startBatch()">确认导入并开始执行</button> <button class="primary" [disabled]="busy() || !p.valid || !selectedCount() || !validRepetitions() || !name.trim() || !formats().length" (click)="startBatch()">确认导入并开始执行</button>
} }
</section> </section>
<section><h3>批次记录</h3> <section><h3>批次记录</h3>
@@ -68,7 +68,7 @@ describe('AutomationPanelComponent batch deletion', () => {
await new Promise<void>(resolve => setTimeout(resolve, 0)); await new Promise<void>(resolve => setTimeout(resolve, 0));
const request = http.post.calls.allArgs().find(args => args[0] === '/api/automation/batches'); const request = http.post.calls.allArgs().find(args => args[0] === '/api/automation/batches');
expect(request?.[1]).toEqual(jasmine.objectContaining({ expect(request?.[1]).toEqual(jasmine.objectContaining({
cases: [{ case_id: '1.1', name: '打开设置', goal: '观察设置', task_config: { model: 'mimo-v2.6-pro' } }], cases: [{ case_id: '1.1', name: '打开设置', goal: '观察设置', task_config: { execution_target: 'android', model: 'mimo-v2.6-pro' } }],
repeat_count: 3, device_serial: 'test-device' repeat_count: 3, device_serial: 'test-device'
})); }));
}); });
@@ -9,6 +9,7 @@ import { renderMarkdownToHtml } from '../../utils/markdown-parser.util';
import { TestResultBadgeComponent } from '../test-result-badge/test-result-badge.component'; import { TestResultBadgeComponent } from '../test-result-badge/test-result-badge.component';
import { TestResultCardComponent } from '../test-result-badge/test-result-card.component'; import { TestResultCardComponent } from '../test-result-badge/test-result-card.component';
import { CaseReviewResponse } from '../../core/models/session.model'; import { CaseReviewResponse } from '../../core/models/session.model';
import { TaskTargetComponent, TaskTargetOptions } from '../task-target/task-target.component';
interface ArchiveProgress { interface ArchiveProgress {
status: 'idle' | 'preparing' | 'ready' | 'failed'; status: 'idle' | 'preparing' | 'ready' | 'failed';
@@ -21,10 +22,11 @@ interface ArchiveProgress {
} }
@Component({ @Component({
selector: 'app-automation-panel', standalone: true, imports: [CommonModule, FormsModule, TestResultBadgeComponent, TestResultCardComponent], selector: 'app-automation-panel', standalone: true, imports: [CommonModule, FormsModule, TestResultBadgeComponent, TestResultCardComponent, TaskTargetComponent],
templateUrl: './automation-panel.component.html', styleUrl: './automation-panel.component.scss' templateUrl: './automation-panel.component.html', styleUrl: './automation-panel.component.scss'
}) })
export class AutomationPanelComponent { export class AutomationPanelComponent {
taskTarget: TaskTargetOptions = { execution_target: 'android' };
queueRequested = output<void>(); queueRequested = output<void>();
agent = inject(AgentService); system = inject(SystemService); http = inject(HttpClient); agent = inject(AgentService); system = inject(SystemService); http = inject(HttpClient);
tab = signal('reports'); error = signal(''); notice = signal(''); busy = signal(false); tab = signal('reports'); error = signal(''); notice = signal(''); busy = signal(false);
@@ -138,9 +140,9 @@ export class AutomationPanelComponent {
if (!this.preview()?.valid) throw Error('请先导入并修正所有错误'); if (!this.preview()?.valid) throw Error('请先导入并修正所有错误');
if (!this.selectedCount()) throw Error('请至少勾选一个用例'); if (!this.selectedCount()) throw Error('请至少勾选一个用例');
if (!this.validRepetitions()) throw Error(`循环次数必须为正整数,执行总数不能超过 ${this.maxExecutions}`); if (!this.validRepetitions()) throw Error(`循环次数必须为正整数,执行总数不能超过 ${this.maxExecutions}`);
if (!this.device) throw Error('请选择执行设备'); const cases = this.preview().tasks.filter((t: any) => this.selectedRows().has(t.row)).map(({row, ...c}: any) => ({...c, task_config: {...this.taskTarget, ...Object.fromEntries(Object.entries(c.task_config || {}).filter(([,v]) => v !== null && v !== undefined))}}));
const cases = this.preview().tasks.filter((t: any) => this.selectedRows().has(t.row)).map(({row, ...c}: any) => c); if (!this.device && cases.some((c: any) => c.task_config.execution_target !== 'computer' && !c.task_config.device_serial)) throw Error('请选择执行设备');
const created: any = await firstValueFrom(this.http.post('/api/automation/batches',{name:this.name.trim(),cases,repeat_count:this.repeatCount,device_serial:this.device,profile:this.profile,stop_on_failure:this.stopOnFailure,timeout_seconds:this.timeout,formats:this.formats()})); const created: any = await firstValueFrom(this.http.post('/api/automation/batches',{name:this.name.trim(),cases,repeat_count:this.repeatCount,device_serial:this.device || null,profile:this.profile,stop_on_failure:this.stopOnFailure,timeout_seconds:this.timeout,formats:this.formats()}));
this.batch.set(created); this.batch.set(created);
await firstValueFrom(this.http.post(`/api/automation/batches/${created.id}/resume`,{})); await firstValueFrom(this.http.post(`/api/automation/batches/${created.id}/resume`,{}));
this.preview.set(null); this.notice.set('批次已开始,可暂停或查看每项结果'); await this.refresh(); this.preview.set(null); this.notice.set('批次已开始,可暂停或查看每项结果'); await this.refresh();
@@ -44,6 +44,9 @@ import { version } from '../../../../package.json';
<span class="material-symbols-outlined tab-icon">space_dashboard</span> <span class="material-symbols-outlined tab-icon">space_dashboard</span>
<span class="tab-label">工作区</span> <span class="tab-label">工作区</span>
</a> </a>
<a routerLink="/plugins" routerLinkActive="active" class="nav-tab-btn" title="管理扩展插件">
<span class="material-symbols-outlined tab-icon">extension</span><span class="tab-label">插件</span>
</a>
<span class="app-version" [title]="'ARTEMIS 版本 ' + version" [attr.aria-label]="'ARTEMIS 版本 ' + version">v{{version}}</span> <span class="app-version" [title]="'ARTEMIS 版本 ' + version" [attr.aria-label]="'ARTEMIS 版本 ' + version">v{{version}}</span>
</nav> </nav>
`, `,
@@ -0,0 +1,24 @@
import { TestBed } from '@angular/core/testing';
import { provideHttpClient } from '@angular/common/http';
import { provideHttpClientTesting, HttpTestingController } from '@angular/common/http/testing';
import { TaskTargetComponent } from './task-target.component';
describe('TaskTargetComponent', () => {
it('defaults to Android without fetching plugins, and distinguishes auto selection from no plugins', () => {
TestBed.configureTestingModule({ imports: [TaskTargetComponent], providers: [provideHttpClient(), provideHttpClientTesting()] });
const fixture = TestBed.createComponent(TaskTargetComponent);
const http = TestBed.inject(HttpTestingController);
const component = fixture.componentInstance;
fixture.detectChanges();
http.expectNone('/api/plugins');
const emitted = jasmine.createSpy('changed');
component.changed.subscribe(emitted);
component.target = 'computer'; component.targetChanged();
expect(emitted).toHaveBeenCalledWith({ execution_target: 'computer' });
component.automatic = false; component.emit();
expect(emitted).toHaveBeenCalledWith({ execution_target: 'computer', plugins: [] });
component.toggle(); http.expectOne('/api/plugins').flush({ plugins: [] });
expect(component.loading()).toBeFalse();
http.verify();
});
});
@@ -0,0 +1,61 @@
import { Component, inject, output, signal } from '@angular/core';
import { FormsModule } from '@angular/forms';
import { InstalledPlugin, PluginService } from '../../services/plugin.service';
export interface TaskTargetOptions {
execution_target: 'android' | 'computer' | 'mixed';
plugins?: { id: string; profile: string; version?: string }[];
}
@Component({
selector: 'app-task-target', standalone: true, imports: [FormsModule],
template: `
<div class="target-row">
<label>执行目标 <select aria-label="执行目标" [(ngModel)]="target" (ngModelChange)="targetChanged()">
<option value="android">Android 手机</option><option value="computer">电脑</option><option value="mixed">手机 + 扩展插件</option>
</select></label>
<button type="button" (click)="toggle()" [attr.aria-expanded]="open()">插件配置 {{ open() ? '收起' : '展开' }}</button>
</div>
@if (open()) {
<div class="plugin-choices">
<label><input type="checkbox" [(ngModel)]="automatic" (ngModelChange)="emit()"> 自动选择已启用且支持此目标的插件</label>
@if (loading()) { <p role="status">正在读取插件…</p> }
@if (error()) { <p role="alert">{{ error() }}</p> }
@if (!automatic) {
@for (plugin of available(); track plugin.manifest.id) {
<label class="choice"><input type="checkbox" [checked]="selected[plugin.manifest.id] !== undefined" (change)="choose(plugin, $event)">
{{ plugin.manifest.name }}
<select [disabled]="selected[plugin.manifest.id] === undefined" [ngModel]="selected[plugin.manifest.id] || 'default'" (ngModelChange)="selected[plugin.manifest.id] = $event; emit()" [attr.aria-label]="plugin.manifest.name + '连接配置'">
@for (profile of plugin.profiles; track profile.name) { @if (profile.authorized) { <option [value]="profile.name">{{ profile.label }}</option> } }
</select>
</label>
} @empty { <p>没有适用的已启用插件,请先在插件页面安装并授权。</p> }
}
<p class="hint">手机任务默认不占电脑资源。电脑任务无需连接 Android;混合任务会同时占用所需资源。</p>
</div>
}
`,
styles: [`:host{display:block;width:100%;font-size:12px;color:var(--text-secondary,#a0a5b4)}.target-row{display:flex;align-items:center;gap:10px;flex-wrap:wrap}.target-row label,.choice{display:flex;align-items:center;gap:8px}select,button{font:inherit;color:inherit;background:var(--bg-tertiary,#252935);border:1px solid var(--border-color,#414653);border-radius:6px;padding:5px 8px}button{cursor:pointer}.plugin-choices{padding:10px 0;display:grid;gap:9px}.choice{justify-content:space-between}p{margin:0;line-height:1.5}.hint{opacity:.8}input{accent-color:#72b9d6}`]
})
export class TaskTargetComponent {
changed = output<TaskTargetOptions>();
private api = inject(PluginService);
target: TaskTargetOptions['execution_target'] = 'android';
automatic = true; selected: Record<string, string> = {};
open = signal(false); loading = signal(false); error = signal('');
plugins = signal<InstalledPlugin[]>([]);
available() { return this.plugins().filter(p => p.enabled && p.manifest.targets.includes(this.target)); }
toggle() { this.open.update(v => !v); if (this.open()) this.load(); }
targetChanged() { this.selected = {}; this.emit(); if (this.open()) this.load(); }
load() { this.loading.set(true); this.error.set(''); this.api.list().subscribe({
next: value => { this.plugins.set(value.plugins); this.loading.set(false); },
error: () => { this.error.set('插件列表暂不可用,请重新展开重试。'); this.loading.set(false); }
}); }
choose(plugin: InstalledPlugin, event: Event) {
if ((event.target as HTMLInputElement).checked) this.selected[plugin.manifest.id] = plugin.profiles.find(p => p.authorized)?.name || 'default';
else delete this.selected[plugin.manifest.id];
this.emit();
}
emit() { this.changed.emit({ execution_target: this.target,
...(this.automatic ? {} : { plugins: Object.entries(this.selected).map(([id, profile]) => ({id, profile})) }) }); }
}
@@ -1379,6 +1379,7 @@
<!-- Task composer --> <!-- Task composer -->
<div class="composer-card" [class.is-pro-active]="selectedProfile() === 'pro'"> <div class="composer-card" [class.is-pro-active]="selectedProfile() === 'pro'">
<app-task-target (changed)="taskTarget.set($event)" />
<!-- Smart Intent Recommendation Strip (if detected) --> <!-- Smart Intent Recommendation Strip (if detected) -->
@if (showIntentSuggestion()) { @if (showIntentSuggestion()) {
@@ -18,6 +18,7 @@ import { ModelSettingsComponent } from '../../components/model-settings/model-se
import { Component, signal, computed, inject, OnInit, OnDestroy, ChangeDetectionStrategy } from '@angular/core'; import { Component, signal, computed, inject, OnInit, OnDestroy, ChangeDetectionStrategy } from '@angular/core';
import { FormsModule } from '@angular/forms'; import { FormsModule } from '@angular/forms';
import { TaskTargetComponent, TaskTargetOptions } from '../../components/task-target/task-target.component';
import { Router } from '@angular/router'; import { Router } from '@angular/router';
import { AgentService } from '../../services/agent.service'; import { AgentService } from '../../services/agent.service';
import { SystemService, formatAdbRestartFeedback } from '../../services/system.service'; import { SystemService, formatAdbRestartFeedback } from '../../services/system.service';
@@ -105,7 +106,7 @@ type AdbGuideTab = 'emulator' | 'usb' | 'wifi' | 'remote';
@Component({ @Component({
selector: 'app-home', selector: 'app-home',
standalone: true, standalone: true,
imports: [FormsModule, ModelSettingsComponent, CapabilitySettingsComponent], imports: [FormsModule, ModelSettingsComponent, CapabilitySettingsComponent, TaskTargetComponent],
templateUrl: './home.component.html', templateUrl: './home.component.html',
changeDetection: ChangeDetectionStrategy.Eager, changeDetection: ChangeDetectionStrategy.Eager,
styleUrl: './home.component.scss' styleUrl: './home.component.scss'
@@ -319,7 +320,8 @@ export class HomeComponent implements OnInit, OnDestroy {
}); });
// Computed helper states delegating to SystemService // Computed helper states delegating to SystemService
public isReady = computed(() => this.systemService.isReady()); taskTarget = signal<TaskTargetOptions>({ execution_target: 'android' });
public isReady = computed(() => this.taskTarget().execution_target === 'computer' || this.systemService.isReady());
public hasReadinessReport = computed(() => this.systemService.hasReadinessReport()); public hasReadinessReport = computed(() => this.systemService.hasReadinessReport());
public isLoading = computed(() => this.systemService.isLoading()); public isLoading = computed(() => this.systemService.isLoading());
public isRestartingAdb = computed(() => this.systemService.isRestartingAdb()); public isRestartingAdb = computed(() => this.systemService.isRestartingAdb());
@@ -810,7 +812,8 @@ export class HomeComponent implements OnInit, OnDestroy {
undefined, undefined,
this.selectedProfile() === 'pro' this.selectedProfile() === 'pro'
? { verificationLevel: this.verificationLevel().id, explorerMode: this.explorerMode().id } ? { verificationLevel: this.verificationLevel().id, explorerMode: this.explorerMode().id }
: undefined : undefined,
{ ...this.taskTarget() }
) )
.subscribe({ .subscribe({
next: () => { next: () => {
@@ -0,0 +1,57 @@
<main class="plugins-page">
<header><div><span class="eyebrow">ARTEMIS · 扩展能力</span><h1>插件</h1><p>为测试接入电脑和其他工具,统一执行并保留证据。</p></div>
<button (click)="refresh(); refreshCatalog()" [disabled]="busy()">刷新</button></header>
@if (error()) { <div class="message error" role="alert">{{error()}}</div> }
@if (notice()) { <div class="message success" role="status">{{notice()}}</div> }
@for (resource of blocked(); track resource[0]) {
<section class="message error" role="alert"><strong>资源保护 · {{resource[0]}}</strong><p>{{resource[1]}}</p><button (click)="confirmingResource.set(resource[0])" [disabled]="busy()">核对并解除保护</button></section>
}
@if (progress(); as job) {
<section class="install-progress" aria-live="polite"><div><span>{{job.stage}}</span><strong>{{job.progress}}%</strong></div>
<progress max="100" [value]="job.progress" [attr.aria-label]="job.stage"></progress></section>
}
<div class="tabs" role="tablist" aria-label="插件来源">
<button role="tab" [attr.aria-selected]="tab() === 'installed'" [class.active]="tab() === 'installed'" (click)="tab.set('installed')">已安装 <span>{{installed().length}}</span></button>
<button role="tab" [attr.aria-selected]="tab() === 'market'" [class.active]="tab() === 'market'" (click)="tab.set('market')">插件市场</button>
</div>
@if (tab() === 'installed') {
@if (loading()) { <p role="status" class="empty">正在读取插件…</p> }
@else if (!installed().length) { <section class="empty"><span class="material-symbols-outlined">extension</span><h2>还没有安装插件</h2><p>从市场安装经过验证的扩展,或在下方导入自己的插件。</p><button class="primary" (click)="tab.set('market')">浏览插件市场</button></section> }
<div class="cards">@for (plugin of installed(); track plugin.manifest.id) {
<article class="card"><div class="card-top"><div class="plugin-icon"><span class="material-symbols-outlined">extension</span></div><div><h2>{{plugin.manifest.name}}</h2><span class="meta">v{{plugin.manifest.version}} · {{plugin.source === 'marketplace' ? '已验证市场插件' : '本地 / 自定义'}}</span></div><span class="badge" [class.enabled]="plugin.enabled">{{plugin.enabled ? '已启用' : '未启用'}}</span></div>
<p>{{plugin.manifest.description || '暂无描述'}}</p><div class="meta">{{plugin.profiles.length}} 套连接配置</div>
<footer><button (click)="edit(plugin)" [disabled]="busy()">配置</button><button (click)="diagnose(plugin)" [disabled]="busy() || !plugin.enabled">检查连接</button><button [class.primary]="!plugin.enabled" (click)="toggle(plugin)" [disabled]="busy()">{{plugin.enabled ? '停用' : '启用'}}</button><button class="text-button" (click)="removing.set(plugin)" [disabled]="busy()">卸载</button></footer>
</article>
}</div>
<details class="advanced"><summary>高级接入 · 本地插件与自定义 MCP</summary><p>本地插件需要包含 plugin.json;导入后配置授权范围,再手动启用。</p>
<div class="inline"><input aria-label="本地插件路径" [(ngModel)]="localPath" placeholder="插件 ZIP 文件或目录的完整路径"><button (click)="install({local_path: localPath})" [disabled]="busy() || !localPath.trim()">导入</button></div>
<details><summary>添加自定义 MCP / Skill</summary><label>插件清单 JSON<textarea rows="12" [(ngModel)]="customManifest" spellcheck="false"></textarea></label><label>Skill 内容(可选)<textarea rows="4" [(ngModel)]="customSkill"></textarea></label><button (click)="addCustom()" [disabled]="busy()">添加插件</button></details>
</details>
} @else {
@if (catalogError()) { <div class="message error" role="alert">{{catalogError()}} <button (click)="refreshCatalog()">重试</button></div> }
@else if (!market().length) { <section class="empty"><h2>首批插件验证中</h2><p>验证通过的插件会出现在这里;不会自动安装未经验证的版本。</p></section> }
<div class="cards">@for (entry of market(); track entry.manifest.id + entry.manifest.version) {
<article class="card"><div class="card-top"><div class="plugin-icon"><span class="material-symbols-outlined">verified</span></div><div><h2>{{entry.manifest.name}}</h2><span class="meta">v{{entry.manifest.version}} · {{entry.manifest.platforms.join(' / ')}}</span></div></div><p>{{entry.manifest.description}}</p><details><summary>验证记录</summary><p>{{entry.verification}}</p></details><footer><button class="primary" (click)="install({plugin_id: entry.manifest.id, version: entry.manifest.version})" [disabled]="busy() || installedVersion(entry.manifest.id) === entry.manifest.version">{{installedVersion(entry.manifest.id) === entry.manifest.version ? '已安装' : installedVersion(entry.manifest.id) ? '安装此版本' : '安装插件'}}</button></footer></article>
}</div>
}
</main>
@if (confirmingResource(); as resource) {
<div class="modal-backdrop"><section class="dialog small" role="dialog" aria-modal="true" aria-labelledby="resource-confirm-title"><h2 id="resource-confirm-title">确认 {{resource}} 已停止操作</h2><p>请先核对目标设备或远程服务,确认之前的插件动作已经停止,并检查实际状态。解除后任务才可再次操作此资源。</p><footer><button (click)="confirmingResource.set(null)" [disabled]="busy()">取消</button><button class="primary" (click)="confirmStopped()" [disabled]="busy()">已核对,解除保护</button></footer></section></div>
}
@if (editing(); as plugin) {
<div class="modal-backdrop"><section class="dialog" role="dialog" aria-modal="true" aria-labelledby="plugin-config-title"><header><div><h2 id="plugin-config-title">{{plugin.manifest.name}}</h2><p>配置连接与允许控制的范围</p></div><button aria-label="关闭配置" (click)="editing.set(null)" [disabled]="busy()">×</button></header>
<div class="inline"><select aria-label="选择连接配置" [ngModel]="profile.name" (ngModelChange)="selectProfile($event)">@for (p of plugin.profiles; track p.name) { <option [value]="p.name">{{p.label}}</option> }</select><button (click)="newProfile()">新增配置</button></div>
<div class="two-columns"><label>配置标识<input [(ngModel)]="profile.name" placeholder="如 lab-router"></label><label>显示名称<input [(ngModel)]="profile.label" placeholder="如实验室路由器"></label></div>
@for (entry of fields(); track entry[0]) { <label>{{entry[1].label}}{{entry[1].required ? ' *' : ''}}
@if (entry[1].secret) { <input type="password" autocomplete="new-password" [(ngModel)]="profile.secrets![entry[0]]" [placeholder]="profile.configured_secrets?.includes(entry[0]) ? '已保存,留空保留' : '仅保存在本机'"> }
@else { <input [(ngModel)]="profile.values[entry[0]]" [placeholder]="entry[1].default"> }
@if (entry[1].description) { <small>{{entry[1].description}}</small> }
</label> }
<details><summary>控制范围与共享资源</summary><label>允许工具(每行一个,留空使用全部工具)<textarea rows="3" [(ngModel)]="allowedTools"></textarea></label><label>参数约束 JSON<textarea rows="4" [(ngModel)]="constraints" placeholder="按工具名称配置允许的参数值"></textarea></label><label>共享资源(同一目标使用相同标识,每行一个)<textarea rows="2" [(ngModel)]="resources" placeholder="router:lab"></textarea></label></details>
<label class="checkbox"><input type="checkbox" [(ngModel)]="profile.is_default">作为自动选择插件时的默认连接配置</label>
<label class="checkbox"><input type="checkbox" [(ngModel)]="profile.authorized">允许 ARTEMIS 使用此配置执行插件工具</label><p class="meta">插件进程使用当前用户的系统权限。工具与参数范围由 ARTEMIS 检查。</p>
@if (error()) { <p class="message error" role="alert">{{error()}}</p> }
<footer><button (click)="editing.set(null)" [disabled]="busy()">取消</button><button class="primary" (click)="save()" [disabled]="busy() || !profile.name.trim()">{{busy() ? '保存中…' : '保存配置'}}</button></footer>
</section></div>
}
@if (removing(); as plugin) { <div class="modal-backdrop"><section class="dialog small" role="dialog" aria-modal="true" aria-labelledby="remove-title"><h2 id="remove-title">卸载 {{plugin.manifest.name}}?</h2><p>此插件及其连接配置将不再供新任务使用。</p><footer><button (click)="removing.set(null)" [disabled]="busy()">取消</button><button class="danger" (click)="remove()" [disabled]="busy()">确认卸载</button></footer></section></div> }
@@ -0,0 +1,24 @@
:host { display: block; color: #1e293b; font-size: 14px; }
.plugins-page { max-width: 1120px; padding: 88px 32px 56px; margin: auto; }
header, .card-top, footer, .inline, .install-progress > div { display: flex; align-items: center; gap: 12px; }
header { justify-content: space-between; margin-bottom: 28px; }
h1 { margin: 8px 0; font-size: 30px; letter-spacing: -.6px; } h2 { margin: 0 0 6px; font-size: 17px; }
p { color: #64748b; line-height: 1.7; white-space: pre-wrap; margin: 8px 0 16px; }
.eyebrow { color: #3b6e9c; font-size: 11px; letter-spacing: 1.4px; font-weight: 600; }
button { border: 1px solid #dce3eb; border-radius: 8px; background: #fff; color: #334155; padding: 8px 14px; font-size: 13px; cursor: pointer; white-space: nowrap; }
button:hover:not(:disabled) { border-color: #9fb7d3; background: #f0f6ff; } button:disabled { opacity: .5; cursor: not-allowed; }
button.primary { color: #fff; background: #2563eb; border-color: #2563eb; } button.danger { color: #b91c1c; border-color: #fecaca; }
.text-button { border: 0; margin-left: auto; color: #64748b; } :focus-visible { outline: 2px solid #3b82f6; outline-offset: 3px; }
.tabs { display: flex; gap: 8px; margin-bottom: 24px; border-bottom: 1px solid #e2e8f0; padding-bottom: 12px; }
.tabs button { border: 0; background: transparent; }.tabs button.active { color: #1d4ed8; background: #eaf1ff; font-weight: 600; }.tabs span { margin-left: 8px; }
.cards { display: grid; grid-template-columns: repeat(auto-fit, minmax(min(100%, 420px), 1fr)); gap: 18px; }
.card { padding: 22px; background: #fff; border: 1px solid #e1e7ef; border-radius: 14px; }.card > p { min-height: 48px; margin-top: 18px; }.card footer { margin-top: 20px; flex-wrap: wrap; }
.plugin-icon { display: grid; place-items: center; width: 44px; height: 44px; flex-shrink: 0; color: #4477ba; background: #eef4ff; border-radius: 12px; }
.meta, small { color: #64748b; font-size: 12px; line-height: 1.6; }.badge { margin-left: auto; white-space: nowrap; padding: 4px 9px; border-radius: 20px; background: #f1f5f9; font-size: 11px; color: #64748b; }.badge.enabled { background: #ecfdf5; color: #047857; }
.empty { padding: 55px 24px; text-align: center; border: 1px dashed #cbd5e1; border-radius: 14px; background: #ffffff80; margin-bottom: 24px; }.empty > span { font-size: 36px; color: #94a3b8; margin-bottom: 16px; }
.message { padding: 12px 16px; border-radius: 8px; margin-bottom: 18px; line-height: 1.6; }.error { color: #b91c1c; background: #fff1f2; }.success { color: #047857; background: #ecfdf5; }
.install-progress { background: #fff; border: 1px solid #dbeafe; padding: 18px; border-radius: 12px; margin-bottom: 20px; }.install-progress > div { justify-content: space-between; }progress { width: 100%; height: 8px; margin-top: 12px; accent-color: #2563eb; }
details { margin: 16px 0; }summary { cursor: pointer; color: #475569; padding: 8px 0; }.advanced { padding: 18px 22px; border: 1px solid #e2e8f0; border-radius: 12px; margin-top: 24px; }
label { display: flex; flex-direction: column; gap: 7px; margin: 16px 0; font-size: 13px; }input, select, textarea { border: 1px solid #d5dde7; background: #fff; border-radius: 8px; padding: 10px 12px; color: #1e293b; font-size: 13px; min-width: 0; width: 100%; }textarea { resize: vertical; font-family: ui-monospace, 'Microsoft YaHei', monospace; line-height: 1.6; }.inline > input, .inline > select { flex: 1; }
.modal-backdrop { position: fixed; inset: 0; background: #0f172a50; backdrop-filter: blur(3px); display: grid; place-items: center; z-index: 1300; padding: 24px; }.dialog { max-width: 620px; width: 100%; max-height: 90vh; overflow-y: auto; background: #fff; padding: 28px; border-radius: 16px; box-shadow: 0 24px 80px #0f172a30; }.dialog.small { max-width: 430px; }.dialog header { margin-bottom: 12px; }.dialog footer { justify-content: flex-end; margin-top: 24px; }.two-columns { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; }.checkbox { flex-direction: row; align-items: center; gap: 10px; }.checkbox input { width: auto; }
@media (max-width: 640px) { .plugins-page { padding: 80px 16px 32px; }.card { padding: 18px; }.dialog { padding: 20px; }.two-columns { grid-template-columns: 1fr; gap: 0; }.card-top { gap: 8px; }h2 { font-size: 15px; } }
@@ -0,0 +1,68 @@
import { TestBed } from '@angular/core/testing';
import { provideHttpClient } from '@angular/common/http';
import { provideHttpClientTesting, HttpTestingController } from '@angular/common/http/testing';
import { PluginsComponent } from './plugins.component';
import { InstalledPlugin } from '../../services/plugin.service';
describe('PluginsComponent', () => {
let http: HttpTestingController;
beforeEach(() => {
TestBed.configureTestingModule({ imports: [PluginsComponent], providers: [provideHttpClient(), provideHttpClientTesting()] });
http = TestBed.inject(HttpTestingController);
});
afterEach(() => http.verify());
function setup() {
const fixture = TestBed.createComponent(PluginsComponent);
fixture.detectChanges();
http.expectOne('/api/plugins').flush({ plugins: [], blocked_resources: { 'router:lab': '操作结果不确定' } });
http.expectOne('/api/plugins/catalog').flush({ plugins: [] });
return fixture;
}
it('shows empty state and requires explicit confirmation before releasing a protected resource', async () => {
const fixture = setup();
await fixture.whenStable(); fixture.detectChanges();
expect(fixture.nativeElement.textContent).toContain('还没有安装插件');
expect(fixture.nativeElement.textContent).toContain('router:lab');
const component = fixture.componentInstance;
await component.confirmStopped();
http.expectNone('/api/plugins/resources/confirm-stopped');
component.confirmingResource.set('router:lab');
const operation = component.confirmStopped();
const request = http.expectOne('/api/plugins/resources/confirm-stopped');
expect(request.request.body).toEqual({ resource: 'router:lab', confirmed_stopped: true });
request.flush({ ok: true });
await Promise.resolve();
http.expectOne('/api/plugins').flush({ plugins: [], blocked_resources: {} });
await operation;
expect(component.blocked()).toEqual([]);
});
it('keeps saved secrets write-only and preserves the configuration form on failure', async () => {
const fixture = setup(); await fixture.whenStable();
const component = fixture.componentInstance;
component.edit({ manifest: { id: 'demo', config: {} }, profiles: [{ name: 'default', label: '默认', values: {}, configured_secrets: ['token'], allowed_tools: null, argument_constraints: {}, resources: [], authorized: true }] } as unknown as InstalledPlugin);
const operation = component.save();
const request = http.expectOne('/api/plugins/demo/profiles');
expect(request.request.body.secrets).toEqual({});
expect(request.request.body.configured_secrets).toBeUndefined();
request.flush({ detail: '插件正被任务使用' }, { status: 409, statusText: 'Conflict' });
await operation;
expect(component.editing()).not.toBeNull();
expect(component.error()).toContain('任务使用');
expect(component.busy()).toBeFalse();
});
it('shows installation failure without pretending that the package was installed', async () => {
const fixture = setup(); await fixture.whenStable();
const operation = fixture.componentInstance.install({ plugin_id: 'demo' });
http.expectOne('/api/plugins/install').flush({ job_id: 'job-1' });
await Promise.resolve();
http.expectOne('/api/plugins/install/job-1').flush({ id: 'job-1', status: 'failed', stage: '校验失败', progress: 75 });
await operation;
expect(fixture.componentInstance.error()).toBe('校验失败');
expect(fixture.componentInstance.busy()).toBeFalse();
expect(fixture.componentInstance.installed()).toEqual([]);
});
});
@@ -0,0 +1,113 @@
import { Component, OnDestroy, OnInit, inject, signal } from '@angular/core';
import { FormsModule } from '@angular/forms';
import { firstValueFrom } from 'rxjs';
import { InstalledPlugin, InstallProgress, MarketPlugin, PluginProfile, PluginService } from '../../services/plugin.service';
@Component({
selector: 'app-plugins', standalone: true, imports: [FormsModule],
templateUrl: './plugins.component.html', styleUrl: './plugins.component.scss'
})
export class PluginsComponent implements OnInit, OnDestroy {
private api = inject(PluginService);
installed = signal<InstalledPlugin[]>([]);
market = signal<MarketPlugin[]>([]);
loading = signal(true);
busy = signal(false);
error = signal('');
catalogError = signal('');
notice = signal('');
blocked = signal<[string, string][]>([]);
confirmingResource = signal<string | null>(null);
tab = signal<'installed' | 'market'>('installed');
progress = signal<InstallProgress | null>(null);
editing = signal<InstalledPlugin | null>(null);
removing = signal<InstalledPlugin | null>(null);
profile: PluginProfile = this.emptyProfile();
resources = ''; allowedTools = ''; constraints = '{}'; localPath = '';
customManifest = JSON.stringify({ schema_version: 1, id: 'my-plugin', name: '我的插件', version: '1.0.0',
description: '插件用途', platforms: ['win32'], server: { transport: 'stdio', command: '程序绝对路径', args: [] } }, null, 2);
customSkill = '';
private timer?: ReturnType<typeof setTimeout>;
private destroyed = false;
ngOnInit() { void this.refresh(); void this.refreshCatalog(); }
ngOnDestroy() { this.destroyed = true; clearTimeout(this.timer); }
emptyProfile(): PluginProfile { return { name: 'default', label: '默认配置', values: {}, secrets: {},
allowed_tools: null, argument_constraints: {}, resources: [], authorized: false }; }
private message(error: any): string { return typeof error?.error?.detail === 'string' ? error.error.detail : '操作失败,请检查连接后重试。'; }
async refresh() {
this.loading.set(true);
try { const result = await firstValueFrom(this.api.list()); this.installed.set(result.plugins); this.blocked.set(Object.entries(result.blocked_resources || {})); }
catch (error) { this.error.set(this.message(error)); }
finally { this.loading.set(false); }
}
async refreshCatalog() {
this.catalogError.set('');
try { this.market.set((await firstValueFrom(this.api.catalog())).plugins); }
catch (error) { this.catalogError.set(this.message(error)); }
}
installedVersion(id: string) { return this.installed().find(p => p.manifest.id === id)?.manifest.version; }
fields() { return Object.entries(this.editing()?.manifest.config || {}); }
edit(plugin: InstalledPlugin) { this.editing.set(plugin); this.selectProfile(plugin.profiles[0]?.name || 'default'); }
selectProfile(name: string) {
const selected = this.editing()?.profiles.find(p => p.name === name);
this.profile = selected ? { ...structuredClone(selected), secrets: {} } : this.emptyProfile();
this.resources = this.profile.resources.join('\n');
this.allowedTools = this.profile.allowed_tools?.join('\n') || '';
this.constraints = JSON.stringify(this.profile.argument_constraints, null, 2);
}
newProfile() { this.profile = { ...this.emptyProfile(), name: '', label: '新配置' }; this.resources = ''; this.allowedTools = ''; this.constraints = '{}'; }
async save() {
const plugin = this.editing(); if (!plugin || this.busy()) return;
this.busy.set(true); this.error.set('');
try {
const profile = { ...this.profile, resources: this.resources.split('\n').map(v => v.trim()).filter(Boolean),
allowed_tools: this.allowedTools.trim() ? this.allowedTools.split('\n').map(v => v.trim()).filter(Boolean) : null,
argument_constraints: JSON.parse(this.constraints) };
await firstValueFrom(this.api.save(plugin.manifest.id, profile));
this.editing.set(null); this.profile = this.emptyProfile(); await this.refresh();
this.notice.set('配置已保存。启用后可供后续任务使用。');
} catch (error) { this.error.set(error instanceof SyntaxError ? '参数约束不是有效的 JSON。' : this.message(error)); }
finally { this.busy.set(false); }
}
async toggle(plugin: InstalledPlugin) { await this.act(async () => { await firstValueFrom(this.api.enable(plugin.manifest.id, !plugin.enabled)); await this.refresh(); }); }
async diagnose(plugin: InstalledPlugin) {
await this.act(async () => { const result = await firstValueFrom(this.api.diagnose(plugin.manifest.id, plugin.profiles[0]?.name || 'default'));
if (result.ok) this.notice.set(result.message); else this.error.set(result.message); });
}
async remove() {
const plugin = this.removing(); if (!plugin) return;
await this.act(async () => { await firstValueFrom(this.api.uninstall(plugin.manifest.id)); this.removing.set(null); await this.refresh(); });
}
async confirmStopped() {
const resource = this.confirmingResource(); if (!resource) return;
await this.act(async () => { await firstValueFrom(this.api.confirmStopped(resource)); this.confirmingResource.set(null); await this.refresh(); this.notice.set('已解除保护。继续任务前将重新观察目标状态。'); });
}
async install(payload: { plugin_id?: string; version?: string; local_path?: string }) {
if (this.busy()) return;
this.busy.set(true); this.error.set(''); this.notice.set('');
this.progress.set({ id: '', status: 'queued', stage: '准备安装', progress: 0 });
try { const { job_id } = await firstValueFrom(this.api.install(payload)); await this.poll(job_id); }
catch (error) { this.error.set(this.message(error)); this.busy.set(false); this.progress.set(null); }
}
private async poll(id: string) {
if (this.destroyed) return;
try {
const progress = await firstValueFrom(this.api.progress(id)); if (this.destroyed) return;
this.progress.set(progress);
if (progress.status === 'failed') { this.error.set(progress.stage); this.busy.set(false); return; }
if (progress.status === 'completed') { this.busy.set(false); this.tab.set('installed'); await this.refresh(); return; }
this.timer = setTimeout(() => void this.poll(id), 750);
} catch (error) { this.error.set(this.message(error)); this.busy.set(false); }
}
async addCustom() {
await this.act(async () => { let manifest: unknown;
try { manifest = JSON.parse(this.customManifest); } catch { this.error.set('插件清单不是有效的 JSON。'); return; }
await firstValueFrom(this.api.custom(manifest, this.customSkill)); await this.refresh(); this.notice.set('已添加。请配置授权范围并手动启用。'); });
}
private async act(operation: () => Promise<void>) {
if (this.busy()) return;
this.busy.set(true); this.error.set(''); this.notice.set('');
try { await operation(); } catch (error) { this.error.set(this.message(error)); } finally { this.busy.set(false); }
}
}
@@ -185,6 +185,7 @@ limitations under the License.
</div> </div>
<!-- Bottom Segmented Controls Row --> <!-- Bottom Segmented Controls Row -->
<app-task-target (changed)="taskTarget = $event" />
<div class="card-footer-row"> <div class="card-footer-row">
<div class="footer-left-group"> <div class="footer-left-group">
<!-- Segmented Architecture Toggle (Flash / Pro) --> <!-- Segmented Architecture Toggle (Flash / Pro) -->
@@ -23,12 +23,14 @@ import { FloatingVideoPlayerComponent } from '../../components/floating-video-pl
import { AgentService } from '../../services/agent.service'; import { AgentService } from '../../services/agent.service';
import { HttpClient } from '@angular/common/http'; import { HttpClient } from '@angular/common/http';
import { firstValueFrom } from 'rxjs'; import { firstValueFrom } from 'rxjs';
import { TaskTargetComponent, TaskTargetOptions } from '../../components/task-target/task-target.component';
@Component({ @Component({
selector: 'app-workspace', selector: 'app-workspace',
standalone: true, standalone: true,
imports: [ imports: [
FormsModule, FormsModule,
TaskTargetComponent,
AgentStreamComponent, AgentStreamComponent,
ChatInterfaceComponent, ChatInterfaceComponent,
FloatingVideoPlayerComponent FloatingVideoPlayerComponent
@@ -38,6 +40,7 @@ import { firstValueFrom } from 'rxjs';
changeDetection: ChangeDetectionStrategy.OnPush changeDetection: ChangeDetectionStrategy.OnPush
}) })
export class WorkspaceComponent implements OnInit { export class WorkspaceComponent implements OnInit {
taskTarget: TaskTargetOptions = { execution_target: 'android' };
public agentService = inject(AgentService); public agentService = inject(AgentService);
private http = inject(HttpClient); private http = inject(HttpClient);
private zone = inject(NgZone); private zone = inject(NgZone);
@@ -325,7 +328,7 @@ export class WorkspaceComponent implements OnInit {
} }
this.isInputFocused.set(false); this.isInputFocused.set(false);
this.agentService.runTask(goal, this.selectedProfile()).subscribe({ this.agentService.runTask(goal, this.selectedProfile(), undefined, undefined, undefined, { ...this.taskTarget }).subscribe({
next: (res) => { next: (res) => {
this.taskInput = ''; this.taskInput = '';
if (this.dockInputRef?.nativeElement) { if (this.dockInputRef?.nativeElement) {
@@ -454,7 +454,8 @@ export class AgentService {
profile: string = 'flash', profile: string = 'flash',
expectedOutput?: string, expectedOutput?: string,
enableOutputter?: boolean, enableOutputter?: boolean,
proTuning?: ProTuningOptions proTuning?: ProTuningOptions,
runtimeOverrides?: Record<string, unknown>
): Observable<any> { ): Observable<any> {
return new Observable((obs) => { return new Observable((obs) => {
const submittedEvent: StartupProgressEvent = { const submittedEvent: StartupProgressEvent = {
@@ -464,6 +465,7 @@ export class AgentService {
}; };
this.pendingStartupProgress.set([submittedEvent]); this.pendingStartupProgress.set([submittedEvent]);
const payload: any = { goal, profile }; const payload: any = { goal, profile };
if (runtimeOverrides) payload.runtime_overrides = runtimeOverrides;
if (expectedOutput && expectedOutput.trim()) { if (expectedOutput && expectedOutput.trim()) {
payload.expected_output = expectedOutput.trim(); payload.expected_output = expectedOutput.trim();
} }
@@ -0,0 +1,40 @@
import { Injectable, inject } from '@angular/core';
import { HttpClient } from '@angular/common/http';
export interface PluginManifest {
id: string; name: string; version: string; description: string;
artemis_version: string; platforms: string[]; source: string;
config: Record<string, { label: string; description: string; secret: boolean; required: boolean; default: string }>;
resources: string[];
targets: ('android' | 'computer' | 'mixed')[];
}
export interface PluginProfile {
name: string; label: string; is_default?: boolean; values: Record<string, string>;
secrets?: Record<string, string>; configured_secrets?: string[];
allowed_tools: string[] | null;
argument_constraints: Record<string, Record<string, string[]>>;
resources: string[]; authorized: boolean;
}
export interface InstalledPlugin { manifest: PluginManifest; enabled: boolean; source: string; profiles: PluginProfile[] }
export interface MarketPlugin { manifest: PluginManifest; verification: string }
export interface InstallProgress { id: string; status: 'queued' | 'running' | 'completed' | 'failed'; stage: string; progress: number }
@Injectable({ providedIn: 'root' })
export class PluginService {
private http = inject(HttpClient);
list() { return this.http.get<{ plugins: InstalledPlugin[]; blocked_resources?: Record<string, string> }>('/api/plugins'); }
confirmStopped(resource: string) { return this.http.post('/api/plugins/resources/confirm-stopped', { resource, confirmed_stopped: true }); }
catalog() { return this.http.get<{ plugins: MarketPlugin[] }>('/api/plugins/catalog'); }
install(payload: { plugin_id?: string; version?: string; local_path?: string }) {
return this.http.post<{ job_id: string }>('/api/plugins/install', payload);
}
progress(id: string) { return this.http.get<InstallProgress>(`/api/plugins/install/${encodeURIComponent(id)}`); }
save(id: string, profile: PluginProfile) {
const { configured_secrets, ...payload } = profile;
return this.http.put<InstalledPlugin>(`/api/plugins/${encodeURIComponent(id)}/profiles`, payload);
}
enable(id: string, enabled: boolean) { return this.http.put<InstalledPlugin>(`/api/plugins/${encodeURIComponent(id)}/enabled`, { enabled }); }
diagnose(id: string, profile: string) { return this.http.post<{ ok: boolean; message: string }>(`/api/plugins/${encodeURIComponent(id)}/diagnose`, { profile }); }
uninstall(id: string) { return this.http.delete(`/api/plugins/${encodeURIComponent(id)}`); }
custom(manifest: unknown, skill_content: string) { return this.http.post('/api/plugins/custom', { manifest, skill_content }); }
}
@@ -9,6 +9,20 @@ import {
} from './action-formatter.util'; } from './action-formatter.util';
describe('action-formatter.util screenshot chaining', () => { describe('action-formatter.util screenshot chaining', () => {
it('keeps plugin observations separate from the phone image chain', () => {
const desktop = { type: 'tool', trace_id: 'desktop', name: 'plug_observe', timestamp: 1001,
payload: { plugin_id: 'desktop-control', post_image_name: 'desktop-screen' } };
const missingDesktopImage = { type: 'tool', trace_id: 'desktop-missing', name: 'plug_observe', timestamp: 1002,
payload: { plugin_id: 'desktop-control' } };
const phone = { action: 'tap', timestamp: 1000 };
const step = { step_id: 'mixed', pre_image_name: 'phone-before', post_image_name: 'phone-after',
action_taken: phone, generic_tools: [desktop, missingDesktopImage] };
expect(getStepPreImageUrl(step, desktop)).toBeNull();
expect(getStepPostImageUrl(step, desktop)).toBe('/images/desktop-screen');
expect(getStepPostImageUrl(step, missingDesktopImage)).toBeNull();
expect(getStepPreImageUrl(step, phone)).toBe('/images/phone-before');
expect(getStepPostImageUrl(step, phone)).toBe('/images/phone-after');
});
it('should chain pre/post images correctly in FailureAnalyzer multi-step recovery', () => { it('should chain pre/post images correctly in FailureAnalyzer multi-step recovery', () => {
const stepData = { const stepData = {
step_id: 'step-06', step_id: 'step-06',
@@ -606,6 +606,12 @@ interface ResolvedImageEntry {
const stepImageMapCache = new WeakMap<any, { key: string; map: Map<any, ResolvedImageEntry> }>(); const stepImageMapCache = new WeakMap<any, { key: string; map: Map<any, ResolvedImageEntry> }>();
function pluginSource(item: any): string | null {
const act = getActionObject(item);
const payload = safeParseJson(act?.payload);
const result = safeParseJson(act?.result || payload?.result);
return act?.plugin_id || payload?.plugin_id || payload?.action?.plugin_id || result?.plugin_id || null;
}
export function extractItemPreImage(item: any): string | null { export function extractItemPreImage(item: any): string | null {
const act = getActionObject(item); const act = getActionObject(item);
@@ -754,13 +760,17 @@ export function resolveStepImageMap(stepData: any): Map<any, ResolvedImageEntry>
switchedToFailureShot = true; switchedToFailureShot = true;
} }
const resolvedPre = explicitPre || currentPreImage; const plugin = pluginSource(act);
const resolvedPre = explicitPre || (plugin ? null : currentPreImage);
let resolvedPost: string | null = null; let resolvedPost: string | null = null;
let hasExplicitPost = false; let hasExplicitPost = false;
if (explicitPost) { if (explicitPost) {
resolvedPost = explicitPost; resolvedPost = explicitPost;
hasExplicitPost = true; hasExplicitPost = true;
} else if (plugin) {
// A phone step image is never evidence of a plugin/desktop operation.
resolvedPost = null;
} else if (e.isPrimary && !hadPrimaryFailure) { } else if (e.isPrimary && !hadPrimaryFailure) {
resolvedPost = stepData.post_image_name || stepData.post_screenshot || null; resolvedPost = stepData.post_image_name || stepData.post_screenshot || null;
} else if (e.item === lastActionItem && !hadPrimaryFailure) { } else if (e.item === lastActionItem && !hadPrimaryFailure) {
@@ -770,7 +780,7 @@ export function resolveStepImageMap(stepData: any): Map<any, ResolvedImageEntry>
} }
// "The next pre-action state is the previous post-action state" // "The next pre-action state is the previous post-action state"
if (resolvedPost) { if (resolvedPost && !plugin) {
currentPreImage = resolvedPost; currentPreImage = resolvedPost;
} }
@@ -782,7 +792,7 @@ export function resolveStepImageMap(stepData: any): Map<any, ResolvedImageEntry>
} }
// Ensure stepData.action_taken cross-resolves to the action images // Ensure stepData.action_taken cross-resolves to the action images
if (stepData.action_taken && lastActionItem) { if (stepData.action_taken && lastActionItem && pluginSource(stepData.action_taken) === pluginSource(lastActionItem)) {
const lastEntry = map.get(lastActionItem); const lastEntry = map.get(lastActionItem);
if (lastEntry) { if (lastEntry) {
map.set(stepData.action_taken, lastEntry); map.set(stepData.action_taken, lastEntry);
@@ -815,6 +825,9 @@ function lookupResolvedImages(stepData: any, actionData?: any): ResolvedImageEnt
const explicitPre = extractItemPreImage(act); const explicitPre = extractItemPreImage(act);
const explicitPost = extractItemPostImage(act); const explicitPost = extractItemPostImage(act);
if (pluginSource(act)) {
return { pre: explicitPre, post: explicitPost, hasExplicitPost: Boolean(explicitPost) };
}
const pre = explicitPre || stepData?.pre_image_name || stepData?.pre_screenshot || null; const pre = explicitPre || stepData?.pre_image_name || stepData?.pre_screenshot || null;
const post = explicitPost || (isGenericToolTrace(actionData, stepData) ? null : (stepData?.post_image_name || stepData?.post_screenshot || null)); const post = explicitPost || (isGenericToolTrace(actionData, stepData) ? null : (stepData?.post_image_name || stepData?.post_screenshot || null));
return { pre, post, hasExplicitPost: Boolean(explicitPost) }; return { pre, post, hasExplicitPost: Boolean(explicitPost) };
+3 -1
View File
@@ -317,7 +317,7 @@ def build_checker_tools(
*get_history_tools(ctx), *get_history_tools(ctx),
get_read_note_tool_pure(ctx), get_read_note_tool_pure(ctx),
] ]
if probes_enabled(ctx): if getattr(ctx, "execution_target", "android") != "computer" and probes_enabled(ctx):
tools.append(get_probe_tool(ctx)) tools.append(get_probe_tool(ctx))
return tools return tools
@@ -758,6 +758,8 @@ async def _capture_final_screen(ctx: ArtemisContext) -> tuple[str | None, str]:
controller = UnifiedMobileController(ctx) controller = UnifiedMobileController(ctx)
device_data = await controller.get_screen_data() device_data = await controller.get_screen_data()
if getattr(device_data, "platform", "android") == "computer":
return device_data.base64 or None, device_data.observation_text
screenshot_b64 = device_data.base64 screenshot_b64 = device_data.base64
xml_hierarchy = device_data.elements xml_hierarchy = device_data.elements
width = device_data.width width = device_data.width
+2 -2
View File
@@ -7,7 +7,7 @@
# ROLE & OBJECTIVE # ROLE & OBJECTIVE
You are an autonomous and highly efficient Android Device Execution Agent. Your goal is to accomplish the user's objective on the device. You are an autonomous UI execution agent. Accomplish the user's objective using the actual available device and plugin tools. Only Android action tools use normalized 0–1000 coordinates; plugin tools use their own documented coordinates and element indices.
**Objective: {{ goal }}** **Objective: {{ goal }}**
@@ -26,7 +26,7 @@ Keep in mind that the mobile device is an asynchronous, constantly evolving envi
{% if "wait_for_delay" in available_tools %}- **Timed Waiting & Transitions**: When facing mid-transition animations, loading spinners, required watch durations, or ad countdown timers, use the precise `wait_for_delay` command to allow time to pass or the UI state to stabilize. Some transition states are subtle and may even resemble a phone bug or system freeze; it is always advisable to wait first, and only consider retrying or choosing an alternative path if the state remains broken.{% endif %} {% if "wait_for_delay" in available_tools %}- **Timed Waiting & Transitions**: When facing mid-transition animations, loading spinners, required watch durations, or ad countdown timers, use the precise `wait_for_delay` command to allow time to pass or the UI state to stabilize. Some transition states are subtle and may even resemble a phone bug or system freeze; it is always advisable to wait first, and only consider retrying or choosing an alternative path if the state remains broken.{% endif %}
- **Time-Sensitive Tasks & Transient UI**: Because of the turn latency above, a target element that you see on the current screenshot (e.g., auto-dismissing overlays, temporary message bars, menus or control bars with short timeouts) may have already vanished by the time your physical click command reaches the device! This is one of the most common reasons why the UI does not change as expected after a single step. We call these **Time-Sensitive Tasks**. - **Time-Sensitive Tasks & Transient UI**: Because of the turn latency above, a target element that you see on the current screenshot (e.g., auto-dismissing overlays, temporary message bars, menus or control bars with short timeouts) may have already vanished by the time your physical click command reaches the device! This is one of the most common reasons why the UI does not change as expected after a single step. We call these **Time-Sensitive Tasks**.
- *Mental Model*: When facing these tasks, any attempt to interact step-by-step (such as tapping once to wake up a menu/bar and waiting for the next turn to tap the target button) WILL fail because the element will auto-dismiss during your decision delay. You MUST combine the summon action on the trigger element (`Trigger`) and the target action (`Target`) into a single atomic sequence. - *Mental Model*: When facing these tasks, any attempt to interact step-by-step (such as tapping once to wake up a menu/bar and waiting for the next turn to tap the target button) WILL fail because the element will auto-dismiss during your decision delay. You MUST combine the summon action on the trigger element (`Trigger`) and the target action (`Target`) into a single atomic sequence.
- *Chained Execution*: Use `click_sequence(sequence=[[trigger_x, trigger_y], [target_x, target_y]])` in `0-1000` normalized scale to execute both taps consecutively with a 50ms delay, catching the transient element before the timeout closes it. {% if "click_sequence" in available_tools %} - *Chained Execution*: Use `click_sequence(sequence=[[trigger_x, trigger_y], [target_x, target_y]])` in `0-1000` normalized scale to execute both taps consecutively with a 50ms delay, catching the transient element before the timeout closes it.{% endif %}
--- ---
+74 -4
View File
@@ -217,6 +217,15 @@ class FlashRunner:
tools.append(VIDEO_ANALYZER_TOOL) tools.append(VIDEO_ANALYZER_TOOL)
tools.append(REPORT_TASK_STATUS_TOOL) tools.append(REPORT_TASK_STATUS_TOOL)
if getattr(self.ctx, "execution_target", None) == "computer":
from artemis.mcp.action_manifest import DEVICE_ACTIONS
tools = [
t
for t in tools
if t.name not in DEVICE_ACTIONS
and t.name not in {"ask_explorer", "video_analyzer", "click_sequence"}
]
# With an actuator installed, drop declarations for device actions the # With an actuator installed, drop declarations for device actions the
# backend does not implement (and append its extension tools); without # backend does not implement (and append its extension tools); without
# one, the full historical declaration set is kept. # one, the full historical declaration set is kept.
@@ -225,6 +234,8 @@ class FlashRunner:
from artemis.mcp.action_manifest import filter_declarations from artemis.mcp.action_manifest import filter_declarations
tools = filter_declarations(tools, actuator, "flash") tools = filter_declarations(tools, actuator, "flash")
if getattr(self.ctx, "plugins", None):
tools.extend(self.ctx.plugins.declarations())
return tools return tools
def _prune_intermediate_screenshots(self, messages: list[BaseMessage]) -> None: def _prune_intermediate_screenshots(self, messages: list[BaseMessage]) -> None:
@@ -305,7 +316,10 @@ class FlashRunner:
prompt_path = Path(__file__).parent / "flash_runner.md" prompt_path = Path(__file__).parent / "flash_runner.md"
prompt_template = prompt_path.read_text(encoding="utf-8") prompt_template = prompt_path.read_text(encoding="utf-8")
available_tools = frozenset(t.name for t in tools_declaration) available_tools = frozenset(t.name for t in tools_declaration)
return Template(prompt_template).render(goal=self.goal, available_tools=available_tools) prompt = Template(prompt_template).render(goal=self.goal, available_tools=available_tools)
if getattr(self.ctx, "plugins", None):
prompt += self.ctx.plugins.prompt()
return prompt
# ------------------------------------------------------------------ # ------------------------------------------------------------------
# Per-turn helpers (observe / think) # Per-turn helpers (observe / think)
@@ -612,6 +626,11 @@ class FlashRunner:
elements. Nothing is captured for non-action tools. elements. Nothing is captured for non-action tools.
""" """
post_img_bytes = exec_result.screenshot_bytes post_img_bytes = exec_result.screenshot_bytes
if (
exec_result.metadata.get("plugin_id")
and getattr(self.ctx, "execution_target", "android") != "computer"
):
return post_img_bytes
if post_img_bytes or name not in action_names: if post_img_bytes or name not in action_names:
return post_img_bytes return post_img_bytes
if state is not None: if state is not None:
@@ -705,6 +724,15 @@ class FlashRunner:
action_dict["normalized_start_coordinates"] = norm_start action_dict["normalized_start_coordinates"] = norm_start
action_dict["normalized_end_coordinates"] = norm_end action_dict["normalized_end_coordinates"] = norm_end
action_dict.update(metadata.get("target_semantics") or {}) action_dict.update(metadata.get("target_semantics") or {})
if metadata.get("plugin_id"):
action_dict.update(
{
key: metadata[key]
for key in ("plugin_id", "plugin_version", "resource_ids", "plugin_status")
}
)
action_dict.pop("coordinates", None)
action_dict.pop(COORDINATE_SPACE_KEY, None)
return action_dict return action_dict
def _record_action_step( def _record_action_step(
@@ -800,10 +828,39 @@ class FlashRunner:
# Dynamic dispatch set: manifest device actions plus any backend # Dynamic dispatch set: manifest device actions plus any backend
# extension tools, so extension steps are recorded like actions. # extension tools, so extension steps are recorded like actions.
action_names = self.executor.action_tool_names action_names = self.executor.action_tool_names
plugin_action = False
try: try:
exec_result = await self.executor.execute( operation = self.executor.execute(
name, args, tc_id, state, index_elements=index_elements name, args, tc_id, state, index_elements=index_elements
) )
from artemis.plugins.runtime import get_plugin_runtime
plugins = get_plugin_runtime(self.ctx)
if plugins and name in plugins.bindings:
plugin_action = True
# A phone observation cannot serve as the before-image for a desktop action.
pre_screenshot_bytes = plugins.last_images.get(plugins.bindings[name][0])
xml_list = ""
control = getattr(self.ctx, "task_control", None)
if plugins and name in plugins.tool_names and isinstance(control, TaskControl):
interrupted, exec_result = await control.run_interruptibly(
operation,
on_pause=getattr(self.ctx, "on_task_pause", None),
on_resume=getattr(self.ctx, "on_task_resume", None),
)
if interrupted:
messages.extend(
tool_result_messages(
tc_id,
"操作被暂停或指导打断,结果不确定。先重新观察,不能重放旧动作。",
name=name,
status="error",
llm=self._llm,
)
)
return None, "操作已打断,必须重新观察。", action_sequence
else:
exec_result = await operation
post_img_bytes = await self._capture_post_screenshot( post_img_bytes = await self._capture_post_screenshot(
exec_result, name, action_names, state exec_result, name, action_names, state
@@ -869,7 +926,11 @@ class FlashRunner:
# device actions always report text — their post screenshot is the # device actions always report text — their post screenshot is the
# next observation tail, never a tool-message image. The image # next observation tail, never a tool-message image. The image
# carrier follows the model's provider (tool_result_messages). # carrier follows the model's provider (tool_result_messages).
raw_blocks = exec_result.raw_result if name not in action_names else None raw_blocks = (
exec_result.raw_result
if name not in action_names or exec_result.metadata.get("plugin_id")
else None
)
content = ( content = (
raw_blocks raw_blocks
if isinstance(raw_blocks, list) and raw_blocks if isinstance(raw_blocks, list) and raw_blocks
@@ -896,6 +957,15 @@ class FlashRunner:
status="error", status="error",
) )
) )
if plugin_action and getattr(self.ctx, "execution_target", "android") != "computer":
# The loop's observation always belongs to Android in a mixed task.
# Plugin images travel in their labelled tool result and step record;
# never pair a desktop bitmap with the phone's element indexes.
shot_path, observation_image, observation_text = await capture_screenshot_and_parse_ui(
self.ctx, state, self.controller
)
state.latest_screenshot = shot_path
return observation_image, observation_text, action_sequence
return pre_screenshot_bytes, xml_list, action_sequence return pre_screenshot_bytes, xml_list, action_sequence
async def _invoke_model(self, llm, current_tools: list, messages: list[BaseMessage]): async def _invoke_model(self, llm, current_tools: list, messages: list[BaseMessage]):
@@ -1156,7 +1226,7 @@ class FlashRunner:
# S + F + A + tail: chunk compression and the scrub edge advance here. # S + F + A + tail: chunk compression and the scrub edge advance here.
messages = ledger.render([tail]) messages = ledger.render([tail])
turn_base = len(messages) - 1 turn_base = len(messages) - 1
current_tools = report_only_tools if is_final else tools_declaration current_tools = report_only_tools if is_final else self._get_tools()
if isinstance(control, TaskControl): if isinstance(control, TaskControl):
interrupted, response = await control.run_interruptibly( interrupted, response = await control.run_interruptibly(
+32 -2
View File
@@ -183,6 +183,8 @@ class OperatorNode:
available_device_actions, available_device_actions,
) )
if getattr(self.ctx, "execution_target", None) == "computer":
return frozenset()
actuator = getattr(self.ctx, "actuator", None) actuator = getattr(self.ctx, "actuator", None)
if actuator is not None and callable(getattr(actuator, "capabilities", None)): if actuator is not None and callable(getattr(actuator, "capabilities", None)):
try: try:
@@ -535,6 +537,15 @@ class OperatorNode:
) )
) )
plugins = getattr(self.ctx, "plugins", None)
if plugins:
traced_tools[:] = [
t
for t in traced_tools
if not t.name.startswith("plug_")
and t.name not in {"plugin_load", "plugin_read_file"}
]
traced_tools.extend(plugins.langchain_tools())
bound_llm = active_llm.bind_tools(tools=traced_tools) bound_llm = active_llm.bind_tools(tools=traced_tools)
interrupted, response = await self._await_or_pause( interrupted, response = await self._await_or_pause(
invoke_llm_with_timeout_message(acomplete(bound_llm, current_messages)) invoke_llm_with_timeout_message(acomplete(bound_llm, current_messages))
@@ -650,6 +661,8 @@ class OperatorNode:
# Manifest-driven, so a backend's extension actions are classified as # Manifest-driven, so a backend's extension actions are classified as
# turn-ending device actions without touching this file. # turn-ending device actions without touching this file.
action_tool_names = sorted(self._available_device_actions()) action_tool_names = sorted(self._available_device_actions())
if plugins:
action_tool_names.extend(plugins.bindings)
def normalize_name(name: str) -> str: def normalize_name(name: str) -> str:
return name.split(":")[-1] if ":" in name else name return name.split(":")[-1] if ":" in name else name
@@ -707,7 +720,9 @@ class OperatorNode:
tool_call_id=tc["id"], tool_call_id=tc["id"],
state=state, state=state,
) )
if normalize_name(tool_name) in INTERRUPTIBLE_HELPERS: if normalize_name(tool_name) in INTERRUPTIBLE_HELPERS or (
plugins and tool_name in plugins.tool_names
):
interrupted, result_obj = await self._await_or_pause(tool_call) interrupted, result_obj = await self._await_or_pause(tool_call)
if interrupted: if interrupted:
span.result = "任务被暂停或指导打断;重新观察屏幕后继续" span.result = "任务被暂停或指导打断;重新观察屏幕后继续"
@@ -1002,7 +1017,10 @@ class OperatorNode:
fused_xml = state.latest_ui_hierarchy if hasattr(state, "latest_ui_hierarchy") else [] fused_xml = state.latest_ui_hierarchy if hasattr(state, "latest_ui_hierarchy") else []
state.latest_screenshot if hasattr(state, "latest_screenshot") else None state.latest_screenshot if hasattr(state, "latest_screenshot") else None
if not latest_screenshot_b64 or xml_hierarchy is None: if (
not latest_screenshot_b64
and getattr(self.ctx, "execution_target", "android") != "computer"
) or xml_hierarchy is None:
logger.error( logger.error(
"Operator requires perception data in state (populated by Perception node)" "Operator requires perception data in state (populated by Perception node)"
) )
@@ -1034,6 +1052,8 @@ class OperatorNode:
except (AttributeError, TypeError): except (AttributeError, TypeError):
pass pass
minimal_list, elements, labels = format_minimal_list_with_elements(fused_xml, width, height) minimal_list, elements, labels = format_minimal_list_with_elements(fused_xml, width, height)
if getattr(self.ctx, "execution_target", "android") == "computer":
minimal_list = operator_raw_data.get("observation_text", "尚未取得电脑观察结果")
# Save transient indexed_points and indexed_elements on the state instance # Save transient indexed_points and indexed_elements on the state instance
state.indexed_points = [el["center"] for el in elements] state.indexed_points = [el["center"] for el in elements]
@@ -1079,6 +1099,10 @@ class OperatorNode:
active_background_tasks=active_background_tasks, active_background_tasks=active_background_tasks,
newly_finished_tasks=newly_finished_tasks, newly_finished_tasks=newly_finished_tasks,
) )
if getattr(self.ctx, "plugins", None):
from langchain_core.messages import SystemMessage
messages.insert(0, SystemMessage(content=self.ctx.plugins.prompt()))
# 8. Invoke LLM Loop # 8. Invoke LLM Loop
new_subagent_calls = [] new_subagent_calls = []
@@ -1226,6 +1250,12 @@ class OperatorNode:
tool_name = tc["name"].split(":")[-1] if ":" in tc["name"] else tc["name"] tool_name = tc["name"].split(":")[-1] if ":" in tc["name"] else tc["name"]
args = tc["args"] args = tc["args"]
from artemis.plugins.runtime import get_plugin_runtime
plugins = get_plugin_runtime(self.ctx)
if plugins and tool_name in plugins.bindings:
return [{"action": tool_name, "args": args}], None
# Get device resolution fallbacks # Get device resolution fallbacks
width = 1080 width = 1080
height = 2400 height = 2400
+5
View File
@@ -380,6 +380,7 @@ class ObservationPromptComponent(PromptComponent):
latest_screenshot_b64 = kwargs.get("latest_screenshot_b64") latest_screenshot_b64 = kwargs.get("latest_screenshot_b64")
minimal_list = kwargs.get("minimal_list") minimal_list = kwargs.get("minimal_list")
if latest_screenshot_b64:
builder.add_human_content("--- Current Screenshot ---") builder.add_human_content("--- Current Screenshot ---")
builder.add_human_content( builder.add_human_content(
{ {
@@ -387,6 +388,10 @@ class ObservationPromptComponent(PromptComponent):
"image_url": {"url": f"data:image/jpeg;base64,{latest_screenshot_b64}"}, "image_url": {"url": f"data:image/jpeg;base64,{latest_screenshot_b64}"},
} }
) )
else:
builder.add_human_content(
"--- No screenshot available: use the text observation and request a target window state before coordinate actions. ---"
)
builder.add_human_content(f"--- Visible UI Elements ---\n{minimal_list}") builder.add_human_content(f"--- Visible UI Elements ---\n{minimal_list}")
+7 -2
View File
@@ -274,7 +274,7 @@ class PlannerNode:
screenshot_b64 = await controller.take_screenshot() screenshot_b64 = await controller.take_screenshot()
# Save to Data Engine if available to keep trace # Save to Data Engine if available to keep trace
if self.ctx.data_engine: if self.ctx.data_engine and screenshot_b64:
img_bytes = base64.b64decode(screenshot_b64) img_bytes = base64.b64decode(screenshot_b64)
image_name = self.ctx.data_engine.get_or_create_image(img_bytes) image_name = self.ctx.data_engine.get_or_create_image(img_bytes)
saved_path = str(self.ctx.data_engine.get_image_path(image_name)) saved_path = str(self.ctx.data_engine.get_image_path(image_name))
@@ -284,7 +284,7 @@ class PlannerNode:
except Exception as e: except Exception as e:
logger.error(f"Failed to take screenshot for Initial Plan: {e}") logger.error(f"Failed to take screenshot for Initial Plan: {e}")
if not screenshot_b64: if not screenshot_b64 and getattr(self.ctx, "execution_target", "android") != "computer":
logger.error("Planner failed to obtain a screenshot.") logger.error("Planner failed to obtain a screenshot.")
raise ValueError("Planner failed to obtain a screenshot.") raise ValueError("Planner failed to obtain a screenshot.")
@@ -296,6 +296,11 @@ class PlannerNode:
human_message = Template(human_content).render(**render_kwargs) human_message = Template(human_content).render(**render_kwargs)
human_message_content = [{"type": "text", "text": human_message}] human_message_content = [{"type": "text", "text": human_message}]
if getattr(self.ctx, "execution_target", "android") == "computer":
screen = await UnifiedMobileController(self.ctx).get_screen_data()
human_message_content.append(
{"type": "text", "text": screen.observation_text + self.ctx.plugins.prompt()}
)
if screenshot_b64: if screenshot_b64:
human_message_content.append( human_message_content.append(
{ {
+16 -1
View File
@@ -84,6 +84,16 @@ class SummarizerNode:
action_name, action_args = self._extract_action(record, state, ctx=self.ctx) action_name, action_args = self._extract_action(record, state, ctx=self.ctx)
result = getattr(state, "last_execution_result", None) result = getattr(state, "last_execution_result", None)
if getattr(self.ctx, "execution_target", "android") == "mixed" and isinstance(result, dict):
if any(
action.get("plugin_id")
for action in result.get("execution", [])
if isinstance(action, dict)
):
# The step began with a mobile observation. Never describe a phone-to-desktop
# image pair as a transition on one screen.
pre_bytes = None
post_bytes = None
# format_result_clean reports only errors/repairs; a clean run keeps # format_result_clean reports only errors/repairs; a clean run keeps
# the controller's bare status word. # the controller's bare status word.
exec_outcome = format_result_clean(result) exec_outcome = format_result_clean(result)
@@ -181,7 +191,12 @@ class SummarizerNode:
device = getattr(ctx, "device", None) if ctx is not None else None device = getattr(ctx, "device", None) if ctx is not None else None
width = getattr(device, "device_width", None) or 1080 width = getattr(device, "device_width", None) or 1080
height = getattr(device, "device_height", None) or 2400 height = getattr(device, "device_height", None) or 2400
actions = normalize_any_structure(actions, width, height) actions = [
action
if str(action.get("action", "")).startswith("plug_")
else normalize_any_structure(action, width, height)
for action in actions
]
first = actions[0] first = actions[0]
action_name = first.get("action") or "unknown" action_name = first.get("action") or "unknown"
action_args = {k: v for k, v in first.items() if k != "action"} action_args = {k: v for k, v in first.items() if k != "action"}
+38 -1
View File
@@ -181,6 +181,33 @@ async def _process_action(
outcome = _ActionOutcome(action_item=action_item) outcome = _ActionOutcome(action_item=action_item)
attempts_log = [] attempts_log = []
from artemis.plugins.runtime import get_plugin_runtime
from artemis.runtime.task_control import TaskControl
plugins = get_plugin_runtime(node.ctx)
if plugins and action_name in plugins.bindings:
operation = plugins.execute(action_name, action_item.get("args", {}), str(uuid.uuid4()))
control = getattr(node.ctx, "task_control", None)
if isinstance(control, TaskControl):
interrupted, result = await control.run_interruptibly(
operation,
on_pause=getattr(node.ctx, "on_task_pause", None),
on_resume=getattr(node.ctx, "on_task_resume", None),
)
if interrupted:
outcome.success = False
outcome.error_msg = "插件操作已打断,结果不确定;继续前必须重新观察。"
return outcome
else:
result = await operation
outcome.success = result.status == "success"
outcome.error_msg = "" if outcome.success else result.text_summary
outcome.action_item.update(result.metadata)
outcome.action_item["result"] = result.text_summary
if result.screenshot_bytes:
outcome.post_screenshot_b64 = base64.b64encode(result.screenshot_bytes).decode()
return outcome
validation_passed = True validation_passed = True
if not burst: if not burst:
# 1. Pre-execution validation (Safety Net) # 1. Pre-execution validation (Safety Net)
@@ -338,6 +365,8 @@ def _model_facing_action(ctx, action_item: dict) -> dict:
The live item carries the controller's pixels; every phrase rendered for The live item carries the controller's pixels; every phrase rendered for
a model is normalized first, and the space marker makes that idempotent. a model is normalized first, and the space marker makes that idempotent.
""" """
if action_item.get("plugin_id"):
return action_item
width = getattr(getattr(ctx, "device", None), "device_width", None) or 1080 width = getattr(getattr(ctx, "device", None), "device_width", None) or 1080
height = getattr(getattr(ctx, "device", None), "device_height", None) or 2400 height = getattr(getattr(ctx, "device", None), "device_height", None) or 2400
return normalize_action_dict(action_item, width, height) return normalize_action_dict(action_item, width, height)
@@ -419,11 +448,19 @@ async def run_validation_loop(node, state: State) -> dict:
execution.append(outcome.action_item) execution.append(outcome.action_item)
post_image_name = None post_image_name = None
if outcome.post_screenshot_b64: plugin_action = bool(outcome.action_item.get("plugin_id"))
mixed_plugin_action = (
plugin_action and getattr(ctx, "execution_target", "android") == "mixed"
)
if outcome.post_screenshot_b64 and not mixed_plugin_action:
post_image_name = _absorb_post_screenshot(ctx, state, outcome.post_screenshot_b64) post_image_name = _absorb_post_screenshot(ctx, state, outcome.post_screenshot_b64)
last_screenshot_b64 = outcome.post_screenshot_b64 last_screenshot_b64 = outcome.post_screenshot_b64
if post_image_name: if post_image_name:
last_screenshot_name = post_image_name last_screenshot_name = post_image_name
elif mixed_plugin_action:
# Keep the Android step observation on its own resource. Plugin
# images remain attached to the tool/action trace and its exports.
post_image_name = outcome.action_item.get("post_image_name")
_record_action_end( _record_action_end(
ctx, ctx,
+9
View File
@@ -86,12 +86,21 @@ class ValidatorNode:
which on Windows was the *only* path that ever ran -- is gone: the server now which on Windows was the *only* path that ever ran -- is gone: the server now
lives in-process, so there is no spawn to fail and exactly one execution path. lives in-process, so there is no spawn to fail and exactly one execution path.
""" """
if getattr(self.ctx, "execution_target", None) == "computer":
from artemis.plugins.desktop import PluginReadSession
return PluginReadSession(self.ctx)
return await get_action_session(self.ctx) return await get_action_session(self.ctx)
def _parse_decisions(self, structured_decisions: str) -> tuple[list[dict] | None, str | None]: def _parse_decisions(self, structured_decisions: str) -> tuple[list[dict] | None, str | None]:
return action_execution.parse_decisions(structured_decisions) return action_execution.parse_decisions(structured_decisions)
async def _get_initial_screenshot(self, session, state: State) -> tuple[str, str | None]: async def _get_initial_screenshot(self, session, state: State) -> tuple[str, str | None]:
if (
getattr(self.ctx, "execution_target", None) == "computer"
and not state.latest_screenshot
):
return "", None
return await action_execution.read_initial_screenshot(state) return await action_execution.read_initial_screenshot(state)
async def _execute_validation_loop(self, state: State): async def _execute_validation_loop(self, state: State):
+11
View File
@@ -86,6 +86,17 @@ def is_placeholder_key(val: str | SecretStr | None) -> bool:
class Settings(BaseSettings): class Settings(BaseSettings):
"""Centralized ARTEMIS runtime settings loaded from environment and .env files.""" """Centralized ARTEMIS runtime settings loaded from environment and .env files."""
ARTEMIS_PLUGIN_MARKETPLACE_URL: str = (
"https://git.playones.com/chenmingxuan/Artemis-Plugins/raw/branch/main/catalog.json"
)
ARTEMIS_PLUGIN_START_TIMEOUT_SECONDS: float = Field(default=30, gt=0, le=300)
ARTEMIS_PLUGIN_TOOL_TIMEOUT_SECONDS: float = Field(default=60, gt=0, le=3600)
ARTEMIS_PLUGIN_SHUTDOWN_TIMEOUT_SECONDS: float = Field(default=15, gt=0, le=60)
ARTEMIS_PLUGIN_DOWNLOAD_MAX_BYTES: int = Field(default=134217728, gt=0)
ARTEMIS_PLUGIN_DOWNLOAD_TIMEOUT_SECONDS: float = Field(default=300, gt=0, le=1800)
ARTEMIS_PLUGIN_EXTRACT_MAX_BYTES: int = Field(default=536870912, gt=0)
ARTEMIS_PLUGIN_RESULT_MAX_BYTES: int = Field(default=16777216, gt=0)
# LLM Provider Authentication # LLM Provider Authentication
OPENAI_API_KEY: SecretStr | None = None OPENAI_API_KEY: SecretStr | None = None
+4
View File
@@ -71,6 +71,7 @@ class DevicePlatform(StrEnum):
"""Mobile device platform enumeration.""" """Mobile device platform enumeration."""
ANDROID = "android" ANDROID = "android"
COMPUTER = "computer"
class DeviceContext(BaseModel): class DeviceContext(BaseModel):
@@ -185,6 +186,7 @@ class ArtemisContext(BaseModel):
) )
device: DeviceContext device: DeviceContext
execution_target: Literal["android", "computer", "mixed"] = "android"
llm_config: LLMConfig | None = None llm_config: LLMConfig | None = None
agent_config: Any = None agent_config: Any = None
adb_client: Any | None = None adb_client: Any | None = None
@@ -251,6 +253,8 @@ class ArtemisContext(BaseModel):
actuator: Any | None = None actuator: Any | None = None
"""Optional actuator backend override (artemis.mcp.actuators). ``None`` selects """Optional actuator backend override (artemis.mcp.actuators). ``None`` selects
the default AdbActuator; installing e.g. a robot-arm backend happens here.""" the default AdbActuator; installing e.g. a robot-arm backend happens here."""
plugins: Any | None = None
"""Task-owned external MCP sessions and pinned plugin configuration."""
step_memory: Any | None = None step_memory: Any | None = None
"""Shared :class:`~artemis.memory.step_memory.StepMemoryService` composition """Shared :class:`~artemis.memory.step_memory.StepMemoryService` composition
+1
View File
@@ -50,6 +50,7 @@ class ScreenDataResponse(BaseModel):
width: int = Field(default=1080, description="Device screen width in pixels") width: int = Field(default=1080, description="Device screen width in pixels")
height: int = Field(default=2400, description="Device screen height in pixels") height: int = Field(default=2400, description="Device screen height in pixels")
platform: str = Field(default="android", description="Platform identifier") platform: str = Field(default="android", description="Platform identifier")
observation_text: str = ""
class MobileDeviceController(ABC): class MobileDeviceController(ABC):
@@ -267,6 +267,7 @@ class UnifiedMobileController:
width=data.width, width=data.width,
height=data.height, height=data.height,
platform=data.platform, platform=data.platform,
observation_text=data.observation_text,
) )
async def find_element( async def find_element(
+29 -2
View File
@@ -134,6 +134,14 @@ class StorageManager:
) )
""") """)
conn.execute("""
CREATE TABLE IF NOT EXISTS plugin_image_references (
session_id TEXT NOT NULL,
image_name TEXT NOT NULL,
PRIMARY KEY (session_id, image_name)
)
""")
conn.execute(""" conn.execute("""
CREATE TABLE IF NOT EXISTS steps ( CREATE TABLE IF NOT EXISTS steps (
step_id TEXT PRIMARY KEY, step_id TEXT PRIMARY KEY,
@@ -1102,6 +1110,7 @@ class StorageManager:
with self._get_connection() as conn: with self._get_connection() as conn:
conn.execute("PRAGMA foreign_keys = OFF") conn.execute("PRAGMA foreign_keys = OFF")
tables = [ tables = [
"plugin_image_references",
"video_analysis_observations", "video_analysis_observations",
"video_analysis_segments", "video_analysis_segments",
"failed_outputs", "failed_outputs",
@@ -1154,6 +1163,15 @@ class StorageManager:
except ValueError: except ValueError:
return False return False
def reference_plugin_image(self, session_id: UUID | str, image_name: str):
"""Keep tool-only images alive until their last owning task is deleted."""
with self._get_connection() as conn:
conn.execute(
"INSERT OR IGNORE INTO plugin_image_references VALUES (?, ?)",
(str(session_id), image_name),
)
conn.commit()
def delete_session(self, session_id: UUID): def delete_session(self, session_id: UUID):
"""Delete all data associated with a session, including files on disk.""" """Delete all data associated with a session, including files on disk."""
session_id_str = str(session_id) session_id_str = str(session_id)
@@ -1173,6 +1191,13 @@ class StorageManager:
for name in row for name in row
if name if name
} }
image_names.update(
row[0]
for row in conn.execute(
"SELECT image_name FROM plugin_image_references WHERE session_id=?",
(session_id_str,),
)
)
cursor = conn.execute( cursor = conn.execute(
"SELECT video_id, local_video_path FROM video_recordings WHERE session_id = ?", "SELECT video_id, local_video_path FROM video_recordings WHERE session_id = ?",
(session_id_str,), (session_id_str,),
@@ -1205,6 +1230,7 @@ class StorageManager:
except sqlite3.OperationalError as e: except sqlite3.OperationalError as e:
logger.warning(f"Failed to delete video memory from {table}: {e}") logger.warning(f"Failed to delete video memory from {table}: {e}")
tables = [ tables = [
"plugin_image_references",
"failed_outputs", "failed_outputs",
"traces", "traces",
"background_tasks", "background_tasks",
@@ -1223,8 +1249,9 @@ class StorageManager:
raise raise
for name in image_names: for name in image_names:
if not conn.execute( if not conn.execute(
"SELECT 1 FROM steps WHERE pre_image_name=? OR post_image_name=? LIMIT 1", "SELECT 1 FROM steps WHERE pre_image_name=? OR post_image_name=? "
(name, name), "UNION ALL SELECT 1 FROM plugin_image_references WHERE image_name=? LIMIT 1",
(name, name, name),
).fetchone(): ).fetchone():
conn.execute("DELETE FROM images WHERE image_name=?", (name,)) conn.execute("DELETE FROM images WHERE image_name=?", (name,))
orphaned_images.append(name) orphaned_images.append(name)
+1
View File
@@ -60,6 +60,7 @@ class ScreenData(BaseModel):
height: int = Field(default=2400, description="Device screen height in pixels") height: int = Field(default=2400, description="Device screen height in pixels")
timestamp: float = Field(default_factory=time.time, description="Capture timestamp") timestamp: float = Field(default_factory=time.time, description="Capture timestamp")
platform: str = Field(default="android", description="Platform identifier") platform: str = Field(default="android", description="Platform identifier")
observation_text: str = ""
model_config = ConfigDict(arbitrary_types_allowed=True) model_config = ConfigDict(arbitrary_types_allowed=True)
+5
View File
@@ -33,6 +33,11 @@ logger = get_logger(__name__)
def create_driver(ctx: "ArtemisContext") -> BaseDeviceDriver: def create_driver(ctx: "ArtemisContext") -> BaseDeviceDriver:
"""Instantiates the appropriate BaseDeviceDriver based on the runtime context.""" """Instantiates the appropriate BaseDeviceDriver based on the runtime context."""
if getattr(ctx, "execution_target", None) == "computer":
from artemis.plugins.desktop import PluginScreenDriver
return PluginScreenDriver(ctx)
# 1. Cloud mode check. Cloud devices are reached through the gateway's # 1. Cloud mode check. Cloud devices are reached through the gateway's
# RemoteUIAutomatorClient; ARTEMIS_HIERARCHY_BACKEND does not apply there # RemoteUIAutomatorClient; ARTEMIS_HIERARCHY_BACKEND does not apply there
# because the Accessibility Helper needs a local adb forward. # because the Accessibility Helper needs a local adb forward.
+2
View File
@@ -1023,6 +1023,8 @@ async def get_graph(ctx: ArtemisContext) -> CompiledStateGraph:
ask_explorer_wrapper, ask_explorer_wrapper,
*HISTORY_TOOL_WRAPPERS, *HISTORY_TOOL_WRAPPERS,
] ]
if getattr(ctx, "execution_target", "android") == "computer":
operator_specialized_wrappers = list(HISTORY_TOOL_WRAPPERS)
# ask_committee does not follow the Operator's pre-decision / turn-ending contract. # ask_committee does not follow the Operator's pre-decision / turn-ending contract.
operator_specialized_tools = get_tools_from_wrappers(ctx, operator_specialized_wrappers) operator_specialized_tools = get_tools_from_wrappers(ctx, operator_specialized_wrappers)
+3 -2
View File
@@ -169,7 +169,7 @@ async def perception_node(state: State, ctx: ArtemisContext) -> dict:
# 3. Perform OCR (if configured, else fallback to pure XML layout) # 3. Perform OCR (if configured, else fallback to pure XML layout)
ocr_results = [] ocr_results = []
if is_ocr_configured(): if is_ocr_configured() and getattr(device_data, "platform", "android") != "computer":
screen_height = device_data.height screen_height = device_data.height
status_bar_height = _detect_status_bar_height(xml_hierarchy, screen_height) status_bar_height = _detect_status_bar_height(xml_hierarchy, screen_height)
try: try:
@@ -201,7 +201,7 @@ async def perception_node(state: State, ctx: ArtemisContext) -> dict:
# 5. Save to Data Engine in background thread # 5. Save to Data Engine in background thread
image_name = None image_name = None
screenshot_path = None screenshot_path = None
if ctx.data_engine: if ctx.data_engine and latest_screenshot_bytes:
image_name = hashlib.sha256(latest_screenshot_bytes).hexdigest() image_name = hashlib.sha256(latest_screenshot_bytes).hexdigest()
screenshot_path = str(ctx.data_engine.get_image_path(image_name)) screenshot_path = str(ctx.data_engine.get_image_path(image_name))
asyncio.create_task( asyncio.create_task(
@@ -224,6 +224,7 @@ async def perception_node(state: State, ctx: ArtemisContext) -> dict:
"ocr_results": ocr_results, "ocr_results": ocr_results,
"width": device_data.width, "width": device_data.width,
"height": device_data.height, "height": device_data.height,
"observation_text": getattr(device_data, "observation_text", ""),
}, },
"injected_instruction": injected_instruction, "injected_instruction": injected_instruction,
# sticky_or reducer: once latched True, later False updates keep it True # sticky_or reducer: once latched True, later False updates keep it True
+57 -3
View File
@@ -33,6 +33,8 @@ import signal
from rich.console import Console from rich.console import Console
from rich.panel import Panel from rich.panel import Panel
import typer import typer
from pydantic import TypeAdapter
from artemis.plugins.models import ExecutionTarget, PluginSelection
logger = get_logger(__name__) logger = get_logger(__name__)
@@ -59,6 +61,8 @@ async def execute_task(
explorer_flash_mode: str | None = None, explorer_flash_mode: str | None = None,
explorer_pro_mode: str | None = None, explorer_pro_mode: str | None = None,
verification_level: str | None = None, verification_level: str | None = None,
execution_target: str | None = None,
plugins: list[PluginSelection] | None = None,
) -> None: ) -> None:
"""Executes a single mobile automation task end-to-end. """Executes a single mobile automation task end-to-end.
@@ -138,7 +142,12 @@ async def execute_task(
target_serial = ( target_serial = (
device_serial or settings.ADB_DEVICE_SERIAL or os.environ.get("ADB_DEVICE_SERIAL") device_serial or settings.ADB_DEVICE_SERIAL or os.environ.get("ADB_DEVICE_SERIAL")
) )
if not target_serial: execution_target = TypeAdapter(ExecutionTarget).validate_python(
execution_target or os.getenv("ARTEMIS_EXECUTION_TARGET", "android")
)
if execution_target == "computer":
target_serial = None
if not target_serial and execution_target != "computer":
try: try:
from artemis.runtime import device_pool from artemis.runtime import device_pool
@@ -158,6 +167,7 @@ async def execute_task(
try: try:
agent = Agent(config=config.build(), session_id=effective_sid) agent = Agent(config=config.build(), session_id=effective_sid)
await agent.init( await agent.init(
execution_target=execution_target,
retry_count=int(os.getenv("ARTEMIS_HEALTH_RETRIES", 5)), retry_count=int(os.getenv("ARTEMIS_HEALTH_RETRIES", 5)),
retry_wait_seconds=int(os.getenv("ARTEMIS_HEALTH_DELAY", 2)), retry_wait_seconds=int(os.getenv("ARTEMIS_HEALTH_DELAY", 2)),
) )
@@ -179,7 +189,14 @@ async def execute_task(
if llm_result_path: if llm_result_path:
task.with_llm_output_saving(path=llm_result_path) task.with_llm_output_saving(path=llm_result_path)
await agent.run_task(request=task.build()) request = task.build()
request.execution_target = execution_target
plugin_choices = os.getenv("ARTEMIS_TASK_PLUGINS")
if plugins is not None:
request.plugins = plugins
elif plugin_choices is not None:
request.plugins = TypeAdapter(list[PluginSelection]).validate_json(plugin_choices)
await agent.run_task(request=request)
finally: finally:
if agent is not None: if agent is not None:
await agent.clean() await agent.clean()
@@ -330,6 +347,17 @@ def run_command(
help="Target specific Android device by serial number (e.g. emulator-5554).", help="Target specific Android device by serial number (e.g. emulator-5554).",
), ),
] = None, ] = None,
execution_target: Annotated[
str | None,
typer.Option("--target", help="执行目标:android、computer 或 mixed。电脑任务无需 ADB。 "),
] = None,
plugins_json: Annotated[
str | None,
typer.Option(
"--plugins-json",
help='插件配置引用 JSON,如 [{"id":"open-computer-use","profile":"default"}];不能提供密钥。',
),
] = None,
session_id: Annotated[ session_id: Annotated[
str | None, str | None,
typer.Option( typer.Option(
@@ -350,6 +378,25 @@ def run_command(
check_ffmpeg_available() check_ffmpeg_available()
console = Console() console = Console()
try:
target = TypeAdapter(ExecutionTarget).validate_python(
execution_target or os.getenv("ARTEMIS_EXECUTION_TARGET", "android")
)
choices_json = (
plugins_json if plugins_json is not None else os.getenv("ARTEMIS_TASK_PLUGINS")
)
choices = (
TypeAdapter(list[PluginSelection]).validate_json(choices_json)
if choices_json is not None
else None
)
except ValueError:
raise typer.BadParameter(
"执行目标或插件配置引用 JSON 无效;只能引用已保存的插件配置。"
) from None
overrides = {"execution_target": target}
if choices is not None:
overrides["plugins"] = [p.model_dump() for p in choices]
is_worker = ( is_worker = (
os.environ.get("ARTEMIS_TASK_WORKER") == "1" os.environ.get("ARTEMIS_TASK_WORKER") == "1"
@@ -386,6 +433,7 @@ def run_command(
session_id=target_sid, session_id=target_sid,
ingress="cli", ingress="cli",
base_url=base_url, base_url=base_url,
runtime_overrides=overrides,
) )
if resp and resp.get("tasks"): if resp and resp.get("tasks"):
console.print( console.print(
@@ -440,9 +488,12 @@ def run_command(
f"[yellow]Daemon routing notice: {exc}. Falling back to local execution...[/yellow]" f"[yellow]Daemon routing notice: {exc}. Falling back to local execution...[/yellow]"
) )
if not is_worker and not is_standalone and (target != "android" or choices):
raise typer.BadParameter("插件任务需要统一队列服务,请检查服务后重试;未自动重复提交。")
adb_client = None adb_client = None
try: try:
if which("adb"): if target != "computer" and which("adb"):
adb_client = AdbClient( adb_client = AdbClient(
host=settings.ADB_HOST or "localhost", host=settings.ADB_HOST or "localhost",
port=settings.ADB_PORT or 5037, port=settings.ADB_PORT or 5037,
@@ -451,6 +502,7 @@ def run_command(
# Optional cosmetic device-status display; run continues without it. # Optional cosmetic device-status display; run continues without it.
logger.debug(f"Could not create ADB client for device status display: {exc}") logger.debug(f"Could not create ADB client for device status display: {exc}")
if target != "computer":
display_device_status(console, adb_client=adb_client) display_device_status(console, adb_client=adb_client)
cancelled = False cancelled = False
@@ -488,6 +540,8 @@ def run_command(
explorer_flash_mode=explorer_flash_mode, explorer_flash_mode=explorer_flash_mode,
explorer_pro_mode=explorer_pro_mode, explorer_pro_mode=explorer_pro_mode,
verification_level=verification_level, verification_level=verification_level,
execution_target=target,
plugins=choices,
) )
) )
except (KeyboardInterrupt, asyncio.CancelledError): except (KeyboardInterrupt, asyncio.CancelledError):
+14 -1
View File
@@ -109,7 +109,14 @@ class McpActionExecutor:
@property @property
def action_tool_names(self) -> frozenset[str]: def action_tool_names(self) -> frozenset[str]:
"""Device actions plus backend extension names -- the dynamic dispatch set.""" """Device actions plus backend extension names -- the dynamic dispatch set."""
return (REQUIRED_ACTIONS | OPTIONAL_ACTIONS) | {e.name for e in self.actuator.extensions()} from artemis.plugins.runtime import get_plugin_runtime
plugins = get_plugin_runtime(self.ctx)
return (
(REQUIRED_ACTIONS | OPTIONAL_ACTIONS)
| {e.name for e in self.actuator.extensions()}
| (set(plugins.bindings) if plugins else set())
)
async def _session_or_start(self) -> ActionSession: async def _session_or_start(self) -> ActionSession:
if self._session is None or not self._session.started: if self._session is None or not self._session.started:
@@ -232,6 +239,12 @@ class McpActionExecutor:
""" """
raw_name = name.split(":")[-1] if ":" in name else name raw_name = name.split(":")[-1] if ":" in name else name
from artemis.plugins.runtime import get_plugin_runtime
plugins = get_plugin_runtime(self.ctx)
if plugins and raw_name in plugins.tool_names:
return await plugins.execute(raw_name, args, tool_call_id)
if raw_name in AGENT_TOOL_NAMES: if raw_name in AGENT_TOOL_NAMES:
return await self._execute_agent_tool(raw_name, name, args, tool_call_id, state) return await self._execute_agent_tool(raw_name, name, args, tool_call_id, state)
+16
View File
@@ -88,6 +88,22 @@ async def observe(
device_data = await asyncio.wait_for( device_data = await asyncio.wait_for(
controller.get_screen_data(), timeout=settings.ARTEMIS_OBSERVATION_TIMEOUT_SECONDS controller.get_screen_data(), timeout=settings.ARTEMIS_OBSERVATION_TIMEOUT_SECONDS
) )
if getattr(device_data, "platform", "android") == "computer":
screenshot = base64.b64decode(device_data.base64) if device_data.base64 else None
path = None
if screenshot and ctx.data_engine:
path = str(
ctx.data_engine.get_image_path(ctx.data_engine.get_or_create_image(screenshot))
)
return ObserveResult(
ok=True,
message="已读取电脑插件观察结果",
screenshot_path=path,
width=device_data.width,
height=device_data.height,
elements_text=device_data.observation_text,
hierarchy_ok=True,
), screenshot
screenshot_bytes = base64.b64decode(device_data.base64) screenshot_bytes = base64.b64decode(device_data.base64)
with Image.open(BytesIO(screenshot_bytes)) as image: with Image.open(BytesIO(screenshot_bytes)) as image:
if min(image.size) <= 1: if min(image.size) <= 1:
+1
View File
@@ -0,0 +1 @@
"""Installed skills and external MCP capabilities for ARTEMIS tasks."""
+93
View File
@@ -0,0 +1,93 @@
"""Read-only screen adapter for plugin-controlled computers. Never fabricates images."""
import base64
from io import BytesIO
from PIL import Image
from artemis.drivers.base import BaseDeviceDriver, ScreenData
class PluginScreenDriver(BaseDeviceDriver):
def __init__(self, ctx):
self.ctx = ctx
@property
def device_id(self):
return self.ctx.device.device_id
@property
def screen_size(self):
return self.ctx.device.device_width, self.ctx.device.device_height
async def connect(self):
return None
async def disconnect(self):
await self.ctx.plugins.aclose()
async def get_screen_data(self, skip_settling=False):
runtime = self.ctx.plugins
candidates = [
(pid, package) for pid, package in runtime.packages.items() if package[0].observation
]
if not candidates:
raise ValueError("电脑任务没有可用的观察插件")
pid, (manifest, _, _) = candidates[0]
loaded = await runtime.load(pid)
if loaded.status != "success":
raise RuntimeError(loaded.text())
spec = manifest.observation
tool_name, arguments = runtime._observation_calls.get(pid, (spec.tool, spec.arguments))
name = next(
(
name
for name, (owner, tool) in runtime.bindings.items()
if owner == pid and tool.name == tool_name
),
None,
)
if not name:
raise ValueError("电脑观察工具未授权或未注册")
outcome = await runtime.call(name, arguments)
if outcome.status != "success":
raise RuntimeError(outcome.text())
image = next((b for b in outcome.content if b.get("type") == "image"), None)
raw = base64.b64decode(image["data"], validate=True) if image else b""
width, height = 0, 0
if raw:
with Image.open(BytesIO(raw)) as picture:
width, height = picture.size
picture.verify()
return ScreenData(
screenshot_bytes=raw,
screenshot_base64=base64.b64encode(raw).decode(),
width=width,
height=height,
platform="computer",
observation_text=f"来源:{manifest.name} {manifest.version} / {tool_name}\n{outcome.text()}",
)
async def _unsupported(self, *args, **kwargs):
raise RuntimeError("此任务通过已授权的电脑插件操作,请使用插件工具")
tap = _unsupported
long_press = _unsupported
swipe = _unsupported
swipe_direction = _unsupported
input_text = _unsupported
press_key = _unsupported
launch_app = _unsupported
stop_app = _unsupported
get_current_package = _unsupported
execute_shell = _unsupported
start_video_recording = _unsupported
stop_video_recording = _unsupported
class PluginReadSession:
def __init__(self, ctx):
self.driver = PluginScreenDriver(ctx)
async def screenshot_b64(self):
return (await self.driver.get_screen_data()).screenshot_base64
+159
View File
@@ -0,0 +1,159 @@
"""Install immutable, verified plugin packages without executing package hooks."""
import hashlib
import io
from pathlib import Path, PurePosixPath
import shutil
import stat
import tempfile
import time
from typing import Callable
import zipfile
import httpx
from artemis.config.settings import settings
from artemis.plugins.models import CatalogEntry, PluginCatalog, PluginManifest
from artemis.plugins.store import PluginStore, contained_path
def download(url: str, progress: Callable[[str, int], None]) -> bytes:
if httpx.URL(url).scheme != "https":
raise ValueError("插件下载必须使用 HTTPS")
deadline = time.monotonic() + settings.ARTEMIS_PLUGIN_DOWNLOAD_TIMEOUT_SECONDS
chunks: list[bytes] = []
size = 0
limit = settings.ARTEMIS_PLUGIN_DOWNLOAD_MAX_BYTES
with httpx.stream("GET", url, follow_redirects=True, timeout=30) as response:
response.raise_for_status()
if response.url.scheme != "https":
raise ValueError("插件下载不能降级为明文 HTTP")
total = int(response.headers.get("content-length", 0))
if total > limit:
raise ValueError("插件下载大小超过限制")
for chunk in response.iter_bytes():
if time.monotonic() > deadline:
raise ValueError("插件下载超时,请检查网络后重试")
size += len(chunk)
if size > limit:
raise ValueError("插件下载大小超过限制")
chunks.append(chunk)
progress("下载中", min(70, int(size / total * 70)) if total else 0)
return b"".join(chunks)
def load_catalog() -> PluginCatalog:
data = download(settings.ARTEMIS_PLUGIN_MARKETPLACE_URL, lambda *_: None)
if len(data) > 2 * 1024 * 1024:
raise ValueError("市场索引过大")
return PluginCatalog.model_validate_json(data)
def _extract(data: bytes, target: Path) -> None:
with zipfile.ZipFile(io.BytesIO(data)) as archive:
members = archive.infolist()
if (
len(members) > 20000
or sum(m.file_size for m in members) > settings.ARTEMIS_PLUGIN_EXTRACT_MAX_BYTES
):
raise ValueError("插件解压大小或文件数量超过限制")
seen: set[str] = set()
for member in members:
relative = member.filename.replace("\\", "/")
parts = PurePosixPath(relative).parts
reserved = {
"con",
"prn",
"aux",
"nul",
*(f"com{i}" for i in range(1, 10)),
*(f"lpt{i}" for i in range(1, 10)),
}
if (
not parts
or relative.startswith("/")
or any(p in {"..", "."} or ":" in p or p.endswith((".", " ")) for p in parts)
or any(p.split(".")[0].casefold() in reserved for p in parts)
or stat.S_ISLNK(member.external_attr >> 16)
):
raise ValueError("插件包包含不安全的路径或链接")
key = relative.rstrip("/").casefold()
if key in seen:
raise ValueError("插件包包含重复文件路径")
seen.add(key)
path = contained_path(target, relative)
if member.is_dir():
path.mkdir(parents=True, exist_ok=True)
else:
path.parent.mkdir(parents=True, exist_ok=True)
with archive.open(member) as source, path.open("wb") as output:
shutil.copyfileobj(source, output)
def install_archive(
store: PluginStore,
data: bytes,
*,
entry: CatalogEntry | None = None,
progress: Callable[[str, int], None] = lambda *_: None,
) -> dict:
if len(data) > settings.ARTEMIS_PLUGIN_DOWNLOAD_MAX_BYTES:
raise ValueError("插件包大小超过限制")
progress("校验安装包", 75)
if entry and hashlib.sha256(data).hexdigest() != entry.sha256:
raise ValueError("安装包 SHA-256 校验失败,原版本保持不变")
with tempfile.TemporaryDirectory(prefix="install-", dir=store.root) as temp:
staged = Path(temp) / "package"
staged.mkdir()
_extract(data, staged)
manifest = PluginManifest.model_validate_json(
(staged / "plugin.json").read_text(encoding="utf-8")
)
manifest.check_compatible()
if entry and manifest != entry.manifest:
raise ValueError("安装包清单与市场索引不一致")
if manifest.skill:
skill = contained_path(staged, manifest.skill)
if not skill.is_file() or skill.stat().st_size > 256 * 1024:
raise ValueError("Skill 文件缺失或超过大小限制")
with store.mutation_lock:
store.assert_idle(manifest.id)
destination = store.package_dir(manifest.id, manifest.version)
if destination.exists():
raise ValueError("此插件版本已经安装;请使用新版本号,不能覆盖任务已固定的文件")
progress("安装中", 90)
destination.parent.mkdir(parents=True, exist_ok=True)
staged.replace(destination)
try:
store.register(manifest, "marketplace" if entry else "local")
except Exception:
# Restore the staging directory; TemporaryDirectory removes this failed install.
destination.replace(staged)
raise
progress("安装完成,请配置并启用", 100)
return store.get(manifest.id)
def install_local(store: PluginStore, path: Path, progress=lambda *_: None) -> dict:
if path.is_file():
if path.stat().st_size > settings.ARTEMIS_PLUGIN_DOWNLOAD_MAX_BYTES:
raise ValueError("插件包大小超过限制")
return install_archive(store, path.read_bytes(), progress=progress)
if not path.is_dir():
raise ValueError("本地插件路径不存在")
output = io.BytesIO()
size = 0
count = 0
with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED) as archive:
for source in path.rglob("*"):
if source.is_symlink():
raise ValueError("本地插件不能包含符号链接")
if source.is_file():
count += 1
size += source.stat().st_size
if count > 20000 or size > settings.ARTEMIS_PLUGIN_EXTRACT_MAX_BYTES:
raise ValueError("本地插件目录过大")
archive.write(source, source.relative_to(path).as_posix())
if output.tell() > settings.ARTEMIS_PLUGIN_DOWNLOAD_MAX_BYTES:
raise ValueError("插件包大小超过限制")
return install_archive(store, output.getvalue(), progress=progress)
+175
View File
@@ -0,0 +1,175 @@
"""Versioned plugin contracts shared by the installer, API and task workers."""
from __future__ import annotations
import re
import sys
import platform
from typing import Any, Annotated, Literal
from urllib.parse import urlsplit
from packaging.specifiers import SpecifierSet
from packaging.version import Version
from pydantic import BaseModel, ConfigDict, Field, SecretStr, field_validator, model_validator
Identifier = Annotated[str, Field(pattern=r"^[a-z][a-z0-9_-]{0,47}$")]
class PluginModel(BaseModel):
model_config = ConfigDict(extra="forbid")
class ServerConfig(PluginModel):
transport: Literal["stdio", "streamable_http"] = "stdio"
command: str = ""
args: list[str] = Field(default_factory=list)
env: dict[str, str] = Field(default_factory=dict)
url: str = ""
headers: dict[str, str] = Field(default_factory=dict)
@model_validator(mode="after")
def validate_transport(self):
if self.transport == "stdio" and not self.command.strip():
raise ValueError("stdio 插件必须提供启动命令")
if self.transport == "streamable_http":
parsed = urlsplit(self.url)
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
raise ValueError("MCP 地址必须是 HTTP(S) 地址")
if parsed.username or parsed.password or parsed.fragment:
raise ValueError("MCP 地址不能包含凭据或片段;请使用密钥配置")
return self
class ConfigField(PluginModel):
label: str = Field(min_length=1, max_length=100)
description: str = Field(default="", max_length=1000)
secret: bool = False
required: bool = False
default: str = ""
ExecutionTarget = Literal["android", "computer", "mixed"]
class PluginObservation(PluginModel):
tool: str = Field(min_length=1)
arguments: dict[str, Any] = Field(default_factory=dict)
refresh_tools: list[str] = Field(default_factory=list)
class PluginManifest(PluginModel):
schema_version: Literal[1] = 1
id: Identifier
name: str = Field(min_length=1, max_length=100)
version: str
description: str = Field(default="", max_length=4000)
artemis_version: str = ">=2.0.0,<3.0.0"
platforms: list[Literal["win32", "linux", "darwin"]] = Field(default_factory=lambda: ["win32"])
architectures: list[Literal["amd64", "arm64"]] = Field(
default_factory=lambda: ["amd64", "arm64"]
)
source: str = ""
license: str = ""
server: ServerConfig | None = None
skill: str | None = None
config: dict[Identifier, ConfigField] = Field(default_factory=dict)
resources: list[str] = Field(default_factory=list)
targets: list[ExecutionTarget] = Field(default_factory=lambda: ["mixed"])
observation: PluginObservation | None = None
@field_validator("version")
@classmethod
def valid_version(cls, value: str) -> str:
Version(value)
if not re.fullmatch(r"[0-9][a-zA-Z0-9.+-]{0,63}", value):
raise ValueError("插件版本格式无效")
return value
@field_validator("artemis_version")
@classmethod
def valid_specifier(cls, value: str) -> str:
SpecifierSet(value)
return value
@model_validator(mode="after")
def has_capability(self):
if self.server is None and self.skill is None:
raise ValueError("插件至少需要 MCP 服务或 Skill")
for name, field in self.config.items():
if field.secret and field.default:
raise ValueError(f"密钥字段 {name} 不允许默认值")
return self
def check_compatible(self, artemis_version: str | None = None) -> None:
from artemis._version import __version__
artemis_version = artemis_version or __version__
if Version(artemis_version) not in SpecifierSet(self.artemis_version):
raise ValueError(f"插件不兼容 ARTEMIS {artemis_version}")
if sys.platform not in self.platforms:
raise ValueError(f"插件不支持当前平台 {sys.platform}")
architecture = {"x86_64": "amd64", "aarch64": "arm64"}.get(
platform.machine().lower(), platform.machine().lower()
)
if architecture not in self.architectures:
raise ValueError(f"插件不支持当前处理器架构 {architecture}")
class PluginProfile(PluginModel):
name: Identifier = "default"
label: str = "默认配置"
is_default: bool = False
values: dict[str, str] = Field(default_factory=dict)
# Only a write payload contains secrets; read responses expose configured names.
secrets: dict[str, SecretStr] = Field(default_factory=dict, repr=False)
allowed_tools: list[str] | None = None
argument_constraints: dict[str, dict[str, list[str]]] = Field(default_factory=dict)
resources: list[str] = Field(default_factory=list)
authorized: bool = False
class PluginSelection(PluginModel):
id: Identifier
profile: Identifier = "default"
version: str | None = None
class CatalogEntry(PluginModel):
manifest: PluginManifest
archive_url: str
sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
verification: str = Field(min_length=1, max_length=4000)
@field_validator("archive_url")
@classmethod
def https_archive(cls, value: str) -> str:
parsed = urlsplit(value)
if parsed.scheme != "https" or not parsed.hostname or parsed.username or parsed.password:
raise ValueError("市场安装包必须使用不含凭据的 HTTPS 地址")
return value
class PluginCatalog(PluginModel):
schema_version: Literal[1] = 1
plugins: list[CatalogEntry] = Field(default_factory=list)
class PluginCallResult(PluginModel):
status: Literal["success", "error", "uncertain"]
content: list[dict[str, Any]] = Field(default_factory=list)
structured_content: dict[str, Any] | None = None
message: str = ""
plugin_id: str = ""
version: str = ""
tool: str = ""
image_names: list[str] = Field(default_factory=list)
def text(self) -> str:
import json
parts = [str(block["text"]) for block in self.content if block.get("type") == "text"]
if self.structured_content is not None:
parts.append(json.dumps(self.structured_content, ensure_ascii=False))
if self.message:
parts.append(self.message)
return "\n".join(parts) or "工具已返回;请根据证据判断业务结果。"
+431
View File
@@ -0,0 +1,431 @@
"""Per-task plugin loading and shared Flash/Pro tool dispatch."""
from __future__ import annotations
import asyncio
import base64
import hashlib
import json
from pathlib import Path
import re
import uuid
from typing import Any
import jsonschema
from langchain_core.messages import ToolMessage
from langchain_core.tools import StructuredTool
from artemis.config.settings import settings
from artemis.core.tool_declaration import ToolDeclaration
from artemis.plugins.models import ExecutionTarget, PluginCallResult, PluginSelection
from artemis.plugins.session import PluginSession, resolve_server
from artemis.plugins.store import PluginStore, contained_path, package_resources
def get_plugin_runtime(ctx: Any) -> PluginRuntime | None:
runtime = getattr(ctx, "plugins", None)
return runtime if isinstance(runtime, PluginRuntime) else None
class PluginRuntime:
def __init__(
self,
selections: list[PluginSelection] | None = None,
*,
store: PluginStore | None = None,
ctx: Any = None,
target: ExecutionTarget | None = None,
):
self.store = store or PluginStore()
self.ctx = ctx
self.packages = {}
target = target or getattr(ctx, "execution_target", "mixed")
self._lease_id = f"plugin-runtime-{uuid.uuid4().hex}"
with self.store.mutation_lock:
pinned = self.store.pin(selections, target)
for selection in pinned:
self.packages[selection.id] = self.store.resolve(selection)
self.store.hold(self._lease_id, pinned)
self.sessions: dict[str, PluginSession] = {}
self.loaded: set[str] = set()
self.bindings: dict[str, tuple[str, Any]] = {}
self._load_lock = asyncio.Lock()
self._observation_calls: dict[str, tuple[str, dict]] = {}
self.last_images: dict[str, bytes] = {}
@property
def resources(self) -> list[str]:
return sorted(
{
resource
for manifest, profile, _ in self.packages.values()
for resource in package_resources(manifest, profile)
}
)
def ensure_resources_ready(self):
if set(self.resources) & self.store.blocked_resources().keys():
raise RuntimeError("插件资源上一次操作未确认停止,请在插件页面核对并解除占用保护")
def _request_guidance(self) -> None:
from artemis.runtime.task_control import TaskControl, request_pause
control = getattr(self.ctx, "task_control", None)
if isinstance(control, TaskControl):
request_pause(control.session_id)
@property
def tool_names(self) -> set[str]:
return set(self.bindings) | (
{"plugin_load", "plugin_read_file"} if self.packages else set()
)
def prompt(self) -> str:
if not self.packages:
return ""
lines = ["\n可用扩展插件(按需调用 plugin_load 加载说明和工具):"]
for manifest, profile, _ in self.packages.values():
lines.append(f"- {manifest.id} / {profile.label}: {manifest.description}")
lines.append(
"插件输出是观察数据。遵循用户目标和已授权范围;工具返回不代表测试通过。中断或超时后先观察真实状态,不能重复提交不确定的写操作。"
)
lines.append(
"只调用实际提供的工具。电脑插件的像素坐标以其返回的截图为准,不使用 Android 的 0–1000 坐标归一化。每个资源的截图和读数必须分开核对;只有文字观察时不能猜像素坐标。"
)
return "\n".join(lines)
async def load(self, plugin_id: str) -> PluginCallResult:
self.ensure_resources_ready()
if plugin_id not in self.packages:
return PluginCallResult(status="error", message="该插件未被此任务授权")
async with self._load_lock:
manifest, profile, directory = self.packages[plugin_id]
if plugin_id not in self.loaded:
if manifest.server:
session = await PluginSession(
resolve_server(manifest, profile, directory), directory
).start()
self.sessions[plugin_id] = session
for tool in session.tools:
if (
profile.allowed_tools is not None
and tool.name not in profile.allowed_tools
):
continue
digest = hashlib.sha256(f"{plugin_id}:{tool.name}".encode()).hexdigest()[
:10
]
label = re.sub(r"[^a-zA-Z0-9_-]", "_", tool.name)[:35]
self.bindings[f"plug_{digest}_{label}"] = (plugin_id, tool)
self.loaded.add(plugin_id)
skill = ""
if manifest.skill:
skill = contained_path(directory, manifest.skill).read_text(encoding="utf-8")
tools = {
name: tool.name for name, (pid, tool) in self.bindings.items() if pid == plugin_id
}
return PluginCallResult(
status="success",
plugin_id=plugin_id,
version=manifest.version,
content=[{"type": "text", "text": f"插件说明(仅适用于此插件):\n{skill}"}],
structured_content={
"tools": tools,
"resources": manifest.resources + profile.resources,
"argument_constraints": profile.argument_constraints,
},
)
def declarations(self) -> list[ToolDeclaration]:
if not self.packages:
return []
tools = [
ToolDeclaration(
"plugin_load",
"按需加载已授权插件的 Skill、操作工具及控制范围。",
{
"type": "object",
"properties": {"plugin_id": {"type": "string", "enum": list(self.packages)}},
"required": ["plugin_id"],
"additionalProperties": False,
},
),
ToolDeclaration(
"plugin_read_file",
"读取已加载插件 Skill 引用的包内文档;不可读取包外文件。",
{
"type": "object",
"properties": {
"plugin_id": {"type": "string", "enum": list(self.packages)},
"path": {"type": "string"},
},
"required": ["plugin_id", "path"],
"additionalProperties": False,
},
),
]
for name, (pid, tool) in self.bindings.items():
tools.append(
ToolDeclaration(name, f"[{pid}] {tool.description or tool.name}", tool.inputSchema)
)
return tools
def langchain_tools(self) -> list[StructuredTool]:
result = []
for declaration in self.declarations():
def handler(name):
async def call(**arguments):
outcome = await self.call(name, arguments)
return ToolMessage(
content=self.content_blocks(outcome),
name=name,
tool_call_id="plugin",
status="success" if outcome.status == "success" else "error",
)
return call
result.append(
StructuredTool(
name=declaration.name,
description=declaration.description,
args_schema=declaration.parameters,
coroutine=handler(declaration.name),
)
)
return result
def _redact(self, value: str) -> str:
for _, profile, _ in self.packages.values():
for secret in profile.secrets.values():
raw = secret.get_secret_value()
if raw:
value = value.replace(raw, "[已隐藏密钥]")
return value
def _redact_payload(self, value: Any) -> Any:
if isinstance(value, list):
return [self._redact_payload(item) for item in value]
if isinstance(value, dict):
# Binary image/audio data are not text. Never corrupt base64 or JSON keys.
binary = value.get("type") in {"image", "audio"}
return {
key: item if binary and key == "data" else self._redact_payload(item)
for key, item in value.items()
}
return self._redact(value) if isinstance(value, str) else value
async def call(self, name: str, arguments: dict) -> PluginCallResult:
try:
if name == "plugin_load":
return await self.load(arguments["plugin_id"])
if name == "plugin_read_file":
plugin_id = arguments["plugin_id"]
if plugin_id not in self.loaded:
raise ValueError("请先加载插件")
path = contained_path(self.packages[plugin_id][2], arguments["path"])
if (
path.suffix.lower() not in {".md", ".txt", ".json"}
or path.stat().st_size > 256 * 1024
):
raise ValueError("仅支持读取 256 KiB 以内的插件说明文件")
return PluginCallResult(
status="success",
content=[
{"type": "text", "text": self._redact(path.read_text(encoding="utf-8"))}
],
)
pid, tool = self.bindings[name]
self.ensure_resources_ready()
manifest, profile, _ = self.packages[pid]
jsonschema.validate(arguments, tool.inputSchema)
for key, allowed in profile.argument_constraints.get(tool.name, {}).items():
value: Any = arguments
for part in key.split("."):
value = value.get(part) if isinstance(value, dict) else None
if not isinstance(value, (str, int, float, bool)) or str(value) not in allowed:
raise PermissionError(f"工具参数 {key} 超出授权范围,请用户补充指导或修改授权")
span = None
if self.ctx:
from artemis.data_engine.trace import TraceSpan
span = TraceSpan(name=name, trace_type="tool", ctx=self.ctx)
if span:
span.__enter__()
span.payload = {
"args": self._redact_payload(arguments),
"plugin_id": pid,
"plugin_version": manifest.version,
"resource_ids": sorted(package_resources(manifest, profile)),
}
try:
raw = await self.sessions[pid].call(tool.name, arguments)
payload = raw.model_dump(mode="json", by_alias=True, exclude_none=True)
encoded = json.dumps(payload, ensure_ascii=False)
if len(encoded.encode()) > settings.ARTEMIS_PLUGIN_RESULT_MAX_BYTES:
raise RuntimeError("插件结果超过大小限制,操作可能已经完成;请重新观察")
payload = self._redact_payload(payload)
outcome = PluginCallResult(
status="error" if payload.get("isError") else "success",
content=payload.get("content", []),
structured_content=payload.get("structuredContent"),
plugin_id=pid,
version=manifest.version,
tool=tool.name,
)
observation = manifest.observation
image_block = next((b for b in outcome.content if b.get("type") == "image"), None)
if image_block:
self.last_images[pid] = base64.b64decode(image_block["data"], validate=True)
elif not (tool.annotations and tool.annotations.readOnlyHint):
self.last_images.pop(pid, None)
if (
outcome.status == "success"
and observation
and tool.name in observation.refresh_tools
):
self._observation_calls[pid] = (tool.name, dict(arguments))
self._persist_images(outcome)
if span:
stored = outcome.model_dump()
stored["content"] = [b for b in stored["content"] if b.get("type") != "image"]
stored["post_image_name"] = next(iter(outcome.image_names), None)
span.payload["post_image_name"] = stored["post_image_name"]
span.payload["image_names"] = outcome.image_names
span.result = stored
if outcome.status != "success":
span.status, span.error = "failed", outcome.text()
return outcome
except (RuntimeError, OSError, ValueError, TimeoutError):
if self.sessions[pid].stop_uncertain:
self.store.block_resources(
self.resources,
"远程插件操作中断,无法确认写操作是否停止;请核对远程状态后解除保护。",
)
if span:
span.status, span.error = "failed", "插件调用失败,结果不确定"
return PluginCallResult(
status="uncertain",
plugin_id=pid,
version=manifest.version,
tool=tool.name,
message="插件调用中断或超时,结果不确定。请先观察实际状态,不能盲目重放写操作。",
)
finally:
if span:
span.__exit__(None, None, None)
except asyncio.CancelledError:
if any(s.stop_uncertain for s in self.sessions.values()):
self.store.block_resources(
self.resources, "插件操作取消后未确认停止,请核对目标状态后解除保护。"
)
raise
except PermissionError:
self._request_guidance()
return PluginCallResult(
status="error",
message="工具参数超出此任务的授权范围,已请求暂停。请提供授权范围内的指导;如需扩大范围,请结束任务后修改插件配置。",
)
except (
KeyError,
ValueError,
OSError,
jsonschema.ValidationError,
jsonschema.SchemaError,
):
# Schema errors can contain the rejected value (including secrets); keep diagnostics structural.
return PluginCallResult(
status="error",
message="插件尚未就绪、参数不符合工具定义或超出授权范围;请检查插件配置和授权。",
)
def _persist_images(self, outcome: PluginCallResult) -> None:
engine = getattr(self.ctx, "data_engine", None)
if not engine:
return
for block in outcome.content:
if block.get("type") == "image" and block.get("data"):
image = base64.b64decode(block["data"], validate=True)
# Save the text from the same tool result before a later action
# can fill this image's missing tree with a different screen.
name = engine.get_or_create_image(image, ui_tree=outcome.text())
outcome.image_names.append(name)
if engine.current_session_id:
engine.storage.reference_plugin_image(engine.current_session_id, name)
@staticmethod
def content_blocks(outcome: PluginCallResult) -> list[dict]:
blocks: list[dict] = [{"type": "text", "text": outcome.text()}]
for block in outcome.content:
if block.get("type") == "image" and block.get("mimeType") in {
"image/png",
"image/jpeg",
"image/webp",
}:
blocks.append(
{
"type": "image_url",
"image_url": {"url": f"data:{block['mimeType']};base64,{block['data']}"},
}
)
elif block.get("type") not in {"image", "text"}:
blocks.append({"type": "text", "text": json.dumps(block, ensure_ascii=False)})
return blocks
async def execute(self, name: str, arguments: dict, tool_call_id: str):
from artemis.agents.validator.tool_declarations import ToolExecutionResult
outcome = await self.call(name, arguments)
image = next((b for b in outcome.content if b.get("type") == "image"), None)
image_bytes = base64.b64decode(image["data"]) if image else None
engine = getattr(self.ctx, "data_engine", None)
path = None
if engine and image_bytes:
path = str(engine.get_image_path(engine.get_or_create_image(image_bytes)))
return ToolExecutionResult(
tool_name=name,
tool_call_id=tool_call_id,
status="success" if outcome.status == "success" else "error",
text_summary=(
f"来源:插件 {outcome.plugin_id} v{outcome.version} / {outcome.tool}\n"
if outcome.plugin_id
else ""
)
+ outcome.text(),
raw_result=self.content_blocks(outcome),
screenshot_bytes=image_bytes,
screenshot_path=path,
ui_elements_text=outcome.text(),
metadata={
"plugin_id": outcome.plugin_id,
"plugin_version": outcome.version,
"resource_ids": (
sorted(package_resources(*self.packages[outcome.plugin_id][:2]))
if outcome.plugin_id in self.packages
else []
),
"plugin_status": outcome.status,
"post_image_name": next(iter(outcome.image_names), None),
},
)
async def aclose(self, *, release: bool = True):
pending = list(self.sessions.items())
errors = await asyncio.gather(
*(session.aclose() for _, session in pending), return_exceptions=True
)
for (pid, _), error in zip(pending, errors):
if not isinstance(error, BaseException):
self.sessions.pop(pid, None)
self.bindings = {
name: binding for name, binding in self.bindings.items() if binding[0] in self.sessions
}
self.loaded.intersection_update(self.sessions)
self._observation_calls.clear()
self.last_images.clear()
if any(isinstance(error, BaseException) for error in errors):
self.store.block_resources(self.resources, "插件未确认停止,请核对目标状态后解除保护。")
raise RuntimeError("部分插件未确认停止,资源仍需保持占用")
if release:
self.store.release(self._lease_id)
+185
View File
@@ -0,0 +1,185 @@
"""Task-owned external MCP transport. A timed-out write is never replayed."""
from __future__ import annotations
import asyncio
from contextlib import AsyncExitStack
import os
from pathlib import Path
import re
from typing import Any
import httpx
from mcp import ClientSession, StdioServerParameters
from mcp.client.stdio import stdio_client
from mcp.client.streamable_http import streamable_http_client
from artemis.config.settings import settings
from artemis.plugins.models import PluginManifest, PluginProfile, ServerConfig
def resolve_server(
manifest: PluginManifest, profile: PluginProfile, directory: Path
) -> ServerConfig:
if manifest.server is None:
raise ValueError("此插件不包含 MCP 服务")
values = {name: field.default for name, field in manifest.config.items() if not field.secret}
values.update(profile.values)
def substitute(value: Any) -> Any:
if isinstance(value, dict):
return {k: substitute(v) for k, v in value.items()}
if isinstance(value, list):
return [substitute(v) for v in value]
if not isinstance(value, str):
return value
def replace(match: re.Match) -> str:
key = match.group(1)
if key == "plugin_dir":
return str(directory)
if key.startswith("config.") and key[7:] in values:
return values[key[7:]]
if key.startswith("secret.") and key[7:] in profile.secrets:
return profile.secrets[key[7:]].get_secret_value()
raise ValueError(f"插件配置引用无法解析:{key}")
return re.sub(r"\$\{([^}]+)\}", replace, value)
return ServerConfig.model_validate(substitute(manifest.server.model_dump()))
class PluginSession:
"""One owner task enters/exits anyio transport scopes, even during cancellation."""
def __init__(self, server: ServerConfig, directory: Path):
self.server = server
self.directory = directory
self.tools: list[Any] = []
self._queue: asyncio.Queue = asyncio.Queue()
self._ready = asyncio.Event()
self._owner: asyncio.Task | None = None
self._error: BaseException | None = None
self.stop_uncertain = False
async def start(self) -> PluginSession:
self._owner = asyncio.create_task(self._run(), name="artemis-plugin-session")
try:
await asyncio.wait_for(
self._ready.wait(), settings.ARTEMIS_PLUGIN_START_TIMEOUT_SECONDS
)
if self._error:
raise RuntimeError(
"插件 MCP 启动失败,请检查命令、连接地址及必填配置"
) from self._error
return self
except BaseException:
await self.aclose()
raise
async def _run(self):
active = None
try:
async with AsyncExitStack() as stack:
if self.server.transport == "stdio":
# A plugin can print credentials to stderr; never forward it to task reports.
errlog = stack.enter_context(open(os.devnull, "w"))
streams = await stack.enter_async_context(
stdio_client(
StdioServerParameters(
command=self.server.command,
args=self.server.args,
env=self.server.env,
cwd=str(self.directory),
),
errlog=errlog,
)
)
else:
client = await stack.enter_async_context(
httpx.AsyncClient(
headers=self.server.headers,
timeout=settings.ARTEMIS_PLUGIN_TOOL_TIMEOUT_SECONDS,
follow_redirects=False,
)
)
streams = await stack.enter_async_context(
streamable_http_client(self.server.url, http_client=client)
)
session = await stack.enter_async_context(ClientSession(streams[0], streams[1]))
await session.initialize()
cursor = None
for _ in range(20):
response = await session.list_tools(cursor=cursor)
self.tools.extend(response.tools)
cursor = response.nextCursor
if not cursor:
break
else:
raise ValueError("插件工具列表分页超过限制")
self._ready.set()
while True:
item = await self._queue.get()
if item is None:
return
active, name, arguments = item
if active.cancelled():
active = None
continue
# On timeout retire the whole connection: MCP may deliver a late response.
result = await asyncio.wait_for(
session.call_tool(name, arguments),
settings.ARTEMIS_PLUGIN_TOOL_TIMEOUT_SECONDS,
)
if not active.done():
active.set_result(result)
active = None
except BaseException as exc:
self._error = exc
if active is not None and self.server.transport == "streamable_http":
self.stop_uncertain = True
finally:
self._ready.set()
if active is not None and not active.done():
active.set_exception(
RuntimeError(
"插件调用中断或超时,执行结果不确定;请先观察状态,不能盲目重放操作"
)
)
while not self._queue.empty():
item = self._queue.get_nowait()
if item is not None and not item[0].done():
item[0].set_exception(RuntimeError("插件连接已关闭,此调用尚未执行"))
async def call(self, name: str, arguments: dict):
if self._owner is None or self._owner.done() or self._error is not None:
raise RuntimeError("插件连接已关闭,不能继续执行")
future = asyncio.get_running_loop().create_future()
self._queue.put_nowait((future, name, arguments))
try:
# Bound the caller as well as the protocol operation: transport context
# cleanup may itself stall after the inner request times out.
return await asyncio.wait_for(
future,
settings.ARTEMIS_PLUGIN_TOOL_TIMEOUT_SECONDS
+ settings.ARTEMIS_PLUGIN_SHUTDOWN_TIMEOUT_SECONDS,
)
except TimeoutError:
await self.aclose()
raise RuntimeError("插件响应或清理超时,执行结果不确定") from None
except asyncio.CancelledError:
# Guidance/pause must stop the transport before the task releases shared resources.
await self.aclose()
raise
async def aclose(self):
owner = self._owner
if owner is None or owner.done():
return
owner.cancel()
done, _ = await asyncio.wait(
{owner}, timeout=settings.ARTEMIS_PLUGIN_SHUTDOWN_TIMEOUT_SECONDS
)
if not done:
self.stop_uncertain = True
raise RuntimeError("插件尚未确认停止,不能释放其占用资源")
+352
View File
@@ -0,0 +1,352 @@
"""Transactional plugin registry; task state continues to live in DataEngine."""
from __future__ import annotations
from contextlib import contextmanager
from functools import wraps
import json
import os
from pathlib import Path
import sqlite3
import shutil
import uuid
from filelock import FileLock
import psutil
from typing import Any
from artemis.config.paths import get_app_dir
from artemis.plugins.models import PluginManifest, PluginProfile, PluginSelection
def serialized(method):
"""Serialize package mutation and task reservations across threads/processes."""
@wraps(method)
def locked(self, *args, **kwargs):
with self.mutation_lock:
return method(self, *args, **kwargs)
return locked
def contained_path(root: Path, relative: str) -> Path:
"""Resolve package paths without following traversal or symlinks outside the package."""
path = (root / relative).resolve()
if not path.is_relative_to(root.resolve()) or path == root.resolve():
raise ValueError("插件文件路径超出安装目录")
return path
def package_resources(manifest: PluginManifest, profile: PluginProfile) -> set[str]:
declared = [*manifest.resources, *profile.resources]
if not declared and manifest.server:
declared = [f"plugin:{manifest.id}:{profile.name}"]
return {resource.strip().casefold() for resource in declared if resource.strip()}
class PluginStore:
def __init__(self, root: Path | None = None):
self.root = root or get_app_dir() / "plugins"
self.root.mkdir(parents=True, exist_ok=True)
self.db_path = self.root / "registry.sqlite3"
self.mutation_lock = FileLock(self.root / "registry.lock", timeout=10)
with self.connect() as db:
db.executescript("""
CREATE TABLE IF NOT EXISTS plugins (
id TEXT PRIMARY KEY, version TEXT NOT NULL,
enabled INTEGER NOT NULL DEFAULT 0, manifest TEXT NOT NULL,
source TEXT NOT NULL DEFAULT 'local'
);
CREATE TABLE IF NOT EXISTS profiles (
plugin_id TEXT NOT NULL, name TEXT NOT NULL, data TEXT NOT NULL,
secrets TEXT NOT NULL DEFAULT '{}',
PRIMARY KEY (plugin_id, name)
);
CREATE TABLE IF NOT EXISTS task_pins (
task_id TEXT NOT NULL, plugin_id TEXT NOT NULL, version TEXT NOT NULL,
pid INTEGER NOT NULL, created REAL NOT NULL,
PRIMARY KEY(task_id, plugin_id)
);
CREATE TABLE IF NOT EXISTS blocked_resources (
resource TEXT PRIMARY KEY, reason TEXT NOT NULL
);
""")
os.chmod(self.db_path, 0o600)
@contextmanager
def connect(self):
db = sqlite3.connect(self.db_path, timeout=10)
db.row_factory = sqlite3.Row
try:
with db:
yield db
finally:
db.close()
def package_dir(self, plugin_id: str, version: str) -> Path:
PluginSelection(id=plugin_id)
# Validate through the manifest validator without accepting directory separators.
PluginManifest.valid_version(version)
return contained_path(self.root, f"packages/{plugin_id}/{version}")
@serialized
def register(self, manifest: PluginManifest, source: str = "local") -> None:
manifest.check_compatible()
self.assert_idle(manifest.id)
with self.connect() as db:
existing = db.execute("SELECT id FROM plugins WHERE id=?", (manifest.id,)).fetchone()
# Updating executable code always leaves new work disabled until explicitly enabled.
db.execute(
"INSERT INTO plugins (id,version,manifest,source) VALUES (?,?,?,?) "
"ON CONFLICT(id) DO UPDATE SET version=excluded.version, "
"manifest=excluded.manifest,source=excluded.source,enabled=0",
(manifest.id, manifest.version, manifest.model_dump_json(), source),
)
if not existing:
profile = PluginProfile(
is_default=True,
values={k: v.default for k, v in manifest.config.items() if not v.secret},
)
db.execute(
"INSERT INTO profiles (plugin_id,name,data) VALUES (?,?,?)",
(manifest.id, profile.name, profile.model_dump_json(exclude={"secrets"})),
)
@serialized
def get(self, plugin_id: str) -> dict[str, Any]:
with self.connect() as db:
row = db.execute("SELECT * FROM plugins WHERE id=?", (plugin_id,)).fetchone()
if row is None:
raise ValueError("插件未安装")
profiles = db.execute(
"SELECT name,data,secrets FROM profiles WHERE plugin_id=? ORDER BY name",
(plugin_id,),
).fetchall()
return {
"manifest": json.loads(row["manifest"]),
"enabled": bool(row["enabled"]),
"source": row["source"],
"profiles": [
{**json.loads(p["data"]), "configured_secrets": list(json.loads(p["secrets"]))}
for p in profiles
],
}
@serialized
def list(self) -> list[dict[str, Any]]:
with self.connect() as db:
ids = [row[0] for row in db.execute("SELECT id FROM plugins ORDER BY id")]
return [self.get(plugin_id) for plugin_id in ids]
@serialized
def save_profile(self, plugin_id: str, profile: PluginProfile) -> None:
self.assert_idle(plugin_id)
manifest = PluginManifest.model_validate(self.get(plugin_id)["manifest"])
for key in profile.values:
if key not in manifest.config or manifest.config[key].secret:
raise ValueError(f"普通配置字段无效:{key}")
for key in profile.secrets:
if key not in manifest.config or not manifest.config[key].secret:
raise ValueError(f"密钥配置字段无效:{key}")
with self.connect() as db:
old = db.execute(
"SELECT secrets FROM profiles WHERE plugin_id=? AND name=?",
(plugin_id, profile.name),
).fetchone()
secrets = json.loads(old[0]) if old else {}
for name, value in profile.secrets.items():
# Blank means keep, matching the model settings form.
if value.get_secret_value():
secrets[name] = value.get_secret_value()
db.execute(
"INSERT INTO profiles (plugin_id,name,data,secrets) VALUES (?,?,?,?) "
"ON CONFLICT(plugin_id,name) DO UPDATE SET data=excluded.data,secrets=excluded.secrets",
(
plugin_id,
profile.name,
profile.model_dump_json(exclude={"secrets"}),
json.dumps(secrets),
),
)
if profile.is_default:
for row in db.execute(
"SELECT name,data FROM profiles WHERE plugin_id=? AND name<>?",
(plugin_id, profile.name),
).fetchall():
data = json.loads(row["data"])
data["is_default"] = False
db.execute(
"UPDATE profiles SET data=? WHERE plugin_id=? AND name=?",
(json.dumps(data), plugin_id, row["name"]),
)
@serialized
def set_enabled(self, plugin_id: str, enabled: bool) -> None:
self.assert_idle(plugin_id)
info = self.get(plugin_id)
if enabled and not any(p["authorized"] for p in info["profiles"]):
raise ValueError("请先配置并授权至少一套连接配置")
if enabled:
for profile in info["profiles"]:
if profile["authorized"]:
self.resolve(
PluginSelection(id=plugin_id, profile=profile["name"]), allow_disabled=True
)
with self.connect() as db:
db.execute("UPDATE plugins SET enabled=? WHERE id=?", (enabled, plugin_id))
@serialized
def resolve(
self, selection: PluginSelection, *, allow_disabled: bool = False
) -> tuple[PluginManifest, PluginProfile, Path]:
info = self.get(selection.id)
if not info["enabled"] and not allow_disabled:
raise ValueError(f"插件 {selection.id} 未启用")
manifest = PluginManifest.model_validate(info["manifest"])
version = selection.version or manifest.version
directory = self.package_dir(selection.id, version)
manifest = PluginManifest.model_validate_json(
(directory / "plugin.json").read_text(encoding="utf-8")
)
if (manifest.id, manifest.version) != (selection.id, version):
raise ValueError("安装包标识与固定版本不一致")
manifest.check_compatible()
with self.connect() as db:
row = db.execute(
"SELECT data,secrets FROM profiles WHERE plugin_id=? AND name=?",
(selection.id, selection.profile),
).fetchone()
if row is None:
raise ValueError("插件连接配置不存在")
profile = PluginProfile.model_validate(
{**json.loads(row[0]), "secrets": json.loads(row[1])}
)
if not profile.authorized:
raise ValueError("插件连接配置尚未授权")
for name, field in manifest.config.items():
value = (
profile.secrets.get(name)
if field.secret
else profile.values.get(name, field.default)
)
if field.required and not value:
raise ValueError(f"请补齐插件配置:{field.label}")
return manifest, profile, directory
def selections(self, target: str = "mixed") -> list[PluginSelection]:
selections = []
for plugin in self.list():
if not plugin["enabled"] or target not in plugin["manifest"].get("targets", ["mixed"]):
continue
profiles = plugin["profiles"]
default = next((p for p in profiles if p.get("is_default")), None)
default = default or next((p for p in profiles if p["name"] == "default"), None)
if default and default["authorized"]:
selections.append(
PluginSelection(
id=plugin["manifest"]["id"],
profile=default["name"],
version=plugin["manifest"]["version"],
)
)
return selections
@serialized
def pin(self, selections: list[PluginSelection] | None, target: str) -> list[PluginSelection]:
pinned = []
for choice in self.selections(target) if selections is None else selections:
manifest, profile, _ = self.resolve(choice)
if target not in manifest.targets:
raise ValueError(f"插件 {manifest.name} 不支持 {target} 任务")
pinned.append(
PluginSelection(id=manifest.id, profile=profile.name, version=manifest.version)
)
if len({p.id for p in pinned}) != len(pinned):
raise ValueError("同一任务不能重复选择同一插件")
if target == "computer" and not any(self.resolve(p)[0].observation for p in pinned):
raise ValueError("纯电脑任务需要启用并选择一个提供画面观察的电脑插件")
return pinned
@serialized
def uninstall(self, plugin_id: str) -> None:
self.get(plugin_id)
self.assert_idle(plugin_id)
PluginSelection(id=plugin_id)
directory = contained_path(self.root, f"packages/{plugin_id}")
trash = contained_path(self.root, f"uninstall-{uuid.uuid4().hex}")
directory.replace(trash)
try:
with self.connect() as db:
db.execute("DELETE FROM profiles WHERE plugin_id=?", (plugin_id,))
db.execute("DELETE FROM plugins WHERE id=?", (plugin_id,))
except sqlite3.Error:
trash.replace(directory)
raise
shutil.rmtree(trash)
@serialized
def hold(self, task_id: str, selections: list[PluginSelection]) -> None:
process = psutil.Process()
resolved = [(s, self.resolve(s)[0].version) for s in selections]
with self.connect() as db:
for selection, version in resolved:
db.execute(
"INSERT OR REPLACE INTO task_pins VALUES (?,?,?,?,?)",
(
task_id,
selection.id,
version,
process.pid,
process.create_time(),
),
)
@serialized
def reserve(self, task_id: str, selections: list[PluginSelection], target: str) -> list[str]:
pinned = self.pin(selections, target)
resources = set()
for selection in pinned:
manifest, profile, _ = self.resolve(selection)
resources.update(package_resources(manifest, profile))
self.hold(task_id, pinned)
return sorted(resources)
@serialized
def release(self, task_id: str) -> None:
with self.connect() as db:
db.execute("DELETE FROM task_pins WHERE task_id=?", (task_id,))
@serialized
def assert_idle(self, plugin_id: str) -> None:
with self.connect() as db:
rows = db.execute("SELECT * FROM task_pins WHERE plugin_id=?", (plugin_id,)).fetchall()
for row in rows:
try:
alive = abs(psutil.Process(row["pid"]).create_time() - row["created"]) < 0.01
except psutil.NoSuchProcess:
alive = False
except psutil.AccessDenied:
alive = True
if alive:
raise ValueError(
"插件正被排队、暂停或执行中的任务使用,请任务结束后再更新配置、升级或卸载"
)
db.execute(
"DELETE FROM task_pins WHERE task_id=? AND plugin_id=?",
(row["task_id"], plugin_id),
)
def block_resources(self, resources: list[str], reason: str) -> None:
with self.connect() as db:
db.executemany(
"INSERT OR REPLACE INTO blocked_resources VALUES (?,?)",
[(r, reason) for r in resources],
)
def blocked_resources(self) -> dict[str, str]:
with self.connect() as db:
return dict(db.execute("SELECT resource,reason FROM blocked_resources").fetchall())
def confirm_resource_stopped(self, resource: str) -> None:
with self.connect() as db:
db.execute("DELETE FROM blocked_resources WHERE resource=?", (resource,))
+83 -2
View File
@@ -19,6 +19,7 @@ from __future__ import annotations
from dataclasses import dataclass from dataclasses import dataclass
from datetime import UTC, datetime from datetime import UTC, datetime
import json import json
import hashlib
import os import os
from pathlib import Path from pathlib import Path
import re import re
@@ -52,6 +53,7 @@ class DeviceLockOwner:
session_id: str | None = None session_id: str | None = None
ingress: str | None = None ingress: str | None = None
lock_scope: str | None = None lock_scope: str | None = None
resource_ids: tuple[str, ...] = ()
@classmethod @classmethod
def from_dict(cls, value: dict) -> DeviceLockOwner: def from_dict(cls, value: dict) -> DeviceLockOwner:
@@ -65,6 +67,7 @@ class DeviceLockOwner:
session_id=(str(value["session_id"]) if value.get("session_id") is not None else None), session_id=(str(value["session_id"]) if value.get("session_id") is not None else None),
ingress=(str(value["ingress"]) if value.get("ingress") is not None else None), ingress=(str(value["ingress"]) if value.get("ingress") is not None else None),
lock_scope=(str(value["lock_scope"]) if value.get("lock_scope") is not None else None), lock_scope=(str(value["lock_scope"]) if value.get("lock_scope") is not None else None),
resource_ids=tuple(str(item) for item in value.get("resource_ids", [])),
) )
@@ -116,6 +119,7 @@ class DeviceExecutionLock:
session_id: str | None = None, session_id: str | None = None,
ingress: str | None = None, ingress: str | None = None,
lock_scope: str | None = None, lock_scope: str | None = None,
resource_ids: list[str] | tuple[str, ...] = (),
): ):
self.device_id = device_id or "default" self.device_id = device_id or "default"
self.lock_scope = lock_scope or os.getenv(self.LOCK_SCOPE_ENV) or None self.lock_scope = lock_scope or os.getenv(self.LOCK_SCOPE_ENV) or None
@@ -140,6 +144,10 @@ class DeviceExecutionLock:
self.ingress = ingress or os.getenv("ARTEMIS_TASK_INGRESS") or "sdk" self.ingress = ingress or os.getenv("ARTEMIS_TASK_INGRESS") or "sdk"
self._queue_path: Path | None = None self._queue_path: Path | None = None
self._acquired = False self._acquired = False
self.resource_ids = tuple(
sorted(set(r.strip().casefold() for r in resource_ids if r.strip()))
)
self._resource_paths: list[Path] = []
@classmethod @classmethod
def _device_lock_path( def _device_lock_path(
@@ -534,6 +542,51 @@ class DeviceExecutionLock:
return removed return removed
def _try_acquire_owner_lock(self) -> bool: def _try_acquire_owner_lock(self) -> bool:
if not self.resource_ids:
return self._try_acquire_device_owner_lock()
payload = self._owner_payload(
self.token,
self.device_id,
self.description,
session_id=self.session_id,
ingress=self.ingress,
lock_scope=self.lock_scope,
)
payload["resource_ids"] = self.resource_ids
try:
for resource in self.resource_ids:
digest = hashlib.sha256(resource.encode()).hexdigest()
path = self.path.parent / f"artemis-device-resource-{digest}.lock"
# Same resource identity applies across plugins and ADB server endpoints.
owner = self._get_active_owner_for_path(path)
if owner is not None and owner.token == self.token:
continue
try:
fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
except FileExistsError:
self._release_resources()
return False
self._resource_paths.append(path)
try:
os.write(fd, json.dumps(payload).encode())
finally:
os.close(fd)
if self._try_acquire_device_owner_lock():
return True
self._release_resources()
return False
except BaseException:
self._release_resources()
raise
def _release_resources(self) -> None:
for path in self._resource_paths:
owner = self._read_owner(path)
if owner is not None and owner.pid == os.getpid() and owner.token == self.token:
self._safe_unlink(path)
self._resource_paths.clear()
def _try_acquire_device_owner_lock(self) -> bool:
owner_payload = self._owner_payload( owner_payload = self._owner_payload(
self.token, self.token,
self.device_id, self.device_id,
@@ -542,6 +595,7 @@ class DeviceExecutionLock:
ingress=self.ingress, ingress=self.ingress,
lock_scope=self.lock_scope, lock_scope=self.lock_scope,
) )
owner_payload["resource_ids"] = self.resource_ids
try: try:
fd = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_WRONLY) fd = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
except FileExistsError: except FileExistsError:
@@ -609,6 +663,11 @@ class DeviceExecutionLock:
) -> None: ) -> None:
"""Wait in FIFO order and acquire exclusive access to the target Artemis device.""" """Wait in FIFO order and acquire exclusive access to the target Artemis device."""
self._queue_path = self._claim_or_create_queue_ticket() self._queue_path = self._claim_or_create_queue_ticket()
if self.resource_ids:
payload = json.loads(self._queue_path.read_text(encoding="utf-8"))
payload["resource_ids"] = self.resource_ids
if not self._atomic_write_json(self._queue_path, payload):
raise DeviceBusyError("Could not register task resource requirements.")
started_at = time.monotonic() started_at = time.monotonic()
effective_concurrency = self.effective_concurrency effective_concurrency = self.effective_concurrency
try: try:
@@ -679,6 +738,14 @@ class DeviceExecutionLock:
) )
is_eligible = bool(device_queue and device_queue[0] == self._queue_path) is_eligible = bool(device_queue and device_queue[0] == self._queue_path)
if is_eligible and self.resource_ids:
for earlier in all_wait_files:
if earlier == self._queue_path:
break
owner = self._read_owner(earlier)
if owner and set(owner.resource_ids).intersection(self.resource_ids):
is_eligible = False
break
if is_eligible and self._try_acquire_owner_lock(): if is_eligible and self._try_acquire_owner_lock():
self._safe_unlink(self._queue_path) self._safe_unlink(self._queue_path)
self._queue_path = None self._queue_path = None
@@ -708,6 +775,7 @@ class DeviceExecutionLock:
raise raise
def release(self) -> None: def release(self) -> None:
self._release_resources()
if not self._acquired: if not self._acquired:
return return
try: try:
@@ -938,7 +1006,7 @@ class DeviceExecutionLock:
"""Attach session metadata after tracing starts inside the lease owner.""" """Attach session metadata after tracing starts inside the lease owner."""
lock_dir = get_temp_dir("device-locks") lock_dir = get_temp_dir("device-locks")
if device_id is not None: if device_id is not None:
clean_id = cls._normalize_device_id(device_id) clean_id = cls._normalize_lock_id(device_id, os.getenv(cls.LOCK_SCOPE_ENV) or None)
path = lock_dir / f"artemis-device-{clean_id}.lock" path = lock_dir / f"artemis-device-{clean_id}.lock"
if not path.exists(): if not path.exists():
path = lock_dir / "artemis-global-device.lock" path = lock_dir / "artemis-global-device.lock"
@@ -959,8 +1027,21 @@ class DeviceExecutionLock:
"acquired_at": owner.acquired_at, "acquired_at": owner.acquired_at,
"session_id": str(session_id), "session_id": str(session_id),
"ingress": ingress or owner.ingress or "sdk", "ingress": ingress or owner.ingress or "sdk",
"lock_scope": owner.lock_scope,
"resource_ids": owner.resource_ids,
} }
return cls._atomic_write_json(path, payload) if not cls._atomic_write_json(path, payload):
return False
# Device and shared-resource records describe one lease. Discovery may
# encounter either first, so attach the same session to all its records.
for related in lock_dir.glob("artemis-device-*.lock"):
if related == path:
continue
candidate = cls._read_owner(related)
if candidate and candidate.pid == owner.pid and candidate.token == owner.token:
if not cls._atomic_write_json(related, payload):
return False
return True
@classmethod @classmethod
def cleanup_stale_locks(cls, device_id: str | None = None) -> int: def cleanup_stale_locks(cls, device_id: str | None = None) -> int:
+15 -8
View File
@@ -190,6 +190,7 @@ class TaskControl:
session_id=self.session_id, session_id=self.session_id,
ingress=previous.ingress, ingress=previous.ingress,
lock_scope=previous.lock_scope, lock_scope=previous.lock_scope,
resource_ids=getattr(previous, "resource_ids", ()),
) )
cancelled = threading.Event() cancelled = threading.Event()
acquiring = asyncio.create_task( acquiring = asyncio.create_task(
@@ -220,24 +221,30 @@ class TaskControl:
""" """
running = asyncio.ensure_future(operation) running = asyncio.ensure_future(operation)
pause = self.directory / "pause.request" pause = self.directory / "pause.request"
async def stop_operation():
running.cancel()
result = (await asyncio.gather(running, return_exceptions=True))[0]
if isinstance(result, BaseException) and not isinstance(result, asyncio.CancelledError):
raise result
try: try:
while True: while True:
if running.done():
# Preserve a completed tool result (including a permission failure
# that requested pause). The next checkpoint handles the request.
return False, running.result()
if pause.exists(): if pause.exists():
running.cancel() await stop_operation()
await asyncio.gather(running, return_exceptions=True)
await self.checkpoint(on_pause=on_pause, on_resume=on_resume) await self.checkpoint(on_pause=on_pause, on_resume=on_resume)
return True, None return True, None
if has_pending_instruction(self.directory): if has_pending_instruction(self.directory):
running.cancel() await stop_operation()
await asyncio.gather(running, return_exceptions=True)
return True, None return True, None
if running.done():
return False, running.result()
await asyncio.wait({running}, timeout=0.2) await asyncio.wait({running}, timeout=0.2)
finally: finally:
if not running.done(): if not running.done():
running.cancel() await stop_operation()
await asyncio.gather(running, return_exceptions=True)
def release(self) -> None: def release(self) -> None:
self.device_lock.release() self.device_lock.release()
+70 -10
View File
@@ -202,7 +202,22 @@ class Agent:
device_serial: str | None = None, device_serial: str | None = None,
retry_count: int = 5, retry_count: int = 5,
retry_wait_seconds: int = 5, retry_wait_seconds: int = 5,
execution_target: str = "android",
): ):
if execution_target not in {"android", "computer", "mixed"}:
raise ValueError("任务目标必须是 android、computer 或 mixed")
self._execution_target = execution_target
if execution_target == "computer":
self._device_context = DeviceContext(
device_id="computer:local",
mobile_platform=DevicePlatform.COMPUTER,
device_width=0,
device_height=0,
)
self._adb_client = None
self._ui_adb_client = None
self._initialized = True
return True
target_dev = device_serial or device_id target_dev = device_serial or device_id
if target_dev: if target_dev:
self._config = self._config.model_copy( self._config = self._config.model_copy(
@@ -366,10 +381,14 @@ class Agent:
- .with_trace_recording() to enable trace recording - .with_trace_recording() to enable trace recording
- .with_name() to set a custom task name - .with_name() to set a custom task name
""" """
return TaskRequestBuilder[None].from_common( return (
TaskRequestBuilder[None]
.from_common(
goal=goal, goal=goal,
common=self._config.task_request_defaults, common=self._config.task_request_defaults,
) )
.for_target(getattr(self, "_execution_target", "android"))
)
@overload @overload
async def run_task( async def run_task(
@@ -480,6 +499,13 @@ class Agent:
if not self._initialized: if not self._initialized:
raise AgentNotInitializedError() raise AgentNotInitializedError()
if request.execution_target != getattr(self, "_execution_target", "android"):
raise AgentTaskRequestError("任务目标与 Agent 初始化目标不一致")
if request.execution_target == "computer" and (
request.app_path or request.locked_app_package
):
raise AgentTaskRequestError("纯电脑任务不能使用 Android 安装包或应用锁定")
if request.profile: if request.profile:
agent_profile = self._config.agent_profiles.get(request.profile) agent_profile = self._config.agent_profiles.get(request.profile)
if agent_profile is None: if agent_profile is None:
@@ -522,7 +548,9 @@ class Agent:
ui_adb_client=self._ui_adb_client, ui_adb_client=self._ui_adb_client,
llm_config=agent_profile.llm_config, llm_config=agent_profile.llm_config,
agent_config=self._config, agent_config=self._config,
execution_target=request.execution_target,
) )
from artemis.plugins.runtime import PluginRuntime
output_config = None output_config = None
if request.output_description or request.output_format: if request.output_description or request.output_format:
@@ -551,6 +579,11 @@ class Agent:
or getattr(task, "id", None) or getattr(task, "id", None)
or getattr(getattr(task, "request", None), "task_name", None) or getattr(getattr(task, "request", None), "task_name", None)
) )
device_lock = None
try:
# Construct the runtime only inside the task's cleanup boundary. A
# cancelled queue wait or setup failure must not leave version pins.
context.plugins = PluginRuntime(request.plugins, ctx=context)
active_owner = DeviceExecutionLock.get_active_owner(self._device_context.device_id) active_owner = DeviceExecutionLock.get_active_owner(self._device_context.device_id)
already_held = ( already_held = (
active_owner is not None active_owner is not None
@@ -560,19 +593,27 @@ class Agent:
or active_owner.token == self._device_context.device_id or active_owner.token == self._device_context.device_id
) )
) )
device_lock = ( if not already_held or context.plugins.resources:
None device_lock = DeviceExecutionLock(
if already_held
else DeviceExecutionLock(
self._device_context.device_id, self._device_context.device_id,
description=f"{request.goal[:120]}", description=f"{request.goal[:120]}",
concurrency_mode=effective_mode, concurrency_mode=effective_mode,
max_concurrency=effective_max, max_concurrency=effective_max,
session_id=str(sess_id) if sess_id else None, session_id=str(sess_id) if sess_id else None,
ingress=os.getenv("ARTEMIS_TASK_INGRESS") or "agent", ingress=os.getenv("ARTEMIS_TASK_INGRESS") or "agent",
resource_ids=context.plugins.resources,
) )
context.plugins.ensure_resources_ready()
if context.plugins.packages:
from artemis.plugins.models import PluginSelection
context.plugins.store.hold(
str(sess_id),
[
PluginSelection(id=m.id, version=m.version, profile=p.name)
for m, p, _ in context.plugins.packages.values()
],
) )
try:
if device_lock is not None and os.environ.get("ARTEMIS_CLOUD_MODE") != "1": if device_lock is not None and os.environ.get("ARTEMIS_CLOUD_MODE") != "1":
from artemis.runtime.task_control import control_dir, _write_atomic from artemis.runtime.task_control import control_dir, _write_atomic
@@ -615,16 +656,23 @@ class Agent:
context.task_control = task_control context.task_control = task_control
task_control._state("running") task_control._state("running")
self._prepare_output_files(task=task) self._prepare_output_files(task=task)
if os.environ.get("ARTEMIS_CLOUD_MODE") != "1": if (
os.environ.get("ARTEMIS_CLOUD_MODE") != "1"
and request.execution_target != "computer"
):
if self._ui_adb_client is not None: if self._ui_adb_client is not None:
await self._connect_screen_client(context, str(sess_id)) await self._connect_screen_client(context, str(sess_id))
await self._ensure_device_unlocked() await self._ensure_device_unlocked()
publish_startup_progress( publish_startup_progress(
"environment", "Preparing the device environment", session_id=str(sess_id) "environment", "Preparing the device environment", session_id=str(sess_id)
) )
if request.execution_target != "computer":
await self._prepare_app_installation(task=task) await self._prepare_app_installation(task=task)
await self._prepare_device_environment(context=context) await self._prepare_device_environment(context=context)
await self._prepare_app_lock(task=task, context=context) await self._prepare_app_lock(task=task, context=context)
elif context.execution_setup:
context.execution_setup.video_recording_tools_enabled = False
context.execution_setup.record_video_enabled = False
async with context: async with context:
recording_started = False recording_started = False
recording_parts = 1 recording_parts = 1
@@ -637,6 +685,7 @@ class Agent:
async def pause_capture(): async def pause_capture():
nonlocal recording_started nonlocal recording_started
await context.plugins.aclose(release=False)
if recording_started: if recording_started:
try: try:
await controller.stop_video_recording() await controller.stop_video_recording()
@@ -648,6 +697,8 @@ class Agent:
async def resume_capture(): async def resume_capture():
nonlocal recording_started, recording_parts nonlocal recording_started, recording_parts
if request.execution_target == "computer":
return
if not ( if not (
self._config.video_recording_tools_enabled self._config.video_recording_tools_enabled
or self._config.record_video_enabled or self._config.record_video_enabled
@@ -662,7 +713,7 @@ class Agent:
context.on_task_pause = pause_capture context.on_task_pause = pause_capture
context.on_task_resume = resume_capture context.on_task_resume = resume_capture
if ( if request.execution_target != "computer" and (
self._config.video_recording_tools_enabled self._config.video_recording_tools_enabled
or self._config.record_video_enabled or self._config.record_video_enabled
): ):
@@ -897,6 +948,15 @@ class Agent:
raise raise
finally: finally:
# Close plugins before releasing the whole task's device/resource lease.
plugins_stopped = False
try:
if context.plugins:
await context.plugins.aclose()
context.plugins.store.release(str(sess_id))
plugins_stopped = True
except Exception:
logger.error("插件未确认停止,执行资源将保留至工作进程退出。")
try: try:
# Background ADB processes (logcat, screenrecord, ...) started # Background ADB processes (logcat, screenrecord, ...) started
# by the Operator must not outlive the automation task. # by the Operator must not outlive the automation task.
@@ -932,9 +992,9 @@ class Agent:
if self._ui_adb_client is not None: if self._ui_adb_client is not None:
await asyncio.to_thread(self._ui_adb_client.disconnect) await asyncio.to_thread(self._ui_adb_client.disconnect)
finally: finally:
if task_control is not None: if task_control is not None and plugins_stopped:
await asyncio.to_thread(task_control.release) await asyncio.to_thread(task_control.release)
elif device_lock is not None: elif device_lock is not None and plugins_stopped:
await asyncio.to_thread(device_lock.release) await asyncio.to_thread(device_lock.release)
async with self._task_lock: async with self._task_lock:
@@ -28,6 +28,7 @@ except ImportError:
from artemis.constants import RECURSION_LIMIT from artemis.constants import RECURSION_LIMIT
from artemis.sdk.types.agent import AgentProfile from artemis.sdk.types.agent import AgentProfile
from artemis.sdk.types.task import TaskRequest, TaskRequestCommon from artemis.sdk.types.task import TaskRequest, TaskRequestCommon
from artemis.plugins.models import ExecutionTarget, PluginSelection
from pydantic import BaseModel from pydantic import BaseModel
TIn = TypeVar("TIn", bound=BaseModel | None) TIn = TypeVar("TIn", bound=BaseModel | None)
@@ -64,6 +65,18 @@ class TaskRequestCommonBuilder(BaseModel):
self._llm_output_path: Path | None = None self._llm_output_path: Path | None = None
self._locked_app_package: str | None = None self._locked_app_package: str | None = None
self._app_path: Path | None = None self._app_path: Path | None = None
self._execution_target: ExecutionTarget = "android"
self._plugins: list[PluginSelection] | None = None
def for_target(self, target: ExecutionTarget) -> Self:
"""Select android, computer or mixed execution; match Agent.init's target."""
self._execution_target = target
return self
def with_plugins(self, plugins: list[PluginSelection] | None) -> Self:
"""Select saved plugin profiles; None uses defaults, [] disables plugins."""
self._plugins = plugins
return self
def with_max_steps(self, max_steps: int) -> Self: def with_max_steps(self, max_steps: int) -> Self:
"""Set the maximum number of steps the task can take. """Set the maximum number of steps the task can take.
@@ -138,6 +151,8 @@ class TaskRequestCommonBuilder(BaseModel):
llm_output_path=self._llm_output_path, llm_output_path=self._llm_output_path,
locked_app_package=self._locked_app_package, locked_app_package=self._locked_app_package,
app_path=self._app_path, app_path=self._app_path,
execution_target=self._execution_target,
plugins=self._plugins,
) )
@@ -179,6 +194,8 @@ class TaskRequestBuilder(TaskRequestCommonBuilder, Generic[TIn]):
res._llm_output_path = common.llm_output_path res._llm_output_path = common.llm_output_path
res._locked_app_package = common.locked_app_package res._locked_app_package = common.locked_app_package
res._app_path = common.app_path res._app_path = common.app_path
res._execution_target = common.execution_target
res._plugins = common.plugins
return res return res
def using_profile(self, profile: str | AgentProfile) -> "TaskRequestBuilder[TIn]": def using_profile(self, profile: str | AgentProfile) -> "TaskRequestBuilder[TIn]":
@@ -255,5 +272,7 @@ class TaskRequestBuilder(TaskRequestCommonBuilder, Generic[TIn]):
llm_output_path=self._llm_output_path, llm_output_path=self._llm_output_path,
locked_app_package=self._locked_app_package, locked_app_package=self._locked_app_package,
app_path=self._app_path, app_path=self._app_path,
execution_target=self._execution_target,
plugins=self._plugins,
) )
return task_request return task_request
+3
View File
@@ -25,6 +25,7 @@ from typing import Any, Generic, TypeVar, overload
from artemis.config import LLMConfig, get_default_llm_config from artemis.config import LLMConfig, get_default_llm_config
from artemis.constants import RECURSION_LIMIT from artemis.constants import RECURSION_LIMIT
from artemis.context import DeviceContext from artemis.context import DeviceContext
from artemis.plugins.models import ExecutionTarget, PluginSelection
from artemis.sdk.types.platform import TaskRunStatus from artemis.sdk.types.platform import TaskRunStatus
from artemis.sdk.utils import load_llm_config_override from artemis.sdk.utils import load_llm_config_override
from pydantic import BaseModel, Field from pydantic import BaseModel, Field
@@ -108,6 +109,8 @@ class TaskRequestCommon(TaskRequestBase):
model_config = {"ignored_types": (CyFunctionDetector,)} model_config = {"ignored_types": (CyFunctionDetector,)}
max_steps: int = RECURSION_LIMIT max_steps: int = RECURSION_LIMIT
plugins: list[PluginSelection] | None = None
execution_target: ExecutionTarget = "android"
locked_app_package: str | None = None locked_app_package: str | None = None
app_path: Path | None = None app_path: Path | None = None
"""Path to an app to install before running the task. """Path to an app to install before running the task.
+4
View File
@@ -55,6 +55,10 @@ def format_action_clean(action_obj) -> str:
act_type = action_obj.get("action") or action_obj.get("name") act_type = action_obj.get("action") or action_obj.get("name")
if not act_type: if not act_type:
return json.dumps(action_obj, ensure_ascii=False) return json.dumps(action_obj, ensure_ascii=False)
if str(act_type).startswith("plug_"):
tool_name = str(act_type).split("_", 2)[-1]
source = action_obj.get("plugin_id") or "扩展插件"
return f"{source} · {tool_name}: {json.dumps(action_obj.get('args') or {}, ensure_ascii=False)}"
# Flash records the tool call verbatim: the action name at the top level # Flash records the tool call verbatim: the action name at the top level
# and every argument (app_name, key, text, direction, ...) under ``args``. # and every argument (app_name, key, text, direction, ...) under ``args``.
+1 -1
View File
@@ -19,7 +19,7 @@
| 工具 | 功能 | | 工具 | 功能 |
| --- | --- | | --- | --- |
| `mobile_diagnose` | 环境、凭据、设备诊断和自带修复 | | `mobile_diagnose` | 环境、凭据、设备诊断和自带修复 |
| `mobile_workspace` | 设备列表与统一队列状态 | | `mobile_workspace` | 设备列表、统一队列、插件配置和资源保护状态 |
| `mobile_config` | 读取或明确保存非敏感模型、功能、报告默认设置 | | `mobile_config` | 读取或明确保存非敏感模型、功能、报告默认设置 |
| `mobile_run_task` | 单任务执行;`model=flash/pro` 表示执行模式 | | `mobile_run_task` | 单任务执行;`model=flash/pro` 表示执行模式 |
| `mobile_manage_task` | 单任务状态、安全暂停、排队恢复、指导提交/领取记录和停止 | | `mobile_manage_task` | 单任务状态、安全暂停、排队恢复、指导提交/领取记录和停止 |
+30 -13
View File
@@ -1,6 +1,6 @@
# ARTEMIS 2.0.0:插件与跨端测试 # ARTEMIS 2.0.0:插件与跨端测试
状态:已确认产品设计,进入开发准备;本文件中的功能尚未全部实现。 状态:核心插件链路、首发 Windows 插件适配已实现;Flash/Pro 纯电脑任务和基础浏览器表单实测通过,跨端与长期稳定性验收仍在进行,不是正式发行。
开发分支:`codex/v2.0.0-plugins`,从 1.5.2 主线建立。版本号为 2.0.0,发布前不创建正式版本标签。1.5.x 的 BUG 修复继续在主线进行,后续正常合并进开发分支。 开发分支:`codex/v2.0.0-plugins`,从 1.5.2 主线建立。版本号为 2.0.0,发布前不创建正式版本标签。1.5.x 的 BUG 修复继续在主线进行,后续正常合并进开发分支。
@@ -70,32 +70,49 @@ Flash 的扩展调用需要完整处理 MCP 错误及多种输出;Pro 的发
- [x] 独立开发分支,版本统一为 2.0.0。 - [x] 独立开发分支,版本统一为 2.0.0。
- [x] 保存确认过的设计和市场仓库地址。 - [x] 保存确认过的设计和市场仓库地址。
- [ ] 形成可运行的插件功能;版本标记不代表功能已经完成。 - [x] 形成可运行的插件功能;版本标记不代表功能已经完成。
### 1. 通用插件基础 ### 1. 通用插件基础
- [ ] 明确清单和市场索引契约,并实现版本兼容与参数校验。 - [x] 明确清单和市场索引契约,并实现版本兼容与参数校验。
- [ ] 支持本地 stdio / 远程 MCP 连接、自定义接入、会话生命周期及有界取消。 - [x] 支持本地 stdio / 远程 MCP 连接、自定义接入、会话生命周期及有界取消。
- [ ] 支持 Skill 按需加载、命名配置、凭据引用和授权范围检查。 - [x] 支持 Skill 按需加载、命名配置、凭据引用和授权范围检查。
- [ ] Flash / Pro 共用工具分派,完整保存错误与图像等结果。 - [x] Flash / Pro 共用工具分派,完整保存错误与图像等结果。
### 2. 插件页面和市场 ### 2. 插件页面和市场
- [ ] 市场查询、安装进度、配置、启停、诊断、更新和卸载。 - [x] 市场查询、安装进度、配置、启停、诊断、更新和卸载。
- [ ] 下载完整性检查、失败恢复、任务版本固定与升级互斥。 - [x] 下载完整性检查、失败恢复、任务版本固定与升级互斥。
- [ ] 仅发布实际验证过的插件版本与验证记录。 - [ ] 仅发布实际验证过的插件版本与验证记录。
### 3. 跨端执行 ### 3. 跨端执行
- [ ] 适配首发电脑控制插件,支持纯电脑和混合任务。 - [x] 适配首发电脑控制插件,支持纯电脑和混合任务入口。
- [ ] 多资源互斥、暂停恢复、指导、超时及共享环境变更处理。 - [x] 多资源互斥、暂停恢复、指导、超时及共享环境变更处理。
- [ ] 跨端证据、验收、人工复验和各类导出保持一致。 - [ ] 跨端证据、验收、人工复验和各类导出保持一致。
- [ ] 同步 HTTP、工作区 MCP 和自动化 Skill。 - [x] 同步 HTTP、工作区 MCP 和自动化 Skill。
### 4. 验收 ### 4. 验收
- [ ] 确定性测试覆盖连接失败、结果不确定、超时、取消、升级及资源互斥。 - [x] 确定性测试覆盖连接失败、结果不确定、超时、取消、升级及资源互斥。
- [ ] Windows 实测窗口遮挡、多窗口、DPI、输入、观察和浏览器控制台操作。 - [ ] Windows 实测窗口遮挡、多窗口、DPI、输入、观察和浏览器控制台操作。
- [ ] 实际跑通电脑修改参数、手机验证、按要求恢复及完整证据导出。 - [x] 实际跑通电脑修改参数、手机验证、按要求恢复及完整证据导出。
- [ ] 验证 Flash / Pro、纯电脑任务、批量执行、暂停恢复和人工指导。 - [ ] 验证 Flash / Pro、纯电脑任务、批量执行、暂停恢复和人工指导。
- [ ] 执行仓库质量检查、前端交互检查,并完成真实安装包验证后发布。 - [ ] 执行仓库质量检查、前端交互检查,并完成真实安装包验证后发布。
### 本轮验证记录(2026-10-14)
- 确定性测试覆盖 stdio、Streamable HTTP、任务固定版本、取消/超时资源保护、图片来源与共享证据清理;网页测试与构建通过。
- 首发包增加窗口自身截图与子控件消息路由,修复实际发现的遮挡画面污染和按钮收不到点击问题。两个临时窗口验证了遮挡下截图来源与实际点击效果。
- 隔离 Chrome 窗口通过 MCP 修改本地表单参数、提交一次并读回确认。
- 使用已配置的 `gpt-6-luna / high` 跑同一个中文输入与结果确认任务:Flash 57 秒、Pro 112 秒,真实控件值和最终通过结果一致。只代表这两个基础样本。
- 真实 Pro 记录成功生成 Word、Excel 与含插件来源命名的截图列表。修复了接口清洗导致来源字段丢失的问题。
- 队列回归复现并修复两处资源遗漏:活动任务元数据丢失插件资源,以及取消后遗留版本占用。运行任务仍等进程退出才释放占用,避免提前允许升级。
- 真实 stdio MCP 的确定性用例验证暂停关闭连接、释放共享资源,恢复等待另一任务完成,再新建连接重新观察;该验证不代替真实模型的批次和指导组合验收。
- 真实 Flash 跨端任务使用选定的 Android 17 模拟器和隔离 Chrome:电脑修改参数、手机刷新验证、电脑恢复、手机再次验证,最终用例通过,耗时约 119 秒;后台仅收到改值与恢复两次提交。Word/Excel 导出包含 11 步、8 条插件操作、23 项截图引用,手机和插件图片分别保留。
- 同一跨端场景实际验证暂停后设备与桌面锁释放、恢复等待另一资源占用者退出;指导已被执行端领取并记录在下一步,未重复提交已完成的修改。该次操作完成但终检发现手机文字层级陈旧,保留为未验证,未把后续修复结果覆盖旧记录。
- 联调修复 Flash 跨端切换时将电脑图像带入手机观察的问题。插件图片保留在带来源的工具结果和步骤记录内,切回手机时重新获取手机截图与元素。
- 插件截图在首次入库时同时保存本次观察文字,回归确认后续动作不会将新文字补写到前一步图片上。
- 联调还复现 Android 辅助服务缓存旧网页文字。辅助 APK 1.2.1 在 Android 13+ 每次观察前清理节点缓存,修复后同一模型任务终检通过;另有不调用模型的动态网页回归,同时验证文字与像素从初始值更新并恢复。Android 12 及更早版本不使用此 API,仍需单独验收。
- 最终默认 Python 测试 2451 通过、2 跳过、8 未选中;Ruff 格式/静态检查、质量基线、核心 Pyright 均通过。前端 187 项测试与生产构建通过,真实 Android 动态网页回归通过。历史检索测试改用同步存储以消除测试自身的线程等待竞争,断言保持不变。
- 复杂浏览器控制台、多 DPI、多显示器、长程运行、真实批次组合及混合 Pro 模式还需要验收。插件市场不提前发布未完成验收的安装包。
+94
View File
@@ -0,0 +1,94 @@
# 插件与跨端任务(2.0.0 开发版)
ARTEMIS 可通过插件连接 stdio 或 Streamable HTTP MCP 服务,并按需加载插件内的 Skill。Flash、Pro、网页任务和批次复用同一套插件执行层。版本仍在开发验证中,市场只收录完成实际验证的安装包。
## 安装和配置
打开网页导航中的「插件」。可以从市场安装,或在「高级接入」导入包含 `plugin.json` 的目录/ZIP,也可以填写自定义 MCP 清单。安装不执行安装钩子,新装或升级后默认未启用。
点击「配置」建立命名连接,例如 `lab-router`。填写字段、工具白名单、允许的参数值和共享资源,勾选授权后保存,再启用插件。密码留空保留已保存的值;页面和 MCP 查询只返回已配置的密钥名称。不要把密钥写进清单、提示词或任务 JSON。
每个插件可设置一套默认连接。自动选择只使用符合任务目标、已启用且默认连接已授权的插件。任务中也可以明确选择插件与连接;空列表表示不使用插件。
正在排队、暂停和执行中的任务会固定插件版本,并阻止修改配置、启停、升级或卸载。操作结束后再修改。卸载会移除插件安装文件与连接配置,历史证据仍保留。
## 执行任务
任务输入区和批次导入区有执行目标选项:
| 目标 | 含义 |
| --- | --- |
| `android` | 手机任务,默认不使用桌面插件 |
| `computer` | 纯电脑任务,无需 Android;必须选择提供观察能力的插件 |
| `mixed` | 手机和扩展插件组合任务 |
单任务 `runtime_overrides`、批次每项 `task_config` 都支持:
```json
{
"execution_target": "computer",
"plugins": [{"id": "open-computer-use", "profile": "default"}],
"timeout_seconds": 1800
}
```
这只是配置引用示例,先安装并启用对应插件。不要从名字推断本机已安装它。`mobile_workspace` 返回实际插件状态和连接配置,纯电脑任务可传入 `include_devices=false` 跳过 ADB 查询;`mobile_run_task` 及 `mobile_batch` 使用现有队列。
CLI:`uv run artemis run "观察目标窗口并验证结果" --target computer --profile flash`。`--plugins-json` 可指定上面的插件引用数组。电脑和混合任务不会在队列服务失败后自动重复提交到另一执行器;`--standalone` 是显式选择本地 SDK 执行。
SDK:先 `await agent.init(execution_target="computer")`,然后 `agent.new_task(goal).with_plugins([...]).build()`。插件引用使用 `artemis.plugins.models.PluginSelection`。`for_target()` 可设置构造器目标,必须与 `Agent.init` 相符。
## 共享资源与中断
同一目标应使用同一个资源名,例如 `desktop:local` 或 `router:lab`。资源名不区分大小写。不同插件操作同一路由器时也要使用相同资源名。无资源声明的 MCP 插件按插件与配置名称保守互斥。
暂停会先取消调用并关闭插件连接,再释放任务资源;恢复排队后重新观察。插件取消/超时的写操作可能已经生效,不能盲目重放。如果远程服务未确认动作停止,页面会显示资源保护及原因,用户核对后点击「已核对,解除保护」。这只解除 ARTEMIS 的保护,不会替用户终止远程操作。
工具返回成功表示调用完成,业务是否通过仍需实际证据。图片、工具文本、插件版本和动作结果进入任务记录;已有报告、截图下载和人工复验入口继续使用。Word 的「扩展插件操作」与 Excel 的「插件操作」保留来源及实际结果,资料包截图名称含插件 ID。同一截图被多个任务引用时,删除最后一个任务后才清理。插件画面与手机截图属于不同来源,不能相互替代。电脑持续录屏不属于首版能力。
## 插件清单
```json
{
"schema_version": 1,
"id": "lab-control",
"name": "实验室控制",
"version": "1.0.0",
"artemis_version": ">=2.0.0,<3.0.0",
"platforms": ["win32"],
"architectures": ["amd64"],
"targets": ["mixed"],
"server": {
"transport": "stdio",
"command": "${plugin_dir}/bin/server.exe",
"args": [],
"env": {"TOKEN": "${secret.token}"}
},
"skill": "SKILL.md",
"config": {"token": {"label": "连接密钥", "secret": true, "required": true}},
"resources": ["router:lab"]
}
```
HTTP 服务使用 `{"transport":"streamable_http","url":"https://example.com/mcp","headers":{"Authorization":"Bearer ${secret.token}"}}`。配置引用支持 `${config.字段}`、`${secret.字段}` 和 `${plugin_dir}`;参数直接以 argv 启动,不经 shell。纯 Skill 可省略 `server`。Skill 的引用文件必须在包内,按需读取支持不超过 256 KiB 的 Markdown、文本和 JSON。
提供电脑画面观察的插件声明 `observation`,其中 `tool` 和 `arguments` 为初始观察调用,`refresh_tools` 为模型选择目标后可复用的只读观察工具名。例如初始列窗口,选择窗口后每步刷新该窗口。观察允许只返回文字;无图时不能伪造截图或推测像素坐标。
清单提供的工具和参数范围检查不是操作系统沙箱;第三方进程以当前用户权限运行。市场包使用 HTTPS、SHA-256、平台/版本检查及安全解压,包内应保留许可证和来源记录。
市场索引为 `{"schema_version":1,"plugins":[...]}`,每项包含 `manifest`、`archive_url`、`sha256`、`verification`。同一版本不可覆盖,先验证新包再使用新版本号。
## 环境参数
| 环境变量 | 默认值 |
| --- | --- |
| `ARTEMIS_PLUGIN_MARKETPLACE_URL` | Artemis-Plugins 仓库的 catalog.json |
| `ARTEMIS_PLUGIN_START_TIMEOUT_SECONDS` | 30 秒 |
| `ARTEMIS_PLUGIN_TOOL_TIMEOUT_SECONDS` | 60 秒 |
| `ARTEMIS_PLUGIN_SHUTDOWN_TIMEOUT_SECONDS` | 15 秒 |
| `ARTEMIS_PLUGIN_DOWNLOAD_TIMEOUT_SECONDS` | 300 秒 |
| `ARTEMIS_PLUGIN_DOWNLOAD_MAX_BYTES` | 128 MiB |
| `ARTEMIS_PLUGIN_EXTRACT_MAX_BYTES` | 512 MiB |
| `ARTEMIS_PLUGIN_RESULT_MAX_BYTES` | 16 MiB |
修改环境参数后重启开发版服务。任务总时限仍由现有执行保护控制,插件不会自行无限重试。
+13 -4
View File
@@ -120,15 +120,24 @@ async def _download(
@mcp.tool() @mcp.tool()
async def mobile_workspace() -> dict: async def mobile_workspace(include_devices: bool = True) -> dict:
"""List connected Android devices and the unified running/queued task state. """List connected Android devices and the unified running/queued task state.
Select the user's device; if multiple devices exist and none is chosen, ask first. Select the user's device; if multiple devices exist and none is chosen, ask first.
This does not submit tasks. On connection/device errors use mobile_diagnose. This does not submit tasks. On connection/device errors use mobile_diagnose.
For computer-only work use include_devices=false to skip ADB discovery entirely.
""" """
devices = await _request("GET", "/api/devices") status, plugins = await asyncio.gather(
status = await _request("GET", "/api/status") _request("GET", "/api/status"), _request("GET", "/api/plugins")
return {"devices": devices.get("devices", []), "queue": status} )
devices = await _request("GET", "/api/devices") if include_devices else {}
return {
"devices": devices.get("devices", []),
"queue": status,
"plugins": plugins.get("plugins", []),
"blocked_resources": plugins.get("blocked_resources", {}),
"execution_targets": ["android", "computer", "mixed"],
}
@mcp.tool() @mcp.tool()
+17 -3
View File
@@ -333,6 +333,11 @@ def mobile_run_task(
`"pro"` (3-turn ReAct), `"ultra"` (deep pixel reasoning; slowest). `"pro"` (3-turn ReAct), `"ultra"` (deep pixel reasoning; slowest).
Ignored for Flash. Ignored for Flash.
runtime_overrides: Optional TaskOptions JSON for this task only. Supports runtime_overrides: Optional TaskOptions JSON for this task only. Supports
execution_target ('android', 'computer', 'mixed') and plugins
([{id, profile}] references to saved authorized configurations; never secrets).
Omitted plugins selects matching enabled defaults; [] disables plugins.
Computer tasks require an observation plugin, not an Android device.
Query mobile_workspace first; installation/authorization is in the plugin page.
timeout_seconds (total execution, default 1800; excludes queue/pause), timeout_seconds (total execution, default 1800; excludes queue/pause),
observation_timeout_seconds (60), observation_max_attempts (3), observation_timeout_seconds (60), observation_max_attempts (3),
device_command_timeout_seconds (10), action_failure_attempts (3), device_command_timeout_seconds (10), action_failure_attempts (3),
@@ -375,7 +380,8 @@ def mobile_run_task(
# authorized. Rejecting here prevents the task from silently running on a # authorized. Rejecting here prevents the task from silently running on a
# different device than the caller asked for. Runs after init_trace so the # different device than the caller asked for. Runs after init_trace so the
# rejection carries a trace_id like every other response of this tool. # rejection carries a trace_id like every other response of this tool.
if device_serial: execution_target = (runtime_overrides or {}).get("execution_target", "android")
if device_serial and execution_target != "computer":
rejection = _validate_device_serial(device_serial) rejection = _validate_device_serial(device_serial)
if rejection: if rejection:
trace_store.update_trace_status(trace_id, "failed", error=rejection["error"]) trace_store.update_trace_status(trace_id, "failed", error=rejection["error"])
@@ -401,8 +407,12 @@ def mobile_run_task(
runtime_overrides=runtime_overrides, runtime_overrides=runtime_overrides,
base_url=base_url, base_url=base_url,
) )
if resp and resp.get("status") == "rejected": if resp and resp.get("status") in {"rejected", "error"}:
rejection_error = resp.get("error") or "Task rejected by Artemis Daemon." rejection_error = (
resp.get("error")
or resp.get("message")
or "Task rejected by Artemis Daemon."
)
trace_store.update_trace_status(trace_id, "failed", error=rejection_error) trace_store.update_trace_status(trace_id, "failed", error=rejection_error)
return { return {
"trace_id": trace_id, "trace_id": trace_id,
@@ -507,6 +517,10 @@ def mobile_run_task(
} }
# 4. Standalone Fallback: Resolve project root, python executable, and background runner module # 4. Standalone Fallback: Resolve project root, python executable, and background runner module
if execution_target != "android" or (runtime_overrides or {}).get("plugins"):
error = "插件及电脑任务需要统一任务服务,请启动 ARTEMIS 网页服务后重试;未启动独立任务。"
trace_store.update_trace_status(trace_id, "failed", error=error)
return {"trace_id": trace_id, "status": "failed", "error": error}
project_root = env_utils.get_project_root() project_root = env_utils.get_project_root()
python_exe = env_utils.resolve_python_executable(project_root) python_exe = env_utils.resolve_python_executable(project_root)
reserve_kwargs: dict[str, Any] = { reserve_kwargs: dict[str, Any] = {
@@ -10,8 +10,8 @@ android {
applicationId = "com.artemis.helper" applicationId = "com.artemis.helper"
minSdk = 24 minSdk = 24
targetSdk = 35 targetSdk = 35
versionCode = 6 versionCode = 7
versionName = "1.2.0" versionName = "1.2.1"
} }
buildTypes { buildTypes {
@@ -395,6 +395,13 @@ public final class HierarchyDumper {
DumpOptions options, DumpOptions options,
DumpStats stats DumpStats stats
) { ) {
// We subscribe only to window-state events. Chromium can update an
// existing DOM node without such an event, leaving the framework's
// cached node text stale even though the bitmap has already changed.
// Invalidate on demand, once per observation, before prefetching roots.
if (Build.VERSION.SDK_INT >= 33) {
service.clearCache();
}
List<RawRootEntry> rawRoots = Collections.emptyList(); List<RawRootEntry> rawRoots = Collections.emptyList();
long[] retryBackoff = new long[]{40L, 80L, 120L, 160L, 220L, 300L}; long[] retryBackoff = new long[]{40L, 80L, 120L, 160L, 220L, 300L};
@@ -1,7 +1,7 @@
{ {
"package": "com.artemis.helper", "package": "com.artemis.helper",
"version_code": 6, "version_code": 7,
"version_name": "1.2.0", "version_name": "1.2.1",
"sha256": "133e373fe03615c251cd74b03a38539c844d09a074954bca2d2dde6684561969", "sha256": "162554d7f8bc7471a7886fb9413d155779f73190e990be229b93a86a76fef88d",
"built_at": "2026-09-10T23:13:11+00:00" "built_at": "2026-10-14T12:08:47+00:00"
} }
+3
View File
@@ -38,6 +38,8 @@ dependencies = [
# Configuration, Models & Validation # Configuration, Models & Validation
"pydantic>=2.7.0", "pydantic>=2.7.0",
"pydantic-settings>=2.10.1", "pydantic-settings>=2.10.1",
"packaging>=24.0",
"jsonschema>=4.20.0",
"python-dotenv>=1.0.1", "python-dotenv>=1.0.1",
"jinja2>=3.1.5", "jinja2>=3.1.5",
"typing-extensions>=4.12.0", "typing-extensions>=4.12.0",
@@ -65,6 +67,7 @@ dependencies = [
"scipy>=1.13.0", "scipy>=1.13.0",
# System, Utilities & Telemetry # System, Utilities & Telemetry
"psutil>=5.9.0", "psutil>=5.9.0",
"filelock>=3.20.0",
"retry>=0.9.2", "retry>=0.9.2",
"posthog>=3.5.0", "posthog>=3.5.0",
"uuid-utils>=0.9.0", "uuid-utils>=0.9.0",
+12 -2
View File
@@ -1,6 +1,6 @@
--- ---
name: artemis-automation name: artemis-automation
description: 使用 ARTEMIS MCP 在 Android 真机或模拟器上完成自然语言测试、批量用例执行、失败诊断、历史检索与 Word/Excel 报告交付。适用于用户希望 AI 操作或测试 Android 应用、导入测试用例或分析 ARTEMIS 执行结果。 description: 使用 ARTEMIS MCP 完成 Android 与已授权插件支持的电脑或跨端测试、批量执行、失败诊断、历史检索与报告交付。适用于用户希望 AI 通过 ARTEMIS 执行测试或分析其结果。
--- ---
# ARTEMIS 自动化 # ARTEMIS 自动化
@@ -9,7 +9,17 @@ description: 使用 ARTEMIS MCP 在 Android 真机或模拟器上完成自然语
Windows 启动脚本只提示源码更新。升级使用 `update.bat`(国内镜像 `update-cn.bat`),会终止该项目的 ARTEMIS 服务、MCP 和运行任务;仅在用户授权升级或终止任务后执行,先告知任务将中断。升级会同步依赖、网页及 Codex MCP,保留配置和历史;完成后启动 ARTEMIS 并重载 Codex MCP。不要通过启动脚本拉取源码或强行覆盖本地修改。 Windows 启动脚本只提示源码更新。升级使用 `update.bat`(国内镜像 `update-cn.bat`),会终止该项目的 ARTEMIS 服务、MCP 和运行任务;仅在用户授权升级或终止任务后执行,先告知任务将中断。升级会同步依赖、网页及 Codex MCP,保留配置和历史;完成后启动 ARTEMIS 并重载 Codex MCP。不要通过启动脚本拉取源码或强行覆盖本地修改。
使用项目提供的 MCP 工具操作 Android。报告由 ARTEMIS 自己生成,不要求宿主 AI 具备 Codex 私有工具、Word 或 Excel 插件。该 Skill 不提供桌面/浏览器自动化,也不提供历史动作脚本回放。 使用项目提供的 MCP 工具发起任务,Android 及已启用插件由 ARTEMIS 自己编排。报告由 ARTEMIS 生成,不要求宿主 AI 具备 Codex 私有工具、Word 或 Excel 插件。电脑/浏览器能力取决于实际安装并授权的插件,不提供历史动作脚本回放。
## 插件与跨端任务
先用 `mobile_workspace` 查看实际插件、连接配置与资源保护状态。插件安装、凭据与授权在网页「插件」页完成;MCP 只引用已保存的配置,不接收密钥。网页显示已启用不等于插件已经通过真实场景验证。
单任务 `runtime_overrides` 和每条批次用例的 `task_config` 支持 `execution_target`(`android`、`computer`、`mixed`)及 `plugins`(如 `[{"id":"open-computer-use","profile":"default"}]`)。省略 plugins 自动选择该目标的已授权默认配置;`[]` 禁用插件。纯电脑任务不要求 Android,也不传 device_serial;混合任务仍需选择手机。完整参数与清单见 [插件任务参数](references/plugins.md)。
同一桌面或路由器上的任务串行。暂停要等插件取消并确认停止后才释放资源;恢复时先观察当前状态。远程操作超时或中断可能已经生效,不自动重试写操作;资源处于保护状态时告知用户核对远程动作后从插件页面解除。插件配置和版本在排队、暂停、运行期间固定,变更范围需结束任务后修改配置。
将电脑与手机证据分别说明。插件调用成功不等于预期效果达成;截图被遮挡、窗口隐藏或观察只有文字时,不假设具备有效像素证据。任务的通过/不通过及人工复验继续沿用现有规则。
## 从需求到执行 ## 从需求到执行
@@ -0,0 +1,21 @@
# 插件任务参数
先调用 `mobile_workspace`,只选实际存在、已启用且配置已授权的插件。纯电脑任务使用 `include_devices=false`,避免不必要的 ADB 查询。通过现有 `mobile_run_task` 发起电脑任务:
```json
{
"task_desc": "观察已打开的测试控制台,确认参数值符合用户预期,并保留证据。",
"model": "Flash",
"runtime_overrides": {
"execution_target": "computer",
"plugins": [{"id": "open-computer-use", "profile": "default"}],
"timeout_seconds": 1800
}
}
```
示例中的插件必须已安装;不要为解决缺插件而调用宿主私有电脑工具代替 ARTEMIS 执行。`computer` 无需 `device_serial`,`mixed` 同时需要手机与插件资源。批次每个用例的 `task_config` 使用相同字段,可混合不同配置和执行目标。
未提供 `plugins` 表示使用该目标下已启用插件的授权默认配置;空数组明确不用插件。只提供插件 ID 和配置名称,版本由服务在入队时固定。安装、授权、连接凭据和资源保护解除由用户在插件页面处理。
插件超时/取消后先查任务状态和错误,不自动重放点击或配置写入。`mobile_workspace.blocked_resources` 非空时说明存在尚未确认停止的资源,不能声称暂停已安全完成。工具返回成功还需从后续画面或结构化读数验证业务结果。
+11
View File
@@ -46,6 +46,17 @@ def mock_driver():
return MockDeviceDriver(device_id="fixture-mock-device", width=1080, height=2400) return MockDeviceDriver(device_id="fixture-mock-device", width=1080, height=2400)
@pytest.fixture(autouse=True)
def isolated_plugin_registry(monkeypatch, tmp_path, request):
"""Unit tests never load or mutate the user's installed plugins and credentials."""
if any(
request.node.get_closest_marker(name)
for name in ("android", "integration", "e2e", "manual")
):
return
monkeypatch.setattr("artemis.plugins.store.get_app_dir", lambda: tmp_path / "app")
def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: def pytest_collection_modifyitems(items: list[pytest.Item]) -> None:
"""Attach test-layer markers according to the owning test directory.""" """Attach test-layer markers according to the owning test directory."""
for item in items: for item in items:
@@ -0,0 +1,115 @@
"""Opt-in cache regression on a selected Android device with Chrome and the helper.
Set ARTEMIS_TEST_DEVICE_SERIAL and run this file with ``pytest -m android``.
Opens a disposable local page via ADB reverse; no model call or system-setting edit.
"""
import base64
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from io import BytesIO
import os
import subprocess
import threading
import time
from uuid import uuid4
from PIL import Image
import pytest
from artemis.clients.accessibility_client import AccessibilityClient
from artemis.runtime.adb_endpoint import adb_command
from artemis.runtime.device_lock import DeviceExecutionLock
pytestmark = [pytest.mark.android, pytest.mark.integration]
def test_helper_refreshes_dynamic_web_text_and_pixels(tmp_path):
serial = os.getenv("ARTEMIS_TEST_DEVICE_SERIAL")
if not serial:
pytest.skip("Select ARTEMIS_TEST_DEVICE_SERIAL explicitly")
marker = "updated-" + uuid4().hex[:8]
state = {"value": "baseline"}
class Handler(BaseHTTPRequestHandler):
def log_message(self, *_args):
pass
def do_GET(self):
if self.path == "/state":
body = state["value"].encode()
else:
body = b"""<!doctype html><meta name="viewport" content="width=device-width">
<title>ARTEMIS dynamic observation fixture</title>
<body style="background:#cc2222;font:24px sans-serif"><p id="result">loading</p>
<script>setInterval(async()=>{const v=await(await fetch('/state')).text();
document.getElementById('result').textContent=v;
document.body.style.background=v==='baseline'?'#cc2222':'#22cc22';},500)</script>"""
self.send_response(200)
self.send_header("Content-Type", "text/html; charset=utf-8")
self.send_header("Cache-Control", "no-store")
self.end_headers()
self.wfile.write(body)
def adb(*arguments, check=True):
return subprocess.run(
adb_command(["-s", serial, *arguments]),
capture_output=True,
check=check,
timeout=15,
)
lease = DeviceExecutionLock(serial, description="Dynamic helper observation regression")
lease.acquire(blocking=False)
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
worker = threading.Thread(target=server.serve_forever, daemon=True)
worker.start()
client = AccessibilityClient(serial, provision_on_connect=False)
device_port = None
try:
client.connect()
# Let ADB allocate an unused reverse port; never replace a user's mapping.
device_port = adb("reverse", "tcp:0", f"tcp:{server.server_port}").stdout.decode().strip()
assert device_port.isdigit(), "ADB did not allocate a reverse port"
adb(
"shell",
"am",
"start",
"-a",
"android.intent.action.VIEW",
"-d",
f"http://127.0.0.1:{device_port}/",
"-p",
"com.android.chrome",
)
def await_observation(expected, green):
deadline = time.monotonic() + 15
observed_text = []
while time.monotonic() < deadline:
observation = client.get_screen_data()
observed_text = [element.get("text") for element in observation.elements]
raw = base64.b64decode(observation.base64)
with Image.open(BytesIO(raw)) as picture:
pixels = list(picture.convert("RGB").resize((32, 32)).getdata())
count = sum(
g > 150 and r < 100 if green else r > 150 and g < 100 for r, g, _ in pixels
)
if expected in observed_text and count > len(pixels) // 4:
(tmp_path / f"{expected}.jpg").write_bytes(raw)
return
time.sleep(0.5)
pytest.fail(f"Pixels and UI hierarchy did not converge to {expected}: {observed_text}")
await_observation("baseline", False)
state["value"] = marker
await_observation(marker, True)
state["value"] = "baseline"
await_observation("baseline", False)
finally:
client.disconnect()
if device_port and device_port.isdigit():
adb("reverse", "--remove", f"tcp:{device_port}", check=False)
server.shutdown()
server.server_close()
worker.join(timeout=5)
lease.release()
@@ -163,7 +163,8 @@ async def test_steps_without_hashes_are_skipped():
assert await _run(steps) == [] assert await _run(steps) == []
def test_record_step_stamps_perceptual_hashes(tmp_path): @pytest.mark.asyncio
async def test_record_step_stamps_perceptual_hashes(tmp_path):
from artemis.context import ArtemisContext from artemis.context import ArtemisContext
from artemis.data_engine.engine import DataEngine from artemis.data_engine.engine import DataEngine
@@ -187,8 +188,7 @@ def test_record_step_stamps_perceptual_hashes(tmp_path):
post_screenshot_bytes=SCREEN_A, post_screenshot_bytes=SCREEN_A,
summary="screen unchanged", summary="screen unchanged",
) )
for t in list(engine._pending_threads): await engine.shutdown()
t.join()
steps = engine.storage.get_steps(engine.current_session_id) steps = engine.storage.get_steps(engine.current_session_id)
meta1 = steps[0].extra_metadata meta1 = steps[0].extra_metadata
+963
View File
@@ -0,0 +1,963 @@
"""Plugin transport/installation tests use local deterministic MCP servers, never devices."""
import asyncio
import base64
import io
import json
from pathlib import Path
import sys
import zipfile
from mcp.types import CallToolResult, ImageContent, TextContent
import pytest
from artemis.plugins.installer import install_archive
from artemis.plugins.models import CatalogEntry, PluginManifest, PluginProfile, PluginSelection
from artemis.plugins.runtime import PluginRuntime
from artemis.plugins.session import resolve_server
from artemis.plugins.store import PluginStore
from artemis.runtime.device_lock import DeviceBusyError, DeviceExecutionLock
def manifest(**updates):
return PluginManifest.model_validate(
{
"id": "test-plugin",
"name": "测试插件",
"version": "1.0.0",
"platforms": [sys.platform],
"skill": "SKILL.md",
"resources": ["desktop:local"],
"config": {"token": {"label": "密钥", "secret": True, "required": True}},
**updates,
}
)
def archive(plugin, extra=None):
buffer = io.BytesIO()
with zipfile.ZipFile(buffer, "w") as output:
output.writestr("plugin.json", plugin.model_dump_json())
output.writestr("SKILL.md", "先观察,再操作;报告真实证据。")
for name, value in (extra or {}).items():
output.writestr(name, value)
return buffer.getvalue()
@pytest.fixture
def store(tmp_path):
return PluginStore(tmp_path / "plugins")
def enable(store, plugin, **profile):
install_archive(store, archive(plugin))
store.save_profile(
plugin.id, PluginProfile(secrets={"token": "private-value"}, authorized=True, **profile)
)
store.set_enabled(plugin.id, True)
def test_profile_secrets_are_write_only_and_blank_preserves(store):
enable(store, manifest())
assert "private-value" not in json.dumps(store.list())
assert store.list()[0]["profiles"][0]["configured_secrets"] == ["token"]
store.save_profile("test-plugin", PluginProfile(secrets={"token": ""}, authorized=True))
_, profile, _ = store.resolve(PluginSelection(id="test-plugin"))
assert profile.secrets["token"].get_secret_value() == "private-value"
def test_untrusted_config_cannot_hide_secrets_in_ordinary_fields(store):
install_archive(store, archive(manifest()))
with pytest.raises(ValueError, match="普通配置"):
store.save_profile("test-plugin", PluginProfile(values={"token": "secret"}))
with pytest.raises(ValueError, match="授权"):
store.set_enabled("test-plugin", True)
def test_package_versions_immutable_and_old_pin_survives_upgrade(store):
enable(store, manifest())
with pytest.raises(ValueError, match="覆盖"):
install_archive(store, archive(manifest()))
install_archive(store, archive(manifest(version="1.1.0")))
assert store.get("test-plugin")["enabled"] is False
store.set_enabled("test-plugin", True)
old, _, _ = store.resolve(PluginSelection(id="test-plugin", version="1.0.0"))
assert old.version == "1.0.0"
@pytest.mark.parametrize(
"path",
["../../escape.txt", "C:/escape.txt", "/escape.txt", "dir/../../../escape", "x/../escape"],
)
def test_archive_rejects_traversal(store, path):
with pytest.raises(ValueError):
install_archive(store, archive(manifest(), {path: "bad"}))
assert store.list() == []
def test_checksum_failure_keeps_previous_version(store):
enable(store, manifest())
entry = CatalogEntry(
manifest=manifest(version="1.1.0"),
archive_url="https://example.com/package.zip",
sha256="0" * 64,
verification="本地确定性验证",
)
with pytest.raises(ValueError, match="SHA-256"):
install_archive(store, archive(entry.manifest), entry=entry)
assert store.get("test-plugin")["manifest"]["version"] == "1.0.0"
assert store.get("test-plugin")["enabled"] is True
def test_server_substitutions_do_not_use_shell_or_expand_unknown_values(tmp_path):
plugin = manifest(
server={
"command": "${plugin_dir}/server.exe",
"args": ["${config.target}"],
"env": {"TOKEN": "${secret.token}"},
},
config={"token": {"label": "密钥", "secret": True}, "target": {"label": "目标"}},
)
profile = PluginProfile(
values={"target": "a b;$(echo forbidden)"}, secrets={"token": "private-value"}
)
config = resolve_server(plugin, profile, tmp_path)
assert config.args == ["a b;$(echo forbidden)"]
assert config.env == {"TOKEN": "private-value"}
@pytest.mark.asyncio
async def test_skill_load_and_reference_cannot_read_outside_package(store):
enable(store, manifest())
runtime = PluginRuntime(store=store)
assert {t.name for t in runtime.declarations()} == {"plugin_load", "plugin_read_file"}
result = await runtime.call("plugin_load", {"plugin_id": "test-plugin"})
assert result.status == "success" and "先观察" in result.text()
assert (
await runtime.call(
"plugin_read_file", {"plugin_id": "test-plugin", "path": "../../../registry.sqlite3"}
)
).status == "error"
@pytest.mark.asyncio
async def test_real_stdio_mcp_errors_images_secrets_and_langchain(store, tmp_path):
server = tmp_path / "server.py"
server.write_text(
"""
import os
from mcp.server.fastmcp import FastMCP
from mcp.types import CallToolResult, TextContent, ImageContent
server = FastMCP("test")
@server.tool()
def observe(app: str) -> CallToolResult:
return CallToolResult(content=[TextContent(type="text", text=os.environ["TOKEN"]),
ImageContent(type="image", mimeType="image/png", data="aGVsbG8=")],
structuredContent={"app": app}, isError=app == "failed")
server.run()
""",
encoding="utf-8",
)
plugin = manifest(
server={
"command": sys.executable,
"args": [str(server)],
"env": {"TOKEN": "${secret.token}"},
}
)
enable(store, plugin, argument_constraints={"observe": {"app": ["ok", "failed"]}})
runtime = PluginRuntime(store=store)
try:
loaded = await runtime.load(plugin.id)
assert loaded.status == "success"
tool_name = next(iter(runtime.bindings))
result = await runtime.execute(tool_name, {"app": "ok"}, "tc-1")
assert result.status == "success"
assert json.loads(json.dumps(result.metadata))["resource_ids"] == ["desktop:local"]
assert result.screenshot_bytes == b"hello"
assert "private-value" not in result.text_summary
assert "[已隐藏密钥]" in result.text_summary
assert any(b["type"] == "image_url" for b in result.raw_result)
failed = await runtime.call(tool_name, {"app": "failed"})
assert failed.status == "error"
assert failed.structured_content == {"app": "failed"}
blocked = await runtime.call(tool_name, {"app": "not-authorized"})
assert blocked.status == "error"
wrapper = next(t for t in runtime.langchain_tools() if t.name == tool_name)
assert (await wrapper.ainvoke({"app": "failed"})).status == "error"
finally:
await runtime.aclose()
assert runtime.sessions == {}
@pytest.mark.asyncio
async def test_stdio_timeout_is_bounded_and_never_replays(store, tmp_path, monkeypatch):
from artemis.config.settings import settings
server = tmp_path / "slow.py"
count = tmp_path / "calls.txt"
server.write_text(
f"""
import asyncio
from pathlib import Path
from mcp.server.fastmcp import FastMCP
server = FastMCP("slow")
@server.tool()
async def write_once() -> str:
with Path({str(count)!r}).open("a") as f: f.write("called\\n")
await asyncio.sleep(60)
return "done"
server.run()
""",
encoding="utf-8",
)
enable(store, manifest(server={"command": sys.executable, "args": [str(server)]}))
runtime = PluginRuntime(store=store)
await runtime.load("test-plugin")
monkeypatch.setattr(settings, "ARTEMIS_PLUGIN_TOOL_TIMEOUT_SECONDS", 0.2)
try:
name = next(iter(runtime.bindings))
result = await asyncio.wait_for(runtime.call(name, {}), timeout=10)
assert result.status == "uncertain"
assert count.read_text().splitlines() == ["called"]
again = await runtime.call(name, {})
assert again.status == "uncertain"
assert count.read_text().splitlines() == ["called"]
finally:
await runtime.aclose()
@pytest.mark.parametrize("scope", ["", "test-adb-endpoint"])
def test_resource_lock_conflicts_across_devices_without_holding_partial_leases(
tmp_path, monkeypatch, scope
):
import artemis.runtime.device_lock as locks
monkeypatch.setattr(locks, "get_temp_dir", lambda *_: tmp_path)
monkeypatch.delenv(DeviceExecutionLock.QUEUE_TICKET_ENV, raising=False)
monkeypatch.setenv(DeviceExecutionLock.LOCK_SCOPE_ENV, scope)
first = DeviceExecutionLock(
"phone-a", session_id="first", resource_ids=["desktop:local"], concurrency_mode="per_device"
)
second = DeviceExecutionLock(
"phone-b",
session_id="second",
resource_ids=["desktop:local", "router:lab"],
concurrency_mode="per_device",
)
other = DeviceExecutionLock(
"phone-c", session_id="other", resource_ids=["router:lab"], concurrency_mode="per_device"
)
first.acquire(blocking=False)
try:
with pytest.raises(DeviceBusyError):
second.acquire(blocking=False)
other.acquire(blocking=False)
assert len(DeviceExecutionLock.get_active_owners()) == 2
assert DeviceExecutionLock.get_active_owners()["phone-a"].resource_ids == ("desktop:local",)
assert DeviceExecutionLock.annotate_active_owner(
session_id="first", device_id="phone-a", ingress="test"
)
assert DeviceExecutionLock.get_active_owners()["phone-a"].resource_ids == ("desktop:local",)
finally:
other.release()
first.release()
second.acquire(blocking=False)
second.release()
assert not DeviceExecutionLock.get_active_owners()
def test_task_reservation_blocks_mutations_and_removes_files_after_release(store):
enable(store, manifest())
selection = PluginSelection(id="test-plugin", version="1.0.0")
assert store.reserve("queued-task", [selection], "mixed") == ["desktop:local"]
for change in (
lambda: store.save_profile("test-plugin", PluginProfile()),
lambda: store.set_enabled("test-plugin", False),
lambda: store.uninstall("test-plugin"),
lambda: install_archive(store, archive(manifest(version="1.1.0"))),
):
with pytest.raises(ValueError, match="任务"):
change()
assert not store.package_dir("test-plugin", "1.1.0").exists()
store.release("queued-task")
store.uninstall("test-plugin")
assert store.list() == []
assert not store.package_dir("test-plugin", "1.0.0").exists()
@pytest.mark.parametrize("stop_all", [False, True])
def test_cancelled_queue_entries_release_plugin_pin_but_live_workers_keep_it(
store, monkeypatch, stop_all
):
from types import SimpleNamespace
from apps.admin_console.services import task_queue_service as queue_module
from apps.admin_console.services import automation_store
enable(store, manifest())
selection = PluginSelection(id="test-plugin", version="1.0.0")
store.reserve("pending", [selection], "mixed")
store.reserve("active", [selection], "mixed")
pending = {"session_id": "pending", "status": "pending"}
active = {"session_id": "active", "status": "running"}
state = SimpleNamespace(
queue_items=[pending, active],
active_connections={},
active_session_id=None,
current_goal=None,
current_profile=None,
wake_event=SimpleNamespace(set=lambda: None),
)
state.clear_queue = lambda: state.queue_items.clear()
monkeypatch.setattr(queue_module, "state", state)
monkeypatch.setattr("artemis.plugins.store.PluginStore", lambda: store)
monkeypatch.setattr(DeviceExecutionLock, "cancel_reservation", lambda *_: None)
monkeypatch.setattr(
automation_store, "store", SimpleNamespace(records=lambda *_: [], job=lambda *_: None)
)
service = queue_module.TaskQueueService
for name in (
"_terminate_all_device_owners",
"_kill_all_local_runs",
"_clear_pause_file",
"ensure_worker_running",
):
monkeypatch.setattr(service, name, lambda: None)
if stop_all:
service._stop_all_tasks()
else:
service._remove_stopped_queue_item("pending", False)
service._remove_stopped_queue_item("active", False)
with store.connect() as db:
assert [row[0] for row in db.execute("SELECT task_id FROM task_pins")] == ["active"]
# Even after stop removes the queue row, the worker finalizer must drop its reservation.
service._remove_task("active")
store.assert_idle("test-plugin")
def test_registry_failure_rolls_back_package_files(store, monkeypatch):
def failed(*_):
raise OSError("disk failure")
monkeypatch.setattr(store, "register", failed)
with pytest.raises(OSError):
install_archive(store, archive(manifest()))
assert not store.package_dir("test-plugin", "1.0.0").exists()
assert store.list() == []
@pytest.mark.parametrize("path", ["NUL", "dir/COM1.txt", "aux.md", "LPT9/data"])
def test_archive_rejects_windows_device_names(store, path):
with pytest.raises(ValueError, match="路径"):
install_archive(store, archive(manifest(), {path: "bad"}))
@pytest.mark.asyncio
async def test_runtime_keeps_version_and_profile_reserved_while_paused(store):
enable(store, manifest())
runtime = PluginRuntime(store=store)
await runtime.aclose(release=False)
with pytest.raises(ValueError, match="任务"):
store.save_profile("test-plugin", PluginProfile())
await runtime.aclose()
store.save_profile("test-plugin", PluginProfile())
def test_plugin_http_api_redacts_invalid_payload_and_requires_resource_confirmation(
store, monkeypatch
):
from fastapi import FastAPI
from fastapi.testclient import TestClient
from apps.admin_console.routers import plugins
enable(store, manifest())
monkeypatch.setattr(plugins, "get_store", lambda: store)
monkeypatch.setattr(DeviceExecutionLock, "get_active_owners", lambda: {})
app = FastAPI()
app.include_router(plugins.router)
with TestClient(app) as client:
response = client.put(
"/api/plugins/test-plugin/profiles",
json={
"name": "default",
"secrets": {"token": {"invalid": "DO-NOT-LEAK"}},
},
)
assert response.status_code == 422
assert "DO-NOT-LEAK" not in response.text
assert "private-value" not in client.get("/api/plugins").text
store.block_resources(["desktop:local"], "动作未确认停止")
assert client.get("/api/plugins").json()["blocked_resources"]
assert (
client.post(
"/api/plugins/resources/confirm-stopped", json={"resource": "desktop:local"}
).status_code
== 422
)
assert store.blocked_resources()
assert (
client.post(
"/api/plugins/resources/confirm-stopped",
json={"resource": "desktop:local", "confirmed_stopped": True},
).status_code
== 200
)
assert store.blocked_resources() == {}
@pytest.mark.asyncio
async def test_pause_cleanup_error_does_not_release_resource(tmp_path, monkeypatch):
from unittest.mock import Mock
from artemis.runtime import task_control
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
lock = Mock()
control = task_control.TaskControl("cleanup-failed", lock)
entered = asyncio.Event()
async def operation():
entered.set()
try:
await asyncio.sleep(30)
except asyncio.CancelledError:
raise RuntimeError("still running remotely") from None
future = asyncio.create_task(control.run_interruptibly(operation()))
await entered.wait()
task_control.request_pause(control.session_id)
with pytest.raises(RuntimeError, match="still running"):
await future
lock.release.assert_not_called()
@pytest.mark.asyncio
async def test_stdio_pause_yields_resources_and_resume_waits_for_other_task(
store, tmp_path, monkeypatch
):
from artemis.runtime import task_control
import artemis.runtime.device_lock as locks
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path / "traces")
monkeypatch.setattr(locks, "get_temp_dir", lambda *_: tmp_path)
monkeypatch.delenv(DeviceExecutionLock.QUEUE_TICKET_ENV, raising=False)
started = tmp_path / "started.txt"
server = tmp_path / "pause_server.py"
server.write_text(
f"""
import asyncio
from pathlib import Path
from mcp.server.fastmcp import FastMCP
server = FastMCP("pause")
@server.tool()
async def observe(slow: bool = False) -> str:
if slow:
Path({str(started)!r}).write_text("started")
await asyncio.sleep(60)
return "fresh observation"
server.run()
""",
encoding="utf-8",
)
enable(store, manifest(server={"command": sys.executable, "args": [str(server)]}))
runtime = PluginRuntime(store=store)
first = DeviceExecutionLock(
"computer-a",
session_id="paused-plugin",
resource_ids=runtime.resources,
concurrency_mode="per_device",
)
other = DeviceExecutionLock(
"computer-b",
session_id="other-task",
resource_ids=runtime.resources,
concurrency_mode="per_device",
)
first.acquire(blocking=False)
control = task_control.TaskControl("paused-plugin", first)
future = None
async def until(predicate):
async with asyncio.timeout(12):
while not predicate():
await asyncio.sleep(0.05)
async def park():
await runtime.aclose(release=False)
async def resume():
await runtime.load("test-plugin")
try:
await runtime.load("test-plugin")
name = next(iter(runtime.bindings))
future = asyncio.create_task(
control.run_interruptibly(
runtime.call(name, {"slow": True}), on_pause=park, on_resume=resume
)
)
await until(started.exists)
task_control.request_pause(control.session_id)
await until(
lambda: task_control.read_state(control.session_id).get("status") == "task_paused"
)
assert not runtime.sessions
with pytest.raises(ValueError, match="任务"):
store.assert_idle("test-plugin")
other.acquire(blocking=False)
task_control.request_resume(control.session_id)
await until(lambda: task_control.read_state(control.session_id).get("status") == "resuming")
assert not future.done()
other.release()
assert await asyncio.wait_for(future, timeout=15) == (True, None)
assert DeviceExecutionLock.get_active_owners()["computer-a"].resource_ids == (
"desktop:local",
)
result = await runtime.call(next(iter(runtime.bindings)), {})
assert result.status == "success"
assert result.structured_content == {"result": "fresh observation"}
finally:
if future and not future.done():
future.cancel()
await asyncio.gather(future, return_exceptions=True)
await runtime.aclose()
control.release()
other.release()
store.assert_idle("test-plugin")
@pytest.mark.asyncio
async def test_computer_observation_and_both_action_pipelines_without_adb(
store, tmp_path, monkeypatch
):
from types import SimpleNamespace
from unittest.mock import Mock
from PIL import Image
from artemis.context import ArtemisContext, DeviceContext, DevicePlatform
from artemis.drivers.factory import create_driver
from artemis.mcp.action_executor import McpActionExecutor
from artemis.agents.validator.execution_loop import _process_action
from artemis.agents.checker.checker import build_checker_tools
picture = io.BytesIO()
Image.new("RGB", (80, 60), "white").save(picture, format="PNG")
encoded = base64.b64encode(picture.getvalue()).decode()
server = tmp_path / "desktop.py"
server.write_text(
f"""
from mcp.server.fastmcp import FastMCP
from mcp.types import CallToolResult, TextContent, ImageContent
server = FastMCP("desktop")
value = "initial"
@server.tool()
def list_apps() -> str:
return "fixture-window"
@server.tool()
def get_app_state(app: str) -> CallToolResult:
return CallToolResult(content=[TextContent(type="text", text=app + ":" + value), ImageContent(type="image", mimeType="image/png", data={encoded!r})])
@server.tool()
def set_value(text: str) -> str:
global value
value = text
return value
server.run()
""",
encoding="utf-8",
)
enable(
store,
manifest(
targets=["computer", "mixed"],
observation={"tool": "list_apps", "refresh_tools": ["get_app_state"]},
server={"command": sys.executable, "args": [str(server)]},
),
)
ctx = ArtemisContext(
device=DeviceContext(mobile_platform=DevicePlatform.COMPUTER, device_id="computer:local"),
execution_target="computer",
)
ctx.plugins = PluginRuntime(store=store, ctx=ctx)
adb = Mock(side_effect=AssertionError("computer tasks cannot create an ADB client"))
monkeypatch.setattr("artemis.drivers.factory.AdbClient", adb)
try:
driver = create_driver(ctx)
first = await driver.get_screen_data()
assert first.screenshot_bytes == b"" and "fixture-window" in first.observation_text
names = {tool.name: name for name, (_, tool) in ctx.plugins.bindings.items()}
await ctx.plugins.call(names["get_app_state"], {"app": "fixture-window"})
executor = McpActionExecutor(ctx)
result = await executor.execute(
names["set_value"], {"text": "flash-value"}, "flash-call", None
)
assert result.status == "success" and result.metadata["plugin_id"] == "test-plugin"
observed = await driver.get_screen_data()
assert observed.width == 80 and "flash-value" in observed.observation_text
node = SimpleNamespace(ctx=ctx)
outcome = await _process_action(
node,
None,
None,
{"action": names["set_value"], "args": {"text": "pro-value"}},
names["set_value"],
"",
burst=False,
)
assert outcome.success and outcome.action_item["plugin_version"] == "1.0.0"
observed = await driver.get_screen_data()
assert "pro-value" in observed.observation_text
assert "probe_device" not in {tool.name for tool in build_checker_tools(ctx, "final")}
adb.assert_not_called()
finally:
await ctx.plugins.aclose()
@pytest.mark.asyncio
@pytest.mark.parametrize("target", ["computer", "mixed"])
async def test_flash_plugin_images_never_replace_the_mixed_phone_observation(monkeypatch, target):
from types import SimpleNamespace
from unittest.mock import Mock, AsyncMock
from artemis.agents.flash import runner as flash
from artemis.agents.validator.tool_declarations import ToolExecutionResult
name = "plug_test_observe"
plugins = SimpleNamespace(
bindings={name: ("test-plugin", None)},
tool_names={name},
last_images={"test-plugin": b"desktop-before"},
)
monkeypatch.setattr("artemis.plugins.runtime.get_plugin_runtime", lambda ctx: plugins)
capture = AsyncMock(return_value=("phone-path", b"phone-after", "phone-tree-after"))
monkeypatch.setattr(flash, "capture_screenshot_and_parse_ui", capture)
runner = flash.FlashRunner.__new__(flash.FlashRunner)
runner.ctx = SimpleNamespace(execution_target=target, data_engine=Mock(), task_control=None)
runner.controller, runner.summarizer, runner._llm = Mock(), None, None
runner.executor = SimpleNamespace(
action_tool_names={name},
execute=AsyncMock(
return_value=ToolExecutionResult(
tool_name=name,
tool_call_id="tc",
status="success",
screenshot_bytes=b"desktop-after",
ui_elements_text="desktop-tree",
text_summary="Desktop observation",
metadata={"plugin_id": "test-plugin"},
)
),
)
runner._build_action_record = Mock(return_value={"action": name})
runner._record_action_step = Mock(return_value="step")
state = SimpleNamespace(latest_screenshot="old-phone-path")
result = await runner._execute_and_record_action(
name,
{},
"tc",
state,
[],
"",
{},
b"phone-before",
"phone-tree-before",
0,
flash._TurnRecord(),
)
assert runner._record_action_step.call_args.args[5:8] == (
b"desktop-before",
"",
b"desktop-after",
)
if target == "mixed":
assert result[:2] == (b"phone-after", "phone-tree-after")
assert state.latest_screenshot == "phone-path"
capture.assert_awaited_once()
else:
assert result[:2] == (b"desktop-after", "desktop-tree")
capture.assert_not_awaited()
def test_named_default_profile_is_used_only_when_authorized(store):
enable(store, manifest())
store.save_profile(
"test-plugin",
PluginProfile(
name="lab", is_default=True, authorized=True, secrets={"token": "lab-secret"}
),
)
assert store.selections()[0].profile == "lab"
store.save_profile("test-plugin", PluginProfile(name="lab", is_default=True, authorized=False))
assert store.selections() == []
@pytest.mark.asyncio
async def test_streamable_http_timeout_protects_resources_and_never_replays(
store, tmp_path, monkeypatch
):
import socket
import subprocess
from artemis.config.settings import settings
with socket.socket() as available:
available.bind(("127.0.0.1", 0))
port = available.getsockname()[1]
counter = tmp_path / "writes.txt"
script = tmp_path / "http_mcp.py"
script.write_text(
f"""
import asyncio
from pathlib import Path
from mcp.server.fastmcp import FastMCP
server = FastMCP("http-test", host="127.0.0.1", port={port})
@server.tool()
async def write_once() -> str:
with Path({str(counter)!r}).open("a") as output: output.write("write\\n")
await asyncio.sleep(30)
return "written"
server.run(transport="streamable-http")
""",
encoding="utf-8",
)
process = subprocess.Popen(
[sys.executable, str(script)],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0),
)
runtime = None
try:
for _ in range(100):
with socket.socket() as probe:
if probe.connect_ex(("127.0.0.1", port)) == 0:
break
assert process.poll() is None
await asyncio.sleep(0.05)
else:
pytest.fail("local MCP fixture did not start")
enable(
store,
manifest(
server={"transport": "streamable_http", "url": f"http://127.0.0.1:{port}/mcp"}
),
)
runtime = PluginRuntime(store=store)
await runtime.load("test-plugin")
monkeypatch.setattr(settings, "ARTEMIS_PLUGIN_TOOL_TIMEOUT_SECONDS", 0.2)
name = next(iter(runtime.bindings))
result = await asyncio.wait_for(runtime.call(name, {}), timeout=10)
assert result.status == "uncertain"
assert "desktop:local" in store.blocked_resources()
assert counter.read_text().splitlines() == ["write"]
with pytest.raises(RuntimeError, match="资源"):
await runtime.load("test-plugin")
assert counter.read_text().splitlines() == ["write"]
finally:
if runtime:
await runtime.aclose()
process.terminate()
process.wait(timeout=10)
@pytest.mark.asyncio
async def test_computer_only_plugin_diagnosis_uses_its_declared_target(store, monkeypatch):
from apps.admin_console.routers import plugins
enable(store, manifest(targets=["computer"], observation={"tool": "observe"}))
monkeypatch.setattr(plugins, "get_store", lambda: store)
result = await plugins.diagnose_plugin("test-plugin", plugins.DiagnoseRequest())
assert result["ok"] is True
store.assert_idle("test-plugin")
@pytest.mark.asyncio
async def test_workspace_computer_discovery_never_queries_adb(monkeypatch):
from mcp_server.tools import automation
routes = []
async def request(method, route):
routes.append(route)
assert route != "/api/devices"
return {"plugins": [{"id": "computer"}]} if route == "/api/plugins" else {}
monkeypatch.setattr(automation, "_request", request)
result = await automation.mobile_workspace(include_devices=False)
assert result["devices"] == []
assert result["plugins"] == [{"id": "computer"}]
assert set(routes) == {"/api/status", "/api/plugins"}
@pytest.mark.asyncio
async def test_sdk_setup_failure_releases_plugin_version_pin(store, monkeypatch):
from artemis.sdk.agent import Agent
import artemis.plugins.runtime as runtime_module
enable(store, manifest(targets=["computer"], observation={"tool": "observe"}))
monkeypatch.setattr(runtime_module, "PluginStore", lambda: store)
monkeypatch.setattr(DeviceExecutionLock, "get_active_owner", lambda _: None)
def fail_before_tracing(*args, **kwargs):
raise ValueError("cannot initialize lock")
monkeypatch.setattr("artemis.sdk.agent.DeviceExecutionLock", fail_before_tracing)
agent = Agent()
await agent.init(execution_target="computer")
request = agent.new_task("fixture").with_plugins([PluginSelection(id="test-plugin")]).build()
# Fail while constructing the lock, before device/trace initialization.
fail_before_tracing.get_active_owner = lambda _: None
with pytest.raises(ValueError, match="initialize lock"):
await agent.run_task(request=request)
store.assert_idle("test-plugin")
def test_plugin_screenshots_export_with_source_and_without_phone_substitution(
tmp_path, monkeypatch
):
from apps.admin_console.services import report_export
phone, desktop = tmp_path / "phone.png", tmp_path / "desktop.png"
phone.write_bytes(b"phone")
desktop.write_bytes(b"desktop")
monkeypatch.setattr(
report_export, "image_path", lambda name: {"phone": phone, "desktop": desktop}.get(name)
)
evidence = {"plugin_id": "desktop-control", "image_names": ["desktop"]}
report = {
"steps": [
{
"step_number": 1,
"pre_image_name": "phone",
"generic_tools": [{"payload": evidence}, {"payload": json.dumps(evidence)}],
}
]
}
images = report_export.session_screenshots(report)
assert [path for path, _ in images] == [phone, desktop]
assert "插件_desktop-control" in images[1][1]
def test_plugin_images_live_until_the_last_referencing_task_is_deleted(tmp_path):
from uuid import uuid4
from artemis.data_engine.storage import StorageManager
storage = StorageManager(tmp_path / "history.db", tmp_path)
name = "f" * 64
image = tmp_path / "images" / f"{name}.jpg"
image.parent.mkdir()
image.write_bytes(b"plugin image")
first, second = uuid4(), uuid4()
storage.reference_plugin_image(first, name)
storage.reference_plugin_image(second, name)
storage.delete_session(first)
assert image.read_bytes() == b"plugin image"
storage.delete_session(second)
assert not image.exists()
def test_plugin_operation_export_retains_source_and_failure_evidence():
from apps.admin_console.services.report_export import plugin_operations
from apps.admin_console.database.repositories.step_repository import StepRepository
operations = plugin_operations(
[
{
"step_number": 3,
"generic_tools": [
{
"name": "plug_click",
"status": "failed",
"payload": {
"plugin_id": "desktop",
"plugin_version": "1.0.0",
"resource_ids": ["desktop:local"],
"error": "窗口已关闭",
"post_image_name": "evidence",
},
},
],
}
]
)
trace = StepRepository()._normalize_display_trace(
{
"name": "plug_click",
"type": "tool",
"status": "failed",
"payload": {
"plugin_id": "desktop",
"plugin_version": "1.0.0",
"resource_ids": ["desktop:local"],
"post_image_name": "evidence",
"image_names": ["evidence"],
"error": "窗口已关闭",
},
}
)
assert trace["payload"]["post_image_name"] == "evidence"
assert plugin_operations([{"step_number": 3, "generic_tools": [trace]}]) == operations
assert operations == [
{
"step": 3,
"plugin": "desktop",
"version": "1.0.0",
"tool": "plug_click",
"resources": "desktop:local",
"status": "failed",
"content": "窗口已关闭",
}
]
@pytest.mark.asyncio
async def test_pro_text_only_observation_never_sends_an_empty_image():
from artemis.agents.operator.prompts import ObservationPromptComponent, PromptBuilder
builder = PromptBuilder()
await ObservationPromptComponent()(
builder, None, None, latest_screenshot_b64="", minimal_list="fixture window"
)
assert all(
not isinstance(part, dict) or part.get("type") != "image_url"
for part in builder.human_parts
)
assert any("fixture window" in part for part in builder.human_parts if isinstance(part, str))
@pytest.mark.asyncio
async def test_plugin_image_text_stays_with_its_own_observation(store, tmp_path):
from types import SimpleNamespace
from unittest.mock import MagicMock
from PIL import Image
from artemis.context import ArtemisContext
from artemis.data_engine.engine import DataEngine
from artemis.plugins.models import PluginCallResult
ctx = MagicMock(spec=ArtemisContext)
ctx.execution_setup = SimpleNamespace(traces_path=str(tmp_path / "traces"))
ctx.device = None
engine = DataEngine(ctx)
engine.start_session("plugin image provenance")
runtime = PluginRuntime([], store=store, ctx=SimpleNamespace(data_engine=engine))
images = []
names = []
try:
for text, color in (("baseline", "white"), ("changed", "green")):
buffer = io.BytesIO()
Image.new("RGB", (40, 40), color).save(buffer, format="PNG")
images.append(buffer.getvalue())
outcome = PluginCallResult(
status="success",
content=[
{"type": "text", "text": text},
{"type": "image", "data": base64.b64encode(images[-1]).decode()},
],
)
runtime._persist_images(outcome)
names.extend(outcome.image_names)
engine.record_step(
pre_screenshot_bytes=images[0],
post_screenshot_bytes=images[1],
ui_tree="changed",
)
await engine.shutdown()
assert engine.storage.get_image(names[0]).ui_tree == "baseline"
assert engine.storage.get_image(names[1]).ui_tree == "changed"
finally:
await runtime.aclose()
+3
View File
@@ -56,6 +56,9 @@ def _make_engine(tmp_path):
mock_ctx.execution_setup = mock_execution_setup mock_ctx.execution_setup = mock_execution_setup
mock_ctx.device = None mock_ctx.device = None
engine = DataEngine(mock_ctx) engine = DataEngine(mock_ctx)
# These tests exercise search contents, not the storage scheduler. Background
# publishers can enqueue another thread while _flush is taking its snapshot.
engine._run_in_background = lambda function, *args: function(*args)
engine.start_session("search test session") engine.start_session("search test session")
return engine return engine
Generated
+6
View File
@@ -213,11 +213,13 @@ dependencies = [
{ name = "artemis-client" }, { name = "artemis-client" },
{ name = "colorama" }, { name = "colorama" },
{ name = "fastapi" }, { name = "fastapi" },
{ name = "filelock" },
{ name = "httpx" }, { name = "httpx" },
{ name = "imageio-ffmpeg" }, { name = "imageio-ffmpeg" },
{ name = "inquirer" }, { name = "inquirer" },
{ name = "ipykernel" }, { name = "ipykernel" },
{ name = "jinja2" }, { name = "jinja2" },
{ name = "jsonschema" },
{ name = "jupyter-client" }, { name = "jupyter-client" },
{ name = "langchain" }, { name = "langchain" },
{ name = "langchain-community" }, { name = "langchain-community" },
@@ -230,6 +232,7 @@ dependencies = [
{ name = "numpy" }, { name = "numpy" },
{ name = "opencv-python" }, { name = "opencv-python" },
{ name = "openpyxl" }, { name = "openpyxl" },
{ name = "packaging" },
{ name = "pillow" }, { name = "pillow" },
{ name = "posthog" }, { name = "posthog" },
{ name = "psutil" }, { name = "psutil" },
@@ -277,11 +280,13 @@ requires-dist = [
{ name = "artemis-client", editable = "packages/artemis-client" }, { name = "artemis-client", editable = "packages/artemis-client" },
{ name = "colorama", specifier = ">=0.4.6" }, { name = "colorama", specifier = ">=0.4.6" },
{ name = "fastapi", specifier = ">=0.111.0" }, { name = "fastapi", specifier = ">=0.111.0" },
{ name = "filelock", specifier = ">=3.20.0" },
{ name = "httpx", specifier = ">=0.28.1" }, { name = "httpx", specifier = ">=0.28.1" },
{ name = "imageio-ffmpeg", specifier = ">=0.5.1" }, { name = "imageio-ffmpeg", specifier = ">=0.5.1" },
{ name = "inquirer", specifier = ">=3.4.0" }, { name = "inquirer", specifier = ">=3.4.0" },
{ name = "ipykernel", specifier = ">=6.29.0" }, { name = "ipykernel", specifier = ">=6.29.0" },
{ name = "jinja2", specifier = ">=3.1.5" }, { name = "jinja2", specifier = ">=3.1.5" },
{ name = "jsonschema", specifier = ">=4.20.0" },
{ name = "jupyter-client", specifier = ">=8.6.0" }, { name = "jupyter-client", specifier = ">=8.6.0" },
{ name = "langchain", specifier = ">=1.0.0" }, { name = "langchain", specifier = ">=1.0.0" },
{ name = "langchain-community", specifier = ">=0.4.1" }, { name = "langchain-community", specifier = ">=0.4.1" },
@@ -294,6 +299,7 @@ requires-dist = [
{ name = "numpy", specifier = ">=1.26.0" }, { name = "numpy", specifier = ">=1.26.0" },
{ name = "opencv-python", specifier = ">=4.9.0.80" }, { name = "opencv-python", specifier = ">=4.9.0.80" },
{ name = "openpyxl", specifier = ">=3.1.5" }, { name = "openpyxl", specifier = ">=3.1.5" },
{ name = "packaging", specifier = ">=24.0" },
{ name = "pillow", specifier = ">=10.0.0" }, { name = "pillow", specifier = ">=10.0.0" },
{ name = "posthog", specifier = ">=3.5.0" }, { name = "posthog", specifier = ">=3.5.0" },
{ name = "pre-commit", marker = "extra == 'dev'", specifier = ">=3.5.0" }, { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=3.5.0" },