From ddfa420945c27355afb325c645fbee60d8abce82 Mon Sep 17 00:00:00 2001 From: somew1nd Date: Sun, 30 Aug 2026 00:34:54 -0700 Subject: [PATCH] feat: enable remote SDK and resilient device orchestration --- Dockerfile | 4 + README.md | 21 +- README_CN.md | 20 +- apps/admin_console/core/security.py | 178 + apps/admin_console/core/state.py | 4 +- .../repositories/session_repository.py | 30 +- apps/admin_console/index.html | 574 +- apps/admin_console/routers/media.py | 85 +- apps/admin_console/routers/sessions.py | 237 +- apps/admin_console/routers/system.py | 209 +- apps/admin_console/routers/tasks.py | 56 + apps/admin_console/server.py | 101 +- apps/admin_console/services/media_service.py | 171 +- apps/admin_console/services/model_service.py | 27 +- .../services/task_queue_service.py | 65 +- apps/showcase_ui/angular.json | 34 +- apps/showcase_ui/package-lock.json | 13178 +++++----------- apps/showcase_ui/package.json | 26 +- .../showcase_ui/src/app/app.component.spec.ts | 4 +- apps/showcase_ui/src/app/app.component.ts | 3 +- apps/showcase_ui/src/app/app.config.ts | 4 +- .../agent-stream/agent-stream.component.html | 1939 ++- .../agent-stream/agent-stream.component.ts | 345 +- .../chat-interface.component.html | 505 +- .../chat-interface.component.ts | 72 +- .../floating-video-player.component.html | 825 +- .../floating-video-player.component.ts | 37 +- .../legacy-agent-stream.component.html | 182 +- .../legacy-agent-stream.component.ts | 3 +- .../nav-switcher/nav-switcher.component.scss | 9 +- .../nav-switcher/nav-switcher.component.ts | 7 +- .../src/app/core/models/system.model.ts | 36 + .../src/app/pages/home/home.component.html | 216 +- .../src/app/pages/home/home.component.scss | 438 +- .../src/app/pages/home/home.component.ts | 181 +- .../legacy-workspace.component.ts | 7 +- .../pages/workspace/workspace.component.html | 140 +- .../pages/workspace/workspace.component.scss | 32 +- .../pages/workspace/workspace.component.ts | 115 +- .../src/app/services/agent.service.ts | 320 +- .../src/app/services/system.service.spec.ts | 105 +- .../src/app/services/system.service.ts | 127 +- .../src/app/utils/stream-aggregator.util.ts | 39 +- .../src/app/utils/tool-formatter.util.ts | 16 + apps/showcase_ui/tsconfig.app.json | 10 +- apps/showcase_ui/tsconfig.spec.json | 10 +- artemis/__init__.py | 6 +- .../core/diagnostics/adb_server_connection.py | 330 + artemis/core/diagnostics/engine.py | 43 +- artemis/core/diagnostics/probes/adb_probe.py | 9 +- artemis/data_engine/engine.py | 6 + artemis/interfaces/cli/commands/batch.py | 1 + artemis/interfaces/cli/commands/run.py | 6 +- .../cli/commands/server_lifecycle.py | 78 +- artemis/interfaces/cli/commands/ui.py | 78 +- artemis/interfaces/sdk/__init__.py | 6 +- artemis/interfaces/sdk/client.py | 441 +- artemis/llm/fake_model.py | 179 + artemis/llm/router.py | 10 + artemis/mcp/action_server.py | 4 +- artemis/runtime/__init__.py | 14 + artemis/runtime/adb_endpoint.py | 168 + artemis/runtime/daemon_client.py | 36 +- artemis/runtime/device_lock.py | 124 +- artemis/runtime/device_pool.py | 271 +- artemis/runtime/server_lifecycle.py | 151 +- artemis/sdk/agent.py | 8 +- docs/design/pro-context-memory-redesign.md | 897 ++ docs/design/video-analysis-memory.md | 606 + mcp_server/background/task_runner.py | 1 + mcp_server/tools/task_manager.py | 7 + mcp_server/tools/task_runner.py | 22 +- mcp_server/utils/trace_store.py | 5 + packages/artemis-client/DESIGN.md | 89 + packages/artemis-client/README.md | 112 + packages/artemis-client/examples/basic.py | 30 + packages/artemis-client/pyproject.toml | 36 + .../src/artemis_client/__init__.py | 45 + .../src/artemis_client/client.py | 365 + .../src/artemis_client/errors.py | 66 + .../src/artemis_client/models.py | 212 + .../src/artemis_client/py.typed | 1 + .../src/artemis_client/transport.py | 134 + packages/artemis-client/tests/test_client.py | 246 + .../artemis-client/tests/test_transport.py | 85 + pyproject.toml | 8 + scripts/run_edge_case_part.py | 10 +- scripts/run_ui.py | 12 +- scripts/start.ps1 | 6 +- scripts/test_all_edge_cases.py | 10 +- scripts/verify_concurrency_and_devices.py | 2 +- start.sh | 4 +- tests/integration/test_thin_sdk_live.py | 128 + .../admin_console/test_security_boundary.py | 212 + .../test_session_status_normalization.py | 60 + .../test_system_emulator_endpoints.py | 34 +- .../test_task_device_readiness.py | 129 + .../admin_console/test_task_queue_service.py | 100 +- tests/unit/core/test_adb_server_connection.py | 296 + tests/unit/data_engine/test_engine.py | 17 + tests/unit/mcp/test_mcp_tools.py | 7 +- tests/unit/mcp/test_task_reconcile.py | 21 + tests/unit/mcp/test_trace_store.py | 13 + tests/unit/runtime/test_daemon_client.py | 20 +- tests/unit/runtime/test_device_lock.py | 27 + tests/unit/runtime/test_device_pool.py | 191 + .../runtime/test_server_process_lifecycle.py | 107 +- tests/unit/test_cli.py | 39 +- tests/unit/test_sdk.py | 263 +- uv.lock | 13 + 110 files changed, 14450 insertions(+), 12954 deletions(-) create mode 100644 apps/admin_console/core/security.py create mode 100644 artemis/core/diagnostics/adb_server_connection.py create mode 100644 artemis/llm/fake_model.py create mode 100644 artemis/runtime/adb_endpoint.py create mode 100644 docs/design/pro-context-memory-redesign.md create mode 100644 docs/design/video-analysis-memory.md create mode 100644 packages/artemis-client/DESIGN.md create mode 100644 packages/artemis-client/README.md create mode 100644 packages/artemis-client/examples/basic.py create mode 100644 packages/artemis-client/pyproject.toml create mode 100644 packages/artemis-client/src/artemis_client/__init__.py create mode 100644 packages/artemis-client/src/artemis_client/client.py create mode 100644 packages/artemis-client/src/artemis_client/errors.py create mode 100644 packages/artemis-client/src/artemis_client/models.py create mode 100644 packages/artemis-client/src/artemis_client/py.typed create mode 100644 packages/artemis-client/src/artemis_client/transport.py create mode 100644 packages/artemis-client/tests/test_client.py create mode 100644 packages/artemis-client/tests/test_transport.py create mode 100644 tests/integration/test_thin_sdk_live.py create mode 100644 tests/unit/admin_console/test_security_boundary.py create mode 100644 tests/unit/admin_console/test_session_status_normalization.py create mode 100644 tests/unit/core/test_adb_server_connection.py diff --git a/Dockerfile b/Dockerfile index ab0d272..885d9b0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,4 +41,8 @@ ENV PORT=8080 EXPOSE 8080 +# The console has no user authentication: publish the port to loopback only, +# e.g. `docker run -p 127.0.0.1:8080:8080 ...`, and reach it remotely through +# a Tailscale/SSH tunnel. (0.0.0.0 here is container-internal and required +# for the port mapping to work.) CMD ["uvicorn", "apps.admin_console.server:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/README.md b/README.md index c658944..9b47fe4 100644 --- a/README.md +++ b/README.md @@ -204,32 +204,31 @@ In Codex, Antigravity, or Claude Code, simply prompt:
-Embed the mobile automation engine into your Python workflows in just a few lines: +Install the zero-runtime-dependency client on the development machine. ADB, +agents, models, and image processing remain on the device host: + +```powershell +uv add "artemis-client @ git+https://github.com/google/artemis.git#subdirectory=packages/artemis-client" +``` ```python import asyncio -from artemis import ArtemisClient, ConcurrencyMode +from artemis_client import ArtemisClient async def main(): - # 1. Initialize client with optional device targeting and concurrency strategy: - # - concurrency_mode="per_device" (default): 1 task per device, allows multi-device parallel execution - # - concurrency_mode="global": 1 task globally across all devices client = ArtemisClient( + "http://artemis-host:8000", device_serial="emulator-5554", # optional: target specific device serial default_profile="flash", # "flash" (fast reactive) or "pro" (deep reasoning) - concurrency_mode="per_device", # or ConcurrencyMode.GLOBAL ) - # 2. Execute natural language end-to-end test case (can also override device per-run) result = await client.run( "Open System Settings, go to 'Battery', verify battery percentage is displayed, and check for any crash dialogs.", - device_serial="emulator-5554", # optional: override target device for this specific run ) - # 3. Structured assertions & execution tracing - assert result.status == "SUCCESS", f"Test failed: {result.error}" - print(f"✅ Test Passed! Device: {result.device_id} | Turns: {result.turns} | Trace ID: {result.trace_id}") + assert result.succeeded, f"Test failed: {result.error or result.status}" + print(f"✅ Test Passed! Device: {result.device_serial} | Trace ID: {result.trace_id}") if __name__ == "__main__": diff --git a/README_CN.md b/README_CN.md index c6824ac..5a13612 100644 --- a/README_CN.md +++ b/README_CN.md @@ -206,32 +206,30 @@ PYTHONPATH = "/path/to/artemis"
-只需几行代码，将移动端自动化智能体无缝嵌入你的 Python 脚本与 CI/CD 流水线中： +开发电脑只需安装零运行时依赖的薄客户端；ADB、Agent、模型与图像处理全部留在设备主机： + +```powershell +uv add "artemis-client @ git+https://github.com/google/artemis.git#subdirectory=packages/artemis-client" +``` ```python import asyncio -from artemis import ArtemisClient, ConcurrencyMode +from artemis_client import ArtemisClient async def main(): - # 1. 初始化测试客户端（支持指定设备序列号与并发策略）： - # - concurrency_mode="per_device"（默认）：单设备排队，跨设备支持多任务并发执行 - # - concurrency_mode="global"：全局单一并发，整个集群跨所有设备同一时刻只执行一个任务 client = ArtemisClient( + "http://artemis-host:8000", device_serial="emulator-5554", # 可选：指定目标设备序列号（不传则自动选择空闲设备） default_profile="flash", # "flash" 极速校验 或 "pro" 深度推理自愈 - concurrency_mode="per_device", # 或 ConcurrencyMode.GLOBAL ) - # 2. 执行端到端任务（也可在调用时动态覆盖指定设备与并发模式） result = await client.run( "打开系统设置，进入『电池』页面，验证是否正常显示电量百分比，确认页面无异常报错弹窗。", - device_serial="emulator-5554", # 可选：覆盖当前调用的目标设备 ) - # 3. 结构化断言与执行追溯 - assert result.status == "SUCCESS", f"测试执行失败: {result.error}" - print(f"✅ 测试通过！设备: {result.device_id} | 步数: {result.turns} | Trace ID: {result.trace_id}") + assert result.succeeded, f"测试执行失败: {result.error or result.status}" + print(f"✅ 测试通过！设备: {result.device_serial} | Trace ID: {result.trace_id}") if __name__ == "__main__": diff --git a/apps/admin_console/core/security.py b/apps/admin_console/core/security.py new file mode 100644 index 0000000..a5b132c --- /dev/null +++ b/apps/admin_console/core/security.py @@ -0,0 +1,178 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Same-origin security boundary for the Artemis console. + +Artemis runs without user accounts: whoever can reach the TCP port is the +operator. For that model to hold for a server that browsers also talk to, +two browser-borne attack vectors must be closed even on a loopback bind: + +- DNS rebinding: a page at ``attacker.example`` re-points its DNS record at + 127.0.0.1 so the victim's browser reads API responses under the attacker's + origin. Blocked by validating the ``Host`` header. +- Cross-site request forgery: any web page can fire side-effectful requests + at ``http://127.0.0.1:8000`` from a visitor's browser. Blocked by requiring + the ``Origin`` header, when a browser sends one, to match the request host. + +Non-browser clients (the SDK, curl, the CLI, MCP) send no ``Origin`` header +and pass through untouched. Remote access is expected to arrive over a +network-level tunnel (Tailscale, SSH port forward); a tunnel hostname that is +not an IP literal can be admitted via ``ARTEMIS_ALLOWED_HOSTS``. +""" + +from __future__ import annotations + +import ipaddress +import json +import os +from urllib.parse import urlsplit + +_SECURITY_HEADERS: tuple[tuple[bytes, bytes], ...] = ( + (b"x-content-type-options", b"nosniff"), + (b"referrer-policy", b"no-referrer"), + (b"x-frame-options", b"DENY"), + (b"cross-origin-opener-policy", b"same-origin"), +) + +# Paths whose responses may hold task data, media, or diagnostics and must not +# land in shared caches or survive on disk after the session. +_NO_STORE_PREFIXES = ("/api/",) + + +def _hostname(netloc: str) -> str: + """Extract the lowercase hostname from a ``host[:port]`` netloc.""" + netloc = netloc.strip().lower() + if netloc.startswith("["): # IPv6 literal, e.g. [::1]:8000 + return netloc.partition("]")[0].lstrip("[") + return netloc.rpartition(":")[0] if netloc.count(":") == 1 else netloc + + +def _is_ip_literal(host: str) -> bool: + """An IP-literal Host cannot be forged through DNS rebinding: a browser + only sends ``Host: 192.168.1.5`` when it actually connected to that IP.""" + try: + ipaddress.ip_address(host) + return True + except ValueError: + return False + + +def allowed_hostnames() -> frozenset[str]: + """Hostnames (not IP literals) accepted in Host and Origin headers.""" + names = {"localhost"} + extra = os.environ.get("ARTEMIS_ALLOWED_HOSTS", "") + names.update(part.strip().lower() for part in extra.split(",") if part.strip()) + return frozenset(names) + + +def host_is_allowed(host_header: str) -> bool: + host = _hostname(host_header) + if not host: + return False + return _is_ip_literal(host) or host in allowed_hostnames() + + +def origin_is_allowed(origin_header: str, host_header: str) -> bool: + """A browser request is acceptable when its page is same-origin with this + server, or was served from an explicitly allowed tunnel hostname.""" + origin = origin_header.strip().lower() + if not origin or origin == "null": + return False + origin_netloc = urlsplit(origin).netloc + if not origin_netloc: + return False + if origin_netloc == host_header.strip().lower(): + return True + return _hostname(origin_netloc) in allowed_hostnames() + + +class SameOriginBoundaryMiddleware: + """Pure ASGI middleware enforcing the Host/Origin boundary and appending + baseline security headers. Pure ASGI (not BaseHTTPMiddleware) so future + WebSocket endpoints cannot slip past the Origin check.""" + + def __init__(self, app): + self.app = app + + async def __call__(self, scope, receive, send): + if scope["type"] not in ("http", "websocket"): + await self.app(scope, receive, send) + return + + if os.environ.get("ARTEMIS_DISABLE_ORIGIN_GUARD", "").lower() in {"1", "true", "yes"}: + await self._forward(scope, receive, send) + return + + headers = {k.decode("latin-1").lower(): v.decode("latin-1") for k, v in scope["headers"]} + host_header = headers.get("host", "") + origin_header = headers.get("origin") + + if not host_is_allowed(host_header): + await self._reject( + scope, + send, + "Unrecognized Host header. Access Artemis via localhost or an IP address, " + "or add your tunnel hostname to ARTEMIS_ALLOWED_HOSTS.", + ) + return + + if origin_header is not None and not origin_is_allowed(origin_header, host_header): + await self._reject( + scope, + send, + "Cross-origin browser requests are not accepted by the Artemis console.", + ) + return + + await self._forward(scope, receive, send) + + async def _forward(self, scope, receive, send): + if scope["type"] != "http": + await self.app(scope, receive, send) + return + + path = scope.get("path", "") + + async def send_with_headers(message): + if message["type"] == "http.response.start": + raw = list(message.get("headers", [])) + existing = {name.lower() for name, _ in raw} + for name, value in _SECURITY_HEADERS: + if name not in existing: + raw.append((name, value)) + if b"cache-control" not in existing and path.startswith(_NO_STORE_PREFIXES): + raw.append((b"cache-control", b"no-store")) + message = {**message, "headers": raw} + await send(message) + + await self.app(scope, receive, send_with_headers) + + @staticmethod + async def _reject(scope, send, detail: str): + if scope["type"] == "websocket": + await send({"type": "websocket.close", "code": 1008}) + return + body = json.dumps({"detail": detail}).encode("utf-8") + await send( + { + "type": "http.response.start", + "status": 403, + "headers": [ + (b"content-type", b"application/json"), + (b"content-length", str(len(body)).encode("latin-1")), + *_SECURITY_HEADERS, + ], + } + ) + await send({"type": "http.response.body", "body": body}) diff --git a/apps/admin_console/core/state.py b/apps/admin_console/core/state.py index b59e946..951b1de 100644 --- a/apps/admin_console/core/state.py +++ b/apps/admin_console/core/state.py @@ -51,7 +51,7 @@ class ServerState: self.ipc_serve_task: asyncio.Task | None = None self.ipc_port: int | None = None self.port: int = 8000 - self.host: str = "0.0.0.0" + self.host: str = "127.0.0.1" self.is_shutting_down: bool = False self.current_process: asyncio.subprocess.Process | None = None @@ -151,7 +151,7 @@ class ServerState: """Device serials currently owned by an in-flight run.""" self.prune_finished_runs() return { - str(run["device_id"]) + str(run.get("lock_key") or run["device_id"]) for run in self.active_runs.values() if run.get("device_id") } diff --git a/apps/admin_console/database/repositories/session_repository.py b/apps/admin_console/database/repositories/session_repository.py index 9350619..4cfe55d 100644 --- a/apps/admin_console/database/repositories/session_repository.py +++ b/apps/admin_console/database/repositories/session_repository.py @@ -21,6 +21,18 @@ except ImportError: from apps.admin_console.database.connection import db_session +def _canonicalize_status(row: dict[str, Any]) -> dict[str, Any]: + """Map the legacy "success" terminal status to the canonical "completed". + + Rows written before the vocabulary was unified may still carry "success"; + API consumers must only ever see "completed". get_session_status is left + raw on purpose so the queue reconcile can detect and persist the rewrite. + """ + if row.get("status") == "success": + row["status"] = "completed" + return row + + class SessionRepository: """Repository handling all database queries and updates for Sessions.""" @@ -42,7 +54,7 @@ class SessionRepository: with db_session(self.db_path) as conn: cursor = conn.cursor() cursor.execute("SELECT * FROM sessions ORDER BY start_time DESC") - return [dict(r) for r in cursor.fetchall()] + return [_canonicalize_status(dict(r)) for r in cursor.fetchall()] def get_video_recordings_map(self) -> dict[str, str]: with db_session(self.db_path) as conn: @@ -240,7 +252,7 @@ class SessionRepository: (str(session_id),), ) row = cursor.fetchone() - return dict(row) if row else None + return _canonicalize_status(dict(row)) if row else None except Exception: return None @@ -267,7 +279,13 @@ class SessionRepository: conn.commit() return count - def cleanup_orphans_on_startup(self) -> int: + def reconcile_orphaned_sessions(self) -> int: + """Mark running sessions with no live worker process as failed. + + This is safe to call both during startup and immediately after a + forced server stop: live workers, including workers owned by another + server instance, are left untouched. + """ try: with db_session(self.db_path) as conn: cursor = conn.cursor() @@ -301,6 +319,10 @@ class SessionRepository: except Exception: return 0 + def cleanup_orphans_on_startup(self) -> int: + """Backward-compatible startup entrypoint for orphan reconciliation.""" + return self.reconcile_orphaned_sessions() + def get_latest_session(self) -> dict[str, Any] | None: try: with db_session(self.db_path) as conn: @@ -309,7 +331,7 @@ class SessionRepository: "SELECT session_id, status FROM sessions ORDER BY start_time DESC LIMIT 1" ) row = cursor.fetchone() - return dict(row) if row else None + return _canonicalize_status(dict(row)) if row else None except Exception: return None diff --git a/apps/admin_console/index.html b/apps/admin_console/index.html index d28f9ea..0d60c4c 100644 --- a/apps/admin_console/index.html +++ b/apps/admin_console/index.html @@ -262,7 +262,7 @@
- +
Loading sessions...
@@ -279,9 +279,9 @@