feat: enable remote SDK and resilient device orchestration

This commit is contained in:
somew1nd committed 2026-08-30 00:41:07 -07:00
1 parent 3cc711df51
commit ddfa420945
110 files changed
+14450 -12954

No files matched your search

+4
View File
@@ -41,4 +41,8 @@ ENV PORT=8080
EXPOSE 8080
# The console has no user authentication: publish the port to loopback only,
# e.g. `docker run -p 127.0.0.1:8080:8080 ...`, and reach it remotely through
# a Tailscale/SSH tunnel. (0.0.0.0 here is container-internal and required
# for the port mapping to work.)
CMD ["uvicorn", "apps.admin_console.server:app", "--host", "0.0.0.0", "--port", "8080"]
+10 -11
View File
@@ -204,32 +204,31 @@ In Codex, Antigravity, or Claude Code, simply prompt:
<br>
Embed the mobile automation engine into your Python workflows in just a few lines:
Install the zero-runtime-dependency client on the development machine. ADB,
agents, models, and image processing remain on the device host:
```powershell
uv add "artemis-client @ git+https://github.com/google/artemis.git#subdirectory=packages/artemis-client"
```
```python
import asyncio
from artemis import ArtemisClient, ConcurrencyMode
from artemis_client import ArtemisClient
async def main():
# 1. Initialize client with optional device targeting and concurrency strategy:
# - concurrency_mode="per_device" (default): 1 task per device, allows multi-device parallel execution
# - concurrency_mode="global": 1 task globally across all devices
client = ArtemisClient(
"http://artemis-host:8000",
device_serial="emulator-5554", # optional: target specific device serial
default_profile="flash", # "flash" (fast reactive) or "pro" (deep reasoning)
concurrency_mode="per_device", # or ConcurrencyMode.GLOBAL
)
# 2. Execute natural language end-to-end test case (can also override device per-run)
result = await client.run(
"Open System Settings, go to 'Battery', verify battery percentage is displayed, and check for any crash dialogs.",
device_serial="emulator-5554", # optional: override target device for this specific run
)
# 3. Structured assertions & execution tracing
assert result.status == "SUCCESS", f"Test failed: {result.error}"
print(f"✅ Test Passed! Device: {result.device_id} | Turns: {result.turns} | Trace ID: {result.trace_id}")
assert result.succeeded, f"Test failed: {result.error or result.status}"
print(f"✅ Test Passed! Device: {result.device_serial} | Trace ID: {result.trace_id}")
if __name__ == "__main__":
+9 -11
View File
@@ -206,32 +206,30 @@ PYTHONPATH = "/path/to/artemis"
<br>
只需几行代码,将移动端自动化智能体无缝嵌入你的 Python 脚本与 CI/CD 流水线中:
开发电脑只需安装零运行时依赖的薄客户端;ADB、Agent、模型与图像处理全部留在设备主机:
```powershell
uv add "artemis-client @ git+https://github.com/google/artemis.git#subdirectory=packages/artemis-client"
```
```python
import asyncio
from artemis import ArtemisClient, ConcurrencyMode
from artemis_client import ArtemisClient
async def main():
# 1. 初始化测试客户端(支持指定设备序列号与并发策略):
# - concurrency_mode="per_device"(默认):单设备排队,跨设备支持多任务并发执行
# - concurrency_mode="global":全局单一并发,整个集群跨所有设备同一时刻只执行一个任务
client = ArtemisClient(
"http://artemis-host:8000",
device_serial="emulator-5554", # 可选:指定目标设备序列号(不传则自动选择空闲设备)
default_profile="flash", # "flash" 极速校验 或 "pro" 深度推理自愈
concurrency_mode="per_device", # 或 ConcurrencyMode.GLOBAL
)
# 2. 执行端到端任务(也可在调用时动态覆盖指定设备与并发模式)
result = await client.run(
"打开系统设置,进入『电池』页面,验证是否正常显示电量百分比,确认页面无异常报错弹窗。",
device_serial="emulator-5554", # 可选:覆盖当前调用的目标设备
)
# 3. 结构化断言与执行追溯
assert result.status == "SUCCESS", f"测试执行失败: {result.error}"
print(f"✅ 测试通过!设备: {result.device_id} | 步数: {result.turns} | Trace ID: {result.trace_id}")
assert result.succeeded, f"测试执行失败: {result.error or result.status}"
print(f"✅ 测试通过!设备: {result.device_serial} | Trace ID: {result.trace_id}")
if __name__ == "__main__":
+178
View File
@@ -0,0 +1,178 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Same-origin security boundary for the Artemis console.
Artemis runs without user accounts: whoever can reach the TCP port is the
operator. For that model to hold for a server that browsers also talk to,
two browser-borne attack vectors must be closed even on a loopback bind:
- DNS rebinding: a page at ``attacker.example`` re-points its DNS record at
127.0.0.1 so the victim's browser reads API responses under the attacker's
origin. Blocked by validating the ``Host`` header.
- Cross-site request forgery: any web page can fire side-effectful requests
at ``http://127.0.0.1:8000`` from a visitor's browser. Blocked by requiring
the ``Origin`` header, when a browser sends one, to match the request host.
Non-browser clients (the SDK, curl, the CLI, MCP) send no ``Origin`` header
and pass through untouched. Remote access is expected to arrive over a
network-level tunnel (Tailscale, SSH port forward); a tunnel hostname that is
not an IP literal can be admitted via ``ARTEMIS_ALLOWED_HOSTS``.
"""
from __future__ import annotations
import ipaddress
import json
import os
from urllib.parse import urlsplit
_SECURITY_HEADERS: tuple[tuple[bytes, bytes], ...] = (
(b"x-content-type-options", b"nosniff"),
(b"referrer-policy", b"no-referrer"),
(b"x-frame-options", b"DENY"),
(b"cross-origin-opener-policy", b"same-origin"),
)
# Paths whose responses may hold task data, media, or diagnostics and must not
# land in shared caches or survive on disk after the session.
_NO_STORE_PREFIXES = ("/api/",)
def _hostname(netloc: str) -> str:
"""Extract the lowercase hostname from a ``host[:port]`` netloc."""
netloc = netloc.strip().lower()
if netloc.startswith("["): # IPv6 literal, e.g. [::1]:8000
return netloc.partition("]")[0].lstrip("[")
return netloc.rpartition(":")[0] if netloc.count(":") == 1 else netloc
def _is_ip_literal(host: str) -> bool:
"""An IP-literal Host cannot be forged through DNS rebinding: a browser
only sends ``Host: 192.168.1.5`` when it actually connected to that IP."""
try:
ipaddress.ip_address(host)
return True
except ValueError:
return False
def allowed_hostnames() -> frozenset[str]:
"""Hostnames (not IP literals) accepted in Host and Origin headers."""
names = {"localhost"}
extra = os.environ.get("ARTEMIS_ALLOWED_HOSTS", "")
names.update(part.strip().lower() for part in extra.split(",") if part.strip())
return frozenset(names)
def host_is_allowed(host_header: str) -> bool:
host = _hostname(host_header)
if not host:
return False
return _is_ip_literal(host) or host in allowed_hostnames()
def origin_is_allowed(origin_header: str, host_header: str) -> bool:
"""A browser request is acceptable when its page is same-origin with this
server, or was served from an explicitly allowed tunnel hostname."""
origin = origin_header.strip().lower()
if not origin or origin == "null":
return False
origin_netloc = urlsplit(origin).netloc
if not origin_netloc:
return False
if origin_netloc == host_header.strip().lower():
return True
return _hostname(origin_netloc) in allowed_hostnames()
class SameOriginBoundaryMiddleware:
"""Pure ASGI middleware enforcing the Host/Origin boundary and appending
baseline security headers. Pure ASGI (not BaseHTTPMiddleware) so future
WebSocket endpoints cannot slip past the Origin check."""
def __init__(self, app):
self.app = app
async def __call__(self, scope, receive, send):
if scope["type"] not in ("http", "websocket"):
await self.app(scope, receive, send)
return
if os.environ.get("ARTEMIS_DISABLE_ORIGIN_GUARD", "").lower() in {"1", "true", "yes"}:
await self._forward(scope, receive, send)
return
headers = {k.decode("latin-1").lower(): v.decode("latin-1") for k, v in scope["headers"]}
host_header = headers.get("host", "")
origin_header = headers.get("origin")
if not host_is_allowed(host_header):
await self._reject(
scope,
send,
"Unrecognized Host header. Access Artemis via localhost or an IP address, "
"or add your tunnel hostname to ARTEMIS_ALLOWED_HOSTS.",
)
return
if origin_header is not None and not origin_is_allowed(origin_header, host_header):
await self._reject(
scope,
send,
"Cross-origin browser requests are not accepted by the Artemis console.",
)
return
await self._forward(scope, receive, send)
async def _forward(self, scope, receive, send):
if scope["type"] != "http":
await self.app(scope, receive, send)
return
path = scope.get("path", "")
async def send_with_headers(message):
if message["type"] == "http.response.start":
raw = list(message.get("headers", []))
existing = {name.lower() for name, _ in raw}
for name, value in _SECURITY_HEADERS:
if name not in existing:
raw.append((name, value))
if b"cache-control" not in existing and path.startswith(_NO_STORE_PREFIXES):
raw.append((b"cache-control", b"no-store"))
message = {**message, "headers": raw}
await send(message)
await self.app(scope, receive, send_with_headers)
@staticmethod
async def _reject(scope, send, detail: str):
if scope["type"] == "websocket":
await send({"type": "websocket.close", "code": 1008})
return
body = json.dumps({"detail": detail}).encode("utf-8")
await send(
{
"type": "http.response.start",
"status": 403,
"headers": [
(b"content-type", b"application/json"),
(b"content-length", str(len(body)).encode("latin-1")),
*_SECURITY_HEADERS,
],
}
)
await send({"type": "http.response.body", "body": body})
+2 -2
View File
@@ -51,7 +51,7 @@ class ServerState:
self.ipc_serve_task: asyncio.Task | None = None
self.ipc_port: int | None = None
self.port: int = 8000
self.host: str = "0.0.0.0"
self.host: str = "127.0.0.1"
self.is_shutting_down: bool = False
self.current_process: asyncio.subprocess.Process | None = None
@@ -151,7 +151,7 @@ class ServerState:
"""Device serials currently owned by an in-flight run."""
self.prune_finished_runs()
return {
str(run["device_id"])
str(run.get("lock_key") or run["device_id"])
for run in self.active_runs.values()
if run.get("device_id")
}
@@ -21,6 +21,18 @@ except ImportError:
from apps.admin_console.database.connection import db_session
def _canonicalize_status(row: dict[str, Any]) -> dict[str, Any]:
"""Map the legacy "success" terminal status to the canonical "completed".
Rows written before the vocabulary was unified may still carry "success";
API consumers must only ever see "completed". get_session_status is left
raw on purpose so the queue reconcile can detect and persist the rewrite.
"""
if row.get("status") == "success":
row["status"] = "completed"
return row
class SessionRepository:
"""Repository handling all database queries and updates for Sessions."""
@@ -42,7 +54,7 @@ class SessionRepository:
with db_session(self.db_path) as conn:
cursor = conn.cursor()
cursor.execute("SELECT * FROM sessions ORDER BY start_time DESC")
return [dict(r) for r in cursor.fetchall()]
return [_canonicalize_status(dict(r)) for r in cursor.fetchall()]
def get_video_recordings_map(self) -> dict[str, str]:
with db_session(self.db_path) as conn:
@@ -240,7 +252,7 @@ class SessionRepository:
(str(session_id),),
)
row = cursor.fetchone()
return dict(row) if row else None
return _canonicalize_status(dict(row)) if row else None
except Exception:
return None
@@ -267,7 +279,13 @@ class SessionRepository:
conn.commit()
return count
def cleanup_orphans_on_startup(self) -> int:
def reconcile_orphaned_sessions(self) -> int:
"""Mark running sessions with no live worker process as failed.
This is safe to call both during startup and immediately after a
forced server stop: live workers, including workers owned by another
server instance, are left untouched.
"""
try:
with db_session(self.db_path) as conn:
cursor = conn.cursor()
@@ -301,6 +319,10 @@ class SessionRepository:
except Exception:
return 0
def cleanup_orphans_on_startup(self) -> int:
"""Backward-compatible startup entrypoint for orphan reconciliation."""
return self.reconcile_orphaned_sessions()
def get_latest_session(self) -> dict[str, Any] | None:
try:
with db_session(self.db_path) as conn:
@@ -309,7 +331,7 @@ class SessionRepository:
"SELECT session_id, status FROM sessions ORDER BY start_time DESC LIMIT 1"
)
row = cursor.fetchone()
return dict(row) if row else None
return _canonicalize_status(dict(row)) if row else None
except Exception:
return None
File diff suppressed because it is too large. Load diff
+58 -27
View File
@@ -12,11 +12,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
from pathlib import Path
from fastapi import APIRouter, HTTPException
from fastapi.responses import FileResponse, HTMLResponse
from artemis.config import IMAGES_DIR, WORKSPACE_ROOT
from artemis.config import IMAGES_DIR, TRACES_PATH, WORKSPACE_ROOT
try:
from admin_console.database.repositories.session_repository import session_repo
@@ -27,6 +28,48 @@ except ImportError:
router = APIRouter(tags=["media"])
_VIDEO_MEDIA_TYPES = {
".mp4": "video/mp4",
".webm": "video/webm",
".mkv": "video/x-matroska",
}
def _allowed_media_roots() -> list[Path]:
"""Directories the media endpoints are permitted to serve from."""
roots = []
for root in (WORKSPACE_ROOT, TRACES_PATH):
try:
roots.append(Path(root).resolve())
except OSError:
continue
return roots
def _resolve_media_path(raw_path: str, allowed_suffixes: set[str]) -> Path:
"""Resolve a client-supplied path strictly inside the allowed media roots.
Every candidate is fully resolved (symlinks, `..`, mixed separators) and
then re-checked against the allowed roots and an extension allowlist, so a
crafted URL can never address source code, databases, or dotfiles.
"""
roots = _allowed_media_roots()
candidate = Path(raw_path)
attempts = [candidate] if candidate.is_absolute() else [root / candidate for root in roots]
for attempt in attempts:
try:
resolved = attempt.resolve(strict=True)
except (OSError, ValueError):
continue
if not resolved.is_file():
continue
if not any(resolved.is_relative_to(root) for root in roots):
continue
if resolved.suffix.lower() not in allowed_suffixes:
raise HTTPException(status_code=403, detail="File type is not served by the media API.")
return resolved
raise HTTPException(status_code=404, detail="Media file not found")
@router.get("/admin", response_class=HTMLResponse)
@router.get("/debug", response_class=HTMLResponse)
@@ -47,8 +90,12 @@ async def get_admin_index():
async def get_image(image_name: str):
if not image_name.endswith(".jpg"):
image_name += ".jpg"
image_path = IMAGES_DIR / image_name
if not image_path.exists():
try:
images_root = IMAGES_DIR.resolve()
image_path = (images_root / image_name).resolve(strict=True)
except (OSError, ValueError):
raise HTTPException(status_code=404, detail="Image not found")
if not image_path.is_file() or not image_path.is_relative_to(images_root):
raise HTTPException(status_code=404, detail="Image not found")
return FileResponse(image_path, media_type="image/jpeg")
@@ -56,34 +103,18 @@ async def get_image(image_name: str):
@router.get("/videos/{video_path:path}")
async def get_video(video_path: str):
path = Path(video_path)
if not path.exists():
if not path.is_absolute():
candidate = WORKSPACE_ROOT / path
if candidate.exists():
path = candidate
else:
candidate_root = Path("/" + video_path.lstrip("/"))
if candidate_root.exists():
path = candidate_root
else:
path = candidate
if not path.exists():
raise HTTPException(status_code=404, detail=f"Video not found at {path}")
suffix = path.suffix.lower()
media_type = "video/mp4"
if suffix == ".webm":
media_type = "video/webm"
elif suffix == ".mkv":
media_type = "video/x-matroska"
return FileResponse(path, media_type=media_type)
path = _resolve_media_path(video_path, set(_VIDEO_MEDIA_TYPES))
return FileResponse(path, media_type=_VIDEO_MEDIA_TYPES[path.suffix.lower()])
@router.get("/api/sessions/{session_id}/video")
async def get_session_video(session_id: str):
# Blocking work (sqlite, filesystem scan, possible ffmpeg conversion) runs
# off the event loop.
return await asyncio.to_thread(_get_session_video_sync, session_id)
def _get_session_video_sync(session_id: str):
video_rec_map = session_repo.get_video_recordings_map()
video_idx = media_service.build_video_index()
row = session_repo.get_session_by_id(session_id)
+123 -114
View File
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import json
import time
from uuid import UUID
@@ -38,125 +39,133 @@ router = APIRouter(tags=["sessions"])
@router.get("/api/sessions")
async def list_sessions():
# The body does blocking work (sqlite queries, filesystem scans, ffmpeg
# conversion on first sight of a new recording), so run it off the event
# loop — the frontend polls this endpoint and it must not stall other
# requests.
try:
rows = session_repo.get_all_sessions()
video_rec_map = session_repo.get_video_recordings_map()
latest_recordings = session_repo.get_latest_video_recordings_map()
agent_names_by_session = session_repo.get_agent_trace_names_map()
video_idx = media_service.build_video_index()
default_model_info = model_service.get_active_model_info()
orphaned_ids = []
unresolved_profiles = []
result = []
try:
from artemis.runtime import DeviceExecutionLock
active_owners = DeviceExecutionLock.get_active_owners()
active_owner_sids = {
str(owner.session_id)
for owner in active_owners.values()
if owner.session_id
}
except Exception:
active_owner_sids = set()
for row_dict in rows:
s_id = str(row_dict.get("session_id"))
recording = latest_recordings.get(s_id)
d_info_raw = row_dict.get("device_info")
device_id = None
if d_info_raw:
try:
d_info = json.loads(d_info_raw) if isinstance(d_info_raw, str) else d_info_raw
if isinstance(d_info, dict):
device_id = d_info.get("device_id") or d_info.get("device_serial")
except Exception:
pass
if not device_id and recording:
device_id = recording.get("device_id")
row_dict["device_id"] = device_id
row_dict["device_serial"] = device_id
if row_dict.get("status") == "running":
is_active = (
s_id in active_owner_sids
or s_id in state.active_connections
or (state.is_running and s_id == str(state.active_session_id))
)
worker_is_alive = session_repo.process_is_alive(row_dict.get("pid"))
if not is_active and not worker_is_alive:
row_dict["status"] = "failed"
row_dict["end_time"] = time.time()
orphaned_ids.append(s_id)
recording_status = str((recording or {}).get("status") or "unavailable")
resolved_v_url = (
media_service.resolve_video_url(row_dict, video_rec_map, video_idx)
if recording_status != "recording"
else None
)
if resolved_v_url:
recording_status = "ready"
row_dict["video_url"] = resolved_v_url
else:
row_dict["video_url"] = None
row_dict["recording_status"] = recording_status
agent_names = agent_names_by_session.get(s_id, [])
sess_profile = model_service.resolve_session_profile(
row_dict, None, state.current_profile, agent_names=agent_names
)
row_dict["model_info"] = (
model_service.get_active_model_info(sess_profile)
if sess_profile
else default_model_info
)
if not sess_profile:
unresolved_profiles.append(row_dict)
result.append(row_dict)
# Most sessions carry a profile in device_info or an agent trace name.
# Only inspect the much larger LLM payload for legacy rows that remain
# ambiguous after those cheap checks.
if unresolved_profiles:
unresolved_ids = [str(row.get("session_id")) for row in unresolved_profiles]
llm_traces_by_session = session_repo.get_llm_traces_for_profiles_map(unresolved_ids)
for row_dict in unresolved_profiles:
s_id = str(row_dict.get("session_id"))
llm_traces = llm_traces_by_session.get(s_id)
if not llm_traces:
continue
sess_profile = model_service.resolve_session_profile(
row_dict, llm_traces, state.current_profile
)
if sess_profile:
row_dict["model_info"] = model_service.get_active_model_info(sess_profile)
if orphaned_ids:
try:
harvested = session_repo.harvest_orphaned_sessions(orphaned_ids)
print(f"[list_sessions] Auto-harvested {harvested} orphaned running session(s).")
try:
from mcp_server.utils import trace_store
for o_id in orphaned_ids:
if trace_store.read_status(str(o_id)):
trace_store.update_trace_status(
str(o_id),
"failed",
error="Process terminated unexpectedly (auto-harvested).",
)
except Exception:
pass
except Exception as e:
print(f"[list_sessions] Failed to update orphaned sessions: {e}")
return result
return await asyncio.to_thread(_list_sessions_sync)
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
def _list_sessions_sync():
rows = session_repo.get_all_sessions()
video_rec_map = session_repo.get_video_recordings_map()
latest_recordings = session_repo.get_latest_video_recordings_map()
agent_names_by_session = session_repo.get_agent_trace_names_map()
video_idx = media_service.build_video_index()
default_model_info = model_service.get_active_model_info()
orphaned_ids = []
unresolved_profiles = []
result = []
try:
from artemis.runtime import DeviceExecutionLock
active_owners = DeviceExecutionLock.get_active_owners()
active_owner_sids = {
str(owner.session_id)
for owner in active_owners.values()
if owner.session_id
}
except Exception:
active_owner_sids = set()
for row_dict in rows:
s_id = str(row_dict.get("session_id"))
recording = latest_recordings.get(s_id)
d_info_raw = row_dict.get("device_info")
device_id = None
if d_info_raw:
try:
d_info = json.loads(d_info_raw) if isinstance(d_info_raw, str) else d_info_raw
if isinstance(d_info, dict):
device_id = d_info.get("device_id") or d_info.get("device_serial")
except Exception:
pass
if not device_id and recording:
device_id = recording.get("device_id")
row_dict["device_id"] = device_id
row_dict["device_serial"] = device_id
if row_dict.get("status") == "running":
is_active = (
s_id in active_owner_sids
or s_id in state.active_connections
or (state.is_running and s_id == str(state.active_session_id))
)
worker_is_alive = session_repo.process_is_alive(row_dict.get("pid"))
if not is_active and not worker_is_alive:
row_dict["status"] = "failed"
row_dict["end_time"] = time.time()
orphaned_ids.append(s_id)
recording_status = str((recording or {}).get("status") or "unavailable")
resolved_v_url = (
media_service.resolve_video_url(row_dict, video_rec_map, video_idx)
if recording_status != "recording"
else None
)
if resolved_v_url:
recording_status = "ready"
row_dict["video_url"] = resolved_v_url
else:
row_dict["video_url"] = None
row_dict["recording_status"] = recording_status
agent_names = agent_names_by_session.get(s_id, [])
sess_profile = model_service.resolve_session_profile(
row_dict, None, state.current_profile, agent_names=agent_names
)
row_dict["model_info"] = (
model_service.get_active_model_info(sess_profile)
if sess_profile
else default_model_info
)
if not sess_profile:
unresolved_profiles.append(row_dict)
result.append(row_dict)
# Most sessions carry a profile in device_info or an agent trace name.
# Only inspect the much larger LLM payload for legacy rows that remain
# ambiguous after those cheap checks.
if unresolved_profiles:
unresolved_ids = [str(row.get("session_id")) for row in unresolved_profiles]
llm_traces_by_session = session_repo.get_llm_traces_for_profiles_map(unresolved_ids)
for row_dict in unresolved_profiles:
s_id = str(row_dict.get("session_id"))
llm_traces = llm_traces_by_session.get(s_id)
if not llm_traces:
continue
sess_profile = model_service.resolve_session_profile(
row_dict, llm_traces, state.current_profile
)
if sess_profile:
row_dict["model_info"] = model_service.get_active_model_info(sess_profile)
if orphaned_ids:
try:
harvested = session_repo.harvest_orphaned_sessions(orphaned_ids)
print(f"[list_sessions] Auto-harvested {harvested} orphaned running session(s).")
try:
from mcp_server.utils import trace_store
for o_id in orphaned_ids:
if trace_store.read_status(str(o_id)):
trace_store.update_trace_status(
str(o_id),
"failed",
error="Process terminated unexpectedly (auto-harvested).",
)
except Exception:
pass
except Exception as e:
print(f"[list_sessions] Failed to update orphaned sessions: {e}")
return result
@router.get("/api/sessions/{session_id}")
async def get_session_details(session_id: str):
"""Retrieve details for a single automation session."""
+174 -35
View File
@@ -14,15 +14,84 @@
"""System Readiness & Diagnostics Router for Artemis Admin Console."""
from fastapi import APIRouter, HTTPException
import ipaddress
import os
import secrets
from urllib.parse import urlsplit
from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel, Field
from artemis.core.diagnostics import readiness_engine
from artemis.core.diagnostics.adb_server_connection import (
InvalidAdbServerEndpoint,
adb_server_connection,
)
from artemis.core.diagnostics.schema import SystemReadinessReport
router = APIRouter(prefix="/api/system", tags=["system"])
def _require_local_admin_request(request: Request) -> None:
"""Keep endpoint probing and mutation on the local administration boundary."""
client_host = request.client.host if request.client else None
allow_remote = os.getenv("ARTEMIS_ALLOW_REMOTE_ADB_CONFIGURATION", "").lower() in {
"1",
"true",
"yes",
}
if client_host and not allow_remote:
try:
is_loopback = ipaddress.ip_address(client_host).is_loopback
except ValueError:
is_loopback = client_host.lower() == "localhost"
if not is_loopback:
raise HTTPException(
status_code=403,
detail=(
"ADB server settings are local-only. Set "
"ARTEMIS_ALLOW_REMOTE_ADB_CONFIGURATION=true to manage them from another "
"computer."
),
)
origin = request.headers.get("origin")
host = request.headers.get("host")
if not origin or not host:
return
origin_host = urlsplit(origin).netloc.lower()
if origin_host != host.lower():
raise HTTPException(
status_code=403,
detail="ADB server settings can only be changed from the Artemis console.",
)
def _require_loopback_request(request: Request, detail: str) -> None:
"""Reject requests whose TCP peer is not the local machine."""
client_host = request.client.host if request.client else None
try:
is_loopback = bool(client_host and ipaddress.ip_address(client_host).is_loopback)
except ValueError:
is_loopback = bool(client_host and client_host.lower() == "localhost")
if not is_loopback:
raise HTTPException(status_code=403, detail=detail)
def _require_local_lifecycle_request(request: Request) -> None:
"""Authorize a process-lifecycle request from the local CLI only."""
_require_loopback_request(request, "Server lifecycle controls are local-only.")
expected = getattr(request.app.state, "lifecycle_token", None)
supplied = request.headers.get("x-artemis-lifecycle-token")
if not (
isinstance(expected, str)
and isinstance(supplied, str)
and secrets.compare_digest(expected, supplied)
):
raise HTTPException(status_code=403, detail="Invalid server lifecycle token.")
class SelectDeviceRequest(BaseModel):
"""Payload to select an active target Android device."""
@@ -94,6 +163,66 @@ async def connect_wireless_adb(request: ConnectAdbRequest):
}
class ConnectAdbServerRequest(BaseModel):
"""Payload to select an ADB server endpoint accessible from this computer."""
host: str = Field(description="Host name or IP address of the ADB server")
port: int = Field(default=5037, ge=1, le=65535, description="ADB server port")
persist: bool = Field(default=True, description="Persist the endpoint for future launches")
@router.get("/adb/server")
async def get_adb_server_status():
"""Return the process-wide ADB server endpoint currently used by Artemis."""
return adb_server_connection.status()
@router.post("/adb/server/connect")
async def connect_adb_server(payload: ConnectAdbServerRequest, request: Request):
"""Validate and activate an ADB server endpoint."""
_require_local_admin_request(request)
try:
connection_result = await adb_server_connection.connect(
payload.host,
payload.port,
persist=payload.persist,
)
except InvalidAdbServerEndpoint as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
response: dict[str, object] = {"connection_result": connection_result}
if connection_result["success"]:
readiness_engine.set_active_device_serial(None)
readiness_engine.invalidate_cache()
response["report"] = await readiness_engine.run_all(force_refresh=True)
return response
@router.post("/adb/server/probe")
async def probe_adb_server(payload: ConnectAdbServerRequest, request: Request):
"""Test an ADB server endpoint without changing the active endpoint."""
_require_local_admin_request(request)
try:
connection_result = await adb_server_connection.probe(payload.host, payload.port)
except InvalidAdbServerEndpoint as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
return {"connection_result": connection_result}
@router.post("/adb/server/local")
async def use_local_adb_server(request: Request, persist: bool = True):
"""Restore the standard local ADB server without touching a remote daemon."""
_require_local_admin_request(request)
connection_result = await adb_server_connection.use_local_server(persist=persist)
readiness_engine.set_active_device_serial(None)
readiness_engine.invalidate_cache()
updated_report = await readiness_engine.run_all(force_refresh=True)
return {
"connection_result": connection_result,
"report": updated_report,
}
class LaunchEmulatorRequest(BaseModel):
"""Payload to launch a local Android Virtual Device (AVD)."""
@@ -157,22 +286,18 @@ class ValidateCredentialsRequest(BaseModel):
@router.get("/credentials")
async def get_credentials():
"""Retrieve current configured API keys for management."""
"""Report which providers have an API key configured.
Secret values never leave the process: this endpoint intentionally returns
presence booleans only. Keys are written via POST /credentials and used
server-side.
"""
from artemis.config import settings
gemini_k = settings.get_api_key("google")
openai_k = settings.get_api_key("openai")
anthropic_k = settings.get_api_key("anthropic")
openrouter_k = settings.get_api_key("openrouter")
ocr_k = settings.get_api_key("ocr")
return {
"google": gemini_k.get_secret_value() if gemini_k else None,
"gemini": gemini_k.get_secret_value() if gemini_k else None,
"openai": openai_k.get_secret_value() if openai_k else None,
"anthropic": anthropic_k.get_secret_value() if anthropic_k else None,
"openrouter": openrouter_k.get_secret_value() if openrouter_k else None,
"ocr": ocr_k.get_secret_value() if ocr_k else None,
}
providers = ("google", "openai", "anthropic", "openrouter", "ocr")
status = {name: bool(settings.get_api_key(name)) for name in providers}
status["gemini"] = status["google"]
return {"providers": [{"name": name, "configured": configured} for name, configured in status.items()]}
@router.post("/credentials/test")
@@ -278,11 +403,13 @@ async def get_model_config_and_env():
return None
def mask_key(k: str | None) -> str | None:
# Expose only enough to recognize which key is active, never a usable
# fragment of the secret itself.
if not k:
return None
if len(k) <= 8:
return "****"
return f"{k[:4]}...{k[-4:]}"
return f"****{k[-4:]}"
gemini_real = get_real_env_value("GEMINI_API_KEY", "google") or get_real_env_value(
"GOOGLE_API_KEY", "google"
@@ -375,18 +502,30 @@ async def get_server_runtime_status():
port = getattr(state, "port", 8000)
status = get_server_status(port=port)
status["current_pid"] = os.getpid()
return status
# Explicit DTO: the raw metadata file additionally holds the lifecycle
# token, cmdline, and filesystem paths, none of which belong on the wire.
return {
"running": status["running"],
"port": status["port"],
"pids": status["pids"],
"active_pid": status["active_pid"],
"uptime_seconds": status["uptime_seconds"],
"url": status["url"],
"admin_url": status["admin_url"],
"current_pid": os.getpid(),
}
@router.post("/restart")
async def restart_server_endpoint():
async def restart_server_endpoint(request: Request):
"""Request a graceful restart of the Artemis server from thin clients/UI."""
import asyncio
import os
import sys
import threading
_require_loopback_request(request, "Server lifecycle controls are local-only.")
try:
from apps.admin_console.core.state import state
except ImportError:
@@ -423,33 +562,33 @@ async def restart_server_endpoint():
}
@router.post("/shutdown")
async def shutdown_server_endpoint():
@router.post("/shutdown", status_code=202)
async def shutdown_server_endpoint(request: Request):
"""Request a graceful shutdown of the Artemis server."""
import os
import signal
import threading
import asyncio
try:
from apps.admin_console.core.state import state
except ImportError:
from admin_console.core.state import state
current_pid = os.getpid()
_require_local_lifecycle_request(request)
server = getattr(request.app.state, "uvicorn_server", None)
if server is None:
raise HTTPException(status_code=503, detail="Server lifecycle controller is unavailable.")
def _shutdown_worker():
import time
async def _shutdown_after_response() -> None:
# Let Starlette flush the accepted response before Uvicorn leaves its
# main loop and invokes the FastAPI shutdown lifecycle.
await asyncio.sleep(0.05)
state.is_shutting_down = True
state.shutdown_event.set()
server.should_exit = True
time.sleep(0.5)
try:
os.kill(current_pid, signal.SIGTERM)
except Exception:
os._exit(0)
threading.Thread(target=_shutdown_worker, daemon=True).start()
asyncio.create_task(_shutdown_after_response())
return {
"status": "shutting_down",
"message": "Artemis server is shutting down.",
"pid": current_pid,
"pid": os.getpid(),
}
+56
View File
@@ -80,6 +80,62 @@ async def run_task(request: RunRequest):
detail="Either 'goal' or 'goals' list must be provided.",
)
# Idempotent SDK retries must never re-run device readiness checks. A task
# can hold the device while its admission response is lost in transit; in
# that state, probing the same device again may fail or block even though
# the original task was accepted successfully.
if request.session_id and len(incoming_goals) == 1:
requested_sid = str(request.session_id)
existing_item = next(
(
item
for item in state.queue_items
if isinstance(item, dict)
and str(item.get("session_id")) == requested_sid
),
None,
)
persisted_session = session_repo.get_session_by_id(requested_sid)
is_active = (
str(state.active_session_id) == requested_sid
or requested_sid in state.active_connections
)
if existing_item or persisted_session or is_active:
task_payload = dict(existing_item or persisted_session or {})
task_payload.setdefault("session_id", requested_sid)
task_payload.setdefault("goal", incoming_goals[0])
task_payload.setdefault("profile", request.profile or "flash")
task_payload.setdefault("device_serial", request.device_serial)
task_payload.setdefault("status", "running" if is_active else "queued")
return {
"status": task_payload["status"],
"tasks": [task_payload],
"enqueued_count": 0,
"total_queued": len(state.queue_tasks),
}
# Reject an explicit unknown/offline target before running the more
# expensive readiness probe. Besides producing a stable SDK response,
# this avoids probing the currently active device for a serial that can
# never be selected. Only a successful, non-empty enumeration may reject:
# an indeterminate one (adb blip, startup) lets the submission queue and
# fail downstream with a clear error instead.
if request.device_serial:
try:
rejection = await device_pool.validate_explicit_serial_async(
request.device_serial
)
except Exception:
rejection = None
if rejection:
return {
"status": "rejected",
"error": rejection,
"tasks": [],
"enqueued_count": 0,
"total_queued": len(state.queue_tasks),
}
# Re-check immediately before enqueueing so a device locked between UI
# polling intervals cannot start through a stale Ready state. Use the
# bounded submission probe: the full diagnostics path also scans packages,
+79 -22
View File
@@ -17,9 +17,11 @@
Modular entrypoint for full trace inspection, step replay, and task execution management.
"""
import argparse
import asyncio
import os
from pathlib import Path
import secrets
import signal
import sys
from types import FrameType
@@ -43,13 +45,13 @@ for _p in (str(_workspace_root), str(_apps_dir), str(_admin_console_dir), str(_c
sys.path.insert(0, _p)
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import FileResponse, HTMLResponse
import uvicorn
from artemis.runtime import (
DeviceExecutionLock,
clear_server_info,
device_pool,
shutdown_awake_service,
start_awake_service,
write_server_info,
@@ -67,6 +69,7 @@ from artemis.config import (
)
try:
from admin_console.core.security import SameOriginBoundaryMiddleware
from admin_console.core.state import ServerState, state
from admin_console.database.connection import db_session, get_db
from admin_console.database.repositories.session_repository import session_repo
@@ -79,6 +82,7 @@ try:
from admin_console.services.model_service import model_service
from admin_console.services.task_queue_service import task_queue_service
except ImportError:
from apps.admin_console.core.security import SameOriginBoundaryMiddleware
from apps.admin_console.core.state import state
from apps.admin_console.database.repositories.session_repository import session_repo
from apps.admin_console.routers import media, replay, sessions, steps, stream, system, tasks
@@ -92,15 +96,13 @@ init_ls_address()
# Initialize FastAPI application
app = FastAPI(title="Artemis Admin & Trace Console")
LIFECYCLE_TOKEN = os.environ.get("ARTEMIS_LIFECYCLE_TOKEN") or secrets.token_urlsafe(32)
app.state.lifecycle_token = LIFECYCLE_TOKEN
# Configure CORS
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
# The console UI is served same-origin from this process, so no CORS grants
# exist at all; the boundary middleware rejects cross-origin browser traffic
# and unrecognized Host headers (DNS rebinding) instead.
app.add_middleware(SameOriginBoundaryMiddleware)
@app.on_event("startup")
@@ -108,11 +110,29 @@ async def on_startup():
"""Startup lifecycle hooks."""
state.is_shutting_down = False
state.shutdown_event.clear()
write_server_info(port=getattr(state, "port", 8000), host=getattr(state, "host", "0.0.0.0"))
write_server_info(
port=getattr(state, "port", 8000),
host=getattr(state, "host", "127.0.0.1"),
lifecycle_token=LIFECYCLE_TOKEN,
)
await asyncio.to_thread(start_awake_service)
cleaned_device_locks = DeviceExecutionLock.cleanup_stale_locks()
if cleaned_device_locks:
print(f"[ServerStartup] Removed {cleaned_device_locks} stale device lock(s).")
# Warm the adb server and complete one enumeration before requests are
# served, so submissions in the first seconds never race an adb cold start
# and get a false "no devices attached" rejection.
try:
adb_warmed = await device_pool.warm_up_async()
except Exception as exc:
adb_warmed = False
print(f"[ServerStartup] ADB warm-up failed: {exc}")
if not adb_warmed:
print(
"[ServerStartup] ADB warm-up did not confirm a responding adb server; "
"device enumeration will retry on demand."
)
asyncio.create_task(asyncio.to_thread(task_queue_service.archive_older_replays_on_launch))
asyncio.create_task(
asyncio.to_thread(task_queue_service.verify_chunks_exist_on_launch, replay_manager)
@@ -178,7 +198,10 @@ async def on_shutdown():
await ipc_service.stop_server()
state.ipc_subscribers.clear()
await asyncio.to_thread(shutdown_awake_service)
clear_server_info()
clear_server_info(
port=getattr(state, "port", 8000),
lifecycle_token=LIFECYCLE_TOKEN,
)
# Mount modular routers
@@ -222,6 +245,18 @@ def _get_showcase_dist() -> Path:
return base_dist / "frontend" / "browser"
def _resolve_static_file(root: Path, relative_path: str) -> Path | None:
"""Return the requested file only when it truly lives under ``root``."""
try:
root = root.resolve()
target = (root / relative_path).resolve()
except (OSError, ValueError):
return None
if target.is_file() and target.is_relative_to(root):
return target
return None
@app.get("/", include_in_schema=False)
async def serve_showcase_root():
"""Explicitly serve the Showcase UI at the root path by default."""
@@ -258,15 +293,18 @@ async def serve_showcase_spa(full_path: str):
showcase_dist = _get_showcase_dist()
# Exact static file match (js, css, images, fonts, favicon, logo)
# Exact static file match (js, css, images, fonts, favicon, logo).
# Resolve and re-anchor under the static root so encoded traversal
# sequences cannot address files outside the build directories.
if clean_path:
target_file = showcase_dist / clean_path
if target_file.is_file():
target_file = _resolve_static_file(showcase_dist, clean_path)
if target_file is not None:
return FileResponse(target_file)
# Fallback for icons/logos if showcase UI hasn't been built yet
public_file = _workspace_root / "apps" / "showcase_ui" / "public" / clean_path
if public_file.is_file():
public_root = _workspace_root / "apps" / "showcase_ui" / "public"
public_file = _resolve_static_file(public_root, clean_path)
if public_file is not None:
return FileResponse(public_file)
# Serve Showcase UI Angular SPA index.html
@@ -403,7 +441,7 @@ def run_ui_server(host: str, port: int, reload: bool = False) -> None:
"""Run the UI server with bounded, signal-aware graceful shutdown."""
state.host = host
state.port = port
write_server_info(port=port, host=host)
write_server_info(port=port, host=host, lifecycle_token=LIFECYCLE_TOKEN)
try:
if reload:
@@ -422,14 +460,33 @@ def run_ui_server(host: str, port: int, reload: bool = False) -> None:
port=port,
timeout_graceful_shutdown=5,
)
ArtemisUvicornServer(config).run()
server = ArtemisUvicornServer(config)
app.state.uvicorn_server = server
server.run()
finally:
clear_server_info()
app.state.uvicorn_server = None
clear_server_info(port=port, lifecycle_token=LIFECYCLE_TOKEN)
def main():
port = int(os.environ.get("ANTIGRAVITY_SIDECAR_WEB_PORT", 8000))
run_ui_server(host="0.0.0.0", port=port)
def main(argv: list[str] | None = None) -> None:
"""Parse the daemon/standalone server bind address and run Uvicorn."""
parser = argparse.ArgumentParser(description="Run the Artemis Admin Console server")
parser.add_argument(
"--host",
default=os.environ.get("ARTEMIS_SERVER_HOST", "127.0.0.1"),
help=(
"Address to bind the HTTP server to. Defaults to loopback; expose "
"remotely via a Tailscale/SSH tunnel rather than a wide bind."
),
)
parser.add_argument(
"--port",
type=int,
default=int(os.environ.get("ANTIGRAVITY_SIDECAR_WEB_PORT", "8000")),
help="TCP port for the HTTP server.",
)
args = parser.parse_args(argv)
run_ui_server(host=args.host, port=args.port)
if __name__ == "__main__":
+114 -57
View File
@@ -18,6 +18,7 @@ import json
import os
from pathlib import Path
import subprocess
import threading
from typing import Any
import urllib.parse
from fastapi import HTTPException
@@ -28,6 +29,12 @@ from artemis.config import IMAGES_DIR, TRACES_PATH, WORKSPACE_ROOT
class MediaService:
"""Service handling media indexing, local file security, payload unwrapping and notes/plans."""
# Conversion results keyed by source path -> (mtime, size, resolved path).
# Failed conversions are cached too (resolved path == source path) so a broken
# file doesn't re-run ffmpeg (up to 45s) on every /api/sessions poll.
_playable_cache: dict[str, tuple[float, int, str]] = {}
_playable_cache_lock = threading.Lock()
@classmethod
def ensure_browser_playable_video(cls, p: Path) -> Path:
"""Ensure the video file is in a browser-supported container (MP4).
@@ -41,59 +48,88 @@ class MediaService:
if suffix == ".mp4":
return p
if suffix in (".mkv", ".webm"):
src_stat = p.stat()
cache_key = str(p)
cached = cls._playable_cache.get(cache_key)
if cached and cached[0] == src_stat.st_mtime and cached[1] == src_stat.st_size:
return Path(cached[2])
mp4_cand = p.with_suffix(".mp4")
if mp4_cand.exists() and mp4_cand.stat().st_size > 0:
return mp4_cand
from artemis.utils.video import get_ffmpeg_path
ffmpeg = get_ffmpeg_path()
# 1. Attempt fast copy remux
subprocess.run(
[
ffmpeg,
"-y",
"-fflags",
"+genpts",
"-i",
str(p),
"-c",
"copy",
"-movflags",
"+faststart",
cls._playable_cache[cache_key] = (
src_stat.st_mtime,
src_stat.st_size,
str(mp4_cand),
],
capture_output=True,
timeout=15,
)
if mp4_cand.exists() and mp4_cand.stat().st_size > 0:
return mp4_cand
# 2. If copy remux failed, fallback to ultrafast re-encode
subprocess.run(
[
ffmpeg,
"-y",
"-fflags",
"+genpts+discardcorrupt",
"-i",
str(p),
"-c:v",
"libx264",
"-preset",
"ultrafast",
"-pix_fmt",
"yuv420p",
"-movflags",
"+faststart",
str(mp4_cand),
],
capture_output=True,
timeout=30,
)
if mp4_cand.exists() and mp4_cand.stat().st_size > 0:
)
return mp4_cand
with cls._playable_cache_lock:
# Another thread may have converted this file while we waited.
cached = cls._playable_cache.get(cache_key)
if cached and cached[0] == src_stat.st_mtime and cached[1] == src_stat.st_size:
return Path(cached[2])
try:
result = cls._convert_to_mp4(p, mp4_cand)
except Exception:
result = p
cls._playable_cache[cache_key] = (
src_stat.st_mtime,
src_stat.st_size,
str(result),
)
return result
except Exception:
pass
return p
@staticmethod
def _convert_to_mp4(p: Path, mp4_cand: Path) -> Path:
from artemis.utils.video import get_ffmpeg_path
ffmpeg = get_ffmpeg_path()
# 1. Attempt fast copy remux
subprocess.run(
[
ffmpeg,
"-y",
"-fflags",
"+genpts",
"-i",
str(p),
"-c",
"copy",
"-movflags",
"+faststart",
str(mp4_cand),
],
capture_output=True,
timeout=15,
)
if mp4_cand.exists() and mp4_cand.stat().st_size > 0:
return mp4_cand
# 2. If copy remux failed, fallback to ultrafast re-encode
subprocess.run(
[
ffmpeg,
"-y",
"-fflags",
"+genpts+discardcorrupt",
"-i",
str(p),
"-c:v",
"libx264",
"-preset",
"ultrafast",
"-pix_fmt",
"yuv420p",
"-movflags",
"+faststart",
str(mp4_cand),
],
capture_output=True,
timeout=30,
)
if mp4_cand.exists() and mp4_cand.stat().st_size > 0:
return mp4_cand
return p
@staticmethod
def path_to_video_url(p: Path) -> str:
resolved = p.resolve()
@@ -287,30 +323,51 @@ class MediaService:
return [MediaService.unwrap_payload(x) for x in obj]
return obj
@staticmethod
def get_safe_local_file(path_str: str) -> tuple[Path, str]:
_LOCAL_FILE_MEDIA_TYPES = {
".jpg": "image/jpeg",
".jpeg": "image/jpeg",
".png": "image/png",
".gif": "image/gif",
".webp": "image/webp",
".mp4": "video/mp4",
".webm": "video/webm",
".mkv": "video/x-matroska",
}
@classmethod
def get_safe_local_file(cls, path_str: str) -> tuple[Path, str]:
"""Serve trace media only: resolved path must sit inside the workspace
or traces tree AND carry a media extension, so this endpoint can never
hand out .env, databases, or source files."""
decoded_path = urllib.parse.unquote(path_str)
if decoded_path.startswith("file://"):
decoded_path = decoded_path[len("file://") :]
p = Path(decoded_path).resolve()
try:
p = Path(decoded_path).resolve(strict=True)
except (OSError, ValueError):
raise HTTPException(status_code=404, detail="File not found")
if not p.is_file():
raise HTTPException(status_code=404, detail="File not found")
if not str(p).startswith(str(WORKSPACE_ROOT)):
allowed_roots = []
for root in (WORKSPACE_ROOT, TRACES_PATH):
try:
allowed_roots.append(Path(root).resolve())
except OSError:
continue
if not any(p.is_relative_to(root) for root in allowed_roots):
raise HTTPException(
status_code=403,
detail="Access denied. Path is outside workspace root.",
)
ext = p.suffix.lower()
media_type = "application/octet-stream"
if ext in (".jpg", ".jpeg"):
media_type = "image/jpeg"
elif ext == ".png":
media_type = "image/png"
elif ext == ".mp4":
media_type = "video/mp4"
media_type = cls._LOCAL_FILE_MEDIA_TYPES.get(p.suffix.lower())
if media_type is None:
raise HTTPException(
status_code=403,
detail="Access denied. Only media files are served.",
)
return p, media_type
+22 -5
View File
@@ -13,18 +13,27 @@
# limitations under the License.
import json
import time
from typing import Any
class ModelService:
"""Service handling agent architecture profile detection and metadata formatting."""
@staticmethod
def get_active_model_info(profile: str | None = None) -> dict[str, str]:
"""Return active architecture and underlying LLM model configuration."""
# 1. Determine underlying LLM model and provider from config
model_id = "gemini-3.7-flash"
# parse_llm_config re-reads and re-validates the config file on every call;
# /api/sessions calls this once per session row, so cache (provider, model_id)
# for a short TTL. Config edits take effect within _LLM_INFO_TTL seconds.
_LLM_INFO_TTL = 10.0
_llm_info_cache: tuple[float, str, str] | None = None
@classmethod
def _get_llm_provider_and_model(cls) -> tuple[str, str]:
cached = cls._llm_info_cache
now = time.monotonic()
if cached and now - cached[0] < cls._LLM_INFO_TTL:
return cached[1], cached[2]
provider = "google"
model_id = "gemini-3.7-flash"
try:
from artemis.config import parse_llm_config
@@ -37,6 +46,14 @@ class ModelService:
model_id = str(llm_cfg.default.model)
except Exception:
pass
cls._llm_info_cache = (now, provider, model_id)
return provider, model_id
@classmethod
def get_active_model_info(cls, profile: str | None = None) -> dict[str, str]:
"""Return active architecture and underlying LLM model configuration."""
# 1. Determine underlying LLM model and provider from config (cached)
provider, model_id = cls._get_llm_provider_and_model()
# 2. Determine agent architecture name (Flash vs Pro)
arch_name = "Flash"
@@ -16,6 +16,7 @@ import asyncio
import codecs
from datetime import datetime
import os
import psutil
import shutil
import subprocess
import sys
@@ -38,7 +39,13 @@ from artemis.config import (
TEST_OUTPUTS_DIR,
WORKSPACE_ROOT,
)
from artemis.runtime import DeviceExecutionLock, process_supervisor
from artemis.runtime import (
AdbEndpoint,
AdbTarget,
DeviceExecutionLock,
current_adb_endpoint,
process_supervisor,
)
class TaskQueueService:
@@ -50,6 +57,17 @@ class TaskQueueService:
# keeps weak references to running tasks).
_run_tasks: set[asyncio.Task] = set()
@staticmethod
def _task_target(task_item: dict[str, Any]) -> AdbTarget:
endpoint_data = task_item.get("adb_endpoint")
endpoint = (
AdbEndpoint.from_mapping(endpoint_data)
if isinstance(endpoint_data, dict)
else current_adb_endpoint()
)
serial = task_item.get("device_serial")
return AdbTarget(endpoint=endpoint, serial=str(serial) if serial else None)
@classmethod
def _broadcast_event(cls, event_type: str, data: Any):
"""Broadcasts an event safely to all registered subscribers."""
@@ -241,8 +259,6 @@ class TaskQueueService:
pid = getattr(proc, "pid", None)
if pid:
try:
import psutil
p = psutil.Process(pid)
if not p.is_running() or p.status() == psutil.STATUS_ZOMBIE:
return proc.returncode if proc.returncode is not None else -15
@@ -345,7 +361,7 @@ class TaskQueueService:
if isinstance(i, dict) and i.get("status") == "running"
]
busy_devices = state.busy_device_ids | {
str(i.get("device_serial")) for i in in_flight if i.get("device_serial")
cls._task_target(i).lock_key for i in in_flight if i.get("device_serial")
}
dispatched_any = False
loop = asyncio.get_running_loop()
@@ -358,19 +374,20 @@ class TaskQueueService:
continue
device = item.get("device_serial")
target = cls._task_target(item)
if mode == "per_device":
# A task without a resolved device may bind to any serial, so it
# only launches on an otherwise idle scheduler; the device lock
# then allocates freely without contending against active runs.
if device is None and (state.active_runs or in_flight or dispatched_any):
continue
if device is not None and str(device) in busy_devices:
if device is not None and target.lock_key in busy_devices:
continue
item["status"] = "running"
dispatched_any = True
if device is not None:
busy_devices.add(str(device))
busy_devices.add(target.lock_key)
run_task = loop.create_task(cls._execute_task_item(item))
# Hold a strong reference: asyncio keeps only weak refs to running
# tasks, and a collected run would strand its queue item forever.
@@ -389,6 +406,8 @@ class TaskQueueService:
proc: asyncio.subprocess.Process | None = None
output_task: asyncio.Task[None] | None = None
try:
if not isinstance(goal, str) or not goal.strip():
raise ValueError("Queued task must contain a non-empty string goal.")
expected_output = task_item.get("expected_output")
enable_outputter = task_item.get("enable_outputter")
locked_app = task_item.get("locked_app_package") or task_item.get("locked_app")
@@ -436,6 +455,9 @@ class TaskQueueService:
env["ARTEMIS_SESSION_ID"] = str(sess_id)
env["ARTEMIS_TASK_INGRESS"] = str(task_item.get("ingress", "frontend"))
env["ARTEMIS_TASK_WORKER"] = "1"
target = cls._task_target(task_item)
target.endpoint.apply_to_environment(env)
env[DeviceExecutionLock.LOCK_SCOPE_ENV] = target.lock_scope
queue_ticket = task_item.get("queue_ticket")
if queue_ticket:
env[DeviceExecutionLock.QUEUE_TICKET_ENV] = str(queue_ticket)
@@ -479,6 +501,8 @@ class TaskQueueService:
state.active_runs[run_key] = {
"process": proc,
"device_id": str(device_serial) if device_serial else None,
"lock_key": target.lock_key if device_serial else None,
"adb_endpoint": target.endpoint.to_dict(),
"goal": goal,
"profile": profile,
}
@@ -503,6 +527,7 @@ class TaskQueueService:
device_id=device_serial or "pending",
session_id=str(sess_id) if sess_id else None,
ingress=ingress_type,
lock_scope=target.lock_scope,
)
# Forward the worker's combined output and tee it into the trace's
# stdout.log so the log paths advertised by the MCP API exist.
@@ -667,6 +692,7 @@ class TaskQueueService:
enqueued_tasks = []
now = time.time()
endpoint = current_adb_endpoint()
# 1. Deduplication by session_id: if session_id is already running or queued, do not re-enqueue
if session_id and len(goals) == 1:
@@ -705,6 +731,7 @@ class TaskQueueService:
and item.get("status") == "pending"
and item.get("goal") == first_goal
and (not device_serial or item.get("device_serial") == device_serial)
and item.get("adb_endpoint", {}).get("identity") == endpoint.identity
and (now - float(item.get("created_at", 0))) < 1.0
),
None,
@@ -719,31 +746,19 @@ class TaskQueueService:
# Strict device binding: reject an explicitly requested serial that is not
# attached and authorized, instead of silently running on another device.
# The shared validator fails open on an indeterminate/empty enumeration so
# the task can proceed and fail downstream with a clear no-device error.
if device_serial:
try:
from artemis.runtime import device_pool
norm = DeviceExecutionLock._normalize_device_id
attached = {
norm(d.serial): d.state
for d in await device_pool.list_devices_async()
}
rejection = await device_pool.validate_explicit_serial_async(device_serial)
except Exception:
attached = None
# Empty enumeration (no adb / transient failure) skips validation so the
# task can proceed and fail downstream with a clear no-device error,
# mirroring mcp_server.tools.task_runner._validate_device_serial.
if attached and attached.get(norm(device_serial)) != "device":
dev_state = attached.get(norm(device_serial))
error_detail = (
f"Device '{device_serial}' is not connected. "
f"Attached devices: {sorted(attached) if attached else 'none'}."
if dev_state is None
else f"Device '{device_serial}' is attached but not ready (state: '{dev_state}')."
)
rejection = None
if rejection:
return {
"status": "rejected",
"error": error_detail,
"error": rejection,
"tasks": [],
"enqueued_count": 0,
"total_queued": len(state.queue_tasks),
@@ -772,6 +787,7 @@ class TaskQueueService:
device_id=assigned_serial or "pending",
session_id=sess_id,
ingress=ingress,
lock_scope=endpoint.identity,
)
task_item = {
"session_id": sess_id,
@@ -782,6 +798,7 @@ class TaskQueueService:
"locked_app_package": locked_app_package,
"app_path": app_path,
"device_serial": assigned_serial,
"adb_endpoint": endpoint.to_dict(),
"ingress": ingress,
"conversation_id": conversation_id,
"status": "pending",
+30 -4
View File
@@ -19,7 +19,7 @@
"prefix": "app",
"architect": {
"build": {
"builder": "@angular-devkit/build-angular:application",
"builder": "@angular/build:application",
"options": {
"outputPath": "dist/frontend",
"index": "src/index.html",
@@ -65,7 +65,7 @@
"defaultConfiguration": "production"
},
"serve": {
"builder": "@angular-devkit/build-angular:dev-server",
"builder": "@angular/build:dev-server",
"configurations": {
"production": {
"buildTarget": "frontend:build:production"
@@ -77,10 +77,10 @@
"defaultConfiguration": "development"
},
"extract-i18n": {
"builder": "@angular-devkit/build-angular:extract-i18n"
"builder": "@angular/build:extract-i18n"
},
"test": {
"builder": "@angular-devkit/build-angular:karma",
"builder": "@angular/build:karma",
"options": {
"polyfills": [
"zone.js",
@@ -102,5 +102,31 @@
}
}
}
},
"schematics": {
"@schematics/angular:component": {
"type": "component"
},
"@schematics/angular:directive": {
"type": "directive"
},
"@schematics/angular:service": {
"type": "service"
},
"@schematics/angular:guard": {
"typeSeparator": "."
},
"@schematics/angular:interceptor": {
"typeSeparator": "."
},
"@schematics/angular:module": {
"typeSeparator": "."
},
"@schematics/angular:pipe": {
"typeSeparator": "."
},
"@schematics/angular:resolver": {
"typeSeparator": "."
}
}
}
+3538 -9640
View File
File diff suppressed because it is too large. Load diff
+13 -13
View File
@@ -10,29 +10,29 @@
},
"private": true,
"dependencies": {
"@angular/common": "^19.2.0",
"@angular/compiler": "^19.2.0",
"@angular/core": "^19.2.0",
"@angular/forms": "^19.2.0",
"@angular/platform-browser": "^19.2.0",
"@angular/platform-browser-dynamic": "^19.2.0",
"@angular/router": "^19.2.0",
"@angular/common": "^22.1.4",
"@angular/compiler": "^22.1.4",
"@angular/core": "^22.1.4",
"@angular/forms": "^22.1.4",
"@angular/platform-browser": "^22.1.4",
"@angular/platform-browser-dynamic": "^22.1.4",
"@angular/router": "^22.1.4",
"rxjs": "~7.8.0",
"tslib": "^2.3.0",
"zone.js": "~0.15.0"
},
"devDependencies": {
"@angular-devkit/build-angular": "^19.2.0",
"@angular/cli": "^19.2.0",
"@angular/compiler-cli": "^19.2.0",
"@angular/build": "^22.1.6",
"@angular/cli": "^22.1.6",
"@angular/compiler-cli": "^22.1.4",
"@types/jasmine": "~5.1.0",
"istanbul-lib-instrument": "^6.0.3",
"jasmine-core": "~5.6.0",
"karma": "~6.4.0",
"karma-chrome-launcher": "~3.2.0",
"karma-coverage": "~2.2.0",
"karma-jasmine": "~5.1.0",
"karma-jasmine-html-reporter": "~2.1.0",
"typescript": "~5.7.2"
},
"packageManager": "pnpm@11.22.0+sha512.1ff870c4c6133dfd88fb2afc46dd13d47f09c9794b438c6fdb47ca98caf3bc16381ee0be93a091b8e3824cf01f889f46d7d9e20910fb0be1ab0fb5baa80dd621"
"typescript": "~6.0.3"
}
}
@@ -16,14 +16,14 @@
import { TestBed } from '@angular/core/testing';
import { provideRouter } from '@angular/router';
import { provideHttpClient } from '@angular/common/http';
import { provideHttpClient, withXhr } from '@angular/common/http';
import { AppComponent } from './app.component';
describe('AppComponent', () => {
beforeEach(async () => {
await TestBed.configureTestingModule({
imports: [AppComponent],
providers: [provideRouter([]), provideHttpClient()]
providers: [provideRouter([]), provideHttpClient(withXhr())]
}).compileComponents();
});
+2 -1
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { Component } from '@angular/core';
import { Component, ChangeDetectionStrategy } from '@angular/core';
import { RouterOutlet } from '@angular/router';
import { NavSwitcherComponent } from './components/nav-switcher/nav-switcher.component';
@@ -22,6 +22,7 @@ import { NavSwitcherComponent } from './components/nav-switcher/nav-switcher.com
selector: 'app-root',
imports: [RouterOutlet, NavSwitcherComponent],
templateUrl: './app.component.html',
changeDetection: ChangeDetectionStrategy.Eager,
styleUrl: './app.component.scss'
})
export class AppComponent {
+2 -2
View File
@@ -16,7 +16,7 @@
import { ApplicationConfig, provideZoneChangeDetection } from '@angular/core';
import { provideRouter } from '@angular/router';
import { provideHttpClient } from '@angular/common/http';
import { provideHttpClient, withXhr } from '@angular/common/http';
import { routes } from './app.routes';
@@ -24,6 +24,6 @@ export const appConfig: ApplicationConfig = {
providers: [
provideZoneChangeDetection({ eventCoalescing: true }),
provideRouter(routes),
provideHttpClient()
provideHttpClient(withXhr())
]
};
File diff suppressed because it is too large. Load diff
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { Component, signal, computed, effect, inject, untracked, DestroyRef, AfterViewInit } from '@angular/core';
import { Component, ChangeDetectionStrategy, NgZone, signal, computed, effect, inject, untracked, DestroyRef, AfterViewInit } from '@angular/core';
import { CommonModule } from '@angular/common';
import { FormsModule } from '@angular/forms';
import { RouterLink } from '@angular/router';
@@ -202,16 +202,28 @@ export type { MarkdownSegment, MarkdownLine, NoteMilestone, ParsedNote };
standalone: true,
imports: [CommonModule, FormsModule],
templateUrl: './agent-stream.component.html',
styleUrl: './agent-stream.component.scss'
styleUrl: './agent-stream.component.scss',
changeDetection: ChangeDetectionStrategy.OnPush
})
export class AgentStreamComponent implements AfterViewInit {
public agentService = inject(AgentService);
private http = inject(HttpClient);
private destroyRef = inject(DestroyRef);
private zone = inject(NgZone);
// Auto-scroll state tracking
public isUserAtBottom = true;
private resizeObserver: ResizeObserver | null = null;
private streamLogsContainer: HTMLElement | null = null;
private autoScrollTimer: ReturnType<typeof setTimeout> | null = null;
private autoScrollStreamBoxes = false;
private onContainerScroll = () => {
const container = this.streamLogsContainer;
if (!container) return;
const threshold = 150;
const distanceToBottom = container.scrollHeight - container.scrollTop - container.clientHeight;
this.isUserAtBottom = distanceToBottom <= threshold;
};
// Dynamic Planning Loader Game-style Phrases
public currentPlanningText = signal<string>(PLANNING_LOADER_PHRASES[0]);
@@ -230,9 +242,29 @@ export class AgentStreamComponent implements AfterViewInit {
public collapsedStreams = signal<Set<string>>(new Set<string>());
private scheduledCollapses = new Set<string>();
// Signals and maps for typewriter effect simulation
// Signals and maps for typewriter effect simulation. All active typewriters
// are driven by ONE requestAnimationFrame loop (run outside the Angular
// zone) that batches every text advance into a single signal update.
public typedTextsSignal = signal<Record<string, string>>({});
private typingTimers = new Map<string, any>();
private typingTargets = new Map<string, {
blockId: string;
isThinking: boolean;
execId?: string;
fallbackText: string;
}>();
private typingRafId: number | null = null;
private lastTypingTimestamp = 0;
// ≈ legacy pace of 8 characters every 12ms interval tick.
private static readonly TYPING_CHARS_PER_SECOND = 667;
private static readonly TYPING_MIN_FRAME_MS = 28;
// Rendered-markdown memo per template slot: returning the identical string
// instance for unchanged text lets the [innerHTML] binding skip re-sanitizing
// and re-parsing the whole fragment on every change-detection pass.
private markdownHtmlCache = new Map<string, { source: string; html: string }>();
// Thinking-text extraction memo (includes the JSON-detection heuristics).
private thinkingTextCache = new WeakMap<any, { native: string | null; raw: string | null }>();
private deviceSerialCache = new WeakMap<Session, string | null>();
// Set to track expanded and collapsed state of action cards
public expandedActionCards = signal<Set<string>>(new Set<string>());
@@ -285,6 +317,13 @@ export class AgentStreamComponent implements AfterViewInit {
return notes['output.md'] || null;
});
// Memoized so the Task Report card is not re-parsed (and its DOM rebuilt)
// on every change-detection pass while streams are typing.
public parsedOutputReport = computed<ParsedNote | null>(() => {
const report = this.outputterReport();
return report ? parseNote(report) : null;
});
public parsedNote = computed<ParsedNote>(() => {
return this.getParsedNote(this.currentNoteContent());
});
@@ -399,21 +438,24 @@ export class AgentStreamComponent implements AfterViewInit {
}
constructor() {
const retryClockInterval = setInterval(() => this.retryClock.set(Date.now()), 1000);
this.destroyRef.onDestroy(() => clearInterval(retryClockInterval));
// The retry clock only needs to tick while something on screen is timing
// (startup progress, LLM retry countdowns). An idle page stays quiet.
this.zone.runOutsideAngular(() => {
const retryClockInterval = setInterval(() => {
const status = this.agentService.agentStatus();
if (status === 'running' || status === 'paused' || this.agentService.isRetrying()) {
this.retryClock.set(Date.now());
}
}, 1000);
this.destroyRef.onDestroy(() => clearInterval(retryClockInterval));
});
// Auto scroll stream box during active text streaming
effect(() => {
const logs = this.filteredLogs();
const activeStream = logs.find(log => log.type === 'llm_stream' && !log.data?.isCompleted);
if (activeStream) {
setTimeout(() => {
const elements = document.getElementsByClassName('stream-box');
for (let i = 0; i < elements.length; i++) {
const el = elements[i] as HTMLElement;
el.scrollTop = el.scrollHeight;
}
}, 30);
this.scheduleAutoScroll(true);
}
});
@@ -421,13 +463,7 @@ export class AgentStreamComponent implements AfterViewInit {
effect(() => {
this.consolidatedBlocks();
this.typedTextsSignal();
setTimeout(() => {
const container = document.querySelector('.stream-logs-content');
if (container && this.isUserAtBottom) {
container.scrollTop = container.scrollHeight;
}
}, 50);
this.scheduleAutoScroll(false);
});
// Typewriter drive logic: triggers typing when block appears or expands
@@ -451,7 +487,7 @@ export class AgentStreamComponent implements AfterViewInit {
}
} else {
const currentVal = currentRecord[blockId] || '';
if (currentVal.length < rawText.length && !this.typingTimers.has(blockId)) {
if (currentVal.length < rawText.length && !this.typingTargets.has(blockId)) {
if (block.data?.isCompleted) {
this.typedTextsSignal.update(r => ({ ...r, [blockId]: rawText }));
} else {
@@ -479,7 +515,7 @@ export class AgentStreamComponent implements AfterViewInit {
}
} else {
const currentVal = currentRecord[nativeKey] || '';
if (currentVal.length < nativeText.length && !this.typingTimers.has(nativeKey)) {
if (currentVal.length < nativeText.length && !this.typingTargets.has(nativeKey)) {
if (block.data?.isCompleted) {
this.typedTextsSignal.update(r => ({ ...r, [nativeKey]: nativeText }));
this.collapsedStreams.update(set => {
@@ -501,9 +537,9 @@ export class AgentStreamComponent implements AfterViewInit {
effect(() => {
this.agentService.currentSessionId();
untracked(() => {
this.typingTimers.forEach((timer) => clearInterval(timer));
this.typingTimers.clear();
this.stopTypingLoop();
this.typedTextsSignal.set({});
this.markdownHtmlCache.clear();
this.scheduledCollapses.clear();
this.collapsedStreams.set(new Set<string>());
this.isUserAtBottom = true;
@@ -547,27 +583,65 @@ export class AgentStreamComponent implements AfterViewInit {
this.resizeObserver.disconnect();
this.resizeObserver = null;
}
this.stopTypingLoop();
if (this.autoScrollTimer) {
clearTimeout(this.autoScrollTimer);
this.autoScrollTimer = null;
}
if (this.streamLogsContainer) {
this.streamLogsContainer.removeEventListener('scroll', this.onContainerScroll);
this.streamLogsContainer = null;
}
});
}
public ngAfterViewInit(): void {
const container = document.querySelector('.stream-logs-content');
if (container && typeof ResizeObserver !== 'undefined') {
const container = document.querySelector('.stream-logs-content') as HTMLElement | null;
this.streamLogsContainer = container;
if (!container) return;
// The scroll listener lives outside the Angular zone: tracking the
// user's scroll position must not trigger change detection per frame.
this.zone.runOutsideAngular(() => {
container.addEventListener('scroll', this.onContainerScroll, { passive: true });
});
if (typeof ResizeObserver !== 'undefined') {
this.resizeObserver = new ResizeObserver(() => {
if (this.isUserAtBottom) {
container.scrollTop = container.scrollHeight;
this.scheduleAutoScroll(false);
}
});
this.resizeObserver.observe(container);
}
}
public onStreamScroll(event: Event): void {
const container = event.target as HTMLElement;
if (!container) return;
const threshold = 150;
const distanceToBottom = container.scrollHeight - container.scrollTop - container.clientHeight;
this.isUserAtBottom = distanceToBottom <= threshold;
/**
* Coalesce every auto-scroll request into one deferred pass that batches all
* DOM reads before all writes, instead of thrashing layout per update.
*/
private scheduleAutoScroll(includeStreamBoxes: boolean): void {
this.autoScrollStreamBoxes = this.autoScrollStreamBoxes || includeStreamBoxes;
if (this.autoScrollTimer) return;
this.zone.runOutsideAngular(() => {
this.autoScrollTimer = setTimeout(() => {
this.autoScrollTimer = null;
const includeBoxes = this.autoScrollStreamBoxes;
this.autoScrollStreamBoxes = false;
const boxes = includeBoxes
? (Array.from(document.getElementsByClassName('stream-box')) as HTMLElement[])
: [];
const boxTargets = boxes.map((el) => el.scrollHeight);
const container = this.streamLogsContainer;
const containerTarget = container && this.isUserAtBottom ? container.scrollHeight : null;
boxes.forEach((el, i) => { el.scrollTop = boxTargets[i]; });
if (container && containerTarget !== null) {
container.scrollTop = containerTarget;
}
}, 50);
});
}
// Top Nav Action Methods
@@ -619,22 +693,26 @@ export class AgentStreamComponent implements AfterViewInit {
* Determine the device serial number for the session
*/
public getDeviceSerial(session: Session): string | null {
if (this.deviceSerialCache.has(session)) {
return this.deviceSerialCache.get(session) ?? null;
}
let resolved: string | null = null;
const serial = session.device_serial || session.device_id;
if (serial && serial !== 'pending' && serial !== 'null' && serial !== 'undefined') {
return serial;
}
if (session.device_info) {
resolved = serial;
} else if (session.device_info) {
try {
const info = typeof session.device_info === 'string' ? JSON.parse(session.device_info) : session.device_info;
const s = info?.device_id || info?.device_serial;
if (s && s !== 'pending' && s !== 'null' && s !== 'undefined') {
return s;
resolved = s;
}
} catch {
// ignore
}
}
return null;
this.deviceSerialCache.set(session, resolved);
return resolved;
}
public selectTask(sessionId: string, event?: Event): void {
@@ -675,40 +753,75 @@ export class AgentStreamComponent implements AfterViewInit {
// Typewriter & Delayed Collapse
private startTyping(key: string, targetText: string, isThinking: boolean = false, execId?: string) {
if (this.typingTimers.has(key)) {
clearInterval(this.typingTimers.get(key));
}
const blockId = key.endsWith('-native') ? key.replace('-native', '') : key;
this.typingTargets.set(key, { blockId, isThinking, execId, fallbackText: targetText });
this.ensureTypingLoop();
}
const interval = 12;
const timer = setInterval(() => {
const currentRecord = this.typedTextsSignal();
const current = currentRecord[key] || '';
const blocks = this.consolidatedBlocks();
const blockId = key.endsWith('-native') ? key.replace('-native', '') : key;
const block = blocks.find(b => b.id === blockId);
let target = targetText;
private ensureTypingLoop(): void {
if (this.typingRafId !== null) return;
this.lastTypingTimestamp = performance.now();
this.zone.runOutsideAngular(() => {
const step = (now: number) => {
this.typingRafId = null;
const elapsed = now - this.lastTypingTimestamp;
if (elapsed >= AgentStreamComponent.TYPING_MIN_FRAME_MS) {
this.lastTypingTimestamp = now;
this.advanceTyping(elapsed);
}
if (this.typingTargets.size > 0) {
this.typingRafId = requestAnimationFrame(step);
}
};
this.typingRafId = requestAnimationFrame(step);
});
}
private stopTypingLoop(): void {
this.typingTargets.clear();
if (this.typingRafId !== null) {
cancelAnimationFrame(this.typingRafId);
this.typingRafId = null;
}
}
/**
* Advance every active typewriter by the elapsed wall time and publish all
* changes as one signal update (one render pass per animation frame at most).
*/
private advanceTyping(elapsedMs: number): void {
const blocks = untracked(() => this.consolidatedBlocks());
const currentRecord = untracked(() => this.typedTextsSignal());
const chars = Math.max(
1,
Math.round(AgentStreamComponent.TYPING_CHARS_PER_SECOND * elapsedMs / 1000)
);
const updates: Record<string, string> = {};
let hasUpdates = false;
for (const [key, typing] of this.typingTargets) {
const block = blocks.find((b) => b.id === typing.blockId);
let target = typing.fallbackText;
if (block) {
target = isThinking ? (this.getNativeThinking(block) || targetText) : (this.getRawThinking(block) || targetText);
target = (typing.isThinking ? this.getNativeThinking(block) : this.getRawThinking(block))
|| typing.fallbackText;
}
const current = currentRecord[key] || '';
if (current.length < target.length) {
const nextLength = Math.min(current.length + 8, target.length);
const newText = target.slice(0, nextLength);
this.typedTextsSignal.update(r => ({ ...r, [key]: newText }));
} else {
if (block?.data?.isCompleted) {
clearInterval(timer);
this.typingTimers.delete(key);
if (isThinking && execId) {
this.triggerDelayedCollapse(execId);
}
updates[key] = target.slice(0, Math.min(current.length + chars, target.length));
hasUpdates = true;
} else if (block?.data?.isCompleted) {
this.typingTargets.delete(key);
if (typing.isThinking && typing.execId) {
this.triggerDelayedCollapse(typing.execId);
}
}
}, interval);
}
this.typingTimers.set(key, timer);
if (hasUpdates) {
this.typedTextsSignal.update((r) => ({ ...r, ...updates }));
}
}
private triggerDelayedCollapse(execId: string) {
@@ -886,6 +999,28 @@ export class AgentStreamComponent implements AfterViewInit {
}
// Generic Tool Delegation Methods
/**
* First save_note call per note key, derived once per blocks change instead
* of re-scanning every block for every rendered tool row.
*/
private firstSaveNoteByKey = computed<Map<string, any>>(() => {
const firstByKey = new Map<string, any>();
for (const block of this.consolidatedBlocks()) {
if (block.type === 'step' && block.data.generic_tools) {
for (const t of this.getUniqueGenericTools(block.data.generic_tools)) {
if (t.name && t.name.toLowerCase() === 'save_note') {
const key = this.getToolKey(t);
if (key && !firstByKey.has(key)) {
firstByKey.set(key, t);
}
}
}
}
}
return firstByKey;
});
public isFirstSaveNoteForKey(tool: any): boolean {
if (!tool || !tool.name || tool.name.toLowerCase() !== 'save_note') {
return false;
@@ -893,26 +1028,11 @@ export class AgentStreamComponent implements AfterViewInit {
const key = this.getToolKey(tool);
if (!key) return true;
const blocks = this.consolidatedBlocks();
const saveNoteCalls: any[] = [];
for (const block of blocks) {
if (block.type === 'step' && block.data.generic_tools) {
const uniqTools = this.getUniqueGenericTools(block.data.generic_tools);
for (const t of uniqTools) {
if (t.name && t.name.toLowerCase() === 'save_note' && this.getToolKey(t) === key) {
saveNoteCalls.push(t);
}
}
}
}
if (saveNoteCalls.length > 0) {
const firstCall = saveNoteCalls[0];
return (tool.trace_id && firstCall.trace_id)
? tool.trace_id === firstCall.trace_id
: tool === firstCall;
}
return true;
const firstCall = this.firstSaveNoteByKey().get(key);
if (!firstCall) return true;
return (tool.trace_id && firstCall.trace_id)
? tool.trace_id === firstCall.trace_id
: tool === firstCall;
}
public getToolDisplayLabel(tool: any): string {
@@ -1013,17 +1133,33 @@ export class AgentStreamComponent implements AfterViewInit {
return getSortedStepEvents(stepData, this.sortedEventsCache);
}
/**
* Memoized per block-data object: templates call these many times per
* change-detection pass, and the raw-thinking path runs JSON-detection
* heuristics over the full text.
*/
private getThinkingTexts(data: any): { native: string | null; raw: string | null } {
let entry = this.thinkingTextCache.get(data);
if (!entry) {
const nativeText = data.operator_native_thinking || (data.stream_type === 'thinking' ? data.text : null);
const rawText = data.operator_raw_thinking || (data.stream_type === 'text' ? data.text : null);
entry = {
native: nativeText && nativeText.trim() ? nativeText : null,
raw: rawText && rawText.trim() && this.isHumanThinking(rawText) ? rawText : null
};
this.thinkingTextCache.set(data, entry);
}
return entry;
}
public getNativeThinking(block: any): string | null {
if (!block || !block.data) return null;
const text = block.data.operator_native_thinking || (block.data.stream_type === 'thinking' ? block.data.text : null);
return text && text.trim() ? text : null;
return this.getThinkingTexts(block.data).native;
}
public getRawThinking(block: any): string | null {
if (!block || !block.data) return null;
const text = block.data.operator_raw_thinking || (block.data.stream_type === 'text' ? block.data.text : null);
if (!text || !text.trim() || !this.isHumanThinking(text)) return null;
return text;
return this.getThinkingTexts(block.data).raw;
}
public hasVisibleContent(block: any): boolean {
@@ -1272,8 +1408,23 @@ export class AgentStreamComponent implements AfterViewInit {
}
}
public renderMarkdown(text: string): string {
return renderMarkdownToHtml(text);
/**
* Render markdown with a per-slot memo. Returning the identical cached
* string instance for unchanged text means the [innerHTML] binding sees the
* same reference and skips DOM/sanitizer work entirely.
*/
public renderMarkdown(text: string, cacheKey?: string): string {
if (!text) return '';
if (!cacheKey) {
return renderMarkdownToHtml(text);
}
const cached = this.markdownHtmlCache.get(cacheKey);
if (cached && cached.source === text) {
return cached.html;
}
const html = renderMarkdownToHtml(text);
this.markdownHtmlCache.set(cacheKey, { source: text, html });
return html;
}
public onToolKeyClick(key: string): void {
@@ -1314,9 +1465,17 @@ export class AgentStreamComponent implements AfterViewInit {
return tool.trace_id || index.toString();
}
public trackStepEvent(index: number, item: { type: string; data: any }): string {
public trackStepEvent(index: number, item: { type: string; data: any; timestamp?: number }): string {
if (!item || !item.data) return index.toString();
return item.type + '-' + (item.data.timestamp || item.data.start_time || item.data.created_at || item.data.id || item.data.action || item.data.name || item.data.text?.length || index);
// A step has at most one thinking and one text event; keying them by type
// alone keeps their DOM stable while the streamed text grows (keying by
// text length would tear down and rebuild the node on every chunk).
if (item.type === 'thinking' || item.type === 'text') {
return item.type;
}
return item.type + '-' + (item.data.trace_id
|| item.data.timestamp || item.data.start_time || item.data.created_at || item.data.id
|| ((item.data.action || item.data.name || '') + '-' + index));
}
public trackParam(index: number, param: { key: string; value: string }): string {
@@ -1,17 +1,17 @@
<!--
Copyright 2026 Google LLC
Copyright 2026 Google LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<div class="chat-interface-container">
@@ -19,267 +19,322 @@
<header class="chat-header">
<!-- Tab Selector Header inside row -->
<div class="tab-selector-bar">
<button
class="tab-selector-btn"
[class.active]="agentService.activeTab() === 'tasks'"
<button
class="tab-selector-btn"
[class.active]="agentService.activeTab() === 'tasks'"
(click)="agentService.activeTab.set('tasks')"
>
>
<span class="material-symbols-outlined tab-icon">list_alt</span>
<span class="tab-text">Task Queue</span>
</button>
<button
class="tab-selector-btn"
[class.active]="agentService.activeTab() === 'notes'"
<button
class="tab-selector-btn"
[class.active]="agentService.activeTab() === 'notes'"
(click)="agentService.activeTab.set('notes')"
>
>
<span class="material-symbols-outlined tab-icon">description</span>
<span class="tab-text">Notes & Plans</span>
</button>
</div>
<div class="header-right">
<button
*ngIf="agentService.sessions().length > 0 && agentService.activeTab() === 'tasks'"
class="btn-clear-history"
(click)="clearHistory()"
title="Clear all tasks and history"
aria-label="Clear all tasks and history"
[disabled]="isSubmitting"
>
<span class="material-symbols-outlined">delete</span>
</button>
@if (agentService.sessions().length > 0 && agentService.activeTab() === 'tasks') {
<button
class="btn-clear-history"
(click)="clearHistory()"
title="Clear all tasks and history"
aria-label="Clear all tasks and history"
[disabled]="isSubmitting()"
>
<span class="material-symbols-outlined">delete</span>
</button>
}
</div>
</header>
<!-- Scrollable Messages/Content Area -->
<main class="chat-messages">
<!-- Tasks Tab Content -->
<ng-container *ngIf="agentService.activeTab() === 'tasks'">
@if (agentService.activeTab() === 'tasks') {
<!-- Active Queue Section -->
<div class="queue-section-header">
<span class="material-symbols-outlined section-icon">bolt</span> Active Queue ({{ activeQueue().length }})
</div>
<div *ngIf="activeQueue().length === 0" class="empty-section-placeholder">
No active or queued tasks.
</div>
<div
*ngFor="let session of activeQueue(); trackBy: trackSession"
class="task-card"
[class.selected]="agentService.currentSessionId() === session.session_id"
[class.running]="getTaskStatus(session) === 'running'"
(click)="selectTask(session.session_id)"
>
<div class="task-card-header">
<span class="task-badge" [ngClass]="getTaskStatus(session)">
<span class="status-dot" *ngIf="getTaskStatus(session) === 'running'"></span>
{{ getTaskStatus(session) | uppercase }}
</span>
<div class="task-card-header-actions">
<span class="task-device" *ngIf="getDeviceSerial(session)" [title]="'Device: ' + getDeviceSerial(session)">
<span class="material-symbols-outlined device-icon">phone_android</span>
<span class="device-name">{{ getDeviceSerial(session) }}</span>
@if (activeQueue().length === 0) {
<div class="empty-section-placeholder">
No active or queued tasks.
</div>
}
@for (session of activeQueue(); track trackSession($index, session)) {
<div
class="task-card"
[class.selected]="agentService.currentSessionId() === session.session_id"
[class.running]="getTaskStatus(session) === 'running'"
(click)="selectTask(session.session_id)"
>
<div class="task-card-header">
<span class="task-badge" [ngClass]="getTaskStatus(session)">
@if (getTaskStatus(session) === 'running') {
<span class="status-dot"></span>
}
{{ getTaskStatus(session) | uppercase }}
</span>
<span class="task-time-divider" *ngIf="getDeviceSerial(session)">·</span>
<span class="task-time">{{ session.start_time * 1000 | date:'shortTime' }}</span>
<button
*ngIf="getTaskStatus(session) === 'running' || getTaskStatus(session) === 'paused'"
type="button"
class="btn-stop-card"
(click)="stopTask(session.session_id, $event)"
title="Stop this task"
aria-label="Stop this task"
>
<span class="material-symbols-outlined">stop_circle</span>
</button>
<button
*ngIf="getTaskStatus(session) !== 'running' && getTaskStatus(session) !== 'paused'"
type="button"
class="btn-delete-card"
(click)="deleteTask(session.session_id, $event)"
title="Delete this task"
aria-label="Delete this task"
[disabled]="isSubmitting"
>
<span class="material-symbols-outlined">delete</span>
</button>
<div class="task-card-header-actions">
@if (getDeviceSerial(session)) {
<span class="task-device" [title]="'Device: ' + getDeviceSerial(session)">
<span class="material-symbols-outlined device-icon">phone_android</span>
<span class="device-name">{{ getDeviceSerial(session) }}</span>
</span>
}
@if (getDeviceSerial(session)) {
<span class="task-time-divider">·</span>
}
<span class="task-time">{{ session.start_time * 1000 | date:'shortTime' }}</span>
@if (getTaskStatus(session) === 'running' || getTaskStatus(session) === 'paused') {
<button
type="button"
class="btn-stop-card"
(click)="stopTask(session.session_id, $event)"
title="Stop this task"
aria-label="Stop this task"
>
<span class="material-symbols-outlined">stop_circle</span>
</button>
}
@if (getTaskStatus(session) !== 'running' && getTaskStatus(session) !== 'paused') {
<button
type="button"
class="btn-delete-card"
(click)="deleteTask(session.session_id, $event)"
title="Delete this task"
aria-label="Delete this task"
[disabled]="isSubmitting()"
>
<span class="material-symbols-outlined">delete</span>
</button>
}
</div>
</div>
<div class="task-goal" title="{{ session.initial_goal }}">
{{ session.initial_goal || 'Unnamed Task' }}
</div>
<div class="task-card-footer">
<span class="task-id" [title]="session.session_id">ID: {{ session.session_id }}</span>
<span class="task-date">{{ session.start_time * 1000 | date:'MM/dd/yy' }}</span>
</div>
</div>
<div class="task-goal" title="{{ session.initial_goal }}">
{{ session.initial_goal || 'Unnamed Task' }}
</div>
<div class="task-card-footer">
<span class="task-id" [title]="session.session_id">ID: {{ session.session_id }}</span>
<span class="task-date">{{ session.start_time * 1000 | date:'MM/dd/yy' }}</span>
</div>
</div>
}
<!-- Collapsible History Section -->
<details class="history-section" open>
<summary class="history-summary">
<span class="material-symbols-outlined section-icon">history</span> Past Runs & History ({{ historyTasks().length }})
</summary>
<div class="history-list-wrapper">
<div *ngIf="historyTasks().length === 0" class="empty-section-placeholder">
No past tasks.
</div>
<div
*ngFor="let session of historyTasks(); trackBy: trackSession"
class="task-card"
[class.selected]="agentService.currentSessionId() === session.session_id"
[class.running]="getTaskStatus(session) === 'running'"
(click)="selectTask(session.session_id)"
>
<div class="task-card-header">
<span class="task-badge" [ngClass]="getTaskStatus(session)">
<span class="status-dot" *ngIf="getTaskStatus(session) === 'running'"></span>
{{ getTaskStatus(session) | uppercase }}
</span>
<div class="task-card-header-actions">
<span class="task-device" *ngIf="getDeviceSerial(session)" [title]="'Device: ' + getDeviceSerial(session)">
<span class="material-symbols-outlined device-icon">phone_android</span>
<span class="device-name">{{ getDeviceSerial(session) }}</span>
@if (historyTasks().length === 0) {
<div class="empty-section-placeholder">
No past tasks.
</div>
}
@for (session of historyTasks(); track trackSession($index, session)) {
<div
class="task-card"
[class.selected]="agentService.currentSessionId() === session.session_id"
[class.running]="getTaskStatus(session) === 'running'"
(click)="selectTask(session.session_id)"
>
<div class="task-card-header">
<span class="task-badge" [ngClass]="getTaskStatus(session)">
@if (getTaskStatus(session) === 'running') {
<span class="status-dot"></span>
}
{{ getTaskStatus(session) | uppercase }}
</span>
<span class="task-time-divider" *ngIf="getDeviceSerial(session)">·</span>
<span class="task-time">{{ session.start_time * 1000 | date:'shortTime' }}</span>
<button
*ngIf="getTaskStatus(session) !== 'running' && getTaskStatus(session) !== 'paused'"
type="button"
class="btn-delete-card"
(click)="deleteTask(session.session_id, $event)"
title="Delete this task"
aria-label="Delete this task"
[disabled]="isSubmitting"
>
<span class="material-symbols-outlined">delete</span>
</button>
<div class="task-card-header-actions">
@if (getDeviceSerial(session)) {
<span class="task-device" [title]="'Device: ' + getDeviceSerial(session)">
<span class="material-symbols-outlined device-icon">phone_android</span>
<span class="device-name">{{ getDeviceSerial(session) }}</span>
</span>
}
@if (getDeviceSerial(session)) {
<span class="task-time-divider">·</span>
}
<span class="task-time">{{ session.start_time * 1000 | date:'shortTime' }}</span>
@if (getTaskStatus(session) !== 'running' && getTaskStatus(session) !== 'paused') {
<button
type="button"
class="btn-delete-card"
(click)="deleteTask(session.session_id, $event)"
title="Delete this task"
aria-label="Delete this task"
[disabled]="isSubmitting()"
>
<span class="material-symbols-outlined">delete</span>
</button>
}
</div>
</div>
<div class="task-goal" title="{{ session.initial_goal }}">
{{ session.initial_goal || 'Unnamed Task' }}
</div>
<div class="task-card-footer">
<span class="task-id" [title]="session.session_id">ID: {{ session.session_id }}</span>
<span class="task-date">{{ session.start_time * 1000 | date:'MM/dd/yy' }}</span>
</div>
</div>
<div class="task-goal" title="{{ session.initial_goal }}">
{{ session.initial_goal || 'Unnamed Task' }}
</div>
<div class="task-card-footer">
<span class="task-id" [title]="session.session_id">ID: {{ session.session_id }}</span>
<span class="task-date">{{ session.start_time * 1000 | date:'MM/dd/yy' }}</span>
</div>
</div>
}
</div>
</details>
</ng-container>
}
<!-- Notes Tab Content -->
<ng-container *ngIf="agentService.activeTab() === 'notes'">
@if (agentService.activeTab() === 'notes') {
<div class="notes-tab-content">
<!-- Notes Selector / Pills (Only shown if there are multiple markdown notes) -->
<div class="notes-selector-container" *ngIf="noteKeys().length > 1">
<div class="notes-selector-list">
<button
*ngFor="let key of noteKeys(); trackBy: trackNoteKey"
class="note-selector-pill"
[class.active]="agentService.selectedNoteKey() === key"
(click)="selectNote(key)"
>
<span class="material-symbols-outlined pill-icon">draft</span>
{{ key }}
</button>
@if (noteKeys().length > 1) {
<div class="notes-selector-container">
<div class="notes-selector-list">
@for (key of noteKeys(); track trackNoteKey($index, key)) {
<button
class="note-selector-pill"
[class.active]="agentService.selectedNoteKey() === key"
(click)="selectNote(key)"
>
<span class="material-symbols-outlined pill-icon">draft</span>
{{ key }}
</button>
}
</div>
</div>
</div>
<div *ngIf="noteKeys().length === 0" class="empty-notes-container">
<div class="empty-notes-icon">
<span class="material-symbols-outlined">description</span>
}
@if (noteKeys().length === 0) {
<div class="empty-notes-container">
<div class="empty-notes-icon">
<span class="material-symbols-outlined">description</span>
</div>
<h4 class="empty-notes-title">No Notes & Plans</h4>
<p class="empty-notes-subtitle">No notes or structured plans were recorded for this session.</p>
</div>
<h4 class="empty-notes-title">No Notes & Plans</h4>
<p class="empty-notes-subtitle">No notes or structured plans were recorded for this session.</p>
</div>
}
<!-- Notes Content Viewer (Clean card blocks matching the second screenshot style) -->
<div class="note-viewer-cards" *ngIf="noteKeys().length > 0">
<!-- Render Note Title if present -->
<h2 class="note-title" *ngIf="parsedNote().title">{{ parsedNote().title }}</h2>
<!-- Render Milestones list as Text Blocks -->
<div class="milestones-container" *ngIf="parsedNote().milestones.length > 0">
<div
*ngFor="let milestone of parsedNote().milestones; trackBy: trackMilestone"
class="milestone-block"
[ngClass]="milestone.type"
>
<!-- Milestone Title Row with status icon -->
<div class="milestone-title-row">
<ng-container [ngSwitch]="milestone.type">
<span *ngSwitchCase="'checked'" class="material-symbols-outlined milestone-status-icon checked">check_box</span>
<span *ngSwitchCase="'progress'" class="material-symbols-outlined milestone-status-icon progress">indeterminate_check_box</span>
<span *ngSwitchCase="'unchecked'" class="material-symbols-outlined milestone-status-icon unchecked">check_box_outline_blank</span>
</ng-container>
<div class="milestone-text">
<span
*ngFor="let seg of milestone.segments"
[class.bold-segment]="seg.bold"
[class.code-segment]="seg.code"
>{{ seg.text }}</span>
</div>
@if (noteKeys().length > 0) {
<div class="note-viewer-cards">
<!-- Render Note Title if present -->
@if (parsedNote().title) {
<h2 class="note-title">{{ parsedNote().title }}</h2>
}
<!-- Render Milestones list as Text Blocks -->
@if (parsedNote().milestones.length > 0) {
<div class="milestones-container">
@for (milestone of parsedNote().milestones; track trackMilestone($index, milestone)) {
<div
class="milestone-block"
[ngClass]="milestone.type"
>
<!-- Milestone Title Row with status icon -->
<div class="milestone-title-row">
@switch (milestone.type) {
@case ('checked') {
<span class="material-symbols-outlined milestone-status-icon checked">check_box</span>
}
@case ('progress') {
<span class="material-symbols-outlined milestone-status-icon progress">indeterminate_check_box</span>
}
@case ('unchecked') {
<span class="material-symbols-outlined milestone-status-icon unchecked">check_box_outline_blank</span>
}
}
<div class="milestone-text">
@for (seg of milestone.segments; track seg) {
<span
[class.bold-segment]="seg.bold"
[class.code-segment]="seg.code"
>{{ seg.text }}</span>
}
</div>
</div>
<!-- Sub-steps list if any -->
@if (milestone.subSteps.length > 0) {
<div class="milestone-substeps">
@for (step of milestone.subSteps; track trackMarkdownLine($index, step)) {
<div
class="substep-row"
[ngClass]="step.type"
[style.padding-left.px]="(step.indent > 2 ? (step.indent - 2) * 12 : 0)"
>
@switch (step.type) {
@case ('checked') {
<span class="material-symbols-outlined check-icon checked">check_box</span>
}
@case ('progress') {
<span class="material-symbols-outlined check-icon progress">indeterminate_check_box</span>
}
@case ('unchecked') {
<span class="material-symbols-outlined check-icon unchecked">check_box_outline_blank</span>
}
@case ('list-item') {
<span class="bullet">•</span>
}
}
@if (step.type !== 'empty') {
<span class="line-text">
@for (seg of step.segments; track seg) {
<span
[class.bold-segment]="seg.bold"
[class.code-segment]="seg.code"
>{{ seg.text }}</span>
}
</span>
}
</div>
}
</div>
}
</div>
}
</div>
<!-- Sub-steps list if any -->
<div class="milestone-substeps" *ngIf="milestone.subSteps.length > 0">
<div
*ngFor="let step of milestone.subSteps; trackBy: trackMarkdownLine"
class="substep-row"
[ngClass]="step.type"
[style.padding-left.px]="(step.indent > 2 ? (step.indent - 2) * 12 : 0)"
>
<ng-container [ngSwitch]="step.type">
<span *ngSwitchCase="'checked'" class="material-symbols-outlined check-icon checked">check_box</span>
<span *ngSwitchCase="'progress'" class="material-symbols-outlined check-icon progress">indeterminate_check_box</span>
<span *ngSwitchCase="'unchecked'" class="material-symbols-outlined check-icon unchecked">check_box_outline_blank</span>
<span *ngSwitchCase="'list-item'" class="bullet">•</span>
</ng-container>
<span class="line-text" *ngIf="step.type !== 'empty'">
<span
*ngFor="let seg of step.segments"
[class.bold-segment]="seg.bold"
[class.code-segment]="seg.code"
>{{ seg.text }}</span>
</span>
</div>
}
<!-- Render any other lines (e.g. text/markdown blocks at the end) -->
@if (parsedNote().otherLines.length > 0) {
<div class="other-lines-container">
@for (line of parsedNote().otherLines; track trackMarkdownLine($index, line)) {
<div
class="md-line"
[ngClass]="line.type"
[style.margin-left.px]="line.indent * 12"
>
@switch (line.type) {
@case ('checked') {
<span class="material-symbols-outlined check-icon checked">check_box</span>
}
@case ('progress') {
<span class="material-symbols-outlined check-icon progress">indeterminate_check_box</span>
}
@case ('unchecked') {
<span class="material-symbols-outlined check-icon unchecked">check_box_outline_blank</span>
}
@case ('list-item') {
<span class="bullet">•</span>
}
}
@if (line.type !== 'empty') {
<span class="line-text">
@for (seg of line.segments; track seg) {
<span
[class.bold-segment]="seg.bold"
[class.code-segment]="seg.code"
>{{ seg.text }}</span>
}
</span>
}
</div>
}
</div>
</div>
}
</div>
<!-- Render any other lines (e.g. text/markdown blocks at the end) -->
<div class="other-lines-container" *ngIf="parsedNote().otherLines.length > 0">
<div
*ngFor="let line of parsedNote().otherLines; trackBy: trackMarkdownLine"
class="md-line"
[ngClass]="line.type"
[style.margin-left.px]="line.indent * 12"
>
<ng-container [ngSwitch]="line.type">
<span *ngSwitchCase="'checked'" class="material-symbols-outlined check-icon checked">check_box</span>
<span *ngSwitchCase="'progress'" class="material-symbols-outlined check-icon progress">indeterminate_check_box</span>
<span *ngSwitchCase="'unchecked'" class="material-symbols-outlined check-icon unchecked">check_box_outline_blank</span>
<span *ngSwitchCase="'list-item'" class="bullet">•</span>
</ng-container>
<span class="line-text" *ngIf="line.type !== 'empty'">
<span
*ngFor="let seg of line.segments"
[class.bold-segment]="seg.bold"
[class.code-segment]="seg.code"
>{{ seg.text }}</span>
</span>
</div>
</div>
</div>
}
</div>
</ng-container>
}
</main>
</div>
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { Component, inject, computed } from '@angular/core';
import { Component, ChangeDetectionStrategy, inject, computed, signal } from '@angular/core';
import { CommonModule } from '@angular/common';
import { FormsModule } from '@angular/forms';
import { AgentService } from '../../services/agent.service';
@@ -29,14 +29,20 @@ export type { MarkdownSegment, MarkdownLine, NoteMilestone, ParsedNote };
standalone: true,
imports: [CommonModule, FormsModule],
templateUrl: './chat-interface.component.html',
styleUrl: './chat-interface.component.scss'
styleUrl: './chat-interface.component.scss',
changeDetection: ChangeDetectionStrategy.OnPush
})
export class ChatInterfaceComponent {
public agentService = inject(AgentService);
public taskInput: string = '';
public isSubmitting: boolean = false;
public errorMessage: string | null = null;
// Signals so async completion handlers refresh this OnPush view.
public isSubmitting = signal<boolean>(false);
public errorMessage = signal<string | null>(null);
// Device-serial resolution can require a JSON.parse of device_info; memoize
// it per session object so template re-evaluation stays cheap.
private deviceSerialCache = new WeakMap<Session, string | null>();
/**
* Filtered computed list of active tasks (running or pending) sorted by status and submission order
@@ -76,23 +82,23 @@ export class ChatInterfaceComponent {
return;
}
this.isSubmitting = true;
this.errorMessage = null;
this.isSubmitting.set(true);
this.errorMessage.set(null);
this.agentService.runTask(goal).subscribe({
next: (res) => {
this.taskInput = '';
this.isSubmitting = false;
this.isSubmitting.set(false);
// Fetch status to refresh sessions list and select new session
this.agentService.fetchStatus();
},
error: (err) => {
console.error('Failed to submit task:', err);
this.isSubmitting = false;
this.errorMessage = err.error?.detail || 'The runner is busy. Please wait for the current task to finish.';
this.isSubmitting.set(false);
this.errorMessage.set(err.error?.detail || 'The runner is busy. Please wait for the current task to finish.');
// Auto-dismiss error banner after 5 seconds
setTimeout(() => {
this.errorMessage = null;
this.errorMessage.set(null);
}, 5000);
}
});
@@ -103,11 +109,11 @@ export class ChatInterfaceComponent {
*/
public stopTask(sessionId: string, event: MouseEvent): void {
event.stopPropagation();
this.isSubmitting = true;
this.errorMessage = null;
this.isSubmitting.set(true);
this.errorMessage.set(null);
this.agentService.stopTask(sessionId, false);
setTimeout(() => {
this.isSubmitting = false;
this.isSubmitting.set(false);
}, 400);
}
@@ -115,11 +121,11 @@ export class ChatInterfaceComponent {
* Stop the currently running task and optionally clear the backend queue
*/
public stopTasks(stopAll: boolean = false): void {
this.isSubmitting = true;
this.errorMessage = null;
this.isSubmitting.set(true);
this.errorMessage.set(null);
this.agentService.stopTask(stopAll);
setTimeout(() => {
this.isSubmitting = false;
this.isSubmitting.set(false);
}, 400);
}
@@ -130,16 +136,16 @@ export class ChatInterfaceComponent {
if (!confirm('Are you sure you want to clear all tasks and history? This cannot be undone.')) {
return;
}
this.isSubmitting = true;
this.errorMessage = null;
this.isSubmitting.set(true);
this.errorMessage.set(null);
this.agentService.clearAllHistory().subscribe({
next: () => {
this.isSubmitting = false;
this.isSubmitting.set(false);
},
error: (err: any) => {
console.error('Failed to clear history:', err);
this.isSubmitting = false;
this.errorMessage = err.error?.detail || 'Failed to clear history.';
this.isSubmitting.set(false);
this.errorMessage.set(err.error?.detail || 'Failed to clear history.');
}
});
}
@@ -152,16 +158,16 @@ export class ChatInterfaceComponent {
if (!confirm(`Are you sure you want to delete this task? This cannot be undone.`)) {
return;
}
this.isSubmitting = true;
this.errorMessage = null;
this.isSubmitting.set(true);
this.errorMessage.set(null);
this.agentService.deleteSession(sessionId).subscribe({
next: () => {
this.isSubmitting = false;
this.isSubmitting.set(false);
},
error: (err: any) => {
console.error(`Failed to delete task ${sessionId}:`, err);
this.isSubmitting = false;
this.errorMessage = err.error?.detail || 'Failed to delete task.';
this.isSubmitting.set(false);
this.errorMessage.set(err.error?.detail || 'Failed to delete task.');
}
});
}
@@ -189,22 +195,26 @@ export class ChatInterfaceComponent {
* Determine the device serial number for the session
*/
public getDeviceSerial(session: Session): string | null {
if (this.deviceSerialCache.has(session)) {
return this.deviceSerialCache.get(session) ?? null;
}
let resolved: string | null = null;
const serial = session.device_serial || session.device_id;
if (serial && serial !== 'pending' && serial !== 'null' && serial !== 'undefined') {
return serial;
}
if (session.device_info) {
resolved = serial;
} else if (session.device_info) {
try {
const info = typeof session.device_info === 'string' ? JSON.parse(session.device_info) : session.device_info;
const s = info?.device_id || info?.device_serial;
if (s && s !== 'pending' && s !== 'null' && s !== 'undefined') {
return s;
resolved = s;
}
} catch {
// ignore
}
}
return null;
this.deviceSerialCache.set(session, resolved);
return resolved;
}
/**
@@ -1,430 +1,433 @@
<!--
Copyright 2026 Google LLC
Copyright 2026 Google LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!-- Backdrop overlay only when in theater / modal mode -->
<div
class="theater-backdrop"
*ngIf="agentService.isVideoWindowOpen() && isTheaterMode()"
(click)="isTheaterMode.set(false)"
></div>
@if (agentService.isVideoWindowOpen() && isTheaterMode()) {
<div
class="theater-backdrop"
(click)="isTheaterMode.set(false)"
></div>
}
<div
class="floating-video-wrapper"
*ngIf="agentService.isVideoWindowOpen()"
[class.minimized]="agentService.isVideoMinimized()"
[class.theater]="isTheaterMode()"
[style.left.px]="!isTheaterMode() ? posX() : null"
[style.top.px]="!isTheaterMode() ? posY() : null"
>
<!-- Window Header (Draggable Handle) -->
<header class="window-header" (mousedown)="startDrag($event)">
<div class="header-left">
<span class="material-symbols-outlined header-icon">
{{ agentService.recordingPlaybackStatus() === 'live' ? 'sensors' : (agentService.playerMode() === 'steps' ? 'slideshow' : (agentService.recordingPlaybackStatus() === 'processing' ? 'progress_activity' : 'smart_display')) }}
</span>
<span class="header-title" [title]="agentService.activeVideoTitle()">
{{ agentService.recordingPlaybackStatus() === 'live' ? 'Live Device Screen' : (agentService.playerMode() === 'steps' ? ('Step Replay (' + (totalStepFrames() > 0 ? (activeStepIndex() + 1) + '/' + totalStepFrames() : '0') + ')') : (agentService.recordingPlaybackStatus() === 'processing' ? 'Preparing Recording' : (agentService.activeVideoTitle() || 'Screen Recording'))) }}
</span>
</div>
<div class="header-actions" (mousedown)="$event.stopPropagation()">
<!-- Mode Toggle Group: Video vs. Step Replay -->
<div class="mode-toggle-group" *ngIf="agentService.recordingPlaybackStatus() === 'ready' && agentService.hasCurrentSessionStepFrames() && !agentService.isVideoMinimized()">
<button
class="mode-btn"
[class.active]="agentService.playerMode() === 'video'"
(click)="agentService.setPlayerMode('video')"
title="Video Recording"
>
<span class="material-symbols-outlined mode-icon">videocam</span>
<span>Video</span>
</button>
<button
class="mode-btn"
[class.active]="agentService.playerMode() === 'steps'"
(click)="agentService.setPlayerMode('steps')"
title="Step Screenshot Replay"
>
<span class="material-symbols-outlined mode-icon">slideshow</span>
<span>Steps</span>
</button>
@if (agentService.isVideoWindowOpen()) {
<div
class="floating-video-wrapper"
[class.minimized]="agentService.isVideoMinimized()"
[class.theater]="isTheaterMode()"
[style.left.px]="!isTheaterMode() ? posX() : null"
[style.top.px]="!isTheaterMode() ? posY() : null"
>
<!-- Window Header (Draggable Handle) -->
<header class="window-header" (mousedown)="startDrag($event)">
<div class="header-left">
<span class="material-symbols-outlined header-icon">
{{ agentService.recordingPlaybackStatus() === 'live' ? 'sensors' : (agentService.playerMode() === 'steps' ? 'slideshow' : (agentService.recordingPlaybackStatus() === 'processing' ? 'progress_activity' : 'smart_display')) }}
</span>
<span class="header-title" [title]="agentService.activeVideoTitle()">
{{ agentService.recordingPlaybackStatus() === 'live' ? 'Live Device Screen' : (agentService.playerMode() === 'steps' ? ('Step Replay (' + (totalStepFrames() > 0 ? (activeStepIndex() + 1) + '/' + totalStepFrames() : '0') + ')') : (agentService.recordingPlaybackStatus() === 'processing' ? 'Preparing Recording' : (agentService.activeVideoTitle() || 'Screen Recording'))) }}
</span>
</div>
<!-- Minimize / Restore Toggle -->
<button
class="header-btn"
(click)="toggleMinimize()"
[title]="agentService.isVideoMinimized() ? 'Restore' : 'Minimize'"
aria-label="Minimize"
>
<span class="material-symbols-outlined">{{ agentService.isVideoMinimized() ? 'expand_less' : 'remove' }}</span>
</button>
<!-- Theater / Fullscreen Toggle -->
<button
class="header-btn"
(click)="toggleTheater()"
[title]="isTheaterMode() ? 'Exit Theater' : 'Theater Mode'"
*ngIf="!agentService.isVideoMinimized()"
aria-label="Theater"
>
<span class="material-symbols-outlined">{{ isTheaterMode() ? 'close_fullscreen' : 'open_in_full' }}</span>
</button>
<!-- Close Window -->
<button
class="header-btn close-btn"
(click)="agentService.closeVideoPlayer()"
title="Close"
aria-label="Close"
>
<span class="material-symbols-outlined">close</span>
</button>
</div>
</header>
<!-- Video Main Content Area -->
<div class="video-body" *ngIf="!agentService.isVideoMinimized()">
<!-- 1. LIVE SCREEN STREAM (AUTOMATICALLY ACTIVE DURING RUNNING TEST) -->
<ng-container *ngIf="agentService.recordingPlaybackStatus() === 'live'">
<div class="live-stream-container" *ngIf="!liveStreamError()">
<img
class="live-screen-img"
[src]="liveStreamUrl()"
(error)="onLiveStreamError()"
alt="Live Device Screen"
/>
<div class="live-status-overlay">
<div class="live-badge">
<span class="pulse-dot"></span>
<span>LIVE</span>
<div class="header-actions" (mousedown)="$event.stopPropagation()">
<!-- Mode Toggle Group: Video vs. Step Replay -->
@if (agentService.recordingPlaybackStatus() === 'ready' && agentService.hasCurrentSessionStepFrames() && !agentService.isVideoMinimized()) {
<div class="mode-toggle-group">
<button
class="mode-btn"
[class.active]="agentService.playerMode() === 'video'"
(click)="agentService.setPlayerMode('video')"
title="Video Recording"
>
<span class="material-symbols-outlined mode-icon">videocam</span>
<span>Video</span>
</button>
<button
class="mode-btn"
[class.active]="agentService.playerMode() === 'steps'"
(click)="agentService.setPlayerMode('steps')"
title="Step Screenshot Replay"
>
<span class="material-symbols-outlined mode-icon">slideshow</span>
<span>Steps</span>
</button>
</div>
<button class="live-refresh-btn" (click)="retryLiveStream()" title="Refresh Live Stream">
<span class="material-symbols-outlined">refresh</span>
</button>
</div>
</div>
<div class="empty-video-fallback" *ngIf="liveStreamError()">
<div class="fallback-icon-wrap">
<span class="material-symbols-outlined fallback-icon">phonelink_off</span>
</div>
<h4 class="fallback-title">Device Stream Connecting</h4>
<p class="fallback-desc">Waiting for ADB device screen stream...</p>
<button class="retry-btn" (click)="retryLiveStream()">
<span class="material-symbols-outlined">refresh</span> Reconnect
</button>
</div>
</ng-container>
<!-- 2. RECORDING PLAYBACK / STEP REPLAY (AUTOMATICALLY ACTIVE WHEN TEST FINISHED) -->
<ng-container *ngIf="agentService.recordingPlaybackStatus() !== 'live'">
<!-- Loading State (when in video mode and processing) -->
<div class="video-loading-state" *ngIf="agentService.playerMode() === 'video' && (agentService.recordingPlaybackStatus() === 'processing' || agentService.isVideoLoading())">
<span class="material-symbols-outlined spinning-icon">progress_activity</span>
<span>{{ agentService.recordingPlaybackMessage() || 'Preparing screen recording...' }}</span>
</div>
<!-- Option A: STEP REPLAY VIEW (ACTIVE IN STEP MODE) -->
<ng-container *ngIf="agentService.playerMode() === 'steps' && totalStepFrames() > 0">
<div class="step-replay-container">
<!-- Fallback info banner if recording had failed -->
<div class="step-replay-notice" *ngIf="agentService.recordingPlaybackStatus() === 'failed' || videoLoadError()">
<span class="material-symbols-outlined notice-icon">info</span>
<span>Screen recording unavailable. Replaying step-by-step screenshots.</span>
</div>
<!-- Step Image Viewport with Action Coordinates Overlay -->
<div
class="step-image-viewport"
(mouseenter)="onViewportMouseEnter()"
(mouseleave)="onViewportMouseLeave()"
}
<!-- Minimize / Restore Toggle -->
<button
class="header-btn"
(click)="toggleMinimize()"
[title]="agentService.isVideoMinimized() ? 'Restore' : 'Minimize'"
aria-label="Minimize"
>
<!-- Loading Skeleton (Clean Light Spinner) -->
<div class="step-image-loading-state" *ngIf="isStepImageLoading()">
<span class="material-symbols-outlined spinning-icon">progress_activity</span>
<span>Loading screenshot...</span>
</div>
<!-- Error / Missing Image Fallback -->
<div class="step-image-error-state" *ngIf="stepImageError() && !isStepImageLoading()">
<span class="material-symbols-outlined broken-icon">broken_image</span>
<span>Screenshot preview unavailable for this step</span>
</div>
<!-- Device Screen Frame & Action Coordinates Overlay -->
<div class="step-image-frame-wrap" *ngIf="!stepImageError()">
<img
#stepImgRef
class="step-frame-img"
[src]="currentStepImageUrl()"
[alt]="currentStepFrame()?.title"
(load)="onStepImageLoad()"
(error)="onStepImageError()"
(click)="openImageInNewTab()"
title="Click to view image in new tab"
/>
<!-- Action Visual Coordinates Overlay (Touch point, swipe arrow, sequence path) -->
<div
#stepOverlayRef
class="overlay-layer"
[style.display]="viewMode() === 'post' ? 'none' : 'block'"
></div>
</div>
<!-- Hideable Floating Detail Card (Appears on hover or when pinned) -->
<div
class="step-frame-badge"
*ngIf="currentStepFrame() as frame"
[class.visible]="isCardVisible()"
(click)="$event.stopPropagation()"
<span class="material-symbols-outlined">{{ agentService.isVideoMinimized() ? 'expand_less' : 'remove' }}</span>
</button>
<!-- Theater / Fullscreen Toggle -->
@if (!agentService.isVideoMinimized()) {
<button
class="header-btn"
(click)="toggleTheater()"
[title]="isTheaterMode() ? 'Exit Theater' : 'Theater Mode'"
aria-label="Theater"
>
<div class="badge-content-left">
<span class="badge-step-no">Step {{ frame.stepNumber }}</span>
<span class="badge-action-desc" [title]="frame.actionText || frame.title">
<span class="material-symbols-outlined action-type-icon" *ngIf="frame.action">
{{ getStepActionIcon(frame.action) }}
</span>
{{ frame.actionText || frame.title }}
</span>
</div>
<div class="badge-actions-right">
<!-- Pre/Post Screenshot Switcher (Only if post-screenshot is available) -->
<div class="pre-post-toggle" *ngIf="frame.postImageUrl">
<button
class="toggle-btn"
[class.active]="viewMode() === 'pre'"
(click)="setStepViewMode('pre', $event)"
title="View screen before action (with touch coordinates)"
>
Pre
</button>
<button
class="toggle-btn"
[class.active]="viewMode() === 'post'"
(click)="setStepViewMode('post', $event)"
title="View resulting screen after action executed"
>
Post
<span class="material-symbols-outlined">{{ isTheaterMode() ? 'close_fullscreen' : 'open_in_full' }}</span>
</button>
}
<!-- Close Window -->
<button
class="header-btn close-btn"
(click)="agentService.closeVideoPlayer()"
title="Close"
aria-label="Close"
>
<span class="material-symbols-outlined">close</span>
</button>
</div>
</header>
<!-- Video Main Content Area -->
@if (!agentService.isVideoMinimized()) {
<div class="video-body">
<!-- 1. LIVE SCREEN STREAM (AUTOMATICALLY ACTIVE DURING RUNNING TEST) -->
@if (agentService.recordingPlaybackStatus() === 'live') {
@if (!liveStreamError()) {
<div class="live-stream-container">
<img
class="live-screen-img"
[src]="liveStreamUrl()"
(error)="onLiveStreamError()"
alt="Live Device Screen"
/>
<div class="live-status-overlay">
<div class="live-badge">
<span class="pulse-dot"></span>
<span>LIVE</span>
</div>
<button class="live-refresh-btn" (click)="retryLiveStream()" title="Refresh Live Stream">
<span class="material-symbols-outlined">refresh</span>
</button>
</div>
<!-- Pin Card Button -->
<button
class="pin-card-btn"
[class.pinned]="isCardPinned()"
(click)="toggleCardPin($event)"
[title]="isCardPinned() ? 'Unpin details card (auto-hide on mouse leave)' : 'Pin details card (keep always visible)'"
>
<span class="material-symbols-outlined">{{ isCardPinned() ? 'push_pin' : 'keep' }}</span>
</div>
}
@if (liveStreamError()) {
<div class="empty-video-fallback">
<div class="fallback-icon-wrap">
<span class="material-symbols-outlined fallback-icon">phonelink_off</span>
</div>
<h4 class="fallback-title">Device Stream Connecting</h4>
<p class="fallback-desc">Waiting for ADB device screen stream...</p>
<button class="retry-btn" (click)="retryLiveStream()">
<span class="material-symbols-outlined">refresh</span> Reconnect
</button>
</div>
</div>
</div>
<!-- Step Replay Controls Footer -->
<div class="video-controls-footer">
<!-- Timeline Scrubber Bar with Step Ticks -->
<div class="scrubber-bar-container">
<input
type="range"
class="scrubber-range"
[min]="0"
[max]="totalStepFrames() - 1"
[step]="1"
[value]="activeStepIndex()"
(input)="onStepScrub($event)"
/>
<div class="scrubber-progress-fill" [style.width.%]="stepProgressPercent()"></div>
<!-- Step Tick Markers -->
<div class="step-ticks-wrapper" *ngIf="totalStepFrames() > 1">
<span
*ngFor="let f of stepFrames(); let i = index"
class="step-tick"
[class.active]="i === activeStepIndex()"
[class.passed]="i < activeStepIndex()"
[style.left.%]="(i / (totalStepFrames() - 1)) * 100"
(click)="seekStepFrame(i); $event.stopPropagation()"
[title]="'Step ' + (i + 1) + ': ' + (f.actionText || f.title)"
></span>
}
}
<!-- 2. RECORDING PLAYBACK / STEP REPLAY (AUTOMATICALLY ACTIVE WHEN TEST FINISHED) -->
@if (agentService.recordingPlaybackStatus() !== 'live') {
<!-- Loading State (when in video mode and processing) -->
@if (agentService.playerMode() === 'video' && (agentService.recordingPlaybackStatus() === 'processing' || agentService.isVideoLoading())) {
<div class="video-loading-state">
<span class="material-symbols-outlined spinning-icon">progress_activity</span>
<span>{{ agentService.recordingPlaybackMessage() || 'Preparing screen recording...' }}</span>
</div>
</div>
<!-- Bottom Controls Row -->
<div class="controls-row">
<div class="controls-left">
<!-- Play / Pause Button -->
<button class="ctrl-btn play-btn" (click)="toggleStepPlay()" [title]="isStepPlaying() ? 'Pause' : 'Play Slideshow'">
<span class="material-symbols-outlined">{{ isStepPlaying() ? 'pause' : 'play_arrow' }}</span>
</button>
<!-- Previous Frame Button -->
<button class="ctrl-btn" (click)="prevStepFrame()" [disabled]="activeStepIndex() <= 0" title="Previous Step">
<span class="material-symbols-outlined">skip_previous</span>
</button>
<!-- Next Frame Button -->
<button class="ctrl-btn" (click)="nextStepFrame()" [disabled]="activeStepIndex() >= totalStepFrames() - 1" title="Next Step">
<span class="material-symbols-outlined">skip_next</span>
</button>
<!-- Step Replay / Reset -->
<button class="ctrl-btn" (click)="restartStep()" title="Replay from start">
<span class="material-symbols-outlined">replay</span>
</button>
<!-- Step Counter Display -->
<span class="time-display">
Step {{ activeStepIndex() + 1 }} / {{ totalStepFrames() }}
</span>
}
<!-- Option A: STEP REPLAY VIEW (ACTIVE IN STEP MODE) -->
@if (agentService.playerMode() === 'steps' && totalStepFrames() > 0) {
<div class="step-replay-container">
<!-- Fallback info banner if recording had failed -->
@if (agentService.recordingPlaybackStatus() === 'failed' || videoLoadError()) {
<div class="step-replay-notice">
<span class="material-symbols-outlined notice-icon">info</span>
<span>Screen recording unavailable. Replaying step-by-step screenshots.</span>
</div>
}
<!-- Step Image Viewport with Action Coordinates Overlay -->
<div
class="step-image-viewport"
(mouseenter)="onViewportMouseEnter()"
(mouseleave)="onViewportMouseLeave()"
>
<!-- Loading Skeleton (Clean Light Spinner) -->
@if (isStepImageLoading()) {
<div class="step-image-loading-state">
<span class="material-symbols-outlined spinning-icon">progress_activity</span>
<span>Loading screenshot...</span>
</div>
}
<!-- Error / Missing Image Fallback -->
@if (stepImageError() && !isStepImageLoading()) {
<div class="step-image-error-state">
<span class="material-symbols-outlined broken-icon">broken_image</span>
<span>Screenshot preview unavailable for this step</span>
</div>
}
<!-- Device Screen Frame & Action Coordinates Overlay -->
@if (!stepImageError()) {
<div class="step-image-frame-wrap">
<img
#stepImgRef
class="step-frame-img"
[src]="currentStepImageUrl()"
[alt]="$safeNavigationMigration(currentStepFrame()?.title)"
(load)="onStepImageLoad()"
(error)="onStepImageError()"
(click)="openImageInNewTab()"
title="Click to view image in new tab"
/>
<!-- Action Visual Coordinates Overlay (Touch point, swipe arrow, sequence path) -->
<div
#stepOverlayRef
class="overlay-layer"
[style.display]="viewMode() === 'post' ? 'none' : 'block'"
></div>
</div>
}
<!-- Hideable Floating Detail Card (Appears on hover or when pinned) -->
@if (currentStepFrame(); as frame) {
<div
class="step-frame-badge"
[class.visible]="isCardVisible()"
(click)="$event.stopPropagation()"
>
<div class="badge-content-left">
<span class="badge-step-no">Step {{ frame.stepNumber }}</span>
<span class="badge-action-desc" [title]="frame.actionText || frame.title">
@if (frame.action) {
<span class="material-symbols-outlined action-type-icon">
{{ getStepActionIcon(frame.action) }}
</span>
}
{{ frame.actionText || frame.title }}
</span>
</div>
<div class="badge-actions-right">
<!-- Pre/Post Screenshot Switcher (Only if post-screenshot is available) -->
@if (frame.postImageUrl) {
<div class="pre-post-toggle">
<button
class="toggle-btn"
[class.active]="viewMode() === 'pre'"
(click)="setStepViewMode('pre', $event)"
title="View screen before action (with touch coordinates)"
>
Pre
</button>
<button
class="toggle-btn"
[class.active]="viewMode() === 'post'"
(click)="setStepViewMode('post', $event)"
title="View resulting screen after action executed"
>
Post
</button>
</div>
}
<!-- Pin Card Button -->
<button
class="pin-card-btn"
[class.pinned]="isCardPinned()"
(click)="toggleCardPin($event)"
[title]="isCardPinned() ? 'Unpin details card (auto-hide on mouse leave)' : 'Pin details card (keep always visible)'"
>
<span class="material-symbols-outlined">{{ isCardPinned() ? 'push_pin' : 'keep' }}</span>
</button>
</div>
</div>
}
</div>
<!-- Step Replay Controls Footer -->
<div class="video-controls-footer">
<!-- Timeline Scrubber Bar with Step Ticks -->
<div class="scrubber-bar-container">
<input
type="range"
class="scrubber-range"
[min]="0"
[max]="totalStepFrames() - 1"
[step]="1"
[value]="activeStepIndex()"
(input)="onStepScrub($event)"
/>
<div class="scrubber-progress-fill" [style.width.%]="stepProgressPercent()"></div>
<!-- Step Tick Markers -->
@if (totalStepFrames() > 1) {
<div class="step-ticks-wrapper">
@for (f of stepFrames(); track trackStepFrame(i, f); let i = $index) {
<span
class="step-tick"
[class.active]="i === activeStepIndex()"
[class.passed]="i < activeStepIndex()"
[style.left.%]="(i / (totalStepFrames() - 1)) * 100"
(click)="seekStepFrame(i); $event.stopPropagation()"
[title]="'Step ' + (i + 1) + ': ' + (f.actionText || f.title)"
></span>
}
</div>
}
</div>
<!-- Bottom Controls Row -->
<div class="controls-row">
<div class="controls-left">
<!-- Play / Pause Button -->
<button class="ctrl-btn play-btn" (click)="toggleStepPlay()" [title]="isStepPlaying() ? 'Pause' : 'Play Slideshow'">
<span class="material-symbols-outlined">{{ isStepPlaying() ? 'pause' : 'play_arrow' }}</span>
</button>
<!-- Previous Frame Button -->
<button class="ctrl-btn" (click)="prevStepFrame()" [disabled]="activeStepIndex() <= 0" title="Previous Step">
<span class="material-symbols-outlined">skip_previous</span>
</button>
<!-- Next Frame Button -->
<button class="ctrl-btn" (click)="nextStepFrame()" [disabled]="activeStepIndex() >= totalStepFrames() - 1" title="Next Step">
<span class="material-symbols-outlined">skip_next</span>
</button>
<!-- Step Replay / Reset -->
<button class="ctrl-btn" (click)="restartStep()" title="Replay from start">
<span class="material-symbols-outlined">replay</span>
</button>
<!-- Step Counter Display -->
<span class="time-display">
Step {{ activeStepIndex() + 1 }} / {{ totalStepFrames() }}
</span>
</div>
<div class="controls-right">
<!-- Speed Button (Cycle 0.5x, 1x, 1.5x, 2x, 4x) -->
<button
class="ctrl-btn speed-pill-btn"
(click)="cycleSpeed()"
[title]="'Speed: ' + playbackRate() + 'x (Click to cycle)'"
>
<span class="speed-label">{{ playbackRate() }}x</span>
</button>
<!-- Loop Toggle -->
<button
class="ctrl-btn"
[class.active]="isLooping()"
(click)="toggleLoop()"
[title]="isLooping() ? 'Loop enabled' : 'Enable loop'"
>
<span class="material-symbols-outlined">repeat</span>
</button>
<!-- Open in new tab -->
<button class="ctrl-btn" (click)="openImageInNewTab()" title="Open image in new tab">
<span class="material-symbols-outlined">open_in_new</span>
</button>
</div>
</div>
</div>
</div>
}
<!-- Option B: ACTIVE VIDEO PLAYBACK VIEW -->
@if (agentService.playerMode() === 'video' && agentService.recordingPlaybackStatus() === 'ready' && !agentService.isVideoLoading() && agentService.activeVideoUrl(); as videoUrl) {
@if (!videoLoadError()) {
<div class="video-container">
<video
#videoRef
[src]="currentVideoUrl()"
playsinline
(loadedmetadata)="onLoadedMetadata()"
(timeupdate)="onTimeUpdate()"
(play)="onPlay()"
(pause)="onPause()"
(ended)="onEnded()"
(error)="onError()"
(click)="togglePlay()"
></video>
</div>
}
<!-- Dedicated Bottom Controls Bar for Video -->
@if (!videoLoadError()) {
<div class="video-controls-footer">
<!-- Timeline Scrubber Bar -->
<div class="scrubber-bar-container">
<input
type="range"
class="scrubber-range"
[min]="0"
[max]="duration() || 100"
[step]="0.1"
[value]="currentTime()"
(input)="onScrub($event)"
/>
<div class="scrubber-progress-fill" [style.width.%]="progressPercent()"></div>
</div>
<!-- Bottom Controls Row -->
<div class="controls-row">
<div class="controls-left">
<!-- Play / Pause Button -->
<button class="ctrl-btn play-btn" (click)="togglePlay()" [title]="isPlaying() ? 'Pause' : 'Play'">
<span class="material-symbols-outlined">{{ isPlaying() ? 'pause' : 'play_arrow' }}</span>
</button>
<!-- Replay / Reset -->
<button class="ctrl-btn" (click)="restart()" title="Replay from start">
<span class="material-symbols-outlined">replay</span>
</button>
<!-- Time Display -->
<span class="time-display">
{{ formatTime(currentTime()) }} / {{ formatTime(duration()) }}
</span>
</div>
<div class="controls-right">
<!-- Speed Button (Cycle 0.5x, 1x, 1.5x, 2x, 4x) -->
<button
class="ctrl-btn speed-pill-btn"
(click)="cycleSpeed()"
[title]="'Speed: ' + playbackRate() + 'x (Click to cycle)'"
>
<span class="speed-label">{{ playbackRate() }}x</span>
</button>
<!-- Loop Toggle -->
<button
class="ctrl-btn"
[class.active]="isLooping()"
(click)="toggleLoop()"
[title]="isLooping() ? 'Loop enabled' : 'Enable loop'"
>
<span class="material-symbols-outlined">repeat</span>
</button>
<!-- Mute / Unmute Toggle -->
<button class="ctrl-btn" (click)="toggleMute()" [title]="isMuted() ? 'Unmute' : 'Mute'">
<span class="material-symbols-outlined">{{ isMuted() ? 'volume_off' : 'volume_up' }}</span>
</button>
<!-- Open in new tab -->
<button class="ctrl-btn" (click)="openInNewTab()" title="Open video in new tab">
<span class="material-symbols-outlined">open_in_new</span>
</button>
</div>
</div>
</div>
}
}
<!-- Option C: EMPTY / FALLBACK STATE (Only when no video AND no step frames) -->
@if (!agentService.isVideoLoading() && (agentService.playerMode() === 'video' || totalStepFrames() === 0) && (agentService.recordingPlaybackStatus() === 'failed' || agentService.recordingPlaybackStatus() === 'unavailable' || videoLoadError()) && totalStepFrames() === 0) {
<div class="empty-video-fallback">
<div class="fallback-icon-wrap">
<span class="material-symbols-outlined fallback-icon">videocam_off</span>
</div>
<h4 class="fallback-title">
{{ agentService.recordingPlaybackStatus() === 'failed' || videoLoadError() ? 'Recording Playback Failed' : 'No Video Recording Available' }}
</h4>
<p class="fallback-desc">
{{ videoLoadError() ? 'The finalized recording could not be played.' : (agentService.recordingPlaybackMessage() || 'No screen recording was found for this task.') }}
</p>
@if (agentService.recordingPlaybackStatus() === 'failed' || videoLoadError()) {
<button class="retry-btn" (click)="videoLoadError.set(false); agentService.retryVideoRecording()">
<span class="material-symbols-outlined">refresh</span> Retry
</button>
}
</div>
}
}
</div>
<div class="controls-right">
<!-- Speed Button (Cycle 0.5x, 1x, 1.5x, 2x, 4x) -->
<button
class="ctrl-btn speed-pill-btn"
(click)="cycleSpeed()"
[title]="'Speed: ' + playbackRate() + 'x (Click to cycle)'"
>
<span class="speed-label">{{ playbackRate() }}x</span>
</button>
<!-- Loop Toggle -->
<button
class="ctrl-btn"
[class.active]="isLooping()"
(click)="toggleLoop()"
[title]="isLooping() ? 'Loop enabled' : 'Enable loop'"
>
<span class="material-symbols-outlined">repeat</span>
</button>
<!-- Open in new tab -->
<button class="ctrl-btn" (click)="openImageInNewTab()" title="Open image in new tab">
<span class="material-symbols-outlined">open_in_new</span>
</button>
</div>
</div>
}
</div>
</div>
</ng-container>
<!-- Option B: ACTIVE VIDEO PLAYBACK VIEW -->
<ng-container *ngIf="agentService.playerMode() === 'video' && agentService.recordingPlaybackStatus() === 'ready' && !agentService.isVideoLoading() && agentService.activeVideoUrl() as videoUrl">
<div class="video-container" *ngIf="!videoLoadError()">
<video
#videoRef
[src]="currentVideoUrl()"
playsinline
(loadedmetadata)="onLoadedMetadata()"
(timeupdate)="onTimeUpdate()"
(play)="onPlay()"
(pause)="onPause()"
(ended)="onEnded()"
(error)="onError()"
(click)="togglePlay()"
></video>
</div>
<!-- Dedicated Bottom Controls Bar for Video -->
<div class="video-controls-footer" *ngIf="!videoLoadError()">
<!-- Timeline Scrubber Bar -->
<div class="scrubber-bar-container">
<input
type="range"
class="scrubber-range"
[min]="0"
[max]="duration() || 100"
[step]="0.1"
[value]="currentTime()"
(input)="onScrub($event)"
/>
<div class="scrubber-progress-fill" [style.width.%]="progressPercent()"></div>
</div>
<!-- Bottom Controls Row -->
<div class="controls-row">
<div class="controls-left">
<!-- Play / Pause Button -->
<button class="ctrl-btn play-btn" (click)="togglePlay()" [title]="isPlaying() ? 'Pause' : 'Play'">
<span class="material-symbols-outlined">{{ isPlaying() ? 'pause' : 'play_arrow' }}</span>
</button>
<!-- Replay / Reset -->
<button class="ctrl-btn" (click)="restart()" title="Replay from start">
<span class="material-symbols-outlined">replay</span>
</button>
<!-- Time Display -->
<span class="time-display">
{{ formatTime(currentTime()) }} / {{ formatTime(duration()) }}
</span>
</div>
<div class="controls-right">
<!-- Speed Button (Cycle 0.5x, 1x, 1.5x, 2x, 4x) -->
<button
class="ctrl-btn speed-pill-btn"
(click)="cycleSpeed()"
[title]="'Speed: ' + playbackRate() + 'x (Click to cycle)'"
>
<span class="speed-label">{{ playbackRate() }}x</span>
</button>
<!-- Loop Toggle -->
<button
class="ctrl-btn"
[class.active]="isLooping()"
(click)="toggleLoop()"
[title]="isLooping() ? 'Loop enabled' : 'Enable loop'"
>
<span class="material-symbols-outlined">repeat</span>
</button>
<!-- Mute / Unmute Toggle -->
<button class="ctrl-btn" (click)="toggleMute()" [title]="isMuted() ? 'Unmute' : 'Mute'">
<span class="material-symbols-outlined">{{ isMuted() ? 'volume_off' : 'volume_up' }}</span>
</button>
<!-- Open in new tab -->
<button class="ctrl-btn" (click)="openInNewTab()" title="Open video in new tab">
<span class="material-symbols-outlined">open_in_new</span>
</button>
</div>
</div>
</div>
</ng-container>
<!-- Option C: EMPTY / FALLBACK STATE (Only when no video AND no step frames) -->
<div class="empty-video-fallback" *ngIf="!agentService.isVideoLoading() && (agentService.playerMode() === 'video' || totalStepFrames() === 0) && (agentService.recordingPlaybackStatus() === 'failed' || agentService.recordingPlaybackStatus() === 'unavailable' || videoLoadError()) && totalStepFrames() === 0">
<div class="fallback-icon-wrap">
<span class="material-symbols-outlined fallback-icon">videocam_off</span>
</div>
<h4 class="fallback-title">
{{ agentService.recordingPlaybackStatus() === 'failed' || videoLoadError() ? 'Recording Playback Failed' : 'No Video Recording Available' }}
</h4>
<p class="fallback-desc">
{{ videoLoadError() ? 'The finalized recording could not be played.' : (agentService.recordingPlaybackMessage() || 'No screen recording was found for this task.') }}
</p>
<button class="retry-btn" *ngIf="agentService.recordingPlaybackStatus() === 'failed' || videoLoadError()" (click)="videoLoadError.set(false); agentService.retryVideoRecording()">
<span class="material-symbols-outlined">refresh</span> Retry
</button>
</div>
</ng-container>
</div>
</div>
}
@@ -16,6 +16,8 @@
import {
Component,
ChangeDetectionStrategy,
NgZone,
inject,
signal,
computed,
@@ -25,7 +27,7 @@ import {
effect,
OnDestroy
} from '@angular/core';
import { CommonModule } from '@angular/common';
import { FormsModule } from '@angular/forms';
import { AgentService } from '../../services/agent.service';
import { StepReplayFrame } from '../../core/models/stream.model';
@@ -35,12 +37,14 @@ import { getActionIcon } from '../../utils/action-formatter.util';
@Component({
selector: 'app-floating-video-player',
standalone: true,
imports: [CommonModule, FormsModule],
imports: [FormsModule],
templateUrl: './floating-video-player.component.html',
styleUrl: './floating-video-player.component.scss'
styleUrl: './floating-video-player.component.scss',
changeDetection: ChangeDetectionStrategy.OnPush
})
export class FloatingVideoPlayerComponent implements OnDestroy {
public agentService = inject(AgentService);
private zone = inject(NgZone);
@ViewChild('videoRef') videoRef?: ElementRef<HTMLVideoElement>;
@ViewChild('stepImgRef') stepImgRef?: ElementRef<HTMLImageElement>;
@@ -217,7 +221,9 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
});
/**
* Start dragging the window from header
* Start dragging the window from header. The move/up listeners are attached
* only for the duration of the drag and live outside the Angular zone, so
* ordinary mouse movement over the page never touches change detection.
*/
public startDrag(event: MouseEvent): void {
if (this.isTheaterMode()) return;
@@ -227,10 +233,14 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
this.initialPosX = this.posX();
this.initialPosY = this.posY();
event.preventDefault();
this.zone.runOutsideAngular(() => {
document.addEventListener('mousemove', this.onDrag);
document.addEventListener('mouseup', this.stopDrag);
});
}
@HostListener('window:mousemove', ['$event'])
public onDrag(event: MouseEvent): void {
private onDrag = (event: MouseEvent): void => {
if (!this.isDragging) return;
const deltaX = event.clientX - this.dragStartX;
const deltaY = event.clientY - this.dragStartY;
@@ -245,12 +255,13 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
this.posX.set(targetX);
this.posY.set(targetY);
}
};
@HostListener('window:mouseup')
public stopDrag(): void {
private stopDrag = (): void => {
this.isDragging = false;
}
document.removeEventListener('mousemove', this.onDrag);
document.removeEventListener('mouseup', this.stopDrag);
};
@HostListener('window:resize')
public onResize(): void {
@@ -593,10 +604,16 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
}
}
public trackStepFrame(index: number, frame: StepReplayFrame): string {
return frame.stepId || String(index);
}
public ngOnDestroy(): void {
if (this.stepTimer) {
clearTimeout(this.stepTimer);
this.stepTimer = null;
}
document.removeEventListener('mousemove', this.onDrag);
document.removeEventListener('mouseup', this.stopDrag);
}
}
@@ -1,17 +1,17 @@
<!--
Copyright 2026 Google LLC
Copyright 2026 Google LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<div class="agent-stream-container">
@@ -29,14 +29,15 @@
[value]="agentService.currentSessionId() || ''"
(change)="onSessionChange($event)"
class="session-dropdown"
>
<option value="" disabled selected>-- Select a Session --</option>
<option
*ngFor="let session of agentService.sessions()"
[value]="session.session_id"
>
{{ session.initial_goal || 'Unnamed Goal' }} ({{ session.device_serial ? session.device_serial + ' · ' : '' }}{{ session.start_time * 1000 | date:'short' }})
</option>
<option value="" disabled selected>-- Select a Session --</option>
@for (session of agentService.sessions(); track session) {
<option
[value]="session.session_id"
>
{{ session.initial_goal || 'Unnamed Goal' }} ({{ session.device_serial ? session.device_serial + ' · ' : '' }}{{ session.start_time * 1000 | date:'short' }})
</option>
}
</select>
<button class="btn-refresh" (click)="agentService.fetchSessions()">🔄 Refresh</button>
</div>
@@ -44,73 +45,86 @@
<!-- Event Logs Section -->
<main class="stream-logs-content">
<div *ngIf="agentService.sessionLogs().length === 0" class="empty-state">
<div class="empty-icon">📡</div>
<h3>No Stream Activity Selected</h3>
<p>Please select a running session from the dropdown above to view real-time log payloads.</p>
</div>
<div *ngIf="agentService.sessionLogs().length > 0" class="logs-list">
<div
*ngFor="let log of agentService.sessionLogs(); let i = index"
class="log-card"
[ngClass]="['type-' + log.type, isStepEvent(log.type) ? 'step-event-card' : 'raw-event-card']"
>
<div class="log-card-header">
<div class="badge-group">
<span class="index-badge">#{{ i + 1 }}</span>
<span class="type-badge" [ngClass]="log.type">{{ log.type }}</span>
</div>
<span class="timestamp">{{ log.timestamp | date:'HH:mm:ss.SSS' }}</span>
</div>
<!-- Render Step Events in a beautiful, structured layout -->
<div *ngIf="isStepEvent(log.type)" class="log-card-body step-body">
<!-- Step Summary (Message to User) -->
<div class="step-summary-section" *ngIf="log.data.summary">
<div class="section-title">💬 Message to User / Summary:</div>
<div class="summary-bubble">{{ log.data.summary }}</div>
</div>
<!-- Model thinking process (collapsible) -->
<details class="thinking-details" *ngIf="log.data.operator_native_thinking">
<summary class="details-summary">🧠 Model Thinking Process (Click to expand)</summary>
<div class="details-content thinking-text">{{ log.data.operator_native_thinking }}</div>
</details>
<!-- Raw thinking process (collapsible) -->
<details class="planning-details" *ngIf="log.data.operator_raw_thinking">
<summary class="details-summary">📝 Raw Plan / Thoughts (Click to expand)</summary>
<div class="details-content planning-text">{{ log.data.operator_raw_thinking }}</div>
</details>
<!-- Tool Call (collapsible) -->
<details class="tool-details" *ngIf="log.data.action_taken" [open]="isShortContent(log.data.action_taken)">
<summary class="details-summary">🛠️ Tool Call Details (Click to expand)</summary>
<div class="details-content">
<pre class="json-code"><code>{{ formatJson(log.data.action_taken) }}</code></pre>
</div>
</details>
<!-- Tool Call Result (collapsible) -->
<details class="result-details" *ngIf="log.data.last_execution_result">
<summary class="details-summary">✓ Execution Result (Click to expand)</summary>
<div class="details-content">
<pre class="json-code"><code>{{ formatJson(log.data.last_execution_result) }}</code></pre>
</div>
</details>
</div>
<!-- Render other event types as raw JSON (collapsible) -->
<div *ngIf="!isStepEvent(log.type)" class="log-card-body raw-body">
<details class="raw-details" [open]="isShortContent(log.data)">
<summary class="details-summary">📄 Raw Payload Data (Click to expand)</summary>
<div class="details-content">
<pre class="json-code"><code>{{ formatJson(log.data) }}</code></pre>
</div>
</details>
</div>
@if (agentService.sessionLogs().length === 0) {
<div class="empty-state">
<div class="empty-icon">📡</div>
<h3>No Stream Activity Selected</h3>
<p>Please select a running session from the dropdown above to view real-time log payloads.</p>
</div>
</div>
}
@if (agentService.sessionLogs().length > 0) {
<div class="logs-list">
@for (log of agentService.sessionLogs(); track log; let i = $index) {
<div
class="log-card"
[ngClass]="['type-' + log.type, isStepEvent(log.type) ? 'step-event-card' : 'raw-event-card']"
>
<div class="log-card-header">
<div class="badge-group">
<span class="index-badge">#{{ i + 1 }}</span>
<span class="type-badge" [ngClass]="log.type">{{ log.type }}</span>
</div>
<span class="timestamp">{{ log.timestamp | date:'HH:mm:ss.SSS' }}</span>
</div>
<!-- Render Step Events in a beautiful, structured layout -->
@if (isStepEvent(log.type)) {
<div class="log-card-body step-body">
<!-- Step Summary (Message to User) -->
@if (log.data.summary) {
<div class="step-summary-section">
<div class="section-title">💬 Message to User / Summary:</div>
<div class="summary-bubble">{{ log.data.summary }}</div>
</div>
}
<!-- Model thinking process (collapsible) -->
@if (log.data.operator_native_thinking) {
<details class="thinking-details">
<summary class="details-summary">🧠 Model Thinking Process (Click to expand)</summary>
<div class="details-content thinking-text">{{ log.data.operator_native_thinking }}</div>
</details>
}
<!-- Raw thinking process (collapsible) -->
@if (log.data.operator_raw_thinking) {
<details class="planning-details">
<summary class="details-summary">📝 Raw Plan / Thoughts (Click to expand)</summary>
<div class="details-content planning-text">{{ log.data.operator_raw_thinking }}</div>
</details>
}
<!-- Tool Call (collapsible) -->
@if (log.data.action_taken) {
<details class="tool-details" [open]="isShortContent(log.data.action_taken)">
<summary class="details-summary">🛠️ Tool Call Details (Click to expand)</summary>
<div class="details-content">
<pre class="json-code"><code>{{ formatJson(log.data.action_taken) }}</code></pre>
</div>
</details>
}
<!-- Tool Call Result (collapsible) -->
@if (log.data.last_execution_result) {
<details class="result-details">
<summary class="details-summary">✓ Execution Result (Click to expand)</summary>
<div class="details-content">
<pre class="json-code"><code>{{ formatJson(log.data.last_execution_result) }}</code></pre>
</div>
</details>
}
</div>
}
<!-- Render other event types as raw JSON (collapsible) -->
@if (!isStepEvent(log.type)) {
<div class="log-card-body raw-body">
<details class="raw-details" [open]="isShortContent(log.data)">
<summary class="details-summary">📄 Raw Payload Data (Click to expand)</summary>
<div class="details-content">
<pre class="json-code"><code>{{ formatJson(log.data) }}</code></pre>
</div>
</details>
</div>
}
</div>
}
</div>
}
</main>
</div>
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { Component, inject } from '@angular/core';
import { Component, inject, ChangeDetectionStrategy } from '@angular/core';
import { CommonModule } from '@angular/common';
import { FormsModule } from '@angular/forms';
import { AgentService } from '../../services/agent.service';
@@ -24,6 +24,7 @@ import { AgentService } from '../../services/agent.service';
standalone: true,
imports: [CommonModule, FormsModule],
templateUrl: './legacy-agent-stream.component.html',
changeDetection: ChangeDetectionStrategy.Eager,
styleUrl: './legacy-agent-stream.component.scss'
})
export class LegacyAgentStreamComponent {
@@ -33,7 +33,9 @@
box-shadow: 0 2px 8px -1px rgba(15, 23, 42, 0.06), 0 1px 2px rgba(0, 0, 0, 0.02), inset 0 1px 1px rgba(255, 255, 255, 0.9);
user-select: none;
font-family: 'Google Sans', 'Google Sans Text', 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
transition: all 0.25s cubic-bezier(0.4, 0, 0.2, 1);
transition: background 0.25s cubic-bezier(0.4, 0, 0.2, 1),
box-shadow 0.25s cubic-bezier(0.4, 0, 0.2, 1),
border-color 0.25s cubic-bezier(0.4, 0, 0.2, 1);
&:hover {
background: rgba(255, 255, 255, 0.85);
@@ -56,7 +58,10 @@
text-decoration: none;
cursor: pointer;
line-height: 1;
transition: all 0.2s cubic-bezier(0.4, 0, 0.2, 1);
transition: background 0.2s cubic-bezier(0.4, 0, 0.2, 1),
color 0.2s cubic-bezier(0.4, 0, 0.2, 1),
box-shadow 0.2s cubic-bezier(0.4, 0, 0.2, 1),
transform 0.2s cubic-bezier(0.4, 0, 0.2, 1);
outline: none;
white-space: nowrap;
@@ -14,14 +14,14 @@
* limitations under the License.
*/
import { Component } from '@angular/core';
import { CommonModule } from '@angular/common';
import { Component, ChangeDetectionStrategy } from '@angular/core';
import { RouterLink, RouterLinkActive } from '@angular/router';
@Component({
selector: 'app-nav-switcher',
standalone: true,
imports: [CommonModule, RouterLink, RouterLinkActive],
imports: [RouterLink, RouterLinkActive],
template: `
<nav class="floating-nav-switcher" aria-label="Main Navigation">
<a
@@ -45,6 +45,7 @@ import { RouterLink, RouterLinkActive } from '@angular/router';
</a>
</nav>
`,
changeDetection: ChangeDetectionStrategy.Eager,
styleUrls: ['./nav-switcher.component.scss']
})
export class NavSwitcherComponent {}
@@ -36,6 +36,42 @@ export interface DeviceInfo {
installed_packages?: string[];
}
export interface AdbServerEndpoint {
host: string;
port: number;
socket: string;
identity: string;
mode: 'local' | 'remote';
is_local_default: boolean;
}
export interface AdbServerDevice {
serial: string;
state: string;
model: string | null;
product: string | null;
}
export interface AdbServerStatus {
endpoint: AdbServerEndpoint;
}
export interface AdbServerConnectionResult {
success: boolean;
message: string;
endpoint: AdbServerEndpoint;
devices: AdbServerDevice[];
persisted?: boolean;
persistence_error?: string | null;
error_code?: string;
output?: string;
}
export interface AdbServerConnectionResponse {
connection_result: AdbServerConnectionResult;
report?: SystemReadinessReport;
}
export interface ProbeResult {
id: string;
category: ProbeCategory;
@@ -674,15 +674,24 @@
<div class="step-title-row">
<h3 class="step-title">Android Device & Emulator Connection</h3>
</div>
<p class="step-desc">Physical Android phone or local Android Virtual Device (AVD)</p>
<p class="step-desc">Local emulator, attached phone, wireless device, or another ADB server</p>
</div>
<button
type="button"
class="btn-restart-adb btn-connection-options"
(click)="toggleConnectionMethods()"
>
<span class="material-symbols-outlined">swap_horiz</span>
<span>{{ showConnectionMethods() ? 'Hide Options' : 'Change Connection' }}</span>
</button>
<button
type="button"
class="btn-restart-adb"
[disabled]="isRestartingAdb()"
(click)="restartAdbServer()"
title="Restart local ADB daemon"
[title]="isRemoteAdbServer() ? 'Refresh devices without stopping the remote ADB server' : 'Restart local ADB daemon'"
>
@if (isRestartingAdb()) {
<span class="material-symbols-outlined spinning">sync</span>
@@ -692,7 +701,7 @@
<span class="feedback-text">{{ adbRestartFeedback() }}</span>
} @else {
<span class="material-symbols-outlined">sync</span>
<span>Restart ADB</span>
<span>{{ isRemoteAdbServer() ? 'Refresh Devices' : 'Restart ADB' }}</span>
}
</button>
</div>
@@ -726,6 +735,12 @@
<span class="meta-dot">·</span>
<span>Android {{ activeDevice()?.android_version }}</span>
}
@if (isRemoteAdbServer()) {
<span class="meta-dot">·</span>
<span class="remote-endpoint-label">
Remote via {{ adbServerStatus()?.endpoint?.host }}:{{ adbServerStatus()?.endpoint?.port }}
</span>
}
</div>
</div>
<div class="device-hero-status">
@@ -919,7 +934,7 @@
<div class="failed-actions-bar">
@if (emulatorLaunchState()?.avd_name) {
<button type="button" class="btn-retry-launch" (click)="launchAvdEmulator(emulatorLaunchState()!.avd_name!)">
<button type="button" class="btn-retry-launch" [disabled]="isRemoteAdbServer()" (click)="launchAvdEmulator(emulatorLaunchState()!.avd_name!)">
<span class="material-symbols-outlined">refresh</span>
<span>Retry Launch ({{ emulatorLaunchState()?.avd_name }})</span>
</button>
@@ -937,7 +952,7 @@
}
<!-- STATE B-3: NO DEVICE DETECTED / READY TO CONNECT GUIDE -->
@if (!isDeviceReady() && !isEmulatorLaunching() && adbProbe()?.summary !== 'Device Booting' && adbProbe()?.summary !== 'Device Unauthorized' && adbProbe()?.summary !== 'Device Locked' && adbProbe()?.summary !== 'Lock State Unknown' && adbProbe()?.status !== 'fail' && emulatorLaunchState()?.status !== 'failed') {
@if (showConnectionMethods() || (!isDeviceReady() && !isEmulatorLaunching() && adbProbe()?.summary !== 'Device Booting' && adbProbe()?.summary !== 'Device Unauthorized' && adbProbe()?.summary !== 'Device Locked' && adbProbe()?.summary !== 'Lock State Unknown' && adbProbe()?.status !== 'fail' && emulatorLaunchState()?.status !== 'failed')) {
<div class="foolproof-guide-container">
<!-- Method Selection Nav Cards -->
<div class="connection-methods-nav">
@@ -954,13 +969,14 @@
<div class="method-info">
<div class="method-title-row">
<span class="method-title">Android Emulator</span>
@if (installedAvds().length > 0) {
<span class="method-badge-pill detected">{{ installedAvds().length }} DETECTED</span>
} @else {
<span class="method-badge-pill recommended">RECOMMENDED</span>
}
</div>
<span class="method-desc">Launch local Android Virtual Device</span>
<span class="method-desc">
@if (installedAvds().length > 0) {
{{ installedAvds().length }} local virtual device{{ installedAvds().length > 1 ? 's' : '' }} available
} @else {
Launch a local Android Virtual Device
}
</span>
</div>
</div>
<div class="method-card-right">
@@ -981,7 +997,6 @@
<div class="method-info">
<div class="method-title-row">
<span class="method-title">USB Cable</span>
<span class="method-badge-pill neutral">4 STEPS</span>
</div>
<span class="method-desc">Connect physical phone via USB</span>
</div>
@@ -1004,7 +1019,6 @@
<div class="method-info">
<div class="method-title-row">
<span class="method-title">Wireless ADB</span>
<span class="method-badge-pill info">WI-FI</span>
</div>
<span class="method-desc">Connect over local Wi-Fi</span>
</div>
@@ -1013,8 +1027,52 @@
<span class="radio-indicator" [class.checked]="activeAdbGuideTab() === 'wifi'"></span>
</div>
</button>
<button
type="button"
class="method-nav-card"
[class.active]="activeAdbGuideTab() === 'remote'"
(click)="setAdbGuideTab('remote')"
>
<div class="method-card-left">
<div class="method-icon-box">
<span class="material-symbols-outlined">devices</span>
</div>
<div class="method-info">
<div class="method-title-row">
<span class="method-title">Remote Computer</span>
</div>
<span class="method-desc">Phone connected to another computer</span>
</div>
</div>
<div class="method-card-right">
<span class="radio-indicator" [class.checked]="activeAdbGuideTab() === 'remote'"></span>
</div>
</button>
</div>
@if (activeAdbGuideTab() !== 'remote' && isRemoteAdbServer()) {
<div class="active-remote-notice">
<div class="active-remote-copy">
<span class="material-symbols-outlined">hub</span>
<span>
Artemis is currently using
<strong>{{ adbServerStatus()?.endpoint?.host }}:{{ adbServerStatus()?.endpoint?.port }}</strong>.
Switch to local ADB before using a local emulator, USB cable, or Wireless ADB.
</span>
</div>
<button
type="button"
class="btn-use-local-adb"
[disabled]="isSwitchingToLocalAdb()"
(click)="switchToLocalAdbServer()"
>
<span class="material-symbols-outlined" [class.spinning]="isSwitchingToLocalAdb()">sync</span>
<span>{{ isSwitchingToLocalAdb() ? 'Switching...' : 'Use Local ADB' }}</span>
</button>
</div>
}
<!-- Method 1: Emulator -->
@if (activeAdbGuideTab() === 'emulator') {
<div class="method-details-panel">
@@ -1049,7 +1107,7 @@
type="button"
class="btn-avd-launch"
[class.launching]="launchingAvd() === avd || (isEmulatorLaunching() && emulatorLaunchState()?.avd_name === avd)"
[disabled]="isEmulatorLaunching()"
[disabled]="isEmulatorLaunching() || isRemoteAdbServer()"
(click)="launchAvdEmulator(avd)"
[title]="isEmulatorLaunching() ? 'An emulator is currently launching...' : 'Launch ' + avd"
>
@@ -1319,7 +1377,7 @@
</div>
<div class="wifi-btn-col">
<button type="button" class="btn-wifi-connect" [disabled]="isConnectingWifi() || !wifiHost().trim()"
<button type="button" class="btn-wifi-connect" [disabled]="isConnectingWifi() || isRemoteAdbServer() || !wifiHost().trim()"
(click)="connectWifiDevice()">
@if (isConnectingWifi()) {
<span class="material-symbols-outlined spinning">sync</span>
@@ -1354,6 +1412,132 @@
</div>
}
<!-- Method 4: ADB server on another computer -->
@if (activeAdbGuideTab() === 'remote') {
<div class="method-details-panel remote-adb-panel">
<div class="remote-guide-intro">
<span class="material-symbols-outlined intro-icon">hub</span>
<span>Enter the host and port of an ADB server endpoint that this computer can access, so Artemis can discover and control a phone physically connected to another computer.</span>
</div>
<div class="remote-adb-interactive-box">
<div class="remote-adb-form-row">
<div class="remote-adb-input-col host-col">
<label class="remote-adb-label" for="remote-adb-host">ADB Server Host</label>
<div class="remote-adb-input-wrap">
<span class="material-symbols-outlined wrap-icon">dns</span>
<input
id="remote-adb-host"
type="text"
class="remote-adb-input"
[ngModel]="remoteAdbHost()"
(ngModelChange)="updateRemoteAdbHost($event)"
placeholder="127.0.0.1"
autocomplete="off"
spellcheck="false"
/>
</div>
</div>
<div class="remote-adb-input-col port-col">
<label class="remote-adb-label" for="remote-adb-port">ADB Server Port</label>
<div class="remote-adb-input-wrap">
<span class="material-symbols-outlined wrap-icon">tag</span>
<input
id="remote-adb-port"
type="text"
inputmode="numeric"
class="remote-adb-input"
[ngModel]="remoteAdbPort()"
(ngModelChange)="updateRemoteAdbPort($event)"
placeholder="5038"
autocomplete="off"
spellcheck="false"
/>
</div>
</div>
<div class="remote-adb-btn-col">
<button
type="button"
class="btn-remote-adb-connect"
[disabled]="isConnectingRemoteAdb() || isActivatingRemoteAdb() || !remoteAdbHost().trim()"
(click)="connectRemoteAdbServer()"
>
@if (isConnectingRemoteAdb()) {
<span class="material-symbols-outlined spinning">sync</span>
<span>Checking...</span>
} @else {
<span class="material-symbols-outlined">bolt</span>
<span>Test Connection</span>
}
</button>
</div>
</div>
<label class="remember-remote-adb">
<input
type="checkbox"
[ngModel]="rememberRemoteAdb()"
(ngModelChange)="rememberRemoteAdb.set($event)"
/>
<span>Remember this endpoint for future Artemis launches</span>
</label>
@if (remoteAdbMessage()) {
<div class="remote-adb-status-alert success">
<span class="material-symbols-outlined">check_circle</span>
<span>{{ remoteAdbMessage() }}</span>
</div>
}
@if (remoteAdbError()) {
<div class="remote-adb-status-alert error" role="alert">
<span class="material-symbols-outlined">error</span>
<span>{{ remoteAdbError() }}</span>
</div>
}
@if (remoteAdbDevices().length > 0) {
<div class="remote-device-results">
@for (device of remoteAdbDevices(); track device.serial) {
<div class="remote-device-row">
<span class="material-symbols-outlined">phone_android</span>
<div class="remote-device-copy">
<strong>{{ device.model || device.serial }}</strong>
<span>{{ device.serial }}</span>
</div>
<span class="remote-device-state" [class.ready]="device.state === 'device'">
{{ device.state === 'device' ? 'Ready' : device.state }}
</span>
</div>
}
</div>
}
@if (remoteAdbProbeResult()?.success && !isProbedRemoteAdbActive()) {
<button
type="button"
class="btn-remote-adb-connect btn-use-tested-endpoint"
[disabled]="isActivatingRemoteAdb()"
(click)="activateRemoteAdbServer()"
>
@if (isActivatingRemoteAdb()) {
<span class="material-symbols-outlined spinning">sync</span>
<span>Connecting...</span>
} @else {
<span class="material-symbols-outlined">check</span>
<span>{{ remoteAdbHasReadyDevice() ? 'Use Connection' : 'Use Anyway' }}</span>
}
</button>
}
</div>
<p class="remote-adb-explanation">
Artemis sends all ADB commands to this server instead of the default local server. Any phone visible to that server becomes available to Artemis.
</p>
</div>
}
</div>
}
@@ -1416,7 +1600,7 @@
</div>
<button type="button" class="btn-retry-auth" (click)="restartAdbServer()">
<span class="material-symbols-outlined">refresh</span>
<span>Restart ADB & Check</span>
<span>{{ isRemoteAdbServer() ? 'Refresh Devices' : 'Restart ADB & Check' }}</span>
</button>
</div>
</div>
@@ -110,12 +110,16 @@ $text-light: #94a3b8;
@mixin glass-primary-btn {
background: linear-gradient(135deg, rgba(26, 115, 232, 0.92) 0%, rgba(37, 99, 235, 0.85) 100%);
backdrop-filter: blur(16px) saturate(180%);
-webkit-backdrop-filter: blur(16px) saturate(180%);
backdrop-filter: blur(10px) saturate(180%);
-webkit-backdrop-filter: blur(10px) saturate(180%);
border: 1px solid rgba(255, 255, 255, 0.28);
box-shadow: 0 4px 14px rgba(26, 115, 232, 0.32), inset 0 1px 1px rgba(255, 255, 255, 0.45);
color: #ffffff !important;
transition: all 0.2s cubic-bezier(0.4, 0, 0.2, 1);
transition: background 0.2s cubic-bezier(0.4, 0, 0.2, 1),
border-color 0.2s cubic-bezier(0.4, 0, 0.2, 1),
box-shadow 0.2s cubic-bezier(0.4, 0, 0.2, 1),
transform 0.2s cubic-bezier(0.4, 0, 0.2, 1),
opacity 0.2s cubic-bezier(0.4, 0, 0.2, 1);
span, .material-symbols-outlined, .run-icon, .action-icon {
color: #ffffff !important;
@@ -146,12 +150,17 @@ $text-light: #94a3b8;
@mixin glass-secondary-btn {
background: rgba(255, 255, 255, 0.72);
backdrop-filter: blur(14px) saturate(180%);
-webkit-backdrop-filter: blur(14px) saturate(180%);
backdrop-filter: blur(10px) saturate(180%);
-webkit-backdrop-filter: blur(10px) saturate(180%);
border: 1px solid rgba(226, 232, 240, 0.88);
box-shadow: 0 2px 8px -1px rgba(15, 23, 42, 0.05), inset 0 1px 1px rgba(255, 255, 255, 0.95);
color: $text-secondary;
transition: all 0.2s cubic-bezier(0.4, 0, 0.2, 1);
transition: background 0.2s cubic-bezier(0.4, 0, 0.2, 1),
border-color 0.2s cubic-bezier(0.4, 0, 0.2, 1),
color 0.2s cubic-bezier(0.4, 0, 0.2, 1),
box-shadow 0.2s cubic-bezier(0.4, 0, 0.2, 1),
transform 0.2s cubic-bezier(0.4, 0, 0.2, 1),
opacity 0.2s cubic-bezier(0.4, 0, 0.2, 1);
&:hover:not(:disabled) {
background: rgba(255, 255, 255, 0.92);
@@ -213,10 +222,13 @@ $text-light: #94a3b8;
opacity: 0.9;
transition: opacity 0.8s ease;
/* The glow orbs are pre-softened radial gradients. A `filter: blur(90px+)`
on top of them looked nearly identical but forced the GPU to re-blur
half-megapixel layers on every frame of the float animations — the
gradients below bake the same softness in for free. */
.glow {
position: absolute;
border-radius: 50%;
filter: blur(90px);
transform: translate3d(0, 0, 0);
will-change: transform, opacity;
pointer-events: none;
@@ -227,7 +239,7 @@ $text-light: #94a3b8;
left: 10%;
width: 520px;
height: 520px;
background: radial-gradient(circle, rgba(59, 130, 246, 0.12) 0%, rgba(99, 102, 241, 0.04) 55%, transparent 75%);
background: radial-gradient(circle, rgba(59, 130, 246, 0.12) 0%, rgba(99, 102, 241, 0.04) 45%, transparent 82%);
animation: floatGlow1 22s ease-in-out infinite alternate;
}
@@ -236,7 +248,7 @@ $text-light: #94a3b8;
right: 12%;
width: 560px;
height: 560px;
background: radial-gradient(circle, rgba(147, 51, 234, 0.08) 0%, rgba(219, 39, 119, 0.03) 55%, transparent 75%);
background: radial-gradient(circle, rgba(147, 51, 234, 0.08) 0%, rgba(219, 39, 119, 0.03) 45%, transparent 82%);
animation: floatGlow2 26s ease-in-out infinite alternate;
}
@@ -245,7 +257,7 @@ $text-light: #94a3b8;
left: 30%;
width: 460px;
height: 460px;
background: radial-gradient(circle, rgba(6, 182, 212, 0.08) 0%, rgba(59, 130, 246, 0.03) 55%, transparent 75%);
background: radial-gradient(circle, rgba(6, 182, 212, 0.08) 0%, rgba(59, 130, 246, 0.03) 45%, transparent 82%);
animation: floatGlow3 20s ease-in-out infinite alternate;
}
@@ -256,9 +268,8 @@ $text-light: #94a3b8;
margin: 0 auto;
width: 720px;
height: 520px;
background: radial-gradient(circle, rgba(245, 158, 11, 0.09) 0%, rgba(217, 119, 6, 0.035) 45%, transparent 75%);
background: radial-gradient(circle, rgba(245, 158, 11, 0.09) 0%, rgba(217, 119, 6, 0.035) 38%, transparent 82%);
opacity: 0;
filter: blur(100px);
transition: opacity 0.8s cubic-bezier(0.4, 0, 0.2, 1);
animation: readyPulseGlow 9s ease-in-out infinite alternate;
pointer-events: none;
@@ -271,9 +282,8 @@ $text-light: #94a3b8;
margin: 0 auto;
width: 700px;
height: 520px;
background: radial-gradient(circle, rgba(16, 185, 129, 0.14) 0%, rgba(6, 182, 212, 0.08) 45%, transparent 75%);
background: radial-gradient(circle, rgba(16, 185, 129, 0.14) 0%, rgba(6, 182, 212, 0.08) 38%, transparent 82%);
opacity: 0;
filter: blur(100px);
transition: opacity 1s cubic-bezier(0.4, 0, 0.2, 1);
animation: readyPulseGlow 7s ease-in-out infinite alternate;
pointer-events: none;
@@ -286,9 +296,8 @@ $text-light: #94a3b8;
margin: 0 auto;
width: 680px;
height: 520px;
background: radial-gradient(circle, rgba(99, 102, 241, 0.18) 0%, rgba(147, 51, 234, 0.08) 45%, transparent 75%);
background: radial-gradient(circle, rgba(99, 102, 241, 0.18) 0%, rgba(147, 51, 234, 0.08) 38%, transparent 82%);
opacity: 0;
filter: blur(95px);
transition: opacity 0.8s cubic-bezier(0.4, 0, 0.2, 1);
pointer-events: none;
}
@@ -383,8 +392,8 @@ $text-light: #94a3b8;
max-width: 520px;
box-sizing: border-box;
background: rgba(255, 255, 255, 0.72);
backdrop-filter: blur(16px) saturate(180%);
-webkit-backdrop-filter: blur(16px) saturate(180%);
backdrop-filter: blur(12px) saturate(180%);
-webkit-backdrop-filter: blur(12px) saturate(180%);
padding: 3px;
border-radius: 9999px;
border: 1px solid rgba(226, 232, 240, 0.85);
@@ -392,7 +401,9 @@ $text-light: #94a3b8;
box-shadow: 0 2px 8px -1px rgba(15, 23, 42, 0.06), 0 1px 2px rgba(0, 0, 0, 0.02), inset 0 1px 1px rgba(255, 255, 255, 0.9);
user-select: none;
font-family: 'Google Sans', 'Google Sans Text', 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
transition: all 0.25s cubic-bezier(0.4, 0, 0.2, 1);
transition: background 0.25s cubic-bezier(0.4, 0, 0.2, 1),
box-shadow 0.25s cubic-bezier(0.4, 0, 0.2, 1),
border-color 0.25s cubic-bezier(0.4, 0, 0.2, 1);
&:hover {
background: rgba(255, 255, 255, 0.85);
@@ -1717,6 +1728,12 @@ $text-light: #94a3b8;
.meta-dot {
color: $text-light;
}
.remote-endpoint-label {
color: $primary;
font-family: $font-mono;
font-size: 11px;
}
}
}
@@ -1739,6 +1756,7 @@ $text-light: #94a3b8;
animation: statusDotPulse 2s infinite cubic-bezier(0.4, 0, 0.6, 1);
}
}
}
.multi-device-switcher {
@@ -2365,10 +2383,14 @@ $text-light: #94a3b8;
.connection-methods-nav {
display: grid;
grid-template-columns: repeat(3, 1fr);
grid-template-columns: repeat(4, minmax(0, 1fr));
gap: 10px;
@media (max-width: 860px) {
@media (max-width: 1100px) {
grid-template-columns: repeat(2, minmax(0, 1fr));
}
@media (max-width: 620px) {
grid-template-columns: 1fr;
}
@@ -2421,37 +2443,12 @@ $text-light: #94a3b8;
.method-title-row {
display: flex;
align-items: center;
gap: 6px;
flex-wrap: wrap;
.method-title {
font-size: 13px;
font-weight: 600;
color: $text-main;
}
.method-badge-pill {
font-size: 9.5px;
font-weight: 700;
padding: 1px 5px;
border-radius: 4px;
text-transform: uppercase;
&.detected, &.info {
background: $primary-light;
color: $primary;
}
&.recommended {
background: $emerald-light;
color: $emerald;
}
&.neutral {
background: #f1f5f9;
color: $text-muted;
}
}
}
.method-desc {
@@ -2511,6 +2508,68 @@ $text-light: #94a3b8;
}
}
.active-remote-notice {
display: flex;
align-items: center;
justify-content: space-between;
gap: 12px;
padding: 10px 12px;
background: #fffbeb;
border: 1px solid $amber-border;
border-radius: 8px;
.active-remote-copy {
display: flex;
align-items: center;
gap: 8px;
min-width: 0;
color: #78350f;
font-size: 11.5px;
line-height: 1.45;
.material-symbols-outlined {
color: $amber;
font-size: 17px;
flex-shrink: 0;
}
strong {
font-family: $font-mono;
font-weight: 600;
}
}
.btn-use-local-adb {
display: inline-flex;
align-items: center;
justify-content: center;
gap: 5px;
min-height: 32px;
padding: 0 12px;
@include glass-secondary-btn;
border-radius: 6px;
font-family: inherit;
font-size: 11.5px;
font-weight: 600;
cursor: pointer;
white-space: nowrap;
.material-symbols-outlined {
font-size: 15px;
}
}
@media (max-width: 700px) {
align-items: stretch;
flex-direction: column;
.btn-use-local-adb {
width: 100%;
}
}
}
.method-details-panel {
background: #ffffff;
border: 1px solid $card-border;
@@ -2852,7 +2911,8 @@ $text-light: #94a3b8;
}
.usb-guide-intro,
.wifi-guide-intro {
.wifi-guide-intro,
.remote-guide-intro {
display: flex;
align-items: center;
gap: 8px;
@@ -3086,6 +3146,258 @@ $text-light: #94a3b8;
margin-top: 1px;
}
}
&.remote-adb-panel {
.remote-adb-interactive-box {
display: flex;
flex-direction: column;
gap: 10px;
padding: 12px 14px;
background: #f8fafc;
border: 1px solid $card-border;
border-radius: 8px;
.remote-adb-form-row {
display: flex;
align-items: flex-end;
gap: 8px;
flex-wrap: wrap;
.remote-adb-input-col {
display: flex;
flex-direction: column;
gap: 3px;
min-width: 0;
&.host-col {
flex: 1 1 220px;
}
&.port-col {
flex: 0 1 120px;
}
.remote-adb-label {
color: $text-muted;
font-size: 11px;
font-weight: 600;
}
.remote-adb-input-wrap {
display: flex;
align-items: center;
height: 36px;
padding: 0 9px;
background: #ffffff;
border: 1px solid $card-border;
border-radius: 6px;
transition: border-color 0.15s ease, box-shadow 0.15s ease;
&:focus-within {
border-color: $primary;
box-shadow: 0 0 0 2px rgba(26, 115, 232, 0.1);
}
.wrap-icon {
margin-right: 6px;
color: $text-light;
font-size: 15px;
flex-shrink: 0;
}
.remote-adb-input {
flex: 1;
min-width: 0;
border: 0;
outline: 0;
background: transparent;
color: $text-main;
font-family: $font-mono;
font-size: 12px;
&::placeholder {
color: $text-light;
}
}
}
}
.remote-adb-btn-col {
flex-shrink: 0;
.btn-remote-adb-connect {
display: inline-flex;
align-items: center;
justify-content: center;
gap: 5px;
height: 36px;
padding: 0 16px;
@include glass-primary-btn;
border-radius: 6px;
font-family: inherit;
font-size: 12px;
font-weight: 600;
cursor: pointer;
white-space: nowrap;
.material-symbols-outlined {
font-size: 16px;
}
}
}
@media (max-width: 640px) {
align-items: stretch;
flex-direction: column;
.remote-adb-input-col,
.remote-adb-input-col.host-col,
.remote-adb-input-col.port-col,
.remote-adb-btn-col {
flex: 1 1 auto;
width: 100%;
}
.remote-adb-btn-col .btn-remote-adb-connect {
width: 100%;
}
}
}
.remember-remote-adb {
display: inline-flex;
align-items: center;
align-self: flex-start;
gap: 7px;
color: $text-secondary;
font-size: 11.5px;
cursor: pointer;
input {
width: 15px;
height: 15px;
margin: 0;
accent-color: $primary;
}
}
.btn-use-tested-endpoint {
display: inline-flex;
align-items: center;
justify-content: center;
align-self: flex-end;
gap: 5px;
min-width: 150px;
height: 36px;
padding: 0 16px;
@include glass-primary-btn;
border-radius: 6px;
font-family: inherit;
font-size: 12px;
font-weight: 600;
cursor: pointer;
.material-symbols-outlined {
font-size: 16px;
}
}
.remote-adb-status-alert {
display: flex;
align-items: flex-start;
gap: 6px;
padding: 7px 10px;
border-radius: 6px;
font-size: 11.5px;
font-weight: 600;
line-height: 1.4;
.material-symbols-outlined {
font-size: 15px;
flex-shrink: 0;
}
&.success {
color: $emerald-dark;
background: $emerald-light;
border: 1px solid $emerald-border;
}
&.error {
color: $rose-dark;
background: $rose-light;
border: 1px solid $rose-border;
}
}
.remote-device-results {
display: flex;
flex-direction: column;
overflow: hidden;
background: #ffffff;
border: 1px solid $card-border;
border-radius: 7px;
.remote-device-row {
display: flex;
align-items: center;
gap: 9px;
min-width: 0;
padding: 8px 10px;
border-bottom: 1px solid #f1f5f9;
&:last-child {
border-bottom: 0;
}
> .material-symbols-outlined {
color: $primary;
font-size: 17px;
}
.remote-device-copy {
display: flex;
flex: 1;
flex-direction: column;
min-width: 0;
strong {
color: $text-main;
font-size: 12px;
font-weight: 600;
}
span {
overflow: hidden;
color: $text-muted;
font-family: $font-mono;
font-size: 10.5px;
text-overflow: ellipsis;
white-space: nowrap;
}
}
.remote-device-state {
color: $amber-dark;
font-size: 10.5px;
font-weight: 600;
text-transform: capitalize;
&.ready {
color: $emerald-dark;
}
}
}
}
}
.remote-adb-explanation {
margin: 0;
color: $text-muted;
font-size: 11.5px;
line-height: 1.5;
}
}
}
}
@@ -3496,13 +3808,15 @@ $text-light: #94a3b8;
/* Unified Sculptured Composer (Linear / Claude / ChatGPT Grade) */
.composer-card {
background: rgba(255, 255, 255, 0.88);
backdrop-filter: blur(28px) saturate(190%);
-webkit-backdrop-filter: blur(28px) saturate(190%);
backdrop-filter: blur(16px) saturate(190%);
-webkit-backdrop-filter: blur(16px) saturate(190%);
border: 1px solid rgba(255, 255, 255, 0.95);
box-shadow: 0 20px 50px -12px rgba(15, 23, 42, 0.08), 0 1px 3px rgba(0, 0, 0, 0.02), inset 0 1px 1px rgba(255, 255, 255, 0.95);
border-radius: 20px;
padding: 18px 20px 14px;
transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1);
transition: border-color 0.3s cubic-bezier(0.4, 0, 0.2, 1),
box-shadow 0.3s cubic-bezier(0.4, 0, 0.2, 1),
background 0.3s cubic-bezier(0.4, 0, 0.2, 1);
position: relative;
&.is-pro-active {
@@ -4125,12 +4439,15 @@ $text-light: #94a3b8;
padding: 14px 16px;
border-radius: 16px;
background: rgba(255, 255, 255, 0.72);
backdrop-filter: blur(20px) saturate(180%);
-webkit-backdrop-filter: blur(20px) saturate(180%);
backdrop-filter: blur(10px) saturate(180%);
-webkit-backdrop-filter: blur(10px) saturate(180%);
border: 1px solid rgba(226, 232, 240, 0.85);
box-shadow: 0 2px 8px -1px rgba(15, 23, 42, 0.04), inset 0 1px 1px rgba(255, 255, 255, 0.95);
cursor: pointer;
transition: all 0.22s cubic-bezier(0.4, 0, 0.2, 1);
transition: background 0.22s cubic-bezier(0.4, 0, 0.2, 1),
transform 0.22s cubic-bezier(0.4, 0, 0.2, 1),
box-shadow 0.22s cubic-bezier(0.4, 0, 0.2, 1),
border-color 0.22s cubic-bezier(0.4, 0, 0.2, 1);
user-select: none;
position: relative;
overflow: hidden;
@@ -4367,18 +4684,17 @@ $text-light: #94a3b8;
}
}
/* Transform-only: animating opacity here overrode the .glow-pending/.glow-ready
fade transitions and kept both state glows permanently visible. */
@keyframes readyPulseGlow {
0% {
transform: scale(0.97);
opacity: 0.7;
}
50% {
transform: scale(1.06);
opacity: 1;
}
100% {
transform: scale(0.98);
opacity: 0.75;
}
}
@@ -4411,3 +4727,13 @@ $text-light: #94a3b8;
opacity: 0.8;
}
}
/* Respect the user's motion preference: freeze the purely decorative ambient
animations (aurora drift, pulses, sparkles). Functional spinners keep going. */
@media (prefers-reduced-motion: reduce) {
.home-container .aurora-background .glow,
.sparkle-icon,
.status-dot {
animation: none !important;
}
}
@@ -14,13 +14,18 @@
* limitations under the License.
*/
import { Component, signal, computed, effect, inject, OnInit, OnDestroy } from '@angular/core';
import { CommonModule } from '@angular/common';
import { Component, signal, computed, effect, inject, OnInit, OnDestroy, ChangeDetectionStrategy } from '@angular/core';
import { FormsModule } from '@angular/forms';
import { Router, RouterLink } from '@angular/router';
import { AgentService } from '../../services/agent.service';
import { SystemService } from '../../services/system.service';
import { DeviceInfo, ProbeResult } from '../../core/models/system.model';
import {
AdbServerConnectionResult,
AdbServerDevice,
DeviceInfo,
ProbeResult
} from '../../core/models/system.model';
import {
AppReference,
SmartSuggestion,
@@ -30,12 +35,15 @@ import { TaskRecommendationService } from '../../core/services/task-recommendati
export type { AppReference, SmartSuggestion, SuggestionCategory };
type AdbGuideTab = 'emulator' | 'usb' | 'wifi' | 'remote';
@Component({
selector: 'app-home',
standalone: true,
imports: [CommonModule, FormsModule],
imports: [FormsModule],
templateUrl: './home.component.html',
changeDetection: ChangeDetectionStrategy.Eager,
styleUrl: './home.component.scss'
})
export class HomeComponent implements OnInit, OnDestroy {
@@ -47,8 +55,8 @@ export class HomeComponent implements OnInit, OnDestroy {
// High-level navigation mode: 'diagnostics' (System Setup Guide) vs 'launcher' (Task Execution)
public activeTab = signal<'diagnostics' | 'launcher'>('launcher');
// Interactive guide sub-tab inside ADB section: 'emulator' | 'usb' | 'wifi'
public activeAdbGuideTab = signal<'emulator' | 'usb' | 'wifi'>('emulator');
// Interactive guide sub-tab inside the ADB section
public activeAdbGuideTab = signal<AdbGuideTab>('emulator');
public emulatorSetupMode = signal<'studio' | 'cli'>('studio');
// Interactive guide tab for LLM / OCR credentials: 'gemini' | 'ocr'
@@ -90,6 +98,29 @@ export class HomeComponent implements OnInit, OnDestroy {
public wifiConnectMessage = signal<string | null>(null);
public wifiConnectError = signal<string | null>(null);
public adbRestartFeedback = signal<string | null>(null);
public showConnectionMethods = signal<boolean>(false);
// ADB server endpoint connection state
public remoteAdbHost = signal<string>('127.0.0.1');
public remoteAdbPort = signal<string>('5038');
public rememberRemoteAdb = signal<boolean>(true);
public isConnectingRemoteAdb = signal<boolean>(false);
public isActivatingRemoteAdb = signal<boolean>(false);
public isSwitchingToLocalAdb = signal<boolean>(false);
public remoteAdbMessage = signal<string | null>(null);
public remoteAdbError = signal<string | null>(null);
public remoteAdbDevices = signal<AdbServerDevice[]>([]);
public remoteAdbProbeResult = signal<AdbServerConnectionResult | null>(null);
public adbServerStatus = computed(() => this.systemService.adbServerStatus());
public isRemoteAdbServer = computed(() => this.systemService.isRemoteAdbServer());
public remoteAdbHasReadyDevice = computed(() =>
this.remoteAdbDevices().some(device => device.state === 'device')
);
public isProbedRemoteAdbActive = computed(() => {
const tested = this.remoteAdbProbeResult()?.endpoint;
const active = this.adbServerStatus()?.endpoint;
return !!tested && !!active && tested.identity === active.identity;
});
public wifiCommand = computed(() => {
const h = this.wifiHost().trim() || '<phone-ip>';
@@ -334,6 +365,16 @@ export class HomeComponent implements OnInit, OnDestroy {
// Initial fetch of system readiness & model configuration
this.systemService.fetchReadiness().subscribe();
this.systemService.fetchModelConfigEnv().subscribe();
this.systemService.fetchAdbServerStatus().subscribe({
next: status => {
if (status.endpoint.mode === 'remote') {
this.remoteAdbHost.set(status.endpoint.host);
this.remoteAdbPort.set(String(status.endpoint.port));
this.activeAdbGuideTab.set('remote');
}
},
error: () => {}
});
window.addEventListener('focus', this.focusListener);
}
@@ -354,8 +395,10 @@ export class HomeComponent implements OnInit, OnDestroy {
this.selectedOs.set(os);
}
public setAdbGuideTab(tab: 'emulator' | 'usb' | 'wifi'): void {
public setAdbGuideTab(tab: AdbGuideTab): void {
this.activeAdbGuideTab.set(tab);
this.remoteAdbError.set(null);
this.remoteAdbMessage.set(null);
}
public setEmulatorSetupMode(mode: 'studio' | 'cli'): void {
@@ -609,7 +652,9 @@ export class HomeComponent implements OnInit, OnDestroy {
this.adbRestartFeedback.set(null);
this.systemService.restartAdb().subscribe({
next: (res) => {
this.adbRestartFeedback.set('ADB Refreshed ✓');
this.adbRestartFeedback.set(
res?.restart_result?.skipped ? 'Devices Refreshed ✓' : 'ADB Refreshed ✓'
);
setTimeout(() => this.adbRestartFeedback.set(null), 2500);
},
error: () => {
@@ -620,6 +665,10 @@ export class HomeComponent implements OnInit, OnDestroy {
}
public connectWifiDevice(): void {
if (this.isRemoteAdbServer()) {
this.wifiConnectError.set('Switch to local ADB before connecting a Wireless ADB device.');
return;
}
const host = this.wifiHost().trim();
const portStr = this.wifiPort().trim() || '5555';
const port = parseInt(portStr, 10) || 5555;
@@ -651,7 +700,123 @@ export class HomeComponent implements OnInit, OnDestroy {
});
}
public connectRemoteAdbServer(): void {
const host = this.remoteAdbHost().trim();
const port = Number(this.remoteAdbPort().trim());
if (!host) {
this.remoteAdbError.set('Enter the host name or IP address of the ADB server.');
return;
}
if (!Number.isInteger(port) || port < 1 || port > 65535) {
this.remoteAdbError.set('Enter a port between 1 and 65535.');
return;
}
this.isConnectingRemoteAdb.set(true);
this.remoteAdbError.set(null);
this.remoteAdbMessage.set(null);
this.remoteAdbDevices.set([]);
this.remoteAdbProbeResult.set(null);
this.systemService.probeAdbServer(host, port).subscribe({
next: response => {
this.isConnectingRemoteAdb.set(false);
const result = response.connection_result;
if (result.success) {
this.remoteAdbProbeResult.set(result);
this.remoteAdbDevices.set(result.devices || []);
this.remoteAdbMessage.set(result.message);
} else {
this.remoteAdbError.set(result.message);
}
},
error: error => {
this.isConnectingRemoteAdb.set(false);
this.remoteAdbError.set(
error?.error?.detail || 'Unable to test the ADB server endpoint.'
);
}
});
}
public activateRemoteAdbServer(): void {
const tested = this.remoteAdbProbeResult();
if (!tested?.success) {
this.remoteAdbError.set('Test the endpoint before using it.');
return;
}
this.isActivatingRemoteAdb.set(true);
this.remoteAdbError.set(null);
this.systemService.connectAdbServer(
tested.endpoint.host,
tested.endpoint.port,
this.rememberRemoteAdb()
).subscribe({
next: response => {
this.isActivatingRemoteAdb.set(false);
const result = response.connection_result;
if (result.success) {
this.remoteAdbProbeResult.set(result);
this.remoteAdbDevices.set(result.devices || []);
this.remoteAdbMessage.set(result.message);
} else {
this.remoteAdbError.set(result.message);
}
},
error: error => {
this.isActivatingRemoteAdb.set(false);
this.remoteAdbError.set(
error?.error?.detail || 'Unable to use the ADB server endpoint.'
);
}
});
}
public updateRemoteAdbHost(value: string): void {
this.remoteAdbHost.set(value);
this.clearRemoteAdbProbe();
}
public updateRemoteAdbPort(value: string): void {
this.remoteAdbPort.set(value);
this.clearRemoteAdbProbe();
}
private clearRemoteAdbProbe(): void {
this.remoteAdbProbeResult.set(null);
this.remoteAdbDevices.set([]);
this.remoteAdbMessage.set(null);
this.remoteAdbError.set(null);
}
public switchToLocalAdbServer(): void {
this.isSwitchingToLocalAdb.set(true);
this.remoteAdbError.set(null);
this.systemService.useLocalAdbServer(true).subscribe({
next: response => {
this.isSwitchingToLocalAdb.set(false);
this.remoteAdbDevices.set([]);
this.remoteAdbMessage.set(response.connection_result.message);
},
error: error => {
this.isSwitchingToLocalAdb.set(false);
this.remoteAdbError.set(
error?.error?.detail || 'Unable to switch back to the local ADB server.'
);
}
});
}
public toggleConnectionMethods(): void {
this.showConnectionMethods.update(value => !value);
}
public launchAvdEmulator(avdName: string): void {
if (this.isRemoteAdbServer()) {
return;
}
this.systemService.launchEmulator(avdName).subscribe();
}
@@ -14,16 +14,17 @@
* limitations under the License.
*/
import { Component, HostListener } from '@angular/core';
import { CommonModule } from '@angular/common';
import { Component, HostListener, ChangeDetectionStrategy } from '@angular/core';
import { LegacyAgentStreamComponent } from '../../components/legacy-agent-stream/legacy-agent-stream.component';
import { ChatInterfaceComponent } from '../../components/chat-interface/chat-interface.component';
@Component({
selector: 'app-legacy-workspace',
standalone: true,
imports: [CommonModule, LegacyAgentStreamComponent, ChatInterfaceComponent],
imports: [LegacyAgentStreamComponent, ChatInterfaceComponent],
templateUrl: './legacy-workspace.component.html',
changeDetection: ChangeDetectionStrategy.Eager,
styleUrl: './legacy-workspace.component.scss'
})
export class LegacyWorkspaceComponent {
@@ -1,20 +1,20 @@
<!--
Copyright 2026 Google LLC
Copyright 2026 Google LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<div class="workspace-container" [class.is-resizing]="isDragging">
<div class="workspace-container" [class.is-resizing]="isDragging()">
<!-- Left Main Panel: Raw Agent Log Stream -->
<main class="left-panel glass-panel">
<app-agent-stream></app-agent-stream>
@@ -23,13 +23,15 @@
<div class="workspace-floating-bar-wrapper" [class.is-expanded]="isBarExpanded()" [class.is-dormant]="!isBarExpanded()"
[class.is-focused]="isInputFocused()" [class.has-text]="!!taskInput.trim()" [class.is-running]="isTaskRunning()">
<!-- Floating Error / Notification Toast -->
<div class="floating-dock-toast error" *ngIf="errorMessage">
<span class="material-symbols-outlined toast-icon">error</span>
<span class="toast-text">{{ errorMessage }}</span>
<button type="button" class="btn-toast-close" (click)="errorMessage = null" aria-label="Close error message">
<span class="material-symbols-outlined">close</span>
</button>
</div>
@if (errorMessage()) {
<div class="floating-dock-toast error">
<span class="material-symbols-outlined toast-icon">error</span>
<span class="toast-text">{{ errorMessage() }}</span>
<button type="button" class="btn-toast-close" (click)="errorMessage.set(null)" aria-label="Close error message">
<span class="material-symbols-outlined">close</span>
</button>
</div>
}
<!-- Dock Row (Contains the Main Card and the Separate Stop Button when running) -->
<div class="floating-dock-row">
@@ -52,9 +54,9 @@
</linearGradient>
</defs>
<path d="M 0,25 Q 300,55 600,20 T 1200,25 Q 1500,55 1800,20 T 2400,25 L 2400,80 L 0,80 Z"
fill="url(#waveGradBack)"></path>
fill="url(#waveGradBack)"></path>
</svg>
<!-- Mid Fluid Wave -->
<svg class="liquid-wave wave-mid" viewBox="0 0 2400 80" preserveAspectRatio="none">
<defs>
@@ -66,9 +68,9 @@
</linearGradient>
</defs>
<path d="M 0,35 Q 300,5 600,45 T 1200,35 Q 1500,5 1800,45 T 2400,35 L 2400,80 L 0,80 Z"
fill="url(#waveGradMid)"></path>
fill="url(#waveGradMid)"></path>
</svg>
<!-- Front Fluid Wave -->
<svg class="liquid-wave wave-front" viewBox="0 0 2400 80" preserveAspectRatio="none">
<defs>
@@ -80,16 +82,16 @@
</linearGradient>
</defs>
<path d="M 0,18 Q 300,50 600,15 T 1200,18 Q 1500,50 1800,15 T 2400,18 L 2400,80 L 0,80 Z"
fill="url(#waveGradFront)"></path>
fill="url(#waveGradFront)"></path>
</svg>
</div>
</div>
<!-- Dormant / Capsule Peek View (Visible when mouse is away and input idle) -->
<div class="capsule-peek-view">
<span class="capsule-placeholder">Start a new task...</span>
</div>
<!-- Expanded Full View Content (Visible on hover, focus, or text) -->
<div class="expanded-card-content">
<div class="card-main-input-row">
@@ -101,7 +103,7 @@
(focus)="onInputFocus()"
(blur)="onInputBlur()"
rows="1"
[disabled]="isSubmitting"
[disabled]="isSubmitting()"
[placeholder]="selectedProfile() === 'pro'
? 'Pro: Multi-agent cognitive graph architecture for complex multi-step tasks'
: 'Flash: Reactive fast loop for standard app workflows within 25 steps'"
@@ -111,30 +113,38 @@
</div>
<!-- Clear Button (visible when text is entered) -->
<button *ngIf="taskInput.trim() && !isSubmitting" type="button" class="dock-circle-btn btn-clear"
(click)="clearInput($event)" title="Clear input text" aria-label="Clear input text">
<span class="material-symbols-outlined">close</span>
</button>
@if (taskInput.trim() && !isSubmitting()) {
<button type="button" class="dock-circle-btn btn-clear"
(click)="clearInput($event)" title="Clear input text" aria-label="Clear input text">
<span class="material-symbols-outlined">close</span>
</button>
}
<!-- Stop Button (Visible when currently viewed task is active) -->
<button *ngIf="isTaskRunning()" type="button" class="dock-circle-btn btn-stop" [disabled]="isSubmitting"
(click)="stopTask($event)"
[title]="stopButtonTitle()"
[attr.aria-label]="stopButtonTitle()"
>
<span class="material-symbols-outlined">stop</span>
</button>
@if (isTaskRunning()) {
<button type="button" class="dock-circle-btn btn-stop" [disabled]="isSubmitting()"
(click)="stopTask($event)"
[title]="stopButtonTitle()"
[attr.aria-label]="stopButtonTitle()"
>
<span class="material-symbols-outlined">stop</span>
</button>
}
<!-- Send Button -->
<button type="button" class="dock-circle-btn btn-send"
[class.has-content]="!!taskInput.trim()"
[disabled]="!taskInput.trim() || isSubmitting"
[disabled]="!taskInput.trim() || isSubmitting()"
(click)="submitTask()"
[title]="isSubmitting ? 'Starting task...' : 'Start task (Enter)'"
[title]="isSubmitting() ? 'Starting task...' : 'Start task (Enter)'"
aria-label="Start task"
>
<span class="material-symbols-outlined spinning-icon" *ngIf="isSubmitting">sync</span>
<span class="material-symbols-outlined" *ngIf="!isSubmitting">arrow_upward</span>
>
@if (isSubmitting()) {
<span class="material-symbols-outlined spinning-icon">sync</span>
}
@if (!isSubmitting()) {
<span class="material-symbols-outlined">arrow_upward</span>
}
</button>
</div>
@@ -162,28 +172,30 @@
</div>
<!-- Separate Standalone Stop Button Beside the Capsule (Visible when currently viewed task is active) -->
<button *ngIf="isTaskRunning()" type="button" class="standalone-stop-btn"
[disabled]="isSubmitting"
(click)="stopTask($event)"
[title]="stopButtonTitle()"
[attr.aria-label]="stopButtonTitle()"
>
<span class="material-symbols-outlined">stop</span>
</button>
@if (isTaskRunning()) {
<button type="button" class="standalone-stop-btn"
[disabled]="isSubmitting()"
(click)="stopTask($event)"
[title]="stopButtonTitle()"
[attr.aria-label]="stopButtonTitle()"
>
<span class="material-symbols-outlined">stop</span>
</button>
}
</div>
</div>
</main>
<!-- Split Pane Drag Resizer -->
<div class="resizer" (mousedown)="onDragStart($event)">
<div class="resizer-handle"></div>
</div>
<!-- Right Sidebar Panel: Chat Interface Mock -->
<aside class="right-panel" [style.width.px]="rightPanelWidth">
<app-chat-interface></app-chat-interface>
</aside>
<!-- Floating Video Replay Player Modal / Window -->
<app-floating-video-player></app-floating-video-player>
<!-- Split Pane Drag Resizer -->
<div class="resizer" (mousedown)="onDragStart($event)">
<div class="resizer-handle"></div>
</div>
<!-- Right Sidebar Panel: Chat Interface Mock -->
<aside class="right-panel" [style.width.px]="rightPanelWidth()">
<app-chat-interface></app-chat-interface>
</aside>
<!-- Floating Video Replay Player Modal / Window -->
<app-floating-video-player></app-floating-video-player>
</div>
@@ -104,7 +104,7 @@ $rose-dark: #dc2626;
height: 40px;
background: rgba(0, 0, 0, 0.15);
border-radius: 4px;
transition: all 0.2s ease;
transition: background 0.2s ease, height 0.2s ease;
}
}
@@ -308,7 +308,11 @@ $rose-dark: #dc2626;
background: radial-gradient(ellipse at center, rgba(168, 85, 247, 0.45) 0%, rgba(236, 72, 153, 0.25) 50%, transparent 80%);
}
/* Wave Tracks Wrapper with Soft Gaussian Blur */
/* Wave Tracks Wrapper. The soft-liquid look comes from blurring each
wave individually: blurring the WRAPPER forced the GPU to re-run the
gaussian pass every animation frame (child transforms invalidate the
parent's filtered layer), while per-wave blur over transform-only
animations is rasterized once and then just composited. */
.wave-tracks-wrapper {
position: absolute;
bottom: 0;
@@ -316,7 +320,6 @@ $rose-dark: #dc2626;
width: 100%;
height: 22px;
overflow: hidden;
filter: blur(7px); /* Soften sharp vector wave edges into silky liquid light */
mask-image: linear-gradient(to top, rgba(0,0,0,0.9) 0%, rgba(0,0,0,0.5) 60%, transparent 100%);
-webkit-mask-image: linear-gradient(to top, rgba(0,0,0,0.9) 0%, rgba(0,0,0,0.5) 60%, transparent 100%);
@@ -327,7 +330,7 @@ $rose-dark: #dc2626;
width: 200%;
height: 100%;
will-change: transform;
filter: blur(3px); /* Secondary soft layer blur */
filter: blur(8px); /* Silky liquid softness, applied per cached wave layer */
&.wave-back {
opacity: 0.45;
@@ -562,7 +565,11 @@ $rose-dark: #dc2626;
cursor: pointer;
outline: none;
flex-shrink: 0;
transition: all 0.18s cubic-bezier(0.16, 1, 0.3, 1);
transition: background 0.18s cubic-bezier(0.16, 1, 0.3, 1),
color 0.18s cubic-bezier(0.16, 1, 0.3, 1),
transform 0.18s cubic-bezier(0.16, 1, 0.3, 1),
box-shadow 0.18s cubic-bezier(0.16, 1, 0.3, 1),
border-color 0.18s cubic-bezier(0.16, 1, 0.3, 1);
.material-symbols-outlined {
font-size: 18px;
@@ -695,7 +702,9 @@ $rose-dark: #dc2626;
font-size: 11px;
font-weight: 550;
cursor: pointer;
transition: all 0.16s cubic-bezier(0.16, 1, 0.3, 1);
transition: color 0.16s cubic-bezier(0.16, 1, 0.3, 1),
background 0.16s cubic-bezier(0.16, 1, 0.3, 1),
box-shadow 0.16s cubic-bezier(0.16, 1, 0.3, 1);
outline: none;
user-select: none;
box-sizing: border-box;
@@ -863,3 +872,14 @@ $rose-dark: #dc2626;
transform: scaleX(1.08);
}
}
/* Respect the user's motion preference: freeze the decorative liquid waves
and pulses while keeping the frosted dock visuals intact. */
@media (prefers-reduced-motion: reduce) {
.liquid-wave,
.wave-glow-ambient,
.pulse-indicator-dot,
.running-pulse-dot {
animation: none !important;
}
}
@@ -14,8 +14,8 @@
* limitations under the License.
*/
import { Component, HostListener, inject, computed, signal, ViewChild, ElementRef, OnInit } from '@angular/core';
import { CommonModule } from '@angular/common';
import { Component, ChangeDetectionStrategy, NgZone, DestroyRef, inject, computed, signal, ViewChild, ElementRef, OnInit } from '@angular/core';
import { FormsModule } from '@angular/forms';
import { AgentStreamComponent } from '../../components/agent-stream/agent-stream.component';
import { ChatInterfaceComponent } from '../../components/chat-interface/chat-interface.component';
@@ -26,26 +26,35 @@ import { AgentService } from '../../services/agent.service';
selector: 'app-workspace',
standalone: true,
imports: [
CommonModule,
FormsModule,
AgentStreamComponent,
ChatInterfaceComponent,
FloatingVideoPlayerComponent
],
],
templateUrl: './workspace.component.html',
styleUrl: './workspace.component.scss'
styleUrl: './workspace.component.scss',
changeDetection: ChangeDetectionStrategy.OnPush
})
export class WorkspaceComponent implements OnInit {
public agentService = inject(AgentService);
private zone = inject(NgZone);
private destroyRef = inject(DestroyRef);
// Default right panel width to 1/3 of the screen (or 450px as fallback)
public rightPanelWidth = typeof window !== 'undefined' ? Math.round(window.innerWidth / 3) : 450;
public isDragging = false;
public rightPanelWidth = signal<number>(
typeof window !== 'undefined' ? Math.round(window.innerWidth / 3) : 450
);
public isDragging = signal<boolean>(false);
private dragWidthRafId: number | null = null;
private pendingDragWidth = 0;
// Floating Command Bar State
public taskInput: string = '';
public isSubmitting: boolean = false;
public errorMessage: string | null = null;
// Floating Command Bar State. taskInput is backed by a signal so computed
// expressions (isBarExpanded) genuinely track it under OnPush.
private taskInputSignal = signal<string>('');
public get taskInput(): string { return this.taskInputSignal(); }
public set taskInput(value: string) { this.taskInputSignal.set(value); }
public isSubmitting = signal<boolean>(false);
public errorMessage = signal<string | null>(null);
public selectedProfile = signal<'flash' | 'pro'>('flash');
// Expand States (Signals for 0-latency reactivity)
@@ -61,6 +70,16 @@ export class WorkspaceComponent implements OnInit {
this.selectedProfile.set(saved);
}
}
// The global ⌘K/Ctrl+K shortcut is registered outside the Angular zone so
// ordinary typing never schedules an extra change-detection pass.
this.zone.runOutsideAngular(() => {
window.addEventListener('keydown', this.onGlobalKeyDown);
});
this.destroyRef.onDestroy(() => {
window.removeEventListener('keydown', this.onGlobalKeyDown);
this.detachDragListeners();
});
}
/**
@@ -179,13 +198,12 @@ export class WorkspaceComponent implements OnInit {
/**
* Global keyboard shortcut (⌘K or Ctrl+K) to focus input bar from anywhere
*/
@HostListener('window:keydown', ['$event'])
public onGlobalKeyDown(event: KeyboardEvent): void {
private onGlobalKeyDown = (event: KeyboardEvent): void => {
if ((event.metaKey || event.ctrlKey) && event.key.toLowerCase() === 'k') {
event.preventDefault();
this.focusInput();
}
}
};
/**
* Handle enter key in floating dock textarea
@@ -202,12 +220,12 @@ export class WorkspaceComponent implements OnInit {
*/
public submitTask(): void {
const goal = this.taskInput.trim();
if (!goal || this.isSubmitting) {
if (!goal || this.isSubmitting()) {
return;
}
this.isSubmitting = true;
this.errorMessage = null;
this.isSubmitting.set(true);
this.errorMessage.set(null);
if (this.dockInputRef?.nativeElement) {
this.dockInputRef.nativeElement.blur();
@@ -220,15 +238,15 @@ export class WorkspaceComponent implements OnInit {
if (this.dockInputRef?.nativeElement) {
this.dockInputRef.nativeElement.style.height = 'auto';
}
this.isSubmitting = false;
this.isSubmitting.set(false);
this.agentService.fetchStatus();
},
error: (err) => {
console.error('Failed to submit task:', err);
this.isSubmitting = false;
this.errorMessage = err.error?.detail || 'The runner is busy. Please wait for current task to finish.';
this.isSubmitting.set(false);
this.errorMessage.set(err.error?.detail || 'The runner is busy. Please wait for current task to finish.');
setTimeout(() => {
this.errorMessage = null;
this.errorMessage.set(null);
}, 5000);
}
});
@@ -245,28 +263,31 @@ export class WorkspaceComponent implements OnInit {
return;
}
const targetSessionId = this.agentService.currentSessionId();
this.isSubmitting = true;
this.errorMessage = null;
this.isSubmitting.set(true);
this.errorMessage.set(null);
this.agentService.stopTask(targetSessionId, false);
setTimeout(() => {
this.isSubmitting = false;
this.isSubmitting.set(false);
}, 400);
}
/**
* Handle mouse down on resizer bar to start dragging
* Handle mouse down on resizer bar to start dragging. The move/up listeners
* are attached only for the duration of the drag and run outside the Angular
* zone: idle mouse movement over the workspace never triggers change
* detection, and drag updates are coalesced to one per animation frame.
*/
public onDragStart(event: MouseEvent): void {
this.isDragging = true;
this.isDragging.set(true);
event.preventDefault();
this.zone.runOutsideAngular(() => {
document.addEventListener('mousemove', this.onMouseMove);
document.addEventListener('mouseup', this.onMouseUp);
});
}
/**
* Listen to global mousemove events to adjust the width dynamically
*/
@HostListener('document:mousemove', ['$event'])
public onMouseMove(event: MouseEvent): void {
if (!this.isDragging) {
private onMouseMove = (event: MouseEvent): void => {
if (!this.isDragging()) {
return;
}
@@ -276,15 +297,27 @@ export class WorkspaceComponent implements OnInit {
// Apply boundary limits to prevent panels from shrinking too much
if (newWidth >= minWidth && newWidth <= maxWidth) {
this.rightPanelWidth = newWidth;
this.pendingDragWidth = newWidth;
if (this.dragWidthRafId === null) {
this.dragWidthRafId = requestAnimationFrame(() => {
this.dragWidthRafId = null;
this.rightPanelWidth.set(this.pendingDragWidth);
});
}
}
};
private onMouseUp = (): void => {
this.isDragging.set(false);
this.detachDragListeners();
};
private detachDragListeners(): void {
document.removeEventListener('mousemove', this.onMouseMove);
document.removeEventListener('mouseup', this.onMouseUp);
if (this.dragWidthRafId !== null) {
cancelAnimationFrame(this.dragWidthRafId);
this.dragWidthRafId = null;
}
}
/**
* Listen to global mouseup events to stop resizing
*/
@HostListener('document:mouseup')
public onMouseUp(): void {
this.isDragging = false;
}
}
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { Injectable, signal, inject, computed } from '@angular/core';
import { Injectable, signal, inject, computed, DestroyRef, NgZone } from '@angular/core';
import { HttpClient } from '@angular/common/http';
import { Observable } from 'rxjs';
@@ -63,6 +63,7 @@ interface SessionVideoResponse {
})
export class AgentService {
private http = inject(HttpClient);
private zone = inject(NgZone);
private activePauseCardKey: string | null = null;
// Signals to expose state to components
@@ -81,21 +82,25 @@ export class AgentService {
const activeList = this.activeTasks();
const sessionMap = new Map<string, Session>();
// Lookup maps keep the per-session merge O(1) instead of scanning the
// pending/active lists multiple times for every raw session.
const pendingById = new Map(pending.map((p) => [p.session_id, p]));
const activeById = new Map(activeList.map((at) => [at.session_id, at]));
// 1. First add raw sessions from DB
raw.forEach((s) => {
const activeMatch = activeById.get(s.session_id);
const pendingMatch = pendingById.get(s.session_id);
const isCurrentActive = ((status === 'running' || status === 'paused') && runId === s.session_id)
|| activeList.some(at => at.session_id === s.session_id);
|| !!activeMatch;
const isTerminal = s.status === 'completed' || s.status === 'success' || s.status === 'failed' || s.status === 'cancelled';
let sStatus = s.status;
if (!isTerminal) {
const isPending = this.pendingQueue().some(p => p.session_id === s.session_id) && !isCurrentActive;
const isPending = !!pendingMatch && !isCurrentActive;
sStatus = isCurrentActive ? (status === 'paused' ? 'paused' : 'running') : (isPending ? 'pending' : s.status);
} else {
sStatus = (s.status === 'success') ? 'completed' : s.status;
}
const activeMatch = activeList.find(at => at.session_id === s.session_id);
const pendingMatch = this.pendingQueue().find(p => p.session_id === s.session_id);
let serial = s.device_serial || s.device_id || activeMatch?.device_id || pendingMatch?.device_serial || null;
if (!serial && s.device_info) {
try {
@@ -122,9 +127,9 @@ export class AgentService {
// 2. Add pending queue sessions if not yet in raw sessions
pending.forEach((p) => {
if (!sessionMap.has(p.session_id)) {
const activeMatch = activeById.get(p.session_id);
const isCurrentRunning = ((status === 'running' || status === 'paused') && runId === p.session_id)
|| activeList.some(at => at.session_id === p.session_id);
const activeMatch = activeList.find(at => at.session_id === p.session_id);
|| !!activeMatch;
const finalPending: Session = {
...p,
status: isCurrentRunning ? (status === 'paused' ? 'paused' : 'running') : (p.status || 'pending'),
@@ -241,11 +246,23 @@ export class AgentService {
private videoRetryTimer: ReturnType<typeof setTimeout> | null = null;
private videoWaitStartedAt = 0;
/**
* Step logs only. Streaming text chunks cannot change replay frames, so the
* custom equality keeps downstream frame extraction from re-running on every
* llm_stream update of the sessionLogs signal.
*/
private stepLogsForReplay = computed<any[]>(
() => this.sessionLogs().filter(
(log) => log && (log.type === 'step_updated' || log.type === 'step_recorded')
),
{ equal: (a, b) => a.length === b.length && a.every((log, i) => log === b[i]) }
);
/**
* Computed step replay frames from current session logs
*/
public currentSessionStepFrames = computed<StepReplayFrame[]>(() => {
return extractStepReplayFrames(this.sessionLogs());
return extractStepReplayFrames(this.stepLogsForReplay());
});
public hasCurrentSessionStepFrames = computed<boolean>(() => {
@@ -328,11 +345,29 @@ export class AgentService {
private sessionSnapshotAppliedId = 0;
private pendingSnapshotRequests = new Set<number>();
// Streaming chunks are coalesced into one signal update per short window so a
// chatty LLM stream cannot force a full recompute pass per SSE chunk.
private pendingStreamChunks = new Map<string, { execId: any; stepId: any; streamType: string; chunk: string }>();
private streamFlushTimer: ReturnType<typeof setTimeout> | null = null;
private lastQueueSignature: string | null = null;
private lastActiveTasksSignature: string | null = null;
private lastPersistedSessionsJson: string | null = null;
private onVisibilityChange = () => {
if (typeof document !== 'undefined' && !document.hidden) {
this.fetchStatus();
this.fetchSessions();
}
};
constructor() {
this.restoreSessionsCache();
this.fetchSessions();
this.startStatusPolling();
this.ensureLiveStream();
if (typeof document !== 'undefined') {
document.addEventListener('visibilitychange', this.onVisibilityChange);
}
inject(DestroyRef).onDestroy(() => this.destroy());
}
/**
@@ -451,20 +486,25 @@ export class AgentService {
this.isPaused.set(false);
this.pausedError.set(null);
this.isRetrying.set(false);
this.invalidateStatusSignatures();
if (effectiveStopAll) {
this.pendingQueue.set([]);
}
// Mark active streaming / pending logs as finished if viewing target session
if (!targetSessionId || this.currentSessionId() === targetSessionId) {
this.sessionLogs.update((logs) =>
logs.map((l) => {
this.flushStreamChunks();
this.sessionLogs.update((logs) => {
if (!logs.some((l) => l.type === 'llm_stream' && !l.data?.isCompleted)) {
return logs;
}
return logs.map((l) => {
if (l.type === 'llm_stream' && !l.data?.isCompleted) {
return { ...l, data: { ...l.data, isCompleted: true } };
}
return l;
})
);
});
});
}
// Optimistically remove from activeTasks
@@ -545,6 +585,7 @@ export class AgentService {
*/
public deleteSession(sessionId: string): Observable<any> {
// 1. Optimistically update local session state immediately
this.invalidateStatusSignatures();
this.rawSessions.update((list) => list.filter((s) => s.session_id !== sessionId));
this.persistSessionsCache(this.rawSessions());
this.pendingQueue.update((list) => list.filter((s) => s.session_id !== sessionId));
@@ -588,6 +629,7 @@ export class AgentService {
*/
public clearAllHistory(): Observable<any> {
// 1. Optimistically clear all local session state immediately
this.invalidateStatusSignatures();
this.userPinnedSessionId.set(null);
this.rawSessions.set([]);
this.clearSessionsCache();
@@ -637,6 +679,7 @@ export class AgentService {
if (!sessionId) {
this.sessionLoadGeneration++;
this.pendingSnapshotRequests.clear();
this.discardPendingStreamChunks();
this.isSessionContentLoading.set(false);
this.currentSessionId.set(null);
this.sessionLogs.set([]);
@@ -650,6 +693,7 @@ export class AgentService {
const loadGeneration = ++this.sessionLoadGeneration;
this.pendingSnapshotRequests.clear();
this.discardPendingStreamChunks();
this.sessionSnapshotAppliedId = 0;
this.isSessionContentLoading.set(true);
this.currentSessionId.set(sessionId);
@@ -681,6 +725,10 @@ export class AgentService {
}
console.log('Establishing persistent unified live stream via /api/stream');
// The stream is registered outside the Angular zone: high-frequency SSE
// callbacks must not schedule a change-detection pass each. Signal writes
// still notify the render scheduler, so the UI stays live.
this.zone.runOutsideAngular(() => {
this.eventSource = new EventSource('/api/stream');
this.eventSource.addEventListener('info', () => {
@@ -717,6 +765,12 @@ export class AgentService {
const parsedData = JSON.parse(event.data);
const evtSessionId = parsedData?.session_id;
// Preserve ordering: buffered stream text must land before any
// non-stream event is appended to the session logs.
if (eventType !== 'llm_stream') {
this.flushStreamChunks();
}
if (eventType === 'recording_ready' || eventType === 'recording_failed') {
if (
this.isVideoWindowOpen()
@@ -765,6 +819,7 @@ export class AgentService {
const endedId = evtSessionId || this.runningSessionId();
if (endedId) {
this.applySessionEndedStatus(endedId, parsedData);
this.invalidateStatusSignatures();
this.activeTasks.update(list => list.filter(at => at.session_id !== endedId));
}
const remaining = this.activeTasks();
@@ -874,62 +929,15 @@ export class AgentService {
}
if (eventType === 'llm_stream') {
const execId = parsedData.execution_id;
const stepId = parsedData.step_id;
const chunk = parsedData.chunk;
const streamType = parsedData.stream_type || 'text';
this.sessionLogs.update((logs) => {
const existingIndex = logs.findIndex(
(l) => l.type === 'llm_stream' &&
l.data.execution_id === execId &&
(l.data.stream_type || 'text') === streamType
);
if (existingIndex > -1) {
const updatedLogs = [...logs];
const existingLog = updatedLogs[existingIndex];
updatedLogs[existingIndex] = {
...existingLog,
data: {
...existingLog.data,
text: existingLog.data.text + chunk,
step_id: stepId || existingLog.data.step_id
}
};
return updatedLogs;
} else {
const updatedLogs = logs.map((l) => {
if (l.type === 'llm_stream' &&
(l.data.stream_type || 'text') === streamType &&
!l.data.isCompleted) {
return {
...l,
data: { ...l.data, isCompleted: true }
};
}
return l;
});
return [
...updatedLogs,
{
type: 'llm_stream',
timestamp: new Date().toISOString(),
data: {
execution_id: execId,
step_id: stepId,
text: chunk,
stream_type: streamType,
isCompleted: false
}
}
];
}
});
this.queueStreamChunk(parsedData);
} else {
this.sessionLogs.update((logs) => {
const updatedLogs = logs.map((l) => {
// Copying every log object on every event is wasted work when no
// stream is open; only rewrite entries that actually change.
const hasOpenStream = logs.some(
(l) => l.type === 'llm_stream' && !l.data.isCompleted
);
const updatedLogs = !hasOpenStream ? [...logs] : logs.map((l) => {
if (l.type === 'llm_stream' && !l.data.isCompleted) {
return {
...l,
@@ -991,6 +999,92 @@ export class AgentService {
this.eventSource.onerror = (err) => {
console.warn('Persistent live stream issue, browser will auto-reconnect:', err);
};
});
}
/**
* Buffer one llm_stream chunk; the buffer is flushed into sessionLogs at most
* once per short window (and immediately before any non-stream event so the
* relative ordering of stream text and step events is preserved).
*/
private queueStreamChunk(parsedData: any): void {
const execId = parsedData.execution_id;
const stepId = parsedData.step_id;
const streamType = parsedData.stream_type || 'text';
const key = `${execId}|${streamType}`;
const pending = this.pendingStreamChunks.get(key);
if (pending) {
pending.chunk += parsedData.chunk;
if (stepId) pending.stepId = stepId;
} else {
this.pendingStreamChunks.set(key, { execId, stepId, streamType, chunk: parsedData.chunk });
}
if (!this.streamFlushTimer) {
const delay = typeof document !== 'undefined' && document.hidden ? 500 : 80;
this.streamFlushTimer = setTimeout(() => this.flushStreamChunks(), delay);
}
}
private discardPendingStreamChunks(): void {
this.pendingStreamChunks?.clear();
if (this.streamFlushTimer) {
clearTimeout(this.streamFlushTimer);
this.streamFlushTimer = null;
}
}
private flushStreamChunks(): void {
if (this.streamFlushTimer) {
clearTimeout(this.streamFlushTimer);
this.streamFlushTimer = null;
}
if (!this.pendingStreamChunks?.size) return;
const batches = Array.from(this.pendingStreamChunks.values());
this.pendingStreamChunks.clear();
this.sessionLogs.update((logs) => {
let next = [...logs];
for (const batch of batches) {
const existingIndex = next.findIndex(
(l) => l.type === 'llm_stream' &&
l.data.execution_id === batch.execId &&
(l.data.stream_type || 'text') === batch.streamType
);
if (existingIndex > -1) {
const existingLog = next[existingIndex];
next[existingIndex] = {
...existingLog,
data: {
...existingLog.data,
text: existingLog.data.text + batch.chunk,
step_id: batch.stepId || existingLog.data.step_id
}
};
} else {
next = next.map((l) => {
if (l.type === 'llm_stream' &&
(l.data.stream_type || 'text') === batch.streamType &&
!l.data.isCompleted) {
return { ...l, data: { ...l.data, isCompleted: true } };
}
return l;
});
next.push({
type: 'llm_stream',
timestamp: new Date().toISOString(),
data: {
execution_id: batch.execId,
step_id: batch.stepId,
text: batch.chunk,
stream_type: batch.streamType,
isCompleted: false
}
});
}
}
return next;
});
}
/**
@@ -1065,6 +1159,7 @@ export class AgentService {
if (!cached) return;
const sessions = JSON.parse(cached);
if (Array.isArray(sessions)) {
this.lastPersistedSessionsJson = cached;
this.rawSessions.set(sessions);
}
} catch {
@@ -1074,13 +1169,17 @@ export class AgentService {
private persistSessionsCache(sessions: Session[]): void {
try {
localStorage.setItem(SESSION_CACHE_KEY, JSON.stringify(sessions));
const serialized = JSON.stringify(sessions);
if (serialized === this.lastPersistedSessionsJson) return;
this.lastPersistedSessionsJson = serialized;
localStorage.setItem(SESSION_CACHE_KEY, serialized);
} catch {
// Storage can be unavailable in private browsing or embedded contexts.
}
}
private clearSessionsCache(): void {
this.lastPersistedSessionsJson = null;
try {
localStorage.removeItem(SESSION_CACHE_KEY);
} catch {
@@ -1217,6 +1316,7 @@ export class AgentService {
private setSessionStatus(sessionId: string | null, status: Session['status']): void {
if (!sessionId) return;
this.invalidateStatusSignatures();
this.rawSessions.update((sessions) => sessions.map((session) =>
session.session_id === sessionId ? { ...session, status } : session
));
@@ -1240,6 +1340,7 @@ export class AgentService {
if (status) {
this.activeSessionTracking.delete(String(sessionId));
this.setSessionStatus(String(sessionId), status);
this.invalidateStatusSignatures();
this.activeTasks.update((list) => list.filter((at) => at.session_id !== String(sessionId)));
}
}
@@ -1249,14 +1350,30 @@ export class AgentService {
*/
private startStatusPolling(): void {
this.fetchStatus();
this.statusInterval = setInterval(() => {
this.fetchStatus();
this.pollCounter++;
// Periodically refresh sessions every 6 seconds (every 3 polling intervals of 2s) to stay in sync with external DB modifications
if (this.pollCounter % 3 === 0) {
this.fetchSessions();
}
}, 2000);
// The interval runs outside the Angular zone so a poll tick alone never
// schedules change detection; only actual signal updates do.
this.zone.runOutsideAngular(() => {
this.statusInterval = setInterval(() => {
if (typeof document !== 'undefined' && document.hidden) {
return; // Paused while the tab is in the background; resumed by visibilitychange.
}
this.fetchStatus();
this.pollCounter++;
// Periodically refresh sessions every 6 seconds (every 3 polling intervals of 2s) to stay in sync with external DB modifications
if (this.pollCounter % 3 === 0) {
this.fetchSessions();
}
}, 2000);
});
}
/**
* Optimistic local mutations of queue/active state must force the next poll
* to re-apply backend data even when its payload has not changed.
*/
private invalidateStatusSignatures(): void {
this.lastQueueSignature = null;
this.lastActiveTasksSignature = null;
}
/**
@@ -1291,26 +1408,37 @@ export class AgentService {
this.activePauseCardKey = null;
}
// Map backend pending queue to real Session / TaskQueueItem objects
const pending: Session[] = (data.queue || []).map((item: any, index: number) => {
if (typeof item === 'object' && item !== null) {
// Skip signal updates when backend payloads did not change: setting a
// fresh array reference every 2s poll would force the whole sessions
// computed chain to re-derive for identical data.
const queueSignature = JSON.stringify(data.queue || []);
if (queueSignature !== this.lastQueueSignature) {
this.lastQueueSignature = queueSignature;
// Map backend pending queue to real Session / TaskQueueItem objects
const pending: Session[] = (data.queue || []).map((item: any, index: number) => {
if (typeof item === 'object' && item !== null) {
return {
session_id: item.session_id || `pending-task-${index}`,
initial_goal: item.goal || '',
start_time: item.start_time || item.created_at || (Date.now() / 1000 + index),
status: item.status || 'pending',
device_serial: item.device_serial || item.device_id || null
};
}
return {
session_id: item.session_id || `pending-task-${index}`,
initial_goal: item.goal || '',
start_time: item.start_time || item.created_at || (Date.now() / 1000 + index),
status: item.status || 'pending',
device_serial: item.device_serial || item.device_id || null
session_id: `task-queued-${index}`,
initial_goal: String(item),
start_time: (Date.now() / 1000) + index,
status: 'pending'
};
}
return {
session_id: `task-queued-${index}`,
initial_goal: String(item),
start_time: (Date.now() / 1000) + index,
status: 'pending'
};
});
this.pendingQueue.set(pending);
this.activeTasks.set(data.active_tasks || []);
});
this.pendingQueue.set(pending);
}
const activeTasksSignature = JSON.stringify(data.active_tasks || []);
if (activeTasksSignature !== this.lastActiveTasksSignature) {
this.lastActiveTasksSignature = activeTasksSignature;
this.activeTasks.set(data.active_tasks || []);
}
if (oldStatus !== data.status || oldRunningSessionId !== data.session_id) {
this.fetchSessions();
@@ -1367,9 +1495,15 @@ export class AgentService {
public destroy(): void {
if (this.statusInterval) {
clearInterval(this.statusInterval);
this.statusInterval = null;
}
if (this.eventSource) {
this.eventSource.close();
this.eventSource = null;
}
this.discardPendingStreamChunks();
if (typeof document !== 'undefined') {
document.removeEventListener('visibilitychange', this.onVisibilityChange);
}
this.cancelVideoRetry();
}
@@ -1,4 +1,4 @@
import { provideHttpClient } from '@angular/common/http';
import { provideHttpClient, withXhr } from '@angular/common/http';
import { HttpTestingController, provideHttpClientTesting } from '@angular/common/http/testing';
import { TestBed } from '@angular/core/testing';
@@ -23,7 +23,7 @@ describe('SystemService readiness polling', () => {
TestBed.configureTestingModule({
providers: [
SystemService,
provideHttpClient(),
provideHttpClient(withXhr()),
provideHttpClientTesting()
]
});
@@ -69,4 +69,105 @@ describe('SystemService readiness polling', () => {
expect(request.request.params.get('force')).toBe('true');
request.flush(report(30));
});
it('loads the active ADB server endpoint', () => {
service.fetchAdbServerStatus().subscribe();
const request = http.expectOne('/api/system/adb/server');
request.flush({
endpoint: {
host: '127.0.0.1',
port: 5038,
socket: 'tcp:127.0.0.1:5038',
mode: 'remote',
is_local_default: false
}
});
expect(service.isRemoteAdbServer()).toBeTrue();
expect(service.adbServerStatus()?.endpoint.port).toBe(5038);
});
it('activates an ADB server endpoint and applies its readiness report', () => {
service.connectAdbServer('127.0.0.1', 5038, true).subscribe();
const request = http.expectOne('/api/system/adb/server/connect');
expect(request.request.body).toEqual({
host: '127.0.0.1',
port: 5038,
persist: true
});
request.flush({
connection_result: {
success: true,
message: 'Connected.',
endpoint: {
host: '127.0.0.1',
port: 5038,
socket: 'tcp:127.0.0.1:5038',
mode: 'remote',
is_local_default: false
},
devices: []
},
report: report(40)
});
expect(service.isRemoteAdbServer()).toBeTrue();
expect(service.readinessReport()?.timestamp).toBe(40);
});
it('probes an ADB server without changing the active endpoint', () => {
service.probeAdbServer('127.0.0.1', 5038).subscribe();
const request = http.expectOne('/api/system/adb/server/probe');
expect(request.request.body).toEqual({
host: '127.0.0.1',
port: 5038,
persist: false
});
request.flush({
connection_result: {
success: true,
message: 'Endpoint is reachable.',
endpoint: {
host: '127.0.0.1',
port: 5038,
socket: 'tcp:127.0.0.1:5038',
identity: 'tcp:127.0.0.1:5038',
mode: 'remote',
is_local_default: false
},
devices: []
}
});
expect(service.adbServerStatus()).toBeNull();
});
it('restores the standard local ADB server explicitly', () => {
service.useLocalAdbServer(true).subscribe();
const request = http.expectOne(req =>
req.url === '/api/system/adb/server/local' && req.params.get('persist') === 'true'
);
request.flush({
connection_result: {
success: true,
message: 'Using local ADB.',
endpoint: {
host: '127.0.0.1',
port: 5037,
socket: 'tcp:127.0.0.1:5037',
mode: 'local',
is_local_default: true
},
devices: []
},
report: report(50)
});
expect(service.isRemoteAdbServer()).toBeFalse();
expect(service.adbServerStatus()?.endpoint.port).toBe(5037);
});
});
@@ -14,10 +14,18 @@
* limitations under the License.
*/
import { Injectable, signal, computed, inject } from '@angular/core';
import { Injectable, signal, computed, inject, DestroyRef, NgZone } from '@angular/core';
import { HttpClient } from '@angular/common/http';
import { Observable, finalize, shareReplay, tap } from 'rxjs';
import { SystemReadinessReport, DeviceInfo, ProbeResult, EmulatorLaunchState, EmulatorLaunchStage } from '../core/models/system.model';
import {
AdbServerConnectionResponse,
AdbServerStatus,
SystemReadinessReport,
DeviceInfo,
ProbeResult,
EmulatorLaunchState,
EmulatorLaunchStage
} from '../core/models/system.model';
@Injectable({
providedIn: 'root'
@@ -30,6 +38,10 @@ export class SystemService {
public hasReadinessReport = computed(() => this.readinessReport() !== null);
public isLoading = signal<boolean>(false);
public isRestartingAdb = signal<boolean>(false);
public adbServerStatus = signal<AdbServerStatus | null>(null);
public isRemoteAdbServer = computed(
() => this.adbServerStatus()?.endpoint.mode === 'remote'
);
public launchingAvd = signal<string | null>(null);
public emulatorLaunchState = signal<EmulatorLaunchState | null>(null);
public isEmulatorLaunching = computed(() => {
@@ -138,24 +150,49 @@ export class SystemService {
private pollingTimer: any = null;
private readinessRequest$: Observable<SystemReadinessReport> | null = null;
private zone = inject(NgZone);
private lastAppliedReportJson: string | null = null;
private onVisibilityChange = () => {
if (typeof document !== 'undefined' && !document.hidden) {
this.fetchReadiness(true).subscribe({ error: () => {} });
}
};
constructor() {
this.startAutoPolling(3000);
if (typeof document !== 'undefined') {
document.addEventListener('visibilitychange', this.onVisibilityChange);
}
inject(DestroyRef).onDestroy(() => {
this.stopAutoPolling();
this.stopEmulatorStatusPolling();
if (typeof document !== 'undefined') {
document.removeEventListener('visibilitychange', this.onVisibilityChange);
}
});
}
/**
* Start periodic auto-polling for system readiness changes
* Start periodic auto-polling for system readiness changes. The interval
* runs outside the Angular zone (a tick alone must not trigger change
* detection) and pauses while the tab is hidden; a visibilitychange listener
* refreshes immediately when the tab becomes visible again.
*/
public startAutoPolling(intervalMs: number = 3000): void {
if (this.pollingTimer) {
clearInterval(this.pollingTimer);
}
this.pollingTimer = setInterval(() => {
// Perform silent background check without disturbing UI loading state
this.fetchReadiness(true).subscribe({
error: () => {} // Silent catch
});
}, intervalMs);
this.zone.runOutsideAngular(() => {
this.pollingTimer = setInterval(() => {
if (typeof document !== 'undefined' && document.hidden) {
return;
}
// Perform silent background check without disturbing UI loading state
this.fetchReadiness(true).subscribe({
error: () => {} // Silent catch
});
}, intervalMs);
});
}
/**
@@ -214,8 +251,15 @@ export class SystemService {
if (current && report.timestamp < current.timestamp) {
return;
}
this.readinessReport.set(report);
this.lastCheckedTime.set(new Date(report.timestamp * 1000));
// A poll usually returns an identical report; setting a new object anyway
// would cascade through every probe computed for no visible change.
const serialized = JSON.stringify({ ...report, timestamp: 0 });
if (serialized === this.lastAppliedReportJson) {
return;
}
this.lastAppliedReportJson = serialized;
this.readinessReport.set(report);
}
/**
@@ -388,6 +432,69 @@ export class SystemService {
);
}
/**
* Fetch the process-wide ADB server endpoint used by Artemis.
*/
public fetchAdbServerStatus(): Observable<AdbServerStatus> {
return this.http.get<AdbServerStatus>('/api/system/adb/server').pipe(
tap(status => this.adbServerStatus.set(status))
);
}
/**
* Test an ADB server endpoint without changing the active connection.
*/
public probeAdbServer(
host: string,
port: number
): Observable<AdbServerConnectionResponse> {
return this.http.post<AdbServerConnectionResponse>(
'/api/system/adb/server/probe',
{ host, port, persist: false }
);
}
/**
* Validate and activate an ADB server endpoint.
*/
public connectAdbServer(
host: string,
port: number,
persist: boolean = true
): Observable<AdbServerConnectionResponse> {
return this.http.post<AdbServerConnectionResponse>(
'/api/system/adb/server/connect',
{ host, port, persist }
).pipe(
tap(response => {
if (response.connection_result.success) {
this.adbServerStatus.set({ endpoint: response.connection_result.endpoint });
if (response.report) {
this.applyReadinessReport(response.report);
}
}
})
);
}
/**
* Switch Artemis back to the standard local ADB server.
*/
public useLocalAdbServer(persist: boolean = true): Observable<AdbServerConnectionResponse> {
return this.http.post<AdbServerConnectionResponse>(
'/api/system/adb/server/local',
{},
{ params: { persist } }
).pipe(
tap(response => {
this.adbServerStatus.set({ endpoint: response.connection_result.endpoint });
if (response.report) {
this.applyReadinessReport(response.report);
}
})
);
}
/**
* Select a specific connected device serial as active target
*/
@@ -283,6 +283,39 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
return blocks;
}
/**
* Cache for the token-estimate fallback. Tool payloads can carry multi-megabyte
* base64 screenshots; serializing them on every recompute would block the main
* thread, so each payload object is measured exactly once.
*/
const payloadSizeCache = new WeakMap<object, { isImage: boolean; length: number }>();
function estimatePayloadSize(payload: any): { isImage: boolean; length: number } {
if (typeof payload === 'string') {
return {
isImage: payload.includes('data:image/') || payload.includes('base64,'),
length: payload.length
};
}
if (payload === null || typeof payload !== 'object') {
return { isImage: false, length: String(payload).length };
}
const cached = payloadSizeCache.get(payload);
if (cached) return cached;
let payloadStr = '';
try {
payloadStr = JSON.stringify(payload) || '';
} catch {
payloadStr = String(payload);
}
const result = {
isImage: payloadStr.includes('data:image/') || payloadStr.includes('base64,'),
length: payloadStr.length
};
payloadSizeCache.set(payload, result);
return result;
}
/**
* Safely extract and aggregate token usage for a step block
*/
@@ -345,11 +378,11 @@ export function extractBlockTokens(block: StepBlock): { total: number; prompt: n
if (data.generic_tools && Array.isArray(data.generic_tools)) {
data.generic_tools.forEach((t: any) => {
if (t.payload) {
const payloadStr = typeof t.payload === 'string' ? t.payload : JSON.stringify(t.payload);
if (payloadStr.includes('data:image/') || payloadStr.includes('base64,')) {
const estimate = estimatePayloadSize(t.payload);
if (estimate.isImage) {
numImages += 1;
} else {
textLen += payloadStr.length;
textLen += estimate.length;
}
}
});
@@ -18,11 +18,27 @@ import { ActionParam } from '../core/models/stream.model';
import { isAndroidAction, isReportStatusAction } from './action-formatter.util';
import { extractNumbersFromCoordinateValue, isPureDirectionString, parseSequenceCoordinates } from './image-overlay.util';
// Tool objects are replaced (not mutated) when a trace is updated, so a
// WeakMap keyed on the tool is a safe memo for the parsed-args lookup that
// templates would otherwise re-run (including JSON.parse) on every render.
const toolArgsCache = new WeakMap<object, any>();
/**
* Extract arguments from tool payload or direct args field
*/
export function getToolArgs(tool: any): any {
if (!tool) return {};
if (typeof tool === 'object') {
const cached = toolArgsCache.get(tool);
if (cached !== undefined) return cached;
const args = computeToolArgs(tool);
toolArgsCache.set(tool, args);
return args;
}
return computeToolArgs(tool);
}
function computeToolArgs(tool: any): any {
if (tool.payload) {
let payloadObj = tool.payload;
if (typeof payloadObj === 'string') {
+9 -1
View File
@@ -11,5 +11,13 @@
],
"include": [
"src/**/*.d.ts"
]
],
"angularCompilerOptions": {
"extendedDiagnostics": {
"checks": {
"nullishCoalescingNotNullable": "suppress",
"optionalChainNotNullable": "suppress"
}
}
}
}
+9 -1
View File
@@ -11,5 +11,13 @@
"include": [
"src/**/*.spec.ts",
"src/**/*.d.ts"
]
],
"angularCompilerOptions": {
"extendedDiagnostics": {
"checks": {
"nullishCoalescingNotNullable": "suppress",
"optionalChainNotNullable": "suppress"
}
}
}
}
+5 -1
View File
@@ -17,7 +17,8 @@
__version__ = "0.2.0"
from artemis.interfaces.sdk.client import ArtemisClient, ConcurrencyMode
from artemis.interfaces.sdk.task import StreamEvent, StreamEventType, Task, TaskResult
from artemis.interfaces.sdk.task import StreamEvent, StreamEventType, Task
from artemis_client import Capabilities, Device, TaskHandle, TaskResult
from artemis.sdk.agent import Agent
from artemis.sdk.builders import Builders
from artemis.agents.base import BaseAgent, AgentRegistry
@@ -27,12 +28,15 @@ from artemis.tools.base import artemis_tool, ToolRegistry
__all__ = [
"__version__",
"ArtemisClient",
"Capabilities",
"ConcurrencyMode",
"Device",
"StreamEvent",
"StreamEventType",
"Agent",
"Builders",
"Task",
"TaskHandle",
"TaskResult",
"BaseAgent",
"AgentRegistry",
@@ -0,0 +1,330 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Process-wide ADB server endpoint discovery and activation."""
from __future__ import annotations
import asyncio
import os
from pathlib import Path
import re
import subprocess
from typing import Any, Callable
from dotenv import set_key
from artemis.config import settings
from artemis.config.paths import ROOT_DIR, get_app_dir
from artemis.runtime.adb_endpoint import (
AdbEndpoint,
AdbSession,
InvalidAdbEndpoint,
current_adb_endpoint,
)
from artemis.toolchain import toolchain
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
AdbServerEndpoint = AdbEndpoint
InvalidAdbServerEndpoint = InvalidAdbEndpoint
class AdbServerConnectionManager:
"""Validate, probe, activate, and persist the selected ADB server endpoint."""
def __init__(
self,
*,
adb_resolver: Callable[[], str | None] | None = None,
env_files: list[Path] | None = None,
) -> None:
self._adb_resolver = adb_resolver or (lambda: toolchain.resolve("adb"))
self._env_files = env_files
self._activation_lock = asyncio.Lock()
@staticmethod
def validate_endpoint(host: str, port: int) -> AdbServerEndpoint:
return AdbEndpoint.create(host, port)
def current_endpoint(self) -> AdbServerEndpoint:
return current_adb_endpoint()
def status(self) -> dict[str, Any]:
return {"endpoint": self.current_endpoint().to_dict()}
def synchronize_environment(self) -> AdbServerEndpoint:
"""Align subprocess and Python ADB clients with the configured endpoint."""
socket_endpoint = self._endpoint_from_socket(os.environ.get("ADB_SERVER_SOCKET"))
has_explicit_host = bool(os.environ.get("ADB_HOST") or os.environ.get("ADB_PORT"))
endpoint = (
socket_endpoint
if socket_endpoint is not None and not has_explicit_host
else self.current_endpoint()
)
self._activate(endpoint)
return endpoint
async def connect(
self,
host: str,
port: int,
*,
persist: bool = True,
) -> dict[str, Any]:
"""Probe an endpoint, then make it the process-wide ADB server on success."""
endpoint = self.validate_endpoint(host, port)
probe_result = await self.probe(host, port)
if not probe_result["success"]:
return probe_result
persisted = False
persistence_error: str | None = None
async with self._activation_lock:
self._activate(endpoint)
if persist:
try:
await asyncio.to_thread(self._persist, endpoint)
persisted = True
except Exception as exc:
persistence_error = str(exc)
logger.warning(f"Unable to persist ADB server endpoint: {exc}")
message = probe_result["message"]
if persist and not persisted:
message = (
f"{message} The connection is active for this session, but Artemis could not "
"save it for the next launch."
)
return {
**probe_result,
"endpoint": endpoint.to_dict(),
"message": message,
"persisted": persisted,
"persistence_error": persistence_error,
}
async def probe(self, host: str, port: int) -> dict[str, Any]:
"""Test an endpoint without changing the process preference."""
endpoint = self.validate_endpoint(host, port)
return await asyncio.to_thread(self._probe_sync, endpoint)
async def use_local_server(self, *, persist: bool = True) -> dict[str, Any]:
"""Restore the standard local ADB server without killing any remote daemon."""
endpoint = AdbEndpoint.local()
persisted = False
persistence_error: str | None = None
async with self._activation_lock:
self._activate(endpoint)
if persist:
try:
await asyncio.to_thread(self._persist, endpoint)
persisted = True
except Exception as exc:
persistence_error = str(exc)
logger.warning(f"Unable to persist local ADB server endpoint: {exc}")
message = "Artemis is now using the local ADB server."
if persist and not persisted:
message += " The setting could not be saved for the next launch."
return {
"success": True,
"endpoint": endpoint.to_dict(),
"persisted": persisted,
"persistence_error": persistence_error,
"message": message,
}
def _probe_sync(self, endpoint: AdbServerEndpoint) -> dict[str, Any]:
adb_path = self._adb_resolver()
if not adb_path:
return {
"success": False,
"error_code": "adb_not_found",
"message": "Android Platform Tools (adb) is not installed or not on PATH.",
"endpoint": endpoint.to_dict(),
"devices": [],
}
session = AdbSession(endpoint, adb_path=adb_path)
command = session.command(["devices", "-l"])
try:
result = subprocess.run(
command,
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
timeout=8,
env=self._probe_environment(),
)
except subprocess.TimeoutExpired:
return {
"success": False,
"error_code": "timeout",
"message": (
"The ADB server did not respond. Check that the host and port are correct "
"and accessible from this computer."
),
"endpoint": endpoint.to_dict(),
"devices": [],
}
except OSError as exc:
logger.warning(f"Failed to execute ADB endpoint probe: {exc}")
return {
"success": False,
"error_code": "adb_execution_failed",
"message": f"Unable to run adb: {exc}",
"endpoint": endpoint.to_dict(),
"devices": [],
}
stdout = result.stdout or ""
stderr = result.stderr or ""
output = "\n".join(part.strip() for part in (stdout, stderr) if part.strip())
devices = self._parse_devices(stdout)
if result.returncode != 0:
return {
"success": False,
"error_code": "server_unreachable",
"message": self._connection_error_message(output),
"output": output,
"endpoint": endpoint.to_dict(),
"devices": [],
}
ready_devices = [device for device in devices if device["state"] == "device"]
if ready_devices:
message = (
f"Connected to {endpoint.host}:{endpoint.port} and found "
f"{len(ready_devices)} ready device{'s' if len(ready_devices) != 1 else ''}."
)
elif devices:
message = (
"The ADB server is reachable, but its devices require attention. "
"Unlock the phone and approve USB debugging if prompted."
)
else:
message = (
"The ADB server is reachable, but no devices are attached to the remote computer."
)
return {
"success": True,
"message": message,
"output": output,
"endpoint": endpoint.to_dict(),
"devices": devices,
}
@staticmethod
def _parse_devices(output: str) -> list[dict[str, str | None]]:
devices: list[dict[str, str | None]] = []
for raw_line in output.splitlines():
line = raw_line.strip()
if not line or line.lower().startswith("list of devices") or line.startswith("*"):
continue
parts = line.split()
if len(parts) < 2:
continue
metadata = {
key: value.replace("_", " ")
for token in parts[2:]
if ":" in token
for key, value in [token.split(":", 1)]
}
devices.append(
{
"serial": parts[0],
"state": parts[1],
"model": metadata.get("model"),
"product": metadata.get("product"),
}
)
return devices
@staticmethod
def _connection_error_message(output: str) -> str:
lowered = output.lower()
if "cannot connect" in lowered or "failed to connect" in lowered:
return (
"No ADB server is listening at this address. Verify that the host and port are "
"correct and accessible from this computer."
)
if "version" in lowered and "mismatch" in lowered:
return (
"The local and remote ADB versions are incompatible. Use matching Android "
"Platform Tools versions on both computers."
)
return output or "Unable to reach the ADB server endpoint."
@staticmethod
def _probe_environment() -> dict[str, str]:
environment = os.environ.copy()
environment.pop("ADB_SERVER_SOCKET", None)
return environment
@classmethod
def _endpoint_from_socket(cls, value: str | None) -> AdbServerEndpoint | None:
if not value or not value.startswith("tcp:"):
return None
address = value.removeprefix("tcp:")
if address.startswith("["):
match = re.fullmatch(r"\[([^]]+)]:(\d+)", address)
else:
match = re.fullmatch(r"(.+):(\d+)", address)
if match is None:
return None
try:
return cls.validate_endpoint(match.group(1), int(match.group(2)))
except (InvalidAdbServerEndpoint, ValueError):
return None
@staticmethod
def _activate(endpoint: AdbServerEndpoint) -> None:
settings.ADB_HOST = endpoint.host
settings.ADB_PORT = endpoint.port
endpoint.apply_to_environment()
logger.info(
f"Activated {endpoint.mode} ADB server endpoint {endpoint.host}:{endpoint.port}"
)
def _persist(self, endpoint: AdbServerEndpoint) -> None:
for env_file in self._environment_files():
env_file.parent.mkdir(parents=True, exist_ok=True)
if not env_file.exists():
env_file.touch()
set_key(str(env_file), "ADB_HOST", endpoint.host, quote_mode="never")
set_key(str(env_file), "ADB_PORT", str(endpoint.port), quote_mode="never")
def _environment_files(self) -> list[Path]:
if self._env_files is not None:
candidates = self._env_files
else:
candidates = [ROOT_DIR / ".env", get_app_dir() / ".env"]
unique_files: list[Path] = []
seen: set[Path] = set()
for candidate in candidates:
resolved = candidate.resolve()
if resolved not in seen:
seen.add(resolved)
unique_files.append(candidate)
return unique_files
adb_server_connection = AdbServerConnectionManager()
adb_server_connection.synchronize_environment()
+36 -7
View File
@@ -15,11 +15,11 @@
"""System Readiness & Diagnostic Orchestration Engine."""
import asyncio
import shutil
import subprocess
import time
from typing import Any
from artemis.core.diagnostics.adb_server_connection import adb_server_connection
from artemis.core.diagnostics.probes.adb_probe import AdbDeviceProbe
from artemis.core.diagnostics.probes.base import BaseProbe
from artemis.core.diagnostics.probes.credentials_probe import (
@@ -38,8 +38,8 @@ from artemis.core.diagnostics.schema import (
ProbeStatus,
SystemReadinessReport,
)
from artemis.platform import platform
from artemis.toolchain import toolchain
from artemis.platform import platform
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
@@ -106,9 +106,7 @@ class ReadinessEngine:
return None
return await probe.probe()
async def run_device_submission_probe(
self, target_serial: str | None = None
) -> ProbeResult:
async def run_device_submission_probe(self, target_serial: str | None = None) -> ProbeResult:
"""Run the bounded device gate used by task submission."""
return await self._adb_probe.probe_submission_readiness(target_serial=target_serial)
@@ -226,7 +224,19 @@ class ReadinessEngine:
"""Execute adb kill-server && adb start-server to recover connectivity, auto-healing corrupted keys if needed."""
from artemis.core.diagnostics.adb_keys import heal_adb_keys, inspect_adb_keys
adb_path = shutil.which("adb") or "adb"
endpoint = adb_server_connection.current_endpoint()
if not endpoint.is_local_default:
return {
"success": True,
"skipped": True,
"message": (
"Remote ADB is active. Artemis refreshed device discovery without stopping "
"the active ADB server endpoint."
),
"endpoint": endpoint.to_dict(),
}
adb_path = toolchain.resolve("adb") or "adb"
def _restart_sync():
# If keys are corrupted, heal them first
@@ -269,6 +279,20 @@ class ReadinessEngine:
async def launch_emulator(self, avd_name: str) -> dict[str, Any]:
"""Launch an Android emulator by AVD name in the background and track its lifecycle."""
if not adb_server_connection.current_endpoint().is_local_default:
return {
"avd_name": avd_name,
"status": "failed",
"pid": None,
"serial": None,
"error": "Switch to local ADB before launching a local emulator.",
"stage_message": "Local ADB is not active",
"progress_percent": 0,
"started_at": None,
"elapsed_seconds": 0,
"logs": [],
"can_retry": True,
}
from artemis.core.diagnostics.emulator_manager import emulator_manager
state = await emulator_manager.launch(avd_name)
@@ -294,11 +318,16 @@ class ReadinessEngine:
async def connect_wireless_adb(self, host: str, port: int = 5555) -> dict[str, Any]:
"""Connect to an Android device over Wi-Fi via adb connect."""
if not adb_server_connection.current_endpoint().is_local_default:
return {
"success": False,
"message": "Switch to local ADB before connecting a Wireless ADB device.",
}
clean_host = host.strip()
if not clean_host:
return {"success": False, "message": "Host IP address cannot be empty"}
target = f"{clean_host}:{port}"
adb_path = shutil.which("adb") or "adb"
adb_path = toolchain.resolve("adb") or "adb"
def _connect_sync():
try:
+7 -2
View File
@@ -351,8 +351,13 @@ class AdbDeviceProbe(BaseProbe):
return states
except TimeoutError:
if proc is not None:
proc.kill()
await proc.communicate()
# The process may have exited between the timeout and the
# kill (frequent under concurrent submissions on Windows).
try:
proc.kill()
await proc.communicate()
except (ProcessLookupError, OSError):
pass
return None
except Exception as exc:
logger.debug(f"Failed submission-time ADB device check: {exc}")
+6
View File
@@ -302,6 +302,12 @@ class DataEngine:
if not self.current_session_id:
return
# Session-level terminal statuses are canonically "completed" /
# "failed" / "cancelled"; "success" is a legacy alias some callers
# still pass and must never reach the sessions table.
if status == "success":
status = "completed"
# Clear pause file if it exists
pause_file = PAUSE_FILE
if pause_file.exists():
+1
View File
@@ -16,6 +16,7 @@
import asyncio
import json
import os
from pathlib import Path
from typing import Annotated
+3 -3
View File
@@ -367,9 +367,9 @@ def run_command(
def on_status(sess_info):
st = sess_info.get("status")
if st == "queued":
console.print(f"[yellow]⏳ Task queued in scheduler, waiting for device...[/yellow]")
console.print("[yellow]⏳ Task queued in scheduler, waiting for device...[/yellow]")
elif st == "running":
console.print(f"[green]▶ Task executing on mobile device...[/green]")
console.print("[green]▶ Task executing on mobile device...[/green]")
try:
final_res = wait_for_daemon_task(
@@ -380,7 +380,7 @@ def run_command(
)
final_st = final_res.get("status")
if final_st in ("completed", "success"):
console.print(f"\n[bold green]✅ Task completed successfully![/bold green]")
console.print("\n[bold green]✅ Task completed successfully![/bold green]")
return
else:
err = final_res.get("error") or final_res.get("explanation") or ""
@@ -17,6 +17,7 @@
from __future__ import annotations
import datetime
import time
from typing import Annotated
import webbrowser
@@ -25,8 +26,8 @@ from rich.panel import Panel
from rich.table import Table
import typer
from artemis.interfaces.cli.commands.ui import ui_command
from artemis.runtime.daemon_client import spawn_daemon
from artemis.interfaces.cli.commands.ui import ensure_showcase_built, ui_command
from artemis.runtime.daemon_client import daemon_log_path, is_daemon_running, spawn_daemon
from artemis.runtime.server_lifecycle import (
find_server_pids,
get_server_status,
@@ -55,8 +56,15 @@ def _format_uptime(seconds: float | None) -> str:
def restart_command(
host: Annotated[
str,
typer.Option("--host", "-H", help="Bind host address for the UI server."),
] = "0.0.0.0",
typer.Option(
"--host",
"-H",
help=(
"Bind host address for the UI server. Defaults to loopback; for remote "
"access prefer a Tailscale/SSH tunnel over a wide bind."
),
),
] = "127.0.0.1",
port: Annotated[
int,
typer.Option("--port", "-p", help="Port to run the unified UI server on."),
@@ -86,14 +94,23 @@ def restart_command(
daemon: Annotated[
bool,
typer.Option(
"--daemon",
"-d",
help="Run the new server as a detached background process instead of foreground.",
"--daemon/--foreground",
"-d/-F",
help=(
"Run the new server as a detached background daemon (default) or attached "
"to the current terminal. The daemon is launched via 'python -m' so the "
"console-script executable is never pinned by a long-lived server "
"(which would block 'uv sync' reinstalls on Windows)."
),
),
] = False,
] = True,
) -> None:
"""Restart the Artemis Web UI server, terminating any existing instance on the port."""
console.print()
if reload and daemon:
# uvicorn auto-reload requires an attached terminal session.
console.print(" [dim]ℹ --reload requested: switching to foreground mode.[/dim]")
daemon = False
console.print(
Panel(
f"[bold cyan]🔄 Artemis Server Restart[/bold cyan]\n"
@@ -111,7 +128,7 @@ def restart_command(
else:
console.print(f" [dim]ℹ No active Artemis server found on port {port}.[/dim]")
success, msg, stopped_pids = stop_server(port=port, force=force, timeout=4.0)
success, msg, stopped_pids = stop_server(port=port, force=force, timeout=12.0)
if stopped_pids:
console.print(f" [green]✓ {msg}[/green]\n")
elif not success:
@@ -122,23 +139,54 @@ def restart_command(
# 2. Launch fresh server instance
local_url = f"http://localhost:{port}"
if daemon:
console.print(f" [cyan]🚀 Starting Artemis in detached background daemon mode...[/cyan]")
ensure_showcase_built(console)
console.print(" [cyan]🚀 Starting Artemis in detached background daemon mode...[/cyan]")
proc = spawn_daemon(host=host, port=port)
if proc:
if not proc:
console.print(" [red]❌ Failed to spawn background daemon.[/red]\n")
raise typer.Exit(code=1)
# Wait until the daemon answers HTTP before declaring success (cold
# start includes DB/trace initialization and can take tens of seconds).
ready = False
deadline = time.monotonic() + 60.0
while time.monotonic() < deadline:
if is_daemon_running(host="127.0.0.1", port=port, timeout=0.5):
ready = True
break
if proc.poll() is not None:
# Parent exited: either it crashed, or it handed the socket off
# to a child worker. Give the worker a short grace period.
grace = time.monotonic() + 5.0
while time.monotonic() < grace:
if is_daemon_running(host="127.0.0.1", port=port, timeout=0.5):
ready = True
break
time.sleep(0.3)
break
time.sleep(0.3)
log_path = daemon_log_path(port)
if ready:
console.print(
f" [bold green]✓ Artemis server running in background (PID: {proc.pid})[/bold green]\n"
f" 🌐 [cyan]{local_url}[/cyan]\n"
f" 🛠️ [cyan]{local_url}/admin[/cyan]\n"
f" 📄 [dim]Logs: {log_path}[/dim]\n"
f" [dim]💡 Stop with [bold]artemis stop[/bold]; run in-terminal with [bold]artemis restart --foreground[/bold].[/dim]\n"
)
if open_browser:
webbrowser.open(local_url)
else:
console.print(" [red]❌ Failed to spawn background daemon.[/red]\n")
console.print(
f" [red]❌ Background daemon did not become ready on port {port}.[/red]\n"
f" 📄 [dim]Check logs: {log_path}[/dim]\n"
)
raise typer.Exit(code=1)
return
# Default foreground mode
console.print(f" [cyan]🚀 Launching unified Artemis Showcase UI in current terminal...[/cyan]\n")
# Foreground mode (--foreground / --reload)
console.print(" [cyan]🚀 Launching unified Artemis Showcase UI in current terminal...[/cyan]\n")
ui_command(host=host, port=port, open_browser=open_browser, reload=reload)
@@ -165,7 +213,7 @@ def stop_command(
else:
console.print(f" [dim]Scanning port {port}...[/dim]")
success, msg, stopped_pids = stop_server(port=port, force=force, timeout=4.0)
success, msg, stopped_pids = stop_server(port=port, force=force, timeout=12.0)
if stopped_pids:
console.print(
Panel(
+59 -19
View File
@@ -29,6 +29,33 @@ from rich.panel import Panel
import typer
def running_via_console_script_shim() -> bool:
"""Return True when this CLI was launched through a console-script ``.exe`` shim.
On Windows, ``uv``/pip install entry points as trampoline executables
(``Scripts/artemis.exe``) that stay resident for the lifetime of the CLI
process and hold a lock on their own file. A long-running foreground server
started through the shim therefore blocks ``uv sync`` from reinstalling the
package ("os error 32") until the server exits. Launching via
``python -m artemis`` avoids the shim entirely.
"""
if sys.platform != "win32":
return False
argv0 = Path(sys.argv[0] or "")
return argv0.suffix.lower() == ".exe" and argv0.name.lower() != "python.exe"
def warn_if_console_script_shim(console: Console) -> None:
"""Print a hint when a foreground server run will pin the console-script exe."""
if running_via_console_script_shim():
console.print(
" [dim]⚠ Running in foreground via the [bold]artemis.exe[/bold] launcher: package "
"syncs (uv sync / uv run) cannot replace it while this server is running.\n"
" Prefer [bold]artemis restart[/bold] (background daemon) or launch with "
"[bold]uv run python -m artemis ui[/bold] to avoid this.[/dim]\n"
)
def _resolve_npm_executable(platform_name: str | None = None) -> str | None:
"""Return a directly executable npm path for the current platform."""
platform_name = platform_name or sys.platform
@@ -57,6 +84,25 @@ def _showcase_build_required(showcase_dir: Path) -> bool:
return any(path.exists() and path.stat().st_mtime > build_time for path in input_paths)
def ensure_showcase_built(console: Console) -> None:
"""Rebuild the Angular Showcase UI when its sources are newer than the dist build."""
from artemis.config.paths import ROOT_DIR
showcase_dir = ROOT_DIR / "apps" / "showcase_ui"
if not _showcase_build_required(showcase_dir):
return
npm_executable = _resolve_npm_executable()
if not npm_executable:
return
console.print(" [yellow]🎨 Showcase UI sources changed. Compiling Angular Showcase UI...[/yellow]")
try:
subprocess.run([npm_executable, "install", "--silent"], cwd=showcase_dir, check=True)
subprocess.run([npm_executable, "run", "build"], cwd=showcase_dir, check=True)
console.print(" [green]✓ Showcase UI built successfully.[/green]\n")
except Exception as e:
console.print(f" [red]⚠ Failed to auto-build Showcase UI: {e}[/red]\n")
def _poll_and_open_browser(url: str, stop_event: threading.Event, timeout: float = 15.0) -> None:
"""Poll the UI server until it responds, then launch the user's default browser."""
start = time.time()
@@ -75,8 +121,15 @@ def _poll_and_open_browser(url: str, stop_event: threading.Event, timeout: float
def ui_command(
host: Annotated[
str,
typer.Option("--host", "-h", help="Bind host address for the UI server."),
] = "0.0.0.0",
typer.Option(
"--host",
"-h",
help=(
"Bind host address for the UI server. Defaults to loopback; for remote "
"access prefer a Tailscale/SSH tunnel over a wide bind."
),
),
] = "127.0.0.1",
port: Annotated[
int,
typer.Option("--port", "-p", help="Port to run the unified UI server on."),
@@ -116,7 +169,7 @@ def ui_command(
if restart:
console.print(f" [yellow]🔄 Restart requested. Recycling port {port}...[/yellow]")
stop_server(port=port, timeout=4.0)
stop_server(port=port, timeout=12.0)
elif is_port_in_use(port):
pids = find_server_pids(port)
pid_str = f" (PID: {', '.join(map(str, pids))})" if pids else ""
@@ -132,7 +185,7 @@ def ui_command(
choice = "1"
if choice == "2":
console.print(f"\n [yellow]🔄 Terminating previous Artemis instance on port {port}...[/yellow]")
stop_server(port=port, timeout=4.0)
stop_server(port=port, timeout=12.0)
elif choice == "3":
return
else:
@@ -178,21 +231,8 @@ def ui_command(
except Exception:
pass
showcase_dir = ROOT_DIR / "apps" / "showcase_ui"
if _showcase_build_required(showcase_dir):
npm_executable = _resolve_npm_executable()
if npm_executable:
console.print(
" [yellow]🎨 Showcase UI sources changed. Compiling Angular Showcase UI...[/yellow]"
)
try:
subprocess.run(
[npm_executable, "install", "--silent"], cwd=showcase_dir, check=True
)
subprocess.run([npm_executable, "run", "build"], cwd=showcase_dir, check=True)
console.print(" [green]✓ Showcase UI built successfully.[/green]\n")
except Exception as e:
console.print(f" [red]⚠ Failed to auto-build Showcase UI: {e}[/red]\n")
ensure_showcase_built(console)
warn_if_console_script_shim(console)
stop_event = threading.Event()
if open_browser:
+5 -1
View File
@@ -15,13 +15,17 @@
"""Python SDK for ARTEMIS."""
from artemis.interfaces.sdk.client import ArtemisClient, ConcurrencyMode
from artemis.interfaces.sdk.task import StreamEvent, StreamEventType, Task, TaskResult
from artemis.interfaces.sdk.task import StreamEvent, StreamEventType, Task
from artemis_client import Capabilities, Device, TaskHandle, TaskResult
__all__ = [
"ArtemisClient",
"Capabilities",
"ConcurrencyMode",
"Device",
"StreamEvent",
"StreamEventType",
"Task",
"TaskHandle",
"TaskResult",
]
+8 -433
View File
@@ -5,440 +5,15 @@
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import os
import uuid
from typing import Any, Literal
from collections.abc import AsyncGenerator, Callable
from artemis.engine.pipeline import Pipeline
from artemis.interfaces.sdk.task import StreamEvent, StreamEventType, Task, TaskResult
from artemis.runtime import (
ConcurrencyMode,
DeviceBusyError,
DeviceExecutionLock,
DeviceStatus,
device_pool,
ensure_daemon_running,
submit_task_to_daemon,
wait_for_daemon_task,
)
from artemis.utils.logger import get_logger
"""Compatibility import for the standalone remote Artemis client.
logger = get_logger(__name__)
The implementation lives in the zero-dependency ``artemis-client`` package.
Keeping this module lets existing full-runtime installations migrate without
maintaining a second client execution path.
"""
from artemis.runtime import ConcurrencyMode
from artemis_client import ArtemisClient
class ArtemisClient:
"""The official high-level developer entrypoint for interacting with ARTEMIS."""
def __init__(
self,
device_id: str | None = None,
device_serial: str | None = None,
default_profile: Literal["flash", "pro"] = "pro",
concurrency_mode: Literal["global", "per_device"] | str = "per_device",
max_concurrency: int | None = None,
standalone: bool = False,
):
self.standalone = standalone or os.environ.get("ARTEMIS_STANDALONE", "").lower() in ("1", "true", "yes")
if not self.standalone:
try:
ensure_daemon_running(timeout=1.5)
except Exception:
pass
target_dev = device_serial or device_id
if target_dev is None or target_dev == "default-device":
self._device_id = device_pool.select_device() or "default-device"
else:
self._device_id = target_dev
self.default_profile = default_profile
self._concurrency_mode = str(concurrency_mode).strip().lower()
self._max_concurrency = max_concurrency
@property
def device_id(self) -> str:
return self._device_id
@device_id.setter
def device_id(self, val: str) -> None:
self._device_id = val
@property
def device_serial(self) -> str:
"""Alias for device_id."""
return self._device_id
@device_serial.setter
def device_serial(self, val: str) -> None:
self._device_id = val
def set_device(self, device_serial: str) -> "ArtemisClient":
"""Set target device serial and return client for chaining."""
self._device_id = device_serial
return self
@property
def concurrency_mode(self) -> str:
"""Concurrency mode: 'global' (1 across all devices) or 'per_device' (1 per device)."""
return self._concurrency_mode
@concurrency_mode.setter
def concurrency_mode(self, mode: Literal["global", "per_device"] | str) -> None:
self._concurrency_mode = str(mode).strip().lower()
@property
def max_concurrency(self) -> int | None:
return self._max_concurrency
@max_concurrency.setter
def max_concurrency(self, val: int | None) -> None:
self._max_concurrency = val
def set_concurrency_mode(self, mode: Literal["global", "per_device"] | str) -> "ArtemisClient":
"""Set concurrency mode ('global' or 'per_device') and return client for chaining."""
self.concurrency_mode = mode
return self
def list_devices(self) -> list[DeviceStatus]:
"""List all connected Android devices with their lock state and availability."""
return device_pool.list_devices()
async def list_devices_async(self) -> list[DeviceStatus]:
"""Asynchronously list all connected Android devices with their lock state."""
return await device_pool.list_devices_async()
def get_idle_devices(self) -> list[DeviceStatus]:
"""Return connected devices currently ready and idle (not holding a task lock)."""
return device_pool.get_idle_devices()
async def run(
self,
goal: str,
profile: Literal["flash", "pro"] | None = None,
device_id: str | None = None,
device_serial: str | None = None,
concurrency_mode: Literal["global", "per_device"] | str | None = None,
max_concurrency: int | None = None,
blocking: bool = True,
lock_timeout: float | None = None,
max_turns: int = 30,
**kwargs: Any,
) -> TaskResult:
"""Executes a single autonomous task on a target device and returns the final TaskResult."""
target_profile = profile or self.default_profile
target_device = device_serial or device_id or self.device_id
if target_device == "default-device":
target_device = device_pool.select_device() or "default-device"
mode = concurrency_mode or self.concurrency_mode
concurrency_limit = max_concurrency if max_concurrency is not None else self.max_concurrency
logger.info(
f"ArtemisClient running task '{goal}' on device '{target_device}' "
f"with profile '{target_profile}' (concurrency_mode='{mode}')..."
)
task_session_id = str(uuid.uuid4())
prev_ingress = os.environ.get("ARTEMIS_TASK_INGRESS")
prev_sess_id = os.environ.get("ARTEMIS_SESSION_ID")
lock = DeviceExecutionLock(
device_id=target_device,
description=f"{goal[:120]}",
concurrency_mode=mode,
max_concurrency=concurrency_limit,
session_id=task_session_id,
ingress="sdk",
)
try:
os.environ["ARTEMIS_TASK_INGRESS"] = "sdk"
os.environ["ARTEMIS_SESSION_ID"] = task_session_id
await asyncio.to_thread(
lock.acquire,
blocking=blocking,
timeout=lock_timeout,
)
# Check if targeting a live connected device
devices = device_pool.list_devices()
connected_serials = {d.serial for d in devices}
is_live_device = target_device in connected_serials and os.environ.get("ARTEMIS_MOCK_DRIVER") != "1"
if is_live_device:
# Route through unified Artemis Daemon scheduler unless standalone explicitly configured
if not self.standalone and os.environ.get("ARTEMIS_STANDALONE") != "1":
try:
is_running, base_url = ensure_daemon_running(timeout=8.0, wait_ready=True)
if is_running and base_url:
dev_serial = target_device if target_device != "default-device" else None
resp = submit_task_to_daemon(
goal=goal,
profile=target_profile,
device_serial=dev_serial,
expected_output=kwargs.get("expected_output"),
enable_outputter=kwargs.get("enable_outputter"),
locked_app_package=kwargs.get("locked_package") or kwargs.get("locked_app_package"),
app_path=kwargs.get("app_path"),
session_id=task_session_id,
ingress="sdk",
base_url=base_url,
)
if resp and resp.get("tasks"):
final_res = wait_for_daemon_task(
task_session_id,
base_url=base_url,
timeout=float(kwargs.get("timeout", 1800.0)),
)
final_st = final_res.get("status", "completed")
err = final_res.get("error")
return TaskResult(
trace_id=task_session_id,
status="completed" if final_st in ("completed", "success") else final_st,
turns=final_res.get("turns", 1),
error=err,
device_id=target_device,
)
except Exception as exc:
logger.debug(f"SDK daemon routing notice: {exc}")
from artemis.sdk.agent import Agent
from artemis.sdk.builders import Builders
from artemis.context import DevicePlatform
cfg_builder = (
Builders.AgentConfig
.for_device(DevicePlatform.ANDROID, target_device)
.with_concurrency_mode(mode)
)
if concurrency_limit is not None:
cfg_builder.with_max_concurrency(concurrency_limit)
agent = Agent(config=cfg_builder.build())
agent._session_id = task_session_id
await agent.init()
try:
task = agent.new_task(goal=goal)
if target_profile:
task.using_profile(target_profile)
if kwargs.get("locked_package"):
task.with_locked_app_package(kwargs["locked_package"])
output = await agent.run_task(request=task.build())
return TaskResult(
trace_id=task_session_id,
status="completed" if output is not None else "success",
turns=getattr(agent, "_turns_completed", 1),
output=output,
device_id=target_device,
)
finally:
await agent.clean()
state = await Pipeline.execute(
goal=goal,
profile=target_profile,
device_id=target_device,
**kwargs,
)
return TaskResult(
trace_id=state.trace_id,
status=state.status.value,
turns=state.current_turn,
error=state.error_message,
device_id=target_device,
)
finally:
if prev_sess_id is not None:
os.environ["ARTEMIS_SESSION_ID"] = prev_sess_id
else:
os.environ.pop("ARTEMIS_SESSION_ID", None)
if prev_ingress is not None:
os.environ["ARTEMIS_TASK_INGRESS"] = prev_ingress
else:
os.environ.pop("ARTEMIS_TASK_INGRESS", None)
if lock._acquired:
await asyncio.to_thread(lock.release)
async def stream_run(
self,
goal: str,
profile: Literal["flash", "pro"] | None = None,
device_id: str | None = None,
device_serial: str | None = None,
concurrency_mode: Literal["global", "per_device"] | str | None = None,
max_concurrency: int | None = None,
blocking: bool = True,
lock_timeout: float | None = None,
max_turns: int = 30,
callbacks: list[Callable[[StreamEvent], Any]] | None = None,
**kwargs: Any,
) -> AsyncGenerator[StreamEvent, None]:
"""Executes an autonomous task while yielding real-time StreamEvents asynchronously."""
target_profile = profile or self.default_profile
target_device = device_serial or device_id or self.device_id
if target_device == "default-device":
target_device = device_pool.select_device() or "default-device"
mode = concurrency_mode or self.concurrency_mode
concurrency_limit = max_concurrency if max_concurrency is not None else self.max_concurrency
logger.info(
f"ArtemisClient streaming task '{goal}' on device '{target_device}' "
f"with profile '{target_profile}' (concurrency_mode='{mode}')..."
)
task_session_id = str(uuid.uuid4())
prev_ingress = os.environ.get("ARTEMIS_TASK_INGRESS")
prev_sess_id = os.environ.get("ARTEMIS_SESSION_ID")
lock = DeviceExecutionLock(
device_id=target_device,
description=f"ArtemisClient streaming task: {goal[:120]}",
concurrency_mode=mode,
max_concurrency=concurrency_limit,
session_id=task_session_id,
ingress="sdk",
)
try:
os.environ["ARTEMIS_TASK_INGRESS"] = "sdk"
os.environ["ARTEMIS_SESSION_ID"] = task_session_id
await asyncio.to_thread(
lock.acquire,
blocking=blocking,
timeout=lock_timeout,
)
# 1. Emit initial starting event
start_event = StreamEvent(
event_type=StreamEventType.STATUS,
step_number=0,
payload={
"status": "starting",
"goal": goal,
"profile": target_profile,
"device_id": target_device,
"concurrency_mode": mode,
},
)
if callbacks:
for cb in callbacks:
try:
cb(start_event)
except Exception as e:
logger.debug(f"Callback error: {e}")
yield start_event
# 2. Execute task pipeline
try:
state = await Pipeline.execute(
goal=goal,
profile=target_profile,
device_id=target_device,
**kwargs,
)
# 3. Emit intermediate step events
for step in state.steps:
ev = StreamEvent(
event_type=StreamEventType.STEP_END,
step_number=step.step_number,
payload={
"thought": step.thought,
"action": step.action_name,
"params": step.action_params,
"result": step.result,
"duration": step.duration_seconds,
},
)
if callbacks:
for cb in callbacks:
try:
cb(ev)
except Exception as e:
logger.debug(f"Callback error: {e}")
yield ev
# 4. Emit completion event
final_ev = StreamEvent(
event_type=StreamEventType.STATUS,
step_number=state.current_turn,
payload={
"status": state.status.value,
"turns": state.current_turn,
"error": state.error_message,
"device_id": target_device,
},
)
if callbacks:
for cb in callbacks:
try:
cb(final_ev)
except Exception as e:
logger.debug(f"Callback error: {e}")
yield final_ev
except Exception as e:
err_ev = StreamEvent(
event_type=StreamEventType.ERROR,
step_number=0,
payload={"error": str(e), "device_id": target_device},
)
if callbacks:
for cb in callbacks:
try:
cb(err_ev)
except Exception:
pass
yield err_ev
finally:
if prev_sess_id is not None:
os.environ["ARTEMIS_SESSION_ID"] = prev_sess_id
else:
os.environ.pop("ARTEMIS_SESSION_ID", None)
if prev_ingress is not None:
os.environ["ARTEMIS_TASK_INGRESS"] = prev_ingress
else:
os.environ.pop("ARTEMIS_TASK_INGRESS", None)
if lock._acquired:
await asyncio.to_thread(lock.release)
async def run_task(
self,
task: Task,
profile: Literal["flash", "pro"] | None = None,
device_id: str | None = None,
device_serial: str | None = None,
concurrency_mode: Literal["global", "per_device"] | str | None = None,
**kwargs: Any,
) -> TaskResult:
"""Executes a Task instance using ArtemisClient."""
target_device = (
device_serial
or device_id
or task.device_serial
or task.device_id
or self.device_id
)
target_profile = profile or task.profile or self.default_profile
target_mode = concurrency_mode or task.concurrency_mode or self.concurrency_mode
return await self.run(
goal=task.goal,
profile=target_profile,
device_id=target_device,
concurrency_mode=target_mode,
max_turns=task.max_turns,
locked_package=task.locked_package,
**kwargs,
)
__all__ = ["ArtemisClient", "ConcurrencyMode"]
+179
View File
@@ -0,0 +1,179 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Fake chat model for infrastructure testing (ARTEMIS_FAKE_LLM=1).
Completes any task on the first LLM invocation so that queueing, entry
points, device locking, and status propagation can be exercised without
real model calls:
- Flash (report_task_status bound): immediately reports status=completed.
- Pro planner (save_note bound): saves a task plan whose only subgoal is
already checked, so convergence_gate ends the graph after planning.
- Anything else: returns a plain "FAKE_LLM_OK" text message.
ARTEMIS_FAKE_LLM_DELAY_S adds an artificial per-call delay so tasks stay
alive long enough to observe queue/concurrency behavior.
"""
import asyncio
import json
import time
from typing import Any, Iterator, AsyncIterator
import uuid
from langchain_core.callbacks import (
AsyncCallbackManagerForLLMRun,
CallbackManagerForLLMRun,
)
from langchain_core.language_models.chat_models import BaseChatModel
from langchain_core.messages import AIMessage, AIMessageChunk, BaseMessage
from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult
FAKE_DONE_TEXT = "FAKE_LLM: task auto-completed for infrastructure testing."
def _tool_name(tool: Any) -> str | None:
name = getattr(tool, "name", None)
if isinstance(name, str):
return name
if isinstance(tool, dict):
if isinstance(tool.get("name"), str):
return tool["name"]
fn = tool.get("function")
if isinstance(fn, dict) and isinstance(fn.get("name"), str):
return fn["name"]
return None
class FakeChatModel(BaseChatModel):
"""Deterministic fake model that finishes every task on the first call."""
bound_tool_names: list[str] = []
delay_s: float = 0.0
@property
def _llm_type(self) -> str:
return "artemis-fake"
def bind_tools(self, tools: Any, **kwargs: Any) -> "FakeChatModel":
names = [n for n in (_tool_name(t) for t in tools) if n]
return FakeChatModel(bound_tool_names=names, delay_s=self.delay_s)
def with_structured_output(self, schema: Any, **kwargs: Any) -> "FakeChatModel":
return self
def _decide(self, messages: list[BaseMessage]) -> AIMessage:
call_id = f"fake-{uuid.uuid4().hex[:12]}"
usage = {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}
if "report_task_status" in self.bound_tool_names:
return AIMessage(
content="Task verified complete on first observation.",
tool_calls=[
{
"name": "report_task_status",
"args": {"status": "completed", "explanation": FAKE_DONE_TEXT},
"id": call_id,
}
],
usage_metadata=usage,
)
if "save_note" in self.bound_tool_names:
already_planned = any(
tc.get("name") == "save_note"
for m in messages
for tc in (getattr(m, "tool_calls", None) or [])
)
if already_planned:
return AIMessage(content="Plan saved. " + FAKE_DONE_TEXT, usage_metadata=usage)
return AIMessage(
content="Saving completed plan.",
tool_calls=[
{
"name": "save_note",
"args": {
"key": "task_plan",
"content": "- [x] " + FAKE_DONE_TEXT,
},
"id": call_id,
}
],
usage_metadata=usage,
)
return AIMessage(content=FAKE_DONE_TEXT, usage_metadata=usage)
def _to_chunk(self, msg: AIMessage) -> ChatGenerationChunk:
tool_call_chunks = [
{
"name": tc["name"],
"args": json.dumps(tc["args"]),
"id": tc["id"],
"index": i,
"type": "tool_call_chunk",
}
for i, tc in enumerate(msg.tool_calls or [])
]
chunk = AIMessageChunk(
content=msg.content,
tool_call_chunks=tool_call_chunks,
usage_metadata=msg.usage_metadata,
)
return ChatGenerationChunk(message=chunk)
def _generate(
self,
messages: list[BaseMessage],
stop: list[str] | None = None,
run_manager: CallbackManagerForLLMRun | None = None,
**kwargs: Any,
) -> ChatResult:
if self.delay_s > 0:
time.sleep(self.delay_s)
return ChatResult(generations=[ChatGeneration(message=self._decide(messages))])
async def _agenerate(
self,
messages: list[BaseMessage],
stop: list[str] | None = None,
run_manager: AsyncCallbackManagerForLLMRun | None = None,
**kwargs: Any,
) -> ChatResult:
if self.delay_s > 0:
await asyncio.sleep(self.delay_s)
return ChatResult(generations=[ChatGeneration(message=self._decide(messages))])
def _stream(
self,
messages: list[BaseMessage],
stop: list[str] | None = None,
run_manager: CallbackManagerForLLMRun | None = None,
**kwargs: Any,
) -> Iterator[ChatGenerationChunk]:
if self.delay_s > 0:
time.sleep(self.delay_s)
yield self._to_chunk(self._decide(messages))
async def _astream(
self,
messages: list[BaseMessage],
stop: list[str] | None = None,
run_manager: AsyncCallbackManagerForLLMRun | None = None,
**kwargs: Any,
) -> AsyncIterator[ChatGenerationChunk]:
if self.delay_s > 0:
await asyncio.sleep(self.delay_s)
yield self._to_chunk(self._decide(messages))
+10
View File
@@ -188,6 +188,16 @@ class ModelFactory:
@classmethod
def create_model(cls, endpoint: ModelEndpoint) -> BaseChatModel:
if os.environ.get("ARTEMIS_FAKE_LLM") == "1":
from artemis.llm.fake_model import FakeChatModel
delay = float(os.environ.get("ARTEMIS_FAKE_LLM_DELAY_S", "0") or 0)
logger.warning(
f"ARTEMIS_FAKE_LLM=1 — returning FakeChatModel (delay={delay}s) "
f"instead of {endpoint.provider}/{endpoint.model_name}"
)
return FakeChatModel(delay_s=delay)
provider = ModelProvider.from_string(endpoint.provider)
if provider == ModelProvider.GOOGLE:
+2 -2
View File
@@ -310,7 +310,7 @@ def build_action_server(actuator: Actuator, name: str = "artemis_actions") -> Fa
coro = asyncio.wait_for(coro, timeout=timeout_ms / 1000.0)
b64 = await coro
payload: dict[str, Any] = {"ok": True, "image_b64": b64}
except (TimeoutError, asyncio.TimeoutError):
except TimeoutError:
payload = {"ok": False, "error": f"screenshot timed out after {timeout_ms}ms"}
except Exception as e:
payload = {"ok": False, "error": str(e)}
@@ -331,7 +331,7 @@ def build_action_server(actuator: Actuator, name: str = "artemis_actions") -> Fa
coro = asyncio.wait_for(coro, timeout=timeout_ms / 1000.0)
elements = await coro
payload: dict[str, Any] = {"ok": True, "elements": elements}
except (TimeoutError, asyncio.TimeoutError):
except TimeoutError:
payload = {"ok": False, "error": f"ui hierarchy fetch timed out after {timeout_ms}ms"}
except Exception as e:
payload = {"ok": False, "error": str(e)}
+14
View File
@@ -19,6 +19,14 @@ from artemis.runtime.device_lock import (
DeviceBusyError,
DeviceExecutionLock,
)
from artemis.runtime.adb_endpoint import (
ADB_ENDPOINT_ID_ENV,
AdbEndpoint,
AdbSession,
AdbTarget,
InvalidAdbEndpoint,
current_adb_endpoint,
)
from artemis.runtime.device_pool import DevicePool, DeviceStatus, device_pool
from artemis.runtime.daemon_client import (
ensure_daemon_running,
@@ -49,12 +57,18 @@ from artemis.runtime.server_lifecycle import (
from artemis.runtime.supervisor import ProcessSupervisor, process_supervisor
__all__ = [
"ADB_ENDPOINT_ID_ENV",
"AdbEndpoint",
"AdbSession",
"AdbTarget",
"ConcurrencyMode",
"DeviceBusyError",
"DeviceExecutionLock",
"DevicePool",
"DeviceStatus",
"InvalidAdbEndpoint",
"clear_server_info",
"current_adb_endpoint",
"device_pool",
"ensure_daemon_running",
"ensure_device_awake",
+168
View File
@@ -0,0 +1,168 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Immutable ADB endpoint and task target primitives.
The selected endpoint is a user preference. An :class:`AdbTarget` is an
execution snapshot. Keeping those concepts separate prevents a queued or
running task from silently moving to another ADB server when the preference
changes in the Admin Console.
"""
from __future__ import annotations
from dataclasses import asdict, dataclass
import os
import re
import subprocess
from typing import Any, Mapping, MutableMapping, Sequence
from artemis.config import settings
from artemis.config.constants import DEFAULT_ADB_HOST, DEFAULT_ADB_PORT
from artemis.toolchain import toolchain
ADB_ENDPOINT_ID_ENV = "ARTEMIS_ADB_ENDPOINT_ID"
_LOCAL_HOSTS = frozenset({"127.0.0.1", "localhost", "::1"})
_SAFE_HOST_PATTERN = re.compile(r"^[A-Za-z0-9._:\-\[\]]+$")
class InvalidAdbEndpoint(ValueError):
"""Raised when an ADB server endpoint is malformed."""
@dataclass(frozen=True, slots=True)
class AdbEndpoint:
"""Network address of one ADB server."""
host: str
port: int
@classmethod
def create(cls, host: str, port: int) -> AdbEndpoint:
clean_host = str(host).strip()
if clean_host.startswith("[") and clean_host.endswith("]"):
clean_host = clean_host[1:-1]
if not clean_host:
raise InvalidAdbEndpoint("ADB server host cannot be empty.")
if len(clean_host) > 253 or not _SAFE_HOST_PATTERN.fullmatch(clean_host):
raise InvalidAdbEndpoint(
"Enter a valid IP address or host name without a URL scheme or path."
)
if not 1 <= int(port) <= 65535:
raise InvalidAdbEndpoint("ADB server port must be between 1 and 65535.")
return cls(host=clean_host, port=int(port))
@classmethod
def from_mapping(cls, value: Mapping[str, Any]) -> AdbEndpoint:
return cls.create(str(value.get("host", "")), int(value.get("port", 0)))
@classmethod
def local(cls) -> AdbEndpoint:
return cls(DEFAULT_ADB_HOST, DEFAULT_ADB_PORT)
@property
def socket(self) -> str:
host = self.host
if ":" in host and not host.startswith("["):
host = f"[{host}]"
return f"tcp:{host}:{self.port}"
@property
def identity(self) -> str:
"""Stable identity used for task snapshots and device-lock scoping."""
return self.socket.lower()
@property
def is_local_default(self) -> bool:
return self.host.lower() in _LOCAL_HOSTS and self.port == DEFAULT_ADB_PORT
@property
def mode(self) -> str:
return "local" if self.is_local_default else "remote"
def to_dict(self) -> dict[str, Any]:
return {
**asdict(self),
"socket": self.socket,
"identity": self.identity,
"mode": self.mode,
"is_local_default": self.is_local_default,
}
def apply_to_environment(
self,
environment: MutableMapping[str, str] | None = None,
) -> MutableMapping[str, str]:
target = environment if environment is not None else os.environ
target["ADB_HOST"] = self.host
target["ADB_PORT"] = str(self.port)
target["ADB_SERVER_SOCKET"] = self.socket
target[ADB_ENDPOINT_ID_ENV] = self.identity
return target
@dataclass(frozen=True, slots=True)
class AdbTarget:
"""A device serial bound to the ADB endpoint that discovered it."""
endpoint: AdbEndpoint
serial: str | None = None
@property
def lock_scope(self) -> str:
return self.endpoint.identity
@property
def lock_key(self) -> str:
return f"{self.lock_scope}/{self.serial or 'pending'}"
def to_dict(self) -> dict[str, Any]:
return {"endpoint": self.endpoint.to_dict(), "serial": self.serial}
class AdbSession:
"""Execute ADB commands against one explicit, immutable endpoint."""
def __init__(self, endpoint: AdbEndpoint, adb_path: str | None = None) -> None:
self.endpoint = endpoint
self.adb_path = adb_path or toolchain.resolve("adb")
def command(self, arguments: Sequence[str]) -> list[str]:
if not self.adb_path:
raise FileNotFoundError("Android Platform Tools (adb) could not be found.")
return [
self.adb_path,
"-H",
self.endpoint.host,
"-P",
str(self.endpoint.port),
*arguments,
]
def environment(self, base: Mapping[str, str] | None = None) -> dict[str, str]:
environment = dict(base if base is not None else os.environ)
self.endpoint.apply_to_environment(environment)
return environment
def run(self, arguments: Sequence[str], **kwargs: Any) -> subprocess.CompletedProcess:
kwargs.setdefault("env", self.environment())
return subprocess.run(self.command(arguments), **kwargs)
def current_adb_endpoint() -> AdbEndpoint:
"""Return the process preference as an immutable endpoint snapshot."""
host = settings.ADB_HOST or os.environ.get("ADB_HOST") or DEFAULT_ADB_HOST
port = settings.ADB_PORT or os.environ.get("ADB_PORT") or DEFAULT_ADB_PORT
return AdbEndpoint.create(str(host), int(port))
+33 -3
View File
@@ -62,11 +62,24 @@ def is_daemon_running(
return False
def daemon_log_path(port: int = DEFAULT_DAEMON_PORT) -> Path:
"""Return the log file path used by a background-spawned Daemon on the given port."""
from artemis.config.paths import get_app_dir
return get_app_dir() / "logs" / f"daemon-{port}.log"
def spawn_daemon(
host: str = DEFAULT_DAEMON_HOST,
port: int = DEFAULT_DAEMON_PORT,
) -> subprocess.Popen | None:
"""Silently spawn the Artemis Daemon server in the background as a detached process."""
"""Silently spawn the Artemis Daemon server in the background as a detached process.
The server is always launched via ``sys.executable -m`` (never through the
``artemis``/``artemis-admin`` console-script shims) so the long-lived process
chain never holds a lock on ``Scripts/artemis.exe`` — on Windows, a resident
shim blocks ``uv sync``/reinstalls with "os error 32" until it exits.
"""
env = os.environ.copy()
env["PYTHONUNBUFFERED"] = "1"
env["PYTHONUTF8"] = "1"
@@ -81,11 +94,21 @@ def spawn_daemon(
str(port),
]
# Persist daemon output for post-mortem debugging (daemon mode is the
# default for `artemis restart`, so silent DEVNULL would hide crashes).
log_handle: Any = subprocess.DEVNULL
try:
log_file = daemon_log_path(port)
log_file.parent.mkdir(parents=True, exist_ok=True)
log_handle = open(log_file, "ab")
except Exception:
log_handle = subprocess.DEVNULL
kwargs: dict[str, Any] = {
"cwd": str(ROOT_DIR),
"env": env,
"stdout": subprocess.DEVNULL,
"stderr": subprocess.DEVNULL,
"stdout": log_handle,
"stderr": subprocess.STDOUT if log_handle is not subprocess.DEVNULL else subprocess.DEVNULL,
"stdin": subprocess.DEVNULL,
}
@@ -105,6 +128,13 @@ def spawn_daemon(
except Exception as exc:
logger.warning(f"Could not auto-spawn Artemis Daemon: {exc}")
return None
finally:
# The child inherited the handle; the parent's copy is no longer needed.
if log_handle is not subprocess.DEVNULL:
try:
log_handle.close()
except Exception:
pass
def ensure_daemon_running(
+107 -17
View File
@@ -50,6 +50,7 @@ class DeviceLockOwner:
acquired_at: str
session_id: str | None = None
ingress: str | None = None
lock_scope: str | None = None
@classmethod
def from_dict(cls, value: dict) -> DeviceLockOwner:
@@ -66,6 +67,11 @@ class DeviceLockOwner:
else None
),
ingress=(str(value["ingress"]) if value.get("ingress") is not None else None),
lock_scope=(
str(value["lock_scope"])
if value.get("lock_scope") is not None
else None
),
)
@@ -84,6 +90,7 @@ class DeviceExecutionLock:
_MALFORMED_LOCK_GRACE_SECONDS = 5.0
_DEFAULT_POLL_INTERVAL_SECONDS = 0.1
QUEUE_TICKET_ENV = "ARTEMIS_DEVICE_QUEUE_TICKET"
LOCK_SCOPE_ENV = "ARTEMIS_ADB_ENDPOINT_ID"
@staticmethod
def _normalize_device_id(device_id: str | None) -> str:
@@ -96,6 +103,16 @@ class DeviceExecutionLock:
clean = re.sub(r"[^a-zA-Z0-9_-]", "_", raw)
return clean or "default"
@classmethod
def _normalize_lock_id(
cls,
device_id: str | None,
lock_scope: str | None = None,
) -> str:
clean_device = cls._normalize_device_id(device_id)
clean_scope = cls._normalize_device_id(lock_scope) if lock_scope else None
return f"{clean_scope}__{clean_device}" if clean_scope else clean_device
def __init__(
self,
device_id: str = "default",
@@ -105,9 +122,11 @@ class DeviceExecutionLock:
concurrency_mode: str | None = None,
session_id: str | None = None,
ingress: str | None = None,
lock_scope: str | None = None,
):
self.device_id = device_id or "default"
self.clean_device_id = self._normalize_device_id(self.device_id)
self.lock_scope = lock_scope or os.getenv(self.LOCK_SCOPE_ENV) or None
self.clean_device_id = self._normalize_lock_id(self.device_id, self.lock_scope)
self.description = description
self.max_concurrency = max_concurrency
self.concurrency_mode = (
@@ -130,8 +149,15 @@ class DeviceExecutionLock:
self._acquired = False
@classmethod
def _device_lock_path(cls, device_id: str | None) -> Path:
clean_id = cls._normalize_device_id(device_id)
def _device_lock_path(
cls,
device_id: str | None,
lock_scope: str | None = None,
) -> Path:
clean_id = cls._normalize_lock_id(
device_id,
lock_scope or os.getenv(cls.LOCK_SCOPE_ENV) or None,
)
return get_temp_dir("device-locks") / f"artemis-device-{clean_id}.lock"
@classmethod
@@ -224,6 +250,7 @@ class DeviceExecutionLock:
*,
session_id: str | None = None,
ingress: str | None = None,
lock_scope: str | None = None,
) -> dict:
payload = {
"pid": os.getpid(),
@@ -237,6 +264,8 @@ class DeviceExecutionLock:
payload["session_id"] = str(session_id)
if ingress:
payload["ingress"] = str(ingress)
if lock_scope:
payload["lock_scope"] = str(lock_scope)
return payload
@classmethod
@@ -247,6 +276,7 @@ class DeviceExecutionLock:
*,
session_id: str | None = None,
ingress: str | None = None,
lock_scope: str | None = None,
) -> str:
"""Reserve a FIFO position in the global queue before a worker process is spawned."""
token = uuid.uuid4().hex
@@ -258,6 +288,7 @@ class DeviceExecutionLock:
description,
session_id=session_id,
ingress=ingress,
lock_scope=lock_scope or os.getenv(cls.LOCK_SCOPE_ENV) or None,
)
for _ in range(8):
path = queue_dir / f"{time.time_ns():020d}-{token}.wait"
@@ -299,6 +330,7 @@ class DeviceExecutionLock:
device_id: str = "pending",
session_id: str | None = None,
ingress: str | None = None,
lock_scope: str | None = None,
) -> bool:
"""Transfer a submission ticket to its worker process."""
queue_dir = get_temp_dir("device-locks") / "artemis-global-device.queue"
@@ -326,6 +358,9 @@ class DeviceExecutionLock:
payload["session_id"] = str(session_id)
if ingress:
payload["ingress"] = str(ingress)
effective_scope = lock_scope or os.getenv(cls.LOCK_SCOPE_ENV) or None
if effective_scope:
payload["lock_scope"] = str(effective_scope)
try:
path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
except OSError:
@@ -352,6 +387,7 @@ class DeviceExecutionLock:
self.description,
session_id=self.session_id,
ingress=self.ingress,
lock_scope=self.lock_scope,
)
path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
return path
@@ -361,6 +397,7 @@ class DeviceExecutionLock:
self.device_id,
session_id=self.session_id,
ingress=self.ingress,
lock_scope=self.lock_scope,
)
path = self._find_reserved_ticket()
if path is None:
@@ -397,6 +434,7 @@ class DeviceExecutionLock:
self.description,
session_id=self.session_id,
ingress=self.ingress,
lock_scope=self.lock_scope,
)
try:
fd = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
@@ -474,7 +512,11 @@ class DeviceExecutionLock:
o
for o in active_owners.values()
if not (
(o.pid == os.getpid() and ((self.session_id and o.session_id == self.session_id) or o.token == self.token))
o.pid == os.getpid()
and (
(self.session_id and o.session_id == self.session_id)
or o.token == self.token
)
)
]
if len(active_other_owners) >= effective_concurrency:
@@ -513,7 +555,18 @@ class DeviceExecutionLock:
if ticket_owner is None:
continue
ticket_dev = self._normalize_device_id(ticket_owner.device_id)
if ticket_dev == target_dev or ticket_dev in ("pending", "any"):
ticket_lock_id = self._normalize_lock_id(
ticket_owner.device_id,
ticket_owner.lock_scope,
)
same_scope = (
not ticket_owner.lock_scope
or not self.lock_scope
or ticket_owner.lock_scope == self.lock_scope
)
if ticket_lock_id == target_dev or (
ticket_dev in ("pending", "any") and same_scope
):
device_queue.append(wait_path)
is_eligible = bool(device_queue and device_queue[0] == self._queue_path)
@@ -585,14 +638,13 @@ class DeviceExecutionLock:
lock_dir = get_temp_dir("device-locks")
if not lock_dir.exists():
return {}
active_map: dict[str, DeviceLockOwner] = {}
seen_tokens = set()
owners: list[DeviceLockOwner] = []
seen_tokens: set[str] = set()
for lock_path in sorted(lock_dir.glob("artemis-device-*.lock")):
owner = cls._get_active_owner_for_path(lock_path)
if owner is not None and owner.token not in seen_tokens:
dev_key = cls._normalize_device_id(owner.device_id)
active_map[dev_key] = owner
owners.append(owner)
seen_tokens.add(owner.token)
# Also check legacy global lock if still present
@@ -600,22 +652,45 @@ class DeviceExecutionLock:
if global_lock.exists():
owner = cls._get_active_owner_for_path(global_lock)
if owner is not None and owner.token not in seen_tokens:
dev_key = cls._normalize_device_id(owner.device_id)
active_map[dev_key] = owner
owners.append(owner)
seen_tokens.add(owner.token)
active_map: dict[str, DeviceLockOwner] = {}
owners_by_device: dict[str, list[DeviceLockOwner]] = {}
for owner in owners:
key = cls._normalize_device_id(owner.device_id)
owners_by_device.setdefault(key, []).append(owner)
for device_key, device_owners in owners_by_device.items():
if len(device_owners) == 1:
active_map[device_key] = device_owners[0]
continue
for owner in device_owners:
scoped_key = cls._normalize_lock_id(owner.device_id, owner.lock_scope)
active_map[scoped_key] = owner
return active_map
@classmethod
def get_active_owner(cls, device_id: str | None = None) -> DeviceLockOwner | None:
def get_active_owner(
cls,
device_id: str | None = None,
lock_scope: str | None = None,
) -> DeviceLockOwner | None:
"""Return the live process owning the lock for a specific device, or any device if None."""
lock_dir = get_temp_dir("device-locks")
if device_id is not None:
clean_id = cls._normalize_device_id(device_id)
effective_scope = lock_scope or os.getenv(cls.LOCK_SCOPE_ENV) or None
clean_id = cls._normalize_lock_id(device_id, effective_scope)
dev_path = lock_dir / f"artemis-device-{clean_id}.lock"
owner = cls._get_active_owner_for_path(dev_path)
if owner is not None:
return owner
if effective_scope:
legacy_path = lock_dir / (
f"artemis-device-{cls._normalize_device_id(device_id)}.lock"
)
owner = cls._get_active_owner_for_path(legacy_path)
if owner is not None:
return owner
# Check legacy global lock as fallback
global_path = lock_dir / "artemis-global-device.lock"
if global_path.exists():
@@ -636,7 +711,10 @@ class DeviceExecutionLock:
if not queue_dir.exists():
return []
clean_target = cls._normalize_device_id(device_id) if device_id else None
target_scope = os.getenv(cls.LOCK_SCOPE_ENV) or None
clean_target = (
cls._normalize_lock_id(device_id, target_scope) if device_id else None
)
active_tokens = {o.token for o in cls.get_active_owners().values()}
queued: list[dict[str, Any]] = []
@@ -652,7 +730,15 @@ class DeviceExecutionLock:
continue
target_dev = cls._normalize_device_id(owner.device_id)
if clean_target is not None and target_dev not in (clean_target, "pending", "any"):
target_lock_id = cls._normalize_lock_id(owner.device_id, owner.lock_scope)
same_scope = (
not target_scope
or not owner.lock_scope
or owner.lock_scope == target_scope
)
if clean_target is not None and target_lock_id != clean_target and not (
target_dev in ("pending", "any") and same_scope
):
continue
try:
@@ -665,6 +751,7 @@ class DeviceExecutionLock:
"goal": owner.description,
"device_id": owner.device_id,
"device_serial": owner.device_id,
"adb_endpoint_id": owner.lock_scope,
"pid": owner.pid,
"token": owner.token,
"ingress": owner.ingress or "unknown",
@@ -681,7 +768,10 @@ class DeviceExecutionLock:
if not lock_dir.exists():
return False
if device_id is not None:
clean_id = cls._normalize_device_id(device_id)
clean_id = cls._normalize_lock_id(
device_id,
os.getenv(cls.LOCK_SCOPE_ENV) or None,
)
return (
lock_dir / f"artemis-device-{clean_id}.lock"
).exists() or (lock_dir / "artemis-global-device.lock").exists()
@@ -694,7 +784,7 @@ class DeviceExecutionLock:
"""Revalidate a previously read owner before performing a destructive action."""
target_dev = device_id or expected.device_id
lock_dir = get_temp_dir("device-locks")
clean_id = cls._normalize_device_id(target_dev)
clean_id = cls._normalize_lock_id(target_dev, expected.lock_scope)
path = lock_dir / f"artemis-device-{clean_id}.lock"
if not path.exists():
path = lock_dir / "artemis-global-device.lock"
+233 -38
View File
@@ -18,8 +18,11 @@ from __future__ import annotations
import asyncio
from dataclasses import dataclass
import os
import shutil
import subprocess
import threading
import time
from typing import Any
from artemis.runtime.device_lock import DeviceExecutionLock
@@ -62,8 +65,28 @@ class DeviceStatus:
class DevicePool:
"""Manages discovery and assignment across all connected Android devices."""
# A snapshot younger than this is returned without touching adb, so a burst
# of concurrent enumerations (UI polling, readiness probes, submissions)
# collapses into a single `adb devices` call.
CACHE_TTL = 1.0
# When enumeration fails outright (adb server restarting, subprocess error),
# the last successful snapshot keeps answering for this long. Real
# disconnects are not failures -- adb reports them as a successful listing
# without the device -- so serving stale here only bridges server-level blips.
STALE_ON_ERROR_TTL = 10.0
# Until one enumeration has succeeded, `adb devices` may need to spawn the
# adb server itself, which routinely exceeds the hot-path budget.
HOT_QUERY_TIMEOUT = 2.0
COLD_QUERY_TIMEOUT = 8.0
def __init__(self, adb_path: str | None = None):
self._adb_path = adb_path
self._cache_lock = threading.Lock()
self._cached_raw: list[tuple[str, str, str | None, str | None]] | None = None
self._cached_at = 0.0
self._warmed = False
self._sync_query_gate = threading.Lock()
self._async_inflight: tuple[asyncio.AbstractEventLoop, asyncio.Task] | None = None
def _resolve_adb(self) -> str | None:
if self._adb_path:
@@ -73,32 +96,41 @@ class DevicePool:
except Exception:
return shutil.which("adb")
def _query_adb_devices_sync(self, timeout: float = 2.0) -> list[tuple[str, str, str | None, str | None]]:
"""Run `adb devices -l` synchronously to parse attached devices."""
def _current_query_timeout(self) -> float:
return self.HOT_QUERY_TIMEOUT if self._warmed else self.COLD_QUERY_TIMEOUT
def _query_adb_devices_sync(
self, timeout: float | None = None
) -> list[tuple[str, str, str | None, str | None]] | None:
"""Run `adb devices -l` synchronously. Returns None when the query
itself failed, as opposed to an empty list of attached devices."""
adb = self._resolve_adb()
if not adb:
return []
return None
try:
res = subprocess.run(
[adb, "devices", "-l"],
stdin=subprocess.DEVNULL,
capture_output=True,
text=True,
timeout=timeout,
timeout=timeout if timeout is not None else self._current_query_timeout(),
check=False,
)
if res.returncode != 0:
return []
return None
return self._parse_device_lines(res.stdout.splitlines())
except Exception as exc:
logger.debug(f"Error querying adb devices: {exc}")
return []
return None
async def _query_adb_devices_async(self, timeout: float = 2.0) -> list[tuple[str, str, str | None, str | None]]:
"""Run `adb devices -l` asynchronously to parse attached devices."""
async def _query_adb_devices_async(
self, timeout: float | None = None
) -> list[tuple[str, str, str | None, str | None]] | None:
"""Run `adb devices -l` asynchronously. Returns None when the query
itself failed, as opposed to an empty list of attached devices."""
adb = self._resolve_adb()
if not adb:
return []
return None
proc = None
try:
proc = await asyncio.create_subprocess_exec(
@@ -108,9 +140,12 @@ class DevicePool:
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=timeout)
stdout, _ = await asyncio.wait_for(
proc.communicate(),
timeout=timeout if timeout is not None else self._current_query_timeout(),
)
if proc.returncode != 0:
return []
return None
return self._parse_device_lines(stdout.decode(errors="replace").splitlines())
except Exception as exc:
logger.debug(f"Error querying adb devices asynchronously: {exc}")
@@ -119,7 +154,7 @@ class DevicePool:
proc.kill()
except Exception:
pass
return []
return None
@staticmethod
def _parse_device_lines(lines: list[str]) -> list[tuple[str, str, str | None, str | None]]:
@@ -143,16 +178,134 @@ class DevicePool:
results.append((serial, state, model, product))
return results
def list_devices(self) -> list[DeviceStatus]:
"""Synchronously list all connected devices along with their active lock state."""
raw_devices = self._query_adb_devices_sync()
def _cached_snapshot(
self, *, allow_stale: bool
) -> list[tuple[str, str, str | None, str | None]] | None:
with self._cache_lock:
if self._cached_raw is None:
return None
age = time.monotonic() - self._cached_at
if age <= self.CACHE_TTL or (allow_stale and age <= self.STALE_ON_ERROR_TTL):
return list(self._cached_raw)
return None
def _store_snapshot(self, raw: list[tuple[str, str, str | None, str | None]]) -> None:
with self._cache_lock:
self._cached_raw = list(raw)
self._cached_at = time.monotonic()
self._warmed = True
def _enumerate_sync(self) -> list[tuple[str, str, str | None, str | None]] | None:
"""Return the raw enumeration, or None when the query failed and no
usable snapshot exists -- never an ambiguous empty list on failure."""
cached = self._cached_snapshot(allow_stale=False)
if cached is not None:
return cached
# The gate serializes concurrent enumerations: followers block until the
# leader finishes, then hit the snapshot it just stored.
with self._sync_query_gate:
cached = self._cached_snapshot(allow_stale=False)
if cached is not None:
return cached
raw = self._query_adb_devices_sync()
if raw is None:
return self._cached_snapshot(allow_stale=True)
self._store_snapshot(raw)
return raw
async def _enumerate_async(self) -> list[tuple[str, str, str | None, str | None]] | None:
cached = self._cached_snapshot(allow_stale=False)
if cached is not None:
return cached
loop = asyncio.get_running_loop()
inflight = self._async_inflight
if inflight is not None and inflight[0] is loop and not inflight[1].done():
# Shield followers so one cancelled waiter cannot abort the shared query.
return await asyncio.shield(inflight[1])
task = loop.create_task(self._enumerate_async_uncached())
self._async_inflight = (loop, task)
try:
return await task
finally:
if self._async_inflight is not None and self._async_inflight[1] is task:
self._async_inflight = None
async def _enumerate_async_uncached(self) -> list[tuple[str, str, str | None, str | None]] | None:
raw = await self._query_adb_devices_async()
if raw is None:
return self._cached_snapshot(allow_stale=True)
self._store_snapshot(raw)
return raw
async def _start_adb_server(self, timeout: float) -> None:
"""Best-effort bounded `adb start-server` so later queries hit a warm daemon."""
adb = self._resolve_adb()
if not adb:
return
proc = None
try:
proc = await asyncio.create_subprocess_exec(
adb,
"start-server",
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
await asyncio.wait_for(proc.communicate(), timeout=timeout)
except Exception as exc:
logger.debug(f"adb start-server warm-up did not complete: {exc}")
if proc is not None:
try:
proc.kill()
except Exception:
pass
async def warm_up_async(
self,
*,
server_timeout: float = 10.0,
settle_timeout: float = 3.0,
poll_interval: float = 0.5,
) -> bool:
"""Start the adb server and complete one successful enumeration.
Meant to run once before an entrypoint starts accepting task
submissions, so the first requests never race an adb cold start.
Devices reconnect asynchronously after the server comes up, so the
settle window keeps polling until at least one device reaches the
ready state (or the window closes). Returns True once any enumeration
succeeded -- zero attached devices is still a warm pool.
"""
if self._resolve_adb() is None:
return False
await self._start_adb_server(timeout=server_timeout)
deadline = time.monotonic() + max(settle_timeout, 0.0)
succeeded = False
while True:
raw = await self._query_adb_devices_async()
if raw is not None:
succeeded = True
self._store_snapshot(raw)
if any(state == "device" for _, state, _, _ in raw):
return True
if time.monotonic() >= deadline:
return succeeded
await asyncio.sleep(poll_interval)
def _build_statuses(
self, raw_devices: list[tuple[str, str, str | None, str | None]]
) -> list[DeviceStatus]:
active_owners = DeviceExecutionLock.get_active_owners()
devices: list[DeviceStatus] = []
for serial, state, model, product in raw_devices:
is_emu = serial.startswith("emulator-") or "127.0.0.1" in serial or "localhost" in serial
clean_id = DeviceExecutionLock._normalize_device_id(serial)
owner = active_owners.get(clean_id)
clean_id = DeviceExecutionLock._normalize_lock_id(
serial,
os.getenv(DeviceExecutionLock.LOCK_SCOPE_ENV) or None,
)
owner = active_owners.get(clean_id) or active_owners.get(
DeviceExecutionLock._normalize_device_id(serial)
)
status = DeviceStatus(
serial=serial,
@@ -169,31 +322,73 @@ class DevicePool:
devices.append(status)
return devices
def list_devices(self) -> list[DeviceStatus]:
"""Synchronously list all connected devices along with their active lock state.
The raw adb enumeration may be served from a short-lived snapshot;
lock ownership is always computed fresh. A failed enumeration collapses
to an empty list -- use try_list_devices when the caller must tell the
two apart.
"""
return self._build_statuses(self._enumerate_sync() or [])
async def list_devices_async(self) -> list[DeviceStatus]:
"""Asynchronously list all connected devices along with their active lock state."""
raw_devices = await self._query_adb_devices_async()
active_owners = DeviceExecutionLock.get_active_owners()
"""Asynchronously list all connected devices along with their active lock state.
devices: list[DeviceStatus] = []
for serial, state, model, product in raw_devices:
is_emu = serial.startswith("emulator-") or "127.0.0.1" in serial or "localhost" in serial
clean_id = DeviceExecutionLock._normalize_device_id(serial)
owner = active_owners.get(clean_id)
The raw adb enumeration may be served from a short-lived snapshot;
lock ownership is always computed fresh. A failed enumeration collapses
to an empty list -- use try_list_devices_async when the caller must tell
the two apart.
"""
return self._build_statuses(await self._enumerate_async() or [])
status = DeviceStatus(
serial=serial,
state=state,
model=model,
product=product,
is_emulator=is_emu,
is_busy=owner is not None,
active_pid=owner.pid if owner else None,
active_task_desc=owner.description if owner else None,
active_session_id=owner.session_id if owner else None,
acquired_at=owner.acquired_at if owner else None,
def try_list_devices(self) -> list[DeviceStatus] | None:
"""Like list_devices, but returns None when enumeration failed and no
usable snapshot exists, so callers can distinguish "could not ask adb"
from "adb answered: no devices attached"."""
raw = self._enumerate_sync()
return None if raw is None else self._build_statuses(raw)
async def try_list_devices_async(self) -> list[DeviceStatus] | None:
"""Async variant of try_list_devices."""
raw = await self._enumerate_async()
return None if raw is None else self._build_statuses(raw)
@staticmethod
def _explicit_serial_error(
requested_serial: str, devices: list[DeviceStatus] | None
) -> str | None:
# Fail open on an indeterminate or empty enumeration (adb missing, server
# blip, devices still handshaking): the submission proceeds and fails
# downstream with a clear no-device error instead of a false hard reject.
if not devices:
return None
norm = DeviceExecutionLock._normalize_device_id
state_by_serial = {norm(d.serial): d.state for d in devices}
dev_state = state_by_serial.get(norm(requested_serial))
if dev_state == "device":
return None
if dev_state is None:
return (
f"Device '{requested_serial}' is not connected. "
f"Attached devices: {sorted(state_by_serial)}."
)
devices.append(status)
return devices
return f"Device '{requested_serial}' is attached but not ready (state: '{dev_state}')."
def validate_explicit_serial(self, requested_serial: str) -> str | None:
"""Return a rejection message for an explicitly requested serial, or None.
Only a successful, non-empty enumeration may reject: it is the single
authority for the strict device-binding check shared by the admission
paths (admin console router/queue and the MCP task runner).
"""
return self._explicit_serial_error(requested_serial, self.try_list_devices())
async def validate_explicit_serial_async(self, requested_serial: str) -> str | None:
"""Async variant of validate_explicit_serial."""
return self._explicit_serial_error(
requested_serial, await self.try_list_devices_async()
)
def get_ready_devices(self) -> list[DeviceStatus]:
"""Return devices in authorized 'device' state."""
+139 -12
View File
@@ -30,6 +30,8 @@ import subprocess
import sys
import time
from typing import Any
import urllib.error
import urllib.request
from artemis.config.paths import ROOT_DIR, get_server_info_file
from artemis.runtime.device_lock import DeviceExecutionLock
@@ -51,8 +53,9 @@ def is_port_in_use(port: int, host: str = "127.0.0.1", timeout: float = 0.5) ->
def write_server_info(
port: int,
host: str = "0.0.0.0",
host: str = "127.0.0.1",
pid: int | None = None,
lifecycle_token: str | None = None,
) -> Path:
"""Persist current Artemis server runtime metadata to well-known path."""
info_file = get_server_info_file()
@@ -67,6 +70,8 @@ def write_server_info(
"sys_executable": sys.executable,
"cmdline": sys.argv,
}
if lifecycle_token:
data["lifecycle_token"] = lifecycle_token
info_file.write_text(json.dumps(data, indent=2), encoding="utf-8")
logger.debug(f"Saved Artemis server info (PID: {data['pid']}, port: {port}) to {info_file}")
except Exception as e:
@@ -91,17 +96,105 @@ def read_server_info() -> dict[str, Any] | None:
return None
def clear_server_info() -> None:
"""Remove the persisted server metadata file if it exists."""
def clear_server_info(
*,
port: int | None = None,
lifecycle_token: str | None = None,
) -> None:
"""Remove persisted metadata, optionally only for the matching server.
Scoped removal keeps ``artemis stop --port <unused>`` and a late shutdown
finalizer from deleting metadata written by a different server instance.
"""
info_file = get_server_info_file()
try:
if info_file.exists():
if port is not None or lifecycle_token is not None:
info = read_server_info()
if not info:
return
if port is not None and info.get("port") != port:
return
if lifecycle_token is not None and info.get("lifecycle_token") != lifecycle_token:
return
info_file.unlink()
logger.debug(f"Cleared server info file at {info_file}")
except Exception as e:
logger.debug(f"Error clearing server info file {info_file}: {e}")
def request_graceful_shutdown(port: int, timeout: float = 1.5) -> bool:
"""Ask a locally managed Artemis server to run its shutdown lifecycle.
The per-process token in the server metadata prevents this control request
from being accepted remotely or accidentally sent to an unrelated service
using the same port. A missing/stale token simply makes the caller fall back
to process termination.
"""
info = read_server_info()
if not info or info.get("port") != port:
return False
token = info.get("lifecycle_token")
if not isinstance(token, str) or not token:
return False
request = urllib.request.Request(
f"http://127.0.0.1:{port}/api/system/shutdown",
method="POST",
headers={
"User-Agent": "Artemis-Lifecycle-Client/1.0",
"X-Artemis-Lifecycle-Token": token,
},
)
try:
with urllib.request.urlopen(request, timeout=timeout) as response:
return response.status in (200, 202)
except (OSError, urllib.error.URLError, urllib.error.HTTPError):
return False
except Exception:
return False
def _reconcile_orphaned_sessions() -> int:
"""Immediately mark sessions whose worker PIDs died during forced stop."""
try:
from apps.admin_console.database.repositories.session_repository import session_repo
return session_repo.reconcile_orphaned_sessions()
except Exception as exc:
logger.debug(f"Could not reconcile orphaned sessions after server stop: {exc}")
return 0
def _any_pid_alive(pids: list[int]) -> bool:
"""Return whether any target process is still running (zombies excluded)."""
if not pids:
return False
try:
import psutil
for pid in pids:
try:
process = psutil.Process(pid)
if process.is_running() and process.status() != psutil.STATUS_ZOMBIE:
return True
except psutil.NoSuchProcess:
continue
except psutil.AccessDenied:
return True
return False
except Exception:
return any(_pid_exists_portably(pid) for pid in pids)
def _pid_exists_portably(pid: int) -> bool:
try:
os.kill(pid, 0)
return True
except OSError:
return False
def find_server_pids(port: int = 8000) -> list[int]:
"""Find all process IDs associated with the Artemis server listening on `port`.
@@ -243,15 +336,15 @@ def get_server_status(port: int = 8000) -> dict[str, Any]:
def stop_server(
port: int = 8000,
timeout: float = 4.0,
timeout: float = 12.0,
force: bool = False,
) -> tuple[bool, str, list[int]]:
"""Gracefully terminate any running Artemis server on `port`.
Args:
port: TCP port to inspect and terminate.
timeout: Maximum seconds to wait for graceful process exit.
force: If True, immediately send SIGKILL without waiting.
timeout: Maximum seconds to wait for the server shutdown lifecycle.
force: If True, skip the shutdown API and terminate the process tree.
Returns:
tuple (success, message, affected_pids)
@@ -259,7 +352,7 @@ def stop_server(
pids = find_server_pids(port)
if not pids and not is_port_in_use(port):
clear_server_info()
clear_server_info(port=port)
DeviceExecutionLock.cleanup_stale_locks()
return True, f"No active server detected on port {port}.", []
@@ -271,15 +364,46 @@ def stop_server(
stopped_pids: list[int] = []
current_pid = os.getpid()
# A normal stop is an application lifecycle request, not an OS signal.
# This matters on Windows where taskkill /F and os.kill(..., SIGTERM) both
# bypass FastAPI's shutdown hook. Give Uvicorn enough time to cancel task
# workers, persist terminal session state, and release IPC/device resources.
graceful_requested = False
if not force:
graceful_requested = request_graceful_shutdown(
port,
timeout=min(2.0, max(0.25, timeout)),
)
if graceful_requested:
deadline = time.time() + max(0.25, timeout)
while time.time() < deadline:
if not is_port_in_use(port) and not _any_pid_alive(pids):
break
time.sleep(0.1)
if not is_port_in_use(port) and not _any_pid_alive(pids):
cleaned_locks = DeviceExecutionLock.cleanup_stale_locks()
clear_server_info(port=port)
pid_str = f" (PID: {', '.join(map(str, pids))})" if pids else ""
msg = f"Artemis server stopped gracefully{pid_str}. Port {port} is released."
if cleaned_locks:
msg += f" Cleaned up {cleaned_locks} device lock(s)."
return True, msg, pids
logger.warning(
f"Artemis server on port {port} did not exit within {timeout}s; "
"falling back to process-tree termination."
)
for pid in pids:
if pid == current_pid:
# Don't terminate self during stop_server call
continue
try:
if force:
ProcessSupervisor.terminate_tree(pid, timeout_seconds=0.5)
else:
ProcessSupervisor.terminate_tree(pid, timeout_seconds=timeout)
ProcessSupervisor.terminate_tree(
pid,
timeout_seconds=0.5 if force or graceful_requested else timeout,
)
stopped_pids.append(pid)
except Exception as e:
logger.warning(f"Failed to terminate process {pid}: {e}")
@@ -304,7 +428,8 @@ def stop_server(
# Cleanup stale device execution locks left by terminated server
cleaned_locks = DeviceExecutionLock.cleanup_stale_locks()
clear_server_info()
reconciled_sessions = _reconcile_orphaned_sessions()
clear_server_info(port=port)
port_free = not is_port_in_use(port)
if port_free:
@@ -312,6 +437,8 @@ def stop_server(
msg = f"Artemis server stopped{pid_str}. Port {port} is released."
if cleaned_locks:
msg += f" Cleaned up {cleaned_locks} device lock(s)."
if reconciled_sessions:
msg += f" Reconciled {reconciled_sessions} interrupted session(s)."
return True, msg, stopped_pids
else:
return (
+6 -2
View File
@@ -234,6 +234,10 @@ class Agent:
async def _prewarm_llm_connections(self, api_key: str | None = None):
"""Pre-warms the HTTP2/gRPC connection pools for both Native GenAI and LangChain clients in the background."""
if os.environ.get("ARTEMIS_FAKE_LLM") == "1":
logger.info("ARTEMIS_FAKE_LLM=1 — skipping real LLM connection pre-warming.")
publish_startup_progress("model_ready", "Model connection is ready (fake LLM)", session_id=self._session_id)
return
publish_startup_progress("model_warmup", "Warming the model connection", session_id=self._session_id)
logger.info("Starting background pre-warming of Gemini API connection pools...")
try:
@@ -646,7 +650,7 @@ class Agent:
logger.info(f"✅ Automation '{task_name}' is success ✅")
await task.finalize(content=output, state=last_state_snapshot)
if context.data_engine:
context.data_engine.end_session("success")
context.data_engine.end_session("completed")
else:
err = (
f"[{task_name}] FlashRunner failed:"
@@ -714,7 +718,7 @@ class Agent:
logger.info(f"✅ Automation '{task_name}' is success ✅")
await task.finalize(content=output, state=last_state_snapshot)
if context.data_engine:
context.data_engine.end_session("success")
context.data_engine.end_session("completed")
return output
finally:
+897
View File
@@ -0,0 +1,897 @@
# ARTEMIS Pro Context, History, and Summarization Redesign
Status: Proposal (2026-08-25)
Scope: ARTEMIS Pro / LangGraph execution mode
Audience: maintainers of Operator, Validator, Summarizer, DataEngine, and model routing
## 1. Executive Summary
This proposal improves long-term history, visually grounded neutral summaries,
context compression, and history recall without weakening Pro mode's current
reasoning continuity. The goal is not to give Operator more raw history by
default. It is to increase information density while preserving the current
`10K-12K` input range and moving all additional image interpretation to a
non-blocking background path.
The design preserves mechanisms that already work:
- the previous Operator step's complete raw or native thinking;
- the exact previous action and Validator result;
- the current screenshot and UI element list;
- complete, auditable raw step records in DataEngine; and
- asynchronous Summarizer scheduling that never blocks the graph's critical path.
It adds the following capabilities:
1. Preserve the previous model message, thought signature, action call, and Validator result as a continuous conversation tail.
2. Keep the current neutral summary contract while allowing the Summarizer to use the decision screenshot and an optional post-action screenshot as context.
3. Bind thinking, actions, available images, validation results, and summaries with a stable `step_id`.
4. Treat the active milestone only as a temporary working-set label; use plan-independent history chunks for long-term compression.
5. Add a bounded, on-demand `recall_history` tool for complete steps, older screenshots, and UI text.
6. Introduce a fixed token budget, value-density history selection, and same-turn tool-output limits.
7. Separate stable system rules from dynamic plans and history to improve prefix caching and multi-turn continuity.
The steady-state Operator context becomes:
```text
Current screenshot/XML
+ complete previous-turn reasoning and action result
+ recent visually grounded step summaries
+ relevant history-chunk summaries
+ cold-history recall on demand
```
The steady-state target remains `10K-12K tokens`, with zero additional model
calls and zero summary waits on the critical path.
## 2. Goals and Constraints
### 2.1 First Priority: Preserve Speed
The engineering definition of preserving speed is:
- no new synchronous model calls in normal steps;
- Operator never waits for summaries, indexes, or history chunks;
- first-call input P50 and P95 remain close to the current baseline;
- background summaries do not compete at the same request priority as Operator;
- normal Operator requests still contain only one current screenshot;
- history improvements replace low-density content instead of expanding normal context; and
- any background failure immediately falls back to the current detailed history.
Absolute zero latency change cannot be guaranteed by adding synchronous input.
This design therefore uses a token-neutral, zero-wait critical path and validates
P50 and P90 through paired A/B tests.
### 2.2 History and Summary Goals
- Preserve the previous strategy, expected result, and unresolved questions.
- Ground older step summaries in available visual evidence instead of guessing from action text.
- Keep important history visible across plan rewrites, milestone renames, and goal rollback.
- Make every compressed record traceable to original steps, screenshots, and tool traces.
- Never discard failures, recovery work, side effects, user constraints, or critical values through ordinary compression.
- Allow Operator to recall older information during long tasks.
### 2.3 Use Model Capacity Effectively
"Maximize" means maximizing useful reasoning information, not filling a one-million-token window:
- the strongest model directly handles current visual state, continuous reasoning, and high-value history;
- available older screenshots help the Summarizer understand the UI, action target, and explicit records;
- long context acts as an exception and deep-recall buffer, not the normal working set;
- native multi-turn messages and thought signatures are preserved;
- stable system prefixes are reused whenever possible; and
- the model does not repeatedly reread duplicate plans, raw logs, or low-value click traces.
### 2.4 Non-Goals
- Do not delete or overwrite raw history in DataEngine.
- Do not send multiple historical screenshots to Operator by default.
- Do not treat plan milestones as permanent fact boundaries.
- Do not replace Validator's objective judgment with a summary.
- Do not require a new model API in the first phase.
- Do not enlarge every step merely to use a larger context window.
## 3. Current Baseline
### 3.1 Pro Graph Path
```text
Planner
-> Convergence
-> Perception
-> Operator
-> Execution Check
-> Validator
-> Summarizer
-> Convergence
```
The Summarizer node creates a background task and returns immediately. This
behavior must remain unchanged.
### 3.2 Current Operator History
`build_plan_and_history` currently:
- renders the latest completed step in detail;
- may include raw/native thinking, actions, tool calls, Validator results, and Failure Analyzer traces;
- renders older summaries when available while retaining exact actions;
- preserves steps under the active subgoal plus the latest three global steps;
- puts the full current plan in the Operator prompt; and
- rebuilds messages on every Operator turn.
The current system already has continuity. Removing complete reasoning from the
latest step would prevent the model from understanding the previous strategy,
expectation, and decision basis.
### 3.3 Remove Separate Short-Term Memory
Remove the existing `<short_term_memory>` extraction, state storage, and next-turn
injection path without introducing a replacement relay object. Cross-turn
continuity should come from the complete previous Operator message, exact action,
Validator result, current plan, long-term Notes, and auditable execution history.
### 3.4 Current Pro Summaries
The current Pro Summarizer receives only text:
- current plan and history;
- current-step thinking;
- action and execution result; and
- Failure Analyzer trace.
It does not load `pre_image_name` or `post_image_name`. Images should reduce
context misunderstanding, not expand the Summarizer's authority to judge results.
### 3.5 Existing Cold History
Complete steps, screenshots, and UI data already exist in DataEngine.
HistoryAnalyzer and Outputter offer partial retrieval, but Operator lacks one
unified direct history-recall tool.
### 3.6 Measured Baseline
Current identifiable Pro Operator calls in `traces/data_engine.db` show:
- 74 direct Operator model calls across 43 Operator turns;
- median input of about 10,221 tokens;
- input P90 of about 11,648 tokens and P95 of about 12,019 tokens;
- maximum input of about 12,747 tokens;
- median model-call latency of about 5.24 seconds and P90 of about 16.42 seconds;
- median full Operator-turn latency of about 20.02 seconds; and
- median observed step interval of about 17.15 seconds.
Input length and model latency show no reliable correlation in the observed
`8.8K-12.7K` range. That result must not be extrapolated to an `80K-100K`
dynamic context.
## 4. Design Principles
### 4.1 Separate Raw Records from the Model View
DataEngine stores complete facts. The prompt contains only the most valuable view
for the current turn. Compression changes what the model sees, never the source record.
### 4.2 Continuous Reasoning Before Compression
The complete previous reasoning, action expectation, action call, and Validator
result are required on the hot path and cannot be replaced by an ordinary summary.
### 4.3 Images Provide Context, Not Judgment
The Summarizer may inspect screenshots that actually exist for a step, but it
must preserve the current prompt's neutral responsibility. It may compress
intent, strategy, explicit verification, iteration progress, missing conditions,
and objective Failure Analyzer intervention. It must not independently decide
whether an action succeeded, failed, completed, or navigated to a new page.
A missing post-action screenshot means only that no independent post-action
visual evidence is available. It never means the screen was unchanged or the
action had no effect.
### 4.4 Plans Are Dynamic Indexes, Not History Keys
The current subgoal may increase relevance but must not control permanent
retention. History identity uses stable `step_id` values, time ranges, and source references.
### 4.5 Validator Facts Take Priority
```text
Explicit user instructions
> objective Validator or Controller results
> activity, package, XML, or OCR differences
> sourced content in visually grounded summaries
> Operator or Summarizer inference
```
### 4.6 Background Enhancements Cannot Become Dependencies
If a visual summary or history chunk is not ready, Operator immediately continues
with the existing detailed record.
## 5. Target Architecture
```text
+-----------------------------+
| DataEngine raw facts |
| steps / traces / images / UI|
+--------------+--------------+
|
Validator completes step ---------------+
|
+--------------v--------------+
| Step Memory Record |
| exact action/result/sources |
+-------+--------------+-------+
| |
immediate | asynchronous
| v
| +--------------------+
| | Context Summarizer |
| | decision image + |
| | optional post/UI |
| +----------+---------+
| |
+-------v----------------v-------+
| Pro Memory Store |
| hot history/summaries/chunks |
+---------------+---------------+
|
Perception current screenshot/XML -------+
v
+-----------------------------+
| Token-Budget Prompt Builder |
+--------------+--------------+
v
Operator
|
recall_history on demand
```
## 6. Layered History Model
### 6.1 L0: Current Observation
Every Operator turn receives the current screenshot, package/activity, compact
XML/OCR element list, screen dimensions and coordinate mapping, user-injected
instructions, active plan path, and background-task state. Historical images do
not enter normal requests.
### 6.2 L1: Hot History and Conversation Tail
```text
Previous AIMessage
- model-visible thinking / thought signature
- raw reasoning text
- turn-ending action tool call
Validator Function/Tool Result
- whether execution occurred
- execution state
- Failure Analyzer repair
- observed post-action state
Current HumanMessage
- current screenshot
- current XML
```
Keep one complete prior Operator turn normally and up to two when the last step
failed, remains uncertain, is recovering, or has an unfinished strategy. Do not
duplicate the same complete reasoning in `Execution History`. Until the provider
can preserve real messages and thought signatures, continue rendering raw/native
thinking verbatim.
### 6.3 L2: Visually Grounded, Neutral Step Summaries
Older steps use the current "summary plus exact action" structure, with a
multimodal summary input:
```text
Step N
Neutral contextual summary
Exact action
Validator outcome
Unresolved/recovery marker
```
The summary never replaces the exact action or overrides Validator. Screenshots
only clarify context; the output does not make a new result judgment.
### 6.4 Current Work Segment
The active milestone may organize recent summaries, loop and candidate progress,
unresolved items, and the most recent entry state. It remains a temporary label
that can change with the plan and never becomes a permanent history identity.
### 6.5 L3: Stable History Chunks
Long-term history is chunked by stable step ranges and token thresholds:
```text
Chunk 1: Steps 1-10
Chunk 2: Steps 11-19
Chunk 3: Steps 20-31
```
Close a chunk after roughly `1.5K-2K tokens` of step summaries or 8-12 steps.
Close early on a major package/activity change, user goal change, Failure Analyzer
boundary, complete strategy change, long work-segment pause, irreversible side
effect, or important external result.
Each chunk preserves its source range and step IDs, attempted path, verified
facts, failed paths, unresolved questions, important entities, entry and exit
states, and raw source references. Later plan rewrites do not alter chunk identity.
### 6.6 L4: Cold History
Retain every screenshot actually saved for a step, raw XML/OCR, raw/native
thinking, complete tool calls and results, ADB and log output, the full Failure
Analyzer trace, all summary versions, Notes, and plan revisions. Cold history is
excluded from normal prompts and loaded only through local matching or
`recall_history`.
## 7. Stable Data Model
### 7.1 StepMemoryRecord
```python
class StepMemoryRecord:
step_id: UUID
session_id: UUID
step_number: int
timestamp: datetime
plan_revision: str | None
current_subgoal_hash: str | None
operator_message_ref: str | None
thought_signature: str | None
raw_thinking_ref: str | None
native_thinking_ref: str | None
action: dict
validator_result: dict | None
failure_recovery: dict | None
pre_image_name: str | None
post_image_name: str | None
pre_ui_ref: str | None
post_ui_ref: str | None
ui_diff: dict | None
summary: str | None
summary_status: str
summary_model: str | None
summary_version: int
summary_decision_image_name: str | None
summary_post_image_name: str | None
summary_used_ui_diff: bool
```
The `summary_status` state machine is:
```text
pending -> visual_ready -> visual_failed -> stale
```
Exact actions, Validator results, and raw sources remain immediately available
regardless of summary state.
### 7.2 SummaryEvidenceMetadata
The summary remains one neutral natural-language paragraph. Deterministic source
metadata is stored separately:
```json
{
"step_id": "...",
"decision_image_name": "image-a",
"post_image_name": null,
"used_ui_diff": false,
"model": "gemini-3.7-flash",
"prompt_version": 2,
"summary_version": 3
}
```
The system derives these fields from actual inputs. A null `post_image_name`
expresses no action result or screen-change judgment.
### 7.3 HistoryChunk
```python
class HistoryChunk:
chunk_id: UUID
session_id: UUID
start_step_id: UUID
end_step_id: UUID
source_step_ids: list[UUID]
digest: str
verified_facts: list[str]
unresolved: list[str]
failed_paths: list[str]
important_entities: list[str]
entry_state: str | None
exit_state: str | None
model: str
version: int
```
Chunk summaries are versioned append-only records and never damage source summaries.
## 8. Visually Grounded Neutral Pro Summarizer
### 8.1 Inputs
Each background summary call uses only inputs that actually exist:
1. the decision screenshot seen by Operator, optionally annotated with action coordinates;
2. an independent post-action screenshot only when Validator captured and saved it;
3. the exact action name and parameters;
4. the Controller result;
5. the final Validator result;
6. Failure Analyzer recovery records;
7. available package/activity information;
8. available UI/OCR differences;
9. current-step thinking; and
10. a small number of recent summaries for loop and strategy continuity.
Do not resend the full plan and ten detailed steps. Build the message dynamically,
never substitute the decision screenshot for a missing post-action screenshot,
and never infer "no screen change" from missing post-action evidence.
### 8.2 Output Rules
`artemis/agents/summarizer/summarizer.md` remains the sole authority for summary
semantics. Preserve its philosophy and one-paragraph output. Use Operator's first
person to compress intent, strategy, explicit progress, and verified facts. Do
not independently claim success, failure, completion, non-completion, or page
navigation. A successful Controller return does not prove task success. Do not
guess unreadable values or judge outcomes by whether two images look similar.
Failure Analyzer intervention is summarized as an objective trigger and action,
without claiming that the repair worked. Step ID, evidence names, model, prompt
version, and summary version are system metadata rather than model-generated text.
The existing prohibition on outcome-judgment terms remains, including
`successfully`, `completed`, `entered`, `navigated to`, `failed`, `unsuccessful`,
`could not`, and `achieved`.
### 8.3 Model Selection
Quality-first default:
```text
Per-step summary: Gemini 3.7 Flash + available screenshots
History-chunk summary: Gemini 3.7 Flash
```
Single-quota, strict-isolation option:
```text
Real-time step summary: Gemini 3.5 Flash-Lite + available screenshots
Idle reprocessing for violations or important omissions: Gemini 3.7 + available screenshots
History-chunk summary: Gemini 3.7 Flash
```
The recommended quality ceiling remains 3.7 with available images. Flash-Lite is
a resource-isolation option, not a replacement for complex strategy, loop, and
recovery reasoning.
### 8.4 Scheduling and Isolation
- Operator never awaits Summarizer.
- Background summary concurrency is limited to one.
- Operator requests have absolute priority.
- Summary timeout or failure never affects graph execution.
- Existing detailed history remains the fallback.
- Prefer a separate endpoint, project, or quota.
- Without resource isolation, do not start a new 3.7 summary while Operator is active.
- Give non-preemptible background calls a short timeout.
- An optional final flush must not delay the next normal step.
## 9. Stable History Chunks
Milestones reflect a temporary interpretation of the task and are not stable fact
boundaries. A plan edit can otherwise hide relevant steps, split one path across
milestones, make rollback lose a prior strategy, or desynchronize summaries from
current labels. Subgoal identity therefore affects relevance only.
```python
if current_open_chunk.summary_tokens >= 1800:
close_chunk()
elif current_open_chunk.step_count >= 12:
close_chunk()
elif important_boundary_event:
close_chunk()
```
Prevent recursive summary loss by generating each chunk from source step
summaries, retaining `source_step_ids`, merging structured fact sets rather than
guessing from prose, and retaining every summary version.
## 10. Operator Cold-History Recall
Expose one unified tool:
```python
recall_history(
query: str,
step_range: list[int] | None = None,
include_details: bool = False,
include_images: bool = False,
max_results: int = 5,
) -> HistoryRecallResult
```
Search visually grounded summaries, history chunks, raw actions, Validator
results, Operator thinking, Notes, XML/OCR text, package/activity data, Failure
Analyzer traces, and image references. Phase one can use SQLite FTS, keywords,
step ranges, package/activity, and screen hashes without another model or vector database.
Return at most five results and 2K tokens by default. Every result includes a
step ID or number. Detailed output remains token-bounded. Image recall returns no
more than the images that actually exist for one step. Large raw results return
only relevant excerpts and references.
Operator should recall history when the current screen resembles an older state,
an old exact value is needed, two consecutive steps make no progress, the same
action is about to be attempted a third time, a summary conflicts with current
observation, the user asks to return to an earlier state, or Failure Analyzer
needs review. Ordinary steps must not recall history speculatively.
Before prompt construction, a local package/activity, screenshot-hash, and UI-
signature match may inject a short hint without a model call or historical image:
```text
Historical state hint: The current screen closely resembles the post-action
screen from Step 17. Use recall_history only if its details are needed.
```
## 11. Token Budget and History Selection
Recommended steady-state budgets:
```text
First Operator call target: 10K-12K
First-call soft limit: 12K
First-call emergency hard limit: 16K
Same-turn soft limit: 20K
Same-turn hard limit: 28K
```
```python
history_budget = (
operator_total_budget
- stable_system_tokens
- tool_schema_tokens
- current_observation_tokens
- active_plan_tokens
- same_turn_reserve
)
```
Always retain the previous complete Operator turn, current user constraints,
active plan path, unresolved items, recent failures and recovery, irreversible
actions or external side effects, the previous chunk's exit state, and current
loop or candidate progress.
Score optional history using current-subgoal relevance, unresolved status,
failure or recovery relevance, screen similarity, entity overlap, path boundary,
explicit user references, recency, resolved status, and duplication. Fill the
remaining budget by `value_density = priority / token_cost`.
When over budget:
1. Remove duplicate renderings.
2. Keep the previous real message and remove its duplicate detailed history text.
3. Replace older detailed steps with visual summaries.
4. Replace groups of old summaries with stable chunks.
5. Remove resolved, unrelated, low-value chunks.
6. Keep only relevant excerpts and references from large tool output.
Never evict hard user constraints, unresolved items, irreversible side effects,
critical values, recent recovery state, or the previous complete reasoning and expectation.
## 12. Prompt and Cache Layout
Keep the stable Operator role, safety rules, cognitive protocol, and output
contract in `SystemMessage`. Put the previous real AI message, thought signature,
action call, and Validator tool result in the conversation tail. Put the current
goal, active plan path, budgeted history, current screenshot, and UI elements in
the dynamic `HumanMessage`. Keep tool schemas and ordering stable when possible.
If the provider adapter supports stateful interactions, preserve one to three
recent Operator interactions with `previous_interaction_id`. Start a new
interaction when the token threshold is reached or the plan changes materially,
seeding it with the stable prefix and chunk summaries. Until then, retain the
current message-rebuild path.
## 13. Same-Turn Context Control
Large ADB, Notes, Explorer, log, or XML results can expand later model calls in
the same Operator turn. Store complete results in a trace or artifact and expose
a local envelope:
```json
{
"status": "success",
"important_lines": ["..."],
"errors": [],
"tail": ["..."],
"full_result_ref": "trace://...",
"truncated": true,
"original_chars": 87321
}
```
Apply character, line, and estimated-token limits; retain matched lines, errors,
and head/tail context; never call another LLM synchronously to summarize tool
output. Compress older results at the soft limit and require a decision instead
of accumulating more exploration at the hard limit.
## 14. Configuration
Add a dedicated Pro memory section instead of reusing Flash `step_summarizer`:
```jsonc
"agent": {
"pro": {
"memory": {
"enabled": true,
"preserve_previous_operator_turn": true,
"max_hot_turns": 2,
"operator_target_input_tokens": 12000,
"operator_hard_input_tokens": 16000,
"same_turn_soft_tokens": 20000,
"same_turn_hard_tokens": 28000,
"visual_step_summary": {
"enabled": true,
"model": "gemini-3.7-flash",
"thinking_level": "low",
"include_pre_image": true,
"include_post_image_when_available": true,
"include_ui_diff": true,
"max_concurrency": 1,
"operator_never_waits": true
},
"history_chunks": {
"enabled": true,
"max_steps": 12,
"target_source_tokens": 1800,
"model": "gemini-3.7-flash"
},
"recall": {
"enabled": true,
"max_results": 5,
"max_text_tokens": 2000,
"max_image_steps": 1
}
}
}
}
```
For a speed-first single-quota deployment, override only the visual step model
with `gemini-3.5-flash-lite`.
## 15. Implementation Scope
### 15.1 Operator
Update Operator, prompt builders, and configuration to add a token-budget memory
assembler, preserve recent real messages and Validator results, avoid duplicate
history rendering, move dynamic state out of the stable system prompt, remove
`<short_term_memory>`, add `recall_history`, inject local screen-similarity hints,
and enforce same-turn output limits.
### 15.2 Summarizer
Load decision and optional post-action images by stable step ID, include available
Validator and UI-diff data, send multimodal messages, preserve the neutral
one-paragraph contract, atomically update versioned evidence metadata, retain old
summaries on failure, enforce concurrency and timeout limits, and never block Operator.
### 15.3 Graph and State
Store the previous Operator `AIMessage`, thought signature, and Validator function
response. Remove the `short_term_memory` field while safely ignoring it in older
tasks. Read ready memory snapshots at the Perception/Operator boundary and retain
the current asynchronous Summarizer semantics.
### 15.4 DataEngine
Add summary status, version, evidence, thought-signature, message-reference, and
UI-diff fields; persist history chunks and local recall indexes; support atomic
updates by step ID; and preserve every older summary version and source.
### 15.5 Task Tree
Separate history visibility from rendering detail, render visual summaries and
stable chunks, use current subgoal only for relevance, avoid repeating the latest
step when it is already present in the real message tail, and retain exact actions
and validation results.
### 15.6 Memory Module
Suggested files:
```text
artemis/memory/pro_memory.py
artemis/memory/history_chunker.py
artemis/memory/history_retriever.py
artemis/memory/token_budget.py
artemis/tools/history_recall.py
```
Responsibilities include memory snapshots, value-based selection, chunk closure
and background summarization, FTS and screen-signature retrieval, the recall tool,
and provider-independent token estimation.
### 15.7 Configuration
Every capability must have an independent switch, default to current Pro fallback
behavior, tolerate missing configuration and older tasks, and support both the
3.7 quality mode and Flash-Lite isolation mode.
## 16. Delivery Phases
### Phase 0: Baseline and Observability
Freeze a representative long-task suite. Record input tokens, TTFT, model latency,
full-turn latency, summary-ready latency, failures, retries, history size,
duplicate-action rate, and Failure Analyzer activation without changing prompts.
### Phase 1: Protect and Improve Hot History
Preserve complete recent reasoning, remove the duplicate short-term-memory path,
avoid duplicate rendering, add the total token threshold, and locally truncate
large tool envelopes. Prove that speed does not regress first.
### Phase 2: Pro Visual Step Summaries
Load actual decision and optional post-action images by stable step ID while
preserving the neutral output contract and text fallback. Compare 3.7 with images,
Flash-Lite with images, and current 3.7 text-only behavior. Confirm no Operator wait.
### Phase 3: Stable History Chunks
Add milestone-independent rolling chunks, generate them from source summaries,
treat milestones as temporary working-set labels, and add source references and versions.
### Phase 4: Operator Cold-History Recall
Reuse step, screenshot, and UI search capabilities behind one bounded
`recall_history` tool, add local screen-similarity hints, and limit text and images.
### Phase 5: Real Multi-Turn Messages and Cache Optimization
Preserve real AI messages, tool calls, and thought signatures; separate stable
system and dynamic human content; evaluate stateful provider APIs; and safely
restart interactions at token thresholds. Begin only after the first four phases stabilize.
## 17. Test Plan
Unit tests must cover hot-history preservation, removal of short-term-memory
parsing, action/Validator pairing, duplicate suppression, two-turn recovery,
image lookup by step ID, optional post-image behavior, neutral-output validation,
Validator precedence, concurrency, timeout, zero-wait behavior, chunk thresholds,
source traceability, recall bounds, image limits, and hard token protections.
Integration tests must include tasks longer than 30 steps, repeated similar
screens, plan rewrites, cross-app flows, dialogs, Toasts, WebViews, Canvas,
Failure Analyzer recovery, repeated ADB exploration, long lists, monitoring loops,
summary timeout or quota exhaustion, and exact-value recall from older screenshots.
Run blinded evaluations on real traces comparing 3.7 text-only, Flash-Lite with
available images, and 3.7 with available images. Reviewers score fidelity to
intent and explicit facts, unsupported visual claims, unauthorized outcome
judgments, anomaly retention, next-step usefulness, generation time, and ready latency.
## 18. Acceptance Criteria
Under the same device, task, model, and comparable service load:
- first-call input P50 increases by no more than 3% and P95 by no more than 5%;
- Operator model-call and full-turn P50/P90 latency regress by no more than 5%;
- observed step-interval P50/P90 regress by no more than 5%;
- Operator waits for summaries exactly zero times;
- normal Operator requests contain zero historical images; and
- images returned by `recall_history(include_images=True)` count only as explicit recall.
Run at least 30 paired tasks and report confidence intervals. Quality must not
regress for previous-strategy recall, overall task success, or Failure Analyzer
recovery. Visual-fact accuracy, important-fact retention across plan edits,
repeat-action rate, history-induced rollback rate, and top-five recall hit rate
must improve. Summary/Validator conflict should approach zero.
## 19. Failure Modes and Fallbacks
- If a visual summary is pending, use complete previous reasoning, exact action, and Validator result.
- If visual summarization fails, set `visual_failed`, preserve old summaries, and do not block for retries.
- If screenshot evidence is incomplete, include only actual evidence and never infer the missing post-action state.
- If chunk generation fails, keep source summaries and retry the pending chunk during an idle window.
- If recall output is too large, return high-scoring excerpts and step/artifact references.
- If background work competes with Operator, switch to Lite, pause new summaries, or use a separate quota.
- If stateful interactions are incompatible, return to prompt reconstruction while preserving raw/native thinking.
## 20. Rollout Flags
```text
pro_memory_enabled
preserve_operator_message_tail
visual_step_summary
history_chunks
operator_history_recall
stateful_operator_interactions
```
Recommended order: enable metrics and token thresholds; run visual summaries in
shadow mode; use them after blinded validation; enable stable chunks; enable
`recall_history`; then evaluate real multi-turn interactions. Disabling any flag
must restore current Pro behavior.
## 21. Improvements Over the Current System
| Capability | Current Pro | Redesigned Pro |
|---|---|---|
| Previous reasoning | Re-rendered as detailed history | Preserved and ready for a real message/thought-signature chain |
| Short-term strategy | Optional `<short_term_memory>` | Carried by the complete previous message, Validator, plan, and Notes |
| Step summary | 3.7 reads text only | Neutral 3.7 summary reads actual available images, UI, and Validator data |
| Exact action | Retained beside summary | Still retained and never replaced by visual summary |
| History pruning | Current subgoal plus latest three steps | Working set plus stable chunks and relevance budget |
| Plan rewrite | May change old-step visibility | Stable step/chunk references survive label changes |
| Old screenshots | No direct Operator recall | One actual step can be recalled on demand |
| Cold history | Primarily for HistoryAnalyzer/Outputter | Locally searchable and bounded for Operator |
| Context control | Mainly a step window | Total token budget, value density, and same-turn limits |
| Cache use | Highly dynamic system prompt | Stable system prefix separated from runtime state |
| Background failure | Summary may be delayed | Explicit state machine and zero-wait fallback |
The objective is not to display more history. It is to preserve recent reasoning,
give older summaries real visual context, decouple long-term identity from plan
milestones, make compressed records precisely recallable, and keep normal input
size and critical-path model-call count unchanged.
## 22. Recommended Defaults
```text
Normal Operator input: 10K-12K tokens
First-call soft/hard limits: 12K / 16K
Same-turn soft/hard limits: 20K / 28K
Complete hot turns: 1 normally, up to 2 for failure/uncertainty
Current-work summaries: budget-controlled, not a fixed count
History chunk: 8-12 steps or about 1.8K source-summary tokens
Historical images in normal input: 0
Summary input images: 0-1 decision, 0-1 independent post-action
Explicit recall images: actual images from at most one step
Preferred step-summary model: Gemini 3.7 Flash + images
Single-quota speed mode: Gemini 3.5 Flash-Lite + images
History-chunk model: Gemini 3.7 Flash
Background summary concurrency: 1
Additional critical-path calls: 0
Critical-path summary waits: 0
```
## 23. Final Decision
Do not remove previous reasoning and replace detailed history with an image-free
capsule. Preserve complete previous-turn reasoning, upgrade older steps to
visually grounded neutral summaries, use milestone-independent rolling chunks,
allow bounded cold-history recall, and enforce a fixed token budget.
The first, highest-value implementation slice is:
1. preserve complete previous reasoning and remove the duplicate short-term-memory path;
2. add the decision screenshot and optional actual post-action screenshot to the current neutral Pro Summarizer;
3. bind summaries with stable step IDs; and
4. establish speed baselines and blinded visual-summary evaluations.
Do not expand the normal history window or remove the current detailed-history
fallback until all four items have passed validation.
+606
View File
@@ -0,0 +1,606 @@
# ARTEMIS Video Analysis Memory and Reliability Design
Status: implemented (2026-08-25)
Audience: ARTEMIS runtime, perception, tracing, and mobile automation maintainers
## 1. Executive summary
The current video analyzer treats its blackboard as state owned by one
`VideoAnalyzer` object. Tool invocations create new analyzer objects, so the
state is lost between calls. A failed parent analysis also discards useful
work completed by child analyzers. Overlap detection is advisory and does not
prevent the same interval from being sent to a video model again.
This design moves video memory out of the agent and into a video-scoped,
persistent blackboard. The blackboard is the source of truth for:
- which intervals have been analyzed for a normalized query and modality;
- observations and evidence extracted from those intervals;
- successful, running, and failed segment attempts;
- resumable partial results across tool calls;
- concurrency leases that prevent duplicate model requests.
The analyzer becomes a coordinator over this state. It calculates the set
difference between a requested interval and completed coverage, analyzes only
the gaps, commits every successful chunk immediately, and returns partial
results without erasing prior work.
The first implementation deliberately uses SQLite and the existing trace
directory. Vector search, semantic graphs, provider-managed context caches,
and entity re-identification are future layers, not prerequisites for fixing
correctness and reliability.
## 2. Goals
### 2.1 Functional goals
1. Share analyzed intervals and results across every `VideoAnalyzer` created
for the same recording.
2. Avoid repeating an identical successful `(video, interval, query,
modality)` analysis.
3. Persist each successful chunk before sibling chunks or the parent agent
finish.
4. Preserve successful chunks when some chunks fail.
5. Expose previous observations to both native Gemini and universal-model
coordinators.
6. Keep evidence tied to recording-relative timestamps and durable files.
7. Prevent concurrent duplicate requests with an expiring lease.
8. Maintain existing public tool signatures.
### 2.2 Reliability goals
- A process or model failure may lose only the currently running chunk.
- An expired running lease is reclaimable.
- Retrying a request analyzes only uncovered or failed chunks.
- A partial result explicitly identifies failed intervals.
- SQLite writes are idempotent and safe under multiple analyzer instances.
- Evidence used by persistent memory is not deleted by temporary-file cleanup.
### 2.3 Non-goals for phase one
- Semantic similarity reuse between materially different queries.
- Cross-recording memory or user identity memory.
- Replacing Gemini or the configured universal VLM.
- Hosting an open-weight video model or accessing model KV caches.
- Full WorldMM semantic graphs, HippoRAG, VLM2Vec, or ReflectWorld ReID.
- Permanent retention of uploaded provider files.
## 3. Current failure modes
### 3.1 Agent-owned state
`VideoAnalyzer.__init__` initializes an empty list. `video_tool.py` creates a
new analyzer for every tool call. Consequently, a later invocation cannot see
the earlier invocation's ledger.
### 3.2 Advisory overlap handling
The analyzer calculates high-overlap warnings but continues with extraction,
compression, upload, and inference. A warning in a prompt is not an
idempotency mechanism.
### 3.3 Commit-at-the-end behavior
Some event observations are appended during child analysis, but they are only
in memory. If the parent fails or cleanup runs, the trace can contain expensive
model activity with no reusable result.
### 3.4 Mutable recording ambiguity
An open-ended interval is currently cached as `(start, None)`. The meaning of
`None` changes while recording continues, so this cannot be a stable cache
identity.
### 3.5 Disposable evidence
Extracted screenshots live in temporary directories registered for cleanup.
The ledger may therefore reference files that no longer exist.
## 4. Design principles
### 4.1 Memory before reasoning
Perception results are written before the coordinator generates its final
answer. Answer generation reads stored evidence; it is not the owner of that
evidence.
### 4.2 Evidence-backed records
Every observation carries an interval, query, confidence, modality, model
metadata where available, and an optional durable screenshot. Summaries never
stand alone without temporal provenance.
### 4.3 Objective observation versus query result
The design has two logical layers:
- **Observations** describe visible or audible events and can help later
queries.
- **Segment results** answer one normalized query over one complete interval
and define exact reusable coverage.
An observation about a button is useful context for many questions, but a
segment result for "did login succeed" must not automatically satisfy a
different query such as "how long did the spinner remain visible".
### 4.4 Immutable interval identity
An analysis segment always has numeric start and end timestamps. Open-ended
requests are resolved to the safe available end before they are claimed or
committed.
### 4.5 At-least-once execution, idempotent commit
Provider requests can be retried. Database constraints and deterministic keys
make commits idempotent. Exact-once provider execution is not promised, but
duplicate execution is strongly suppressed by leases.
## 5. Architecture
```text
video_analyzer tool
|
v
VideoAnalyzer coordinator ---------> persistent blackboard ledger
| | |
| | +-- observations/evidence
| +-- segment coverage and attempts
|
+-- resolve stable numeric interval
+-- subtract completed coverage
+-- claim uncovered chunks with leases
+-- run child analysis
+-- commit each successful chunk immediately
+-- mark failed chunks retryable
+-- synthesize cached + new + partial results
```
The blackboard is acquired from `ArtemisContext`, not constructed as private
agent state. Multiple analyzers in the same context share the object, while
SQLite makes the data available to later instances and process restarts.
## 6. Recording identity
The preferred blackboard key is the active `RecordingSession.video_id`:
```text
video:<uuid>
```
If no active recording is available, the fallback is the DataEngine session:
```text
session:<uuid>
```
The context retains the resolved blackboard instance so analysis after a
recording stops continues to use the same key. Extracted recording artifacts
also carry an explicit source revision:
```text
<video-uuid>:<generation>:<sealed-end>
```
A generation advances whenever the recorder rolls a physical segment. The
recorder tracks the sealed numeric prefix, and every `end=None` analysis is
resolved to the current safe numeric end before it is claimed.
## 7. Persistence model
### 7.1 `video_analysis_segments`
One row represents a complete query-specific analysis of a numeric interval.
| Column | Meaning |
| --- | --- |
| `board_key` | Recording/session namespace |
| `query_key` | SHA-256 of the normalized query |
| `query_text` | Original query for debugging and retrieval |
| `modality` | `video` or `audio` |
| `start_ms`, `end_ms` | Stable integer interval |
| `status` | `running`, `succeeded`, `retryable_failed`, `permanent_failed` |
| `lease_owner`, `lease_expires_at` | Duplicate-work suppression |
| `attempt_count` | Number of acquired attempts |
| `summary`, `analysis` | Reusable completed result |
| `error` | Last structured failure text |
| `created_at`, `updated_at` | Audit timestamps |
The unique key is:
```text
(board_key, query_key, modality, start_ms, end_ms)
```
### 7.2 `video_analysis_observations`
One row represents a timestamped event or fact.
| Column | Meaning |
| --- | --- |
| `observation_id` | Deterministic content fingerprint |
| `board_key` | Recording/session namespace |
| `start_ms`, `end_ms` | Event interval |
| `query_text` | Query that caused extraction |
| `summary` | Concise observation |
| `confidence` | Model-reported confidence |
| `modality` | `video` or `audio` |
| `screenshot_path` | Durable evidence path |
| `extra_json` | Forward-compatible metadata |
| `created_at` | Audit timestamp |
The deterministic observation ID makes repeated commits safe.
### 7.3 Evidence layout
```text
<trace-session>/video_blackboard/evidence/<content-id>.jpg
```
Temporary screenshots are copied into this directory before their source is
registered for deletion. Stored records reference the durable copy.
## 8. Segment state machine
```text
lease expires
running ------------------------------+
| |
| success v
+----------> succeeded retryable_failed
| |
| retryable error | claim again
+----------> retryable_failed ------+
|
| non-retryable/budget exhausted
+----------> permanent_failed
```
Claim behavior is transactional:
1. A missing row is inserted as `running`.
2. `succeeded` returns its cached result without acquiring work.
3. `running` with a live lease returns `in_progress`.
4. An expired lease or retryable failure is atomically reclaimed and increments
`attempt_count`.
Exhausted failures store a normalized category and category-derived
retryability. Permanent failure is reserved for validated bad input,
authentication/policy errors, cancellation, or explicit decisions.
## 9. Coverage and planning
Coverage is query- and modality-specific. Given requested interval `R` and
the union of successful intervals `S`, the work planner calculates:
```text
missing = R - union(S)
```
Each missing interval is split into bounded chunks. Completed chunks are
included in the returned result; only gaps are sent to the provider.
Interval math uses integer milliseconds internally to prevent floating-point
key drift. API-facing timestamps remain seconds.
Observations from other queries are still placed in the ledger as context, but
they do not count as completed coverage for the new query.
## 10. Execution flow
### 10.1 Tool invocation
1. Acquire the context-scoped blackboard.
2. Refresh the local ledger from persistent observations.
3. Give the coordinator the full ledger snapshot.
4. Coordinator requests segment analysis.
### 10.2 Child analysis
1. Resolve a numeric interval from recording metadata.
2. Calculate missing intervals.
3. For every chunk, acquire a lease.
4. Reuse `succeeded`; skip live `running`; execute only `claimed`.
5. Extract/compress/upload and invoke the configured VLM.
6. Persist timeline observations as soon as they are parsed.
7. Persist the complete segment result before returning to the parent.
8. On final failure, mark that exact chunk failed and keep all sibling results.
### 10.3 Parent synthesis
The parent receives cached and newly completed results in the same format. If
some chunks failed, it receives an explicit partial-result marker and failed
intervals. It may answer with qualified confidence or request a targeted retry.
## 11. Retry and degradation policy
The desired production order is:
1. retry transient upload/poll/generation failures with exponential backoff
and jitter;
2. retry the exact chunk without expanding its identity;
3. split a repeatedly failing chunk into smaller children;
4. use the configured fallback video model;
5. fall back to high-frequency keyframes, OCR, and UI hierarchy around known
action timestamps;
6. return partial evidence with machine-readable failed intervals.
The runtime implements this order with structured failure classes, bounded
bisection, configured fallback models, and a primary-model circuit breaker.
## 12. Provider and sampling strategy
Provider context caching is an optimization, not the system of record. ARTEMIS
must remain correct if a provider cache misses or expires.
Default 1 FPS video sampling is supplemented by exact, dense seeks around
DataEngine action timestamps. Quiet intervals remain sparse while taps,
transitions, toasts, and spinners receive a higher sampling budget.
## 13. Concurrency
- SQLite uses WAL mode and short `BEGIN IMMEDIATE` claim transactions.
- The unique segment key prevents duplicate rows.
- A UUID lease owner identifies one execution attempt.
- Leases expire so crashes do not permanently block work.
- Existing API and transcode semaphores still bound local concurrency.
- Conflict cleaning affects an analyzer's presentation ledger; it never
deletes persistent observations written by another analyzer.
## 14. Cleanup and retention
Three artifact classes have different policies:
1. Trimmed/compressed chunks: temporary, deleted after the call.
2. Provider-uploaded files: temporary, deleted best-effort after inference.
3. Blackboard evidence: durable for the trace/session retention period.
Deleting a trace/session removes blackboard rows through DataEngine and
evidence through the session directory lifecycle.
## 15. Observability
Required counters and trace fields:
- requested duration;
- cached duration;
- newly analyzed duration;
- number of claimed, cached, in-progress, succeeded, and failed chunks;
- provider attempts and error class;
- blackboard key and query key;
- partial versus complete outcome;
- evidence paths attached to observations.
No prompt or trace status should report success merely because an error string
was returned as normal text.
## 16. Compatibility and migration
- Public `video_analyzer` and `video_analyzer_pure` arguments do not change.
- Existing in-memory ledger dictionaries remain the presentation format.
- When DataEngine persistence is unavailable, the blackboard falls back to a
context-scoped in-memory store. This preserves tests and embedded usages but
does not promise process-restart recovery.
- Databases are migrated with `CREATE TABLE IF NOT EXISTS`; no destructive
migration is required.
- Existing traces without blackboard rows continue to work and start empty.
## 17. Security and privacy
- Evidence remains local under the trace directory.
- Provider files are deleted best-effort after requests.
- Blackboard rows must not store credentials or entire model prompts.
- Retention follows trace retention.
- Future cross-session or identity memory requires a separate consent and
privacy design; it is outside this scope.
## 18. Testing strategy
### 18.1 Unit tests
- persistence across two blackboard instances using the same SQLite file;
- deterministic observation deduplication;
- interval union and missing-range calculation;
- exact successful segment reuse;
- live lease suppression and expired lease reclamation;
- failed chunk remains retryable;
- evidence is copied to durable storage;
- two `VideoAnalyzer` instances share prior ledger state;
- repeated `exec_spawn_sub_agent` does not invoke child inference again;
- mixed success/failure returns partial output and preserves success.
### 18.2 Integration tests
- analyze overlapping intervals against a fixture recording;
- interrupt after one successful chunk and resume;
- concurrent identical requests result in one provider call;
- active recording resolves open-ended requests to numeric endpoints;
- evidence remains readable after analyzer cleanup.
### 18.3 Device tests
Per ARTEMIS mobile testing rules, end-to-end mobile flows must be explored and
verified on a connected device before production test scripts are finalized.
The refactor's deterministic unit tests do not require a device; live timing,
fast UI transitions, and recording alignment do.
## 19. Rollout
1. **Complete:** SQLite blackboard, sharing, observation persistence, exact
coverage reuse, leases, and tests.
2. **Complete:** failure classification, jitter, bounded bisection, fallback,
and circuit breaking.
3. **Complete:** recording generations, sealed-prefix metadata, and
generation-scoped caches.
4. **Complete:** action-aware sampling and real audio delivery.
5. **Deferred optimization:** provider file/context caching, gated by an
explicit retention policy.
6. **Future accuracy layer:** semantic/event hierarchical retrieval after
production traces provide a representative evaluation corpus.
## 20. Acceptance criteria
- Two analyzer instances in one task see the same ledger.
- Repeating an identical successful interval/query causes zero new child VLM
calls.
- If one of several chunks fails, successful chunks are available to the next
invocation and are not rerun.
- A crashed `running` chunk becomes reclaimable after lease expiry.
- Persistent evidence survives normal analyzer temporary-file cleanup.
- Existing video-analyzer unit tests and new blackboard tests pass.
Additional reliability criteria:
- timeout/media failures recover through bounded recursive subdivision;
- provider-wide failures do not trigger request fan-out;
- primary-model failures select the configured `utils.video_analyzer` fallback;
- the primary circuit opens after a configured consecutive-failure threshold;
- audio analysis is independently resumable and reusable;
- recording generation changes invalidate stale trim-cache identities;
- frames are sampled densely around recorded mobile actions;
- traces include blackboard status, attempts, failure categories, successful
duration, observation count, and circuit state.
## 21. Implemented production architecture
The completed refactor extends the phase-one blackboard with the following
production controls:
1. Failures are normalized as rate limit, provider outage, timeout,
connection, authentication, bad request, media processing, cancellation,
or unknown. The class determines retryability, bisection, and fallback.
2. Timeout, media, oversized, and unknown failures can recursively bisect a
chunk down to configured size/depth bounds. Provider outages, connections,
and rate limits do not bisect because fan-out would amplify load.
3. Coordinator, video child, and audio child use the configured utility-model
fallback. A context-scoped circuit breaker bypasses an unhealthy primary.
4. Recording results expose `video_id`, `generation`, `sealed_until`, and
`source_revision`. Trim caches are scoped by video UUID and generation;
open-ended requests are resolved to numeric safe ends.
5. Universal video analysis receives sparse baseline frames, dense frames
around DataEngine mobile-action timestamps, and the real audio track.
Audio-only analysis sends a standard base64 audio content block. Models
that reject audio degrade only that video chunk to visual-only analysis.
6. Audio uses the same lease, exact reuse, completion, and failure state
machine as video.
7. DataEngine session deletion removes blackboard rows by session/video key;
evidence is deleted with the trace directory. Universal-path temporary
artifacts are cleaned on early exit.
The persistence schema adds `error_category`, `model_name`, and
`source_generation` to every segment attempt. Migrations are additive and
safe for existing SQLite traces.
## 22. State-of-the-art review and adoption decision
The relevant frontier is not “send the full video to a bigger model.” Recent
systems converge on four architectural ideas:
1. **Agentic coarse-to-fine localization.** [VideoAgent](https://arxiv.org/abs/2403.10517)
iteratively retrieves a small number of frames; [VideoTree](https://arxiv.org/abs/2405.19209)
builds a query-adaptive temporal hierarchy.
2. **Persistent hierarchical multimodal memory.** The memory-augmented
[VideoAgent](https://arxiv.org/abs/2403.11481) separates memory construction
and query-time tool use. [VideoARM](https://arxiv.org/abs/2512.12360) uses an
observe-think-act-memorize loop over hierarchical memory.
3. **Adaptive redundancy reduction.** [LongVU](https://arxiv.org/abs/2410.17434)
removes redundant frames and preserves query-relevant temporal/spatial
detail instead of sampling uniformly.
4. **Retrieval over very long media.** [VideoRAG](https://arxiv.org/abs/2502.01549)
combines graph/text grounding with multimodal retrieval. This is useful for
many-hour corpora, but heavier than one mobile-test recording.
Provider guidance supports the same separation: Gemini's
[video-understanding documentation](https://ai.google.dev/gemini-api/docs/video-understanding)
describes direct audio/video processing and reusable Files API inputs, while
[context caching](https://ai.google.dev/gemini-api/docs/caching) is an
optimization for repeated common input. Neither is a durable work ledger.
### 22.1 Adopted directly
- VideoARM/VideoAgent's observe → reason → act → memorize loop;
- query-scoped coarse-to-fine temporal analysis;
- separation of objective observations and answered query spans;
- adaptive sampling, with mobile action timestamps as the domain-specific
relevance signal;
- independent visual/audio tools and evidence-grounded synthesis.
### 22.2 Not copied wholesale
The research implementations assume offline benchmark videos, often require
GPU feature extractors, and optimize multiple-choice VideoQA. ARTEMIS operates
on a growing recorder stream, must survive provider/process failures, prevent
duplicate paid calls, and tie evidence to test-relative action time. Importing
VideoARM or VideoRAG would add a second orchestration runtime without solving
leases, idempotent commit, active-recording identity, retention, or mobile
alignment.
ARTEMIS therefore ports their architectural invariants, not their
benchmark-specific code, and supplies the production controls the prototypes
do not provide.
## 23. Runtime configuration
All controls live under `video_analyzer` and flow through the SDK builder:
| Setting | Default | Purpose |
| --- | ---: | --- |
| `chunk_size_seconds` | 60 | Initial coarse chunk size |
| `min_chunk_seconds` | 4 | Smallest bisection leaf |
| `max_split_depth` | 4 | Bound failure-driven fan-out |
| `circuit_breaker_threshold` | 3 | Failures before primary circuit opens |
| `circuit_breaker_cooldown_seconds` | 60 | Half-open delay |
| `action_window_seconds` | 2 | Dense window on either side of an action |
| `dense_action_fps` | 4 | Sampling rate in action windows |
| `max_dense_action_frames` | 24 | Dense-frame budget per chunk |
| `native_max_retries` | 1 | Native-provider retries before universal fallback |
| `model_call_timeout_seconds` | 120 | Hard bound for one model response |
The configured `llm_config.utils.video_analyzer.fallback` supplies the model
fallback. Public video tool signatures remain unchanged.
## 24. Verification status
Local verification covers persistence, exact reuse, leases, partial commit,
structured failures, circuit breaking, fallback selection, timeout bisection,
audio reuse, real audio content blocks, action-aware sampling, exact frame
seeking on an MP4 fixture, generation-scoped caches, configuration
propagation, and existing analyzer/controller behavior.
### 24.1 Real-device acceptance (2026-08-25)
A connected Pixel 10 (`59260DLCR00360`) completed a deterministic Google
Calculator recording with independently observed UI ground truth:
- `12+34=46` at 21.15s;
- `7×8=56` at 32.44s;
- history opened at 39.01s and closed at 42.12s;
- the final screen returned to result `56`.
Three query-independent video analyses all matched that ground truth. Each
query was then replayed twice with the identical interval and query. All six
replays returned `CACHED VIDEO ANALYSIS` in roughly 3ms, with no new child
analysis. Final blackboard state was three succeeded segments, 127.8 covered
query-seconds, and eight timestamped observations. The machine-readable
report and retained MP4 are produced by
`scripts/real_device_video_acceptance.py`.
The run also exercised real provider degradation: native Gemini returned a
503 and later stalled a stream. The affected query preserved prior successes,
retried only its own uncovered work, then completed through the configured
universal fallback. Based on that trace, native attempts now default to two
total calls with a 120-second response bound before fallback.
Device exploration also exposed two non-video runtime faults that affect
perceived stability and were fixed as part of the gate:
1. UIAutomator may return current string/dictionary bounds with a stale cached
center. Dynamic taps now always derive their center from fresh bounds and
use the cached center only when bounds are unavailable.
2. A stale desktop IPC port previously blocked every DataEngine event for one
second. IPC connects now use a short localhost timeout plus exponential
backoff, and detached agent initialization has a 30-second hard bound.
The acceptance result demonstrates 100% accuracy and reuse for this defined
fixture and run. It is not a mathematical guarantee for every future app,
video, device, or provider outage; production SLOs require a larger repeated
corpus and ongoing monitoring.
+1
View File
@@ -146,6 +146,7 @@ async def run_task(
target_serial = device_serial
try:
from artemis.config import settings
from artemis.sdk import Agent
from artemis.sdk.builders import Builders
from artemis.sdk.types import AgentProfile
+7
View File
@@ -92,6 +92,13 @@ def _reconcile_task_state(
pid = status_data.get("pid")
dirty = False
# "success" is a legacy alias for the canonical "completed" terminal
# status; consumers of this reconcile must only ever see "completed".
if current_status == "success":
current_status = "completed"
status_data["status"] = "completed"
dirty = True
db_status: str | None = None
db_pid: int | None = None
db_path = _find_data_engine_db()
+4 -18
View File
@@ -137,28 +137,14 @@ def _validate_device_serial(device_serial: str) -> dict[str, Any] | None:
"""
try:
from artemis.runtime import device_pool
from artemis.runtime.device_lock import DeviceExecutionLock
norm = DeviceExecutionLock._normalize_device_id
attached = {norm(d.serial): d.state for d in device_pool.list_devices()}
# The shared validator fails open on an indeterminate/empty enumeration:
# the task proceeds and fails downstream with a clear no-device error.
detail = device_pool.validate_explicit_serial(device_serial)
except Exception:
return None
if not attached:
# Enumeration unavailable (no adb / transient failure): let the task
# proceed and fail downstream with a clear no-device error instead.
if detail is None:
return None
dev_state = attached.get(norm(device_serial))
if dev_state == "device":
return None
if dev_state is None:
detail = (
f"Device '{device_serial}' is not connected. "
f"Attached devices: {sorted(attached) if attached else 'none'}."
)
else:
detail = f"Device '{device_serial}' is attached but not ready (state: '{dev_state}')."
return {
"status": "failed",
"error": detail,
+5
View File
@@ -113,6 +113,11 @@ def update_trace_status(
if not data:
return None
# "success" is a legacy alias for the canonical "completed" terminal
# status and must never be persisted into status.json.
if status == "success":
status = "completed"
data["status"] = status
if status in ("completed", "failed", "cancelled"):
data["end_time"] = time.time()
+89
View File
@@ -0,0 +1,89 @@
# Artemis Thin Client Design
## Goals
Developer machines install only a small remote SDK, while the host connected to
Android devices runs the full Artemis stack. The SDK must:
- have no dependency on ADB, device drivers, agents, LLMs, web services, or image libraries;
- never attempt to start a local Artemis daemon;
- remain compatible with the existing server API;
- allow the server to add agent and device capabilities independently; and
- provide explicit capability negotiation and errors as the protocol evolves.
## Boundary
The client owns request modeling, HTTP communication, status polling, error
normalization, and result models. Device discovery, task scheduling, concurrency
control, model selection, execution, logs, videos, and artifact generation all
belong to the server.
The distribution is named `artemis-client`, while the Python import is
`artemis_client`. This prevents it from shadowing the full runtime's top-level
`artemis` package. The full runtime can depend on this package later and
re-export the client from `artemis` for compatibility.
## Phase 1: Compatibility Adapter
The first release calls the current `/api/*` administration endpoints directly.
The client generates a task ID and maps it to the server's existing `session_id`
to make retries idempotent. While a task is queued but not yet visible in the
session database, the client reconstructs its state from `queue` and
`active_tasks` in `/api/status`.
Experimental parameters live in `options`. Older servers ignore unknown fields,
while newer servers can consume them without requiring an immediate SDK release.
## Phase 2: Stable Remote API
The server should expose versioned endpoints dedicated to remote clients:
```text
POST /api/v1/tasks
GET /api/v1/tasks/{task_id}
DELETE /api/v1/tasks/{task_id}
GET /api/v1/tasks/{task_id}/events
GET /api/v1/tasks/{task_id}/artifacts
GET /api/v1/devices
GET /api/v1/capabilities
```
The administration API and remote SDK API should remain separate. The remote API
uses a consistent error envelope:
```json
{
"error": {
"code": "DEVICE_OFFLINE",
"message": "The selected device is offline",
"retryable": false
}
}
```
## Compatibility Rules
1. `/api/v1` may add optional request and response fields, but must not change existing field semantics.
2. The client ignores unknown response fields and preserves the original response in each model's `raw` attribute.
3. Every new request field must have a server-side default.
4. The SDK checks `/api/v1/capabilities` before using a new feature.
5. Breaking changes ship under `/api/v2`; they are never applied in place.
6. The SDK and server follow semantic versioning independently.
## Security
The current daemon exposes administration and credential-related endpoints and
must not listen on a public interface without protection. Use an SSH or VPN tunnel
in the short term. In production, place a reverse proxy in front of the daemon,
expose only the remote API, and enforce HTTPS, bearer-token authentication,
request-size limits, rate limits, and audit logging. Tokens are accepted only
through arguments or environment variables and are never persisted by the SDK.
## Future Work
- SSE event subscriptions with reconnection support;
- artifact manifests and streaming downloads;
- batch task submission;
- server and API version constraints;
- retries limited to network failures and idempotent requests; and
- OpenAPI contract tests to prevent server and SDK drift.
+112
View File
@@ -0,0 +1,112 @@
# Artemis Client
`artemis-client` is the remote Python SDK that runs on a developer machine. It
communicates with the Artemis daemon on a device host over HTTP and contains no
ADB, device-driver, agent, LLM, or image-processing code.
Runtime dependencies: **zero**. HTTP, JSON, and asynchronous adapters use only
the Python standard library.
## Installation
Install from a local checkout:
```powershell
uv add --editable C:\path\to\artemis\packages\artemis-client
```
Install from the Git repository subdirectory:
```powershell
uv add "artemis-client @ git+https://github.com/google/artemis.git#subdirectory=packages/artemis-client"
```
## Connect to a Device Host
The current Artemis daemon administration API does not include built-in remote
authentication. Do not expose its port directly to the public internet. During
the initial rollout, connect through an SSH tunnel:
```powershell
ssh -L 8000:127.0.0.1:8000 user@artemis-host
```
Then connect the SDK to the local tunnel endpoint:
```python
from artemis_client import ArtemisClient
client = ArtemisClient("http://127.0.0.1:8000")
```
For production, place a reverse proxy in front of the daemon that enforces HTTPS
and bearer-token authentication and exposes only task-related APIs.
## Usage
```python
import asyncio
import os
from artemis_client import ArtemisClient
async def main() -> None:
client = ArtemisClient(
os.environ["ARTEMIS_BASE_URL"],
token=os.environ.get("ARTEMIS_TOKEN"),
)
handle = await client.submit(
"Open the Battery page in Settings and verify the battery percentage",
profile="flash",
device_serial="emulator-5554",
)
print("queued:", handle.task_id)
result = await client.wait_for_task(handle.task_id, timeout=600)
if not result.succeeded:
raise RuntimeError(result.error or result.status)
asyncio.run(main())
```
You can also submit a task and wait for it in one call:
```python
result = await client.run("Open Android Settings and check the Wi-Fi status")
```
Main methods:
- `health()` quickly checks whether the server and scheduler are alive.
- `readiness()` runs full device and toolchain diagnostics and may take longer.
- `capabilities()` negotiates API features and falls back for older servers.
- `list_devices()` lists devices available on the host.
- `submit()` submits an idempotent task and returns immediately.
- `get_task()` retrieves a task.
- `wait_for_task()` waits for a terminal state.
- `run()` submits a task and waits for completion.
- `stop()` stops one task.
By default, `submit()` generates a UUID on the client and sends it as the current
server's `session_id`. Network retries reuse the same `task_id`, preventing the
server from creating duplicate tasks.
## Compatibility
Version 0.1 supports these current Artemis daemon endpoints:
- `POST /api/run`
- `GET /api/sessions/{session_id}`
- `GET /api/status`
- `POST /api/stop`
- `GET /api/devices`
- `GET /api/system/readiness`
Capability discovery first attempts the future `GET /api/v1/capabilities`
endpoint. If an older server returns 404, the SDK uses its known baseline
capability set.
See [DESIGN.md](DESIGN.md) for the architecture and protocol evolution plan.
+30
View File
@@ -0,0 +1,30 @@
"""Run one task against an Artemis host."""
import asyncio
import os
from artemis_client import ArtemisClient
async def main() -> None:
client = ArtemisClient(
os.environ.get("ARTEMIS_BASE_URL", "http://127.0.0.1:8000"),
token=os.environ.get("ARTEMIS_TOKEN"),
)
devices = await client.list_devices()
if not devices:
raise RuntimeError("The Artemis host did not report an Android device")
result = await client.run(
"Open the Battery page in Android Settings and verify the battery percentage.",
profile="flash",
device_serial=devices[0].serial,
)
if not result.succeeded:
raise RuntimeError(result.error or f"Task ended with status {result.status}")
print(f"Task {result.task_id} completed on {result.device_serial}")
if __name__ == "__main__":
asyncio.run(main())
+36
View File
@@ -0,0 +1,36 @@
[build-system]
requires = ["setuptools>=68"]
build-backend = "setuptools.build_meta"
[project]
name = "artemis-client"
version = "0.1.0"
description = "Zero-runtime-dependency Python client for a remote Artemis host"
readme = "README.md"
requires-python = ">=3.10"
license = "Apache-2.0"
authors = [
{ name = "Artemis contributors" },
]
dependencies = []
classifiers = [
"Development Status :: 3 - Alpha",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3 :: Only",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
]
[tool.setuptools.packages.find]
where = ["src"]
[tool.setuptools.package-data]
artemis_client = ["py.typed"]
[tool.pyright]
include = ["src", "tests", "examples"]
extraPaths = ["src"]
pythonVersion = "3.10"
typeCheckingMode = "standard"
@@ -0,0 +1,45 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
"""Public API for the lightweight Artemis remote client."""
from artemis_client.client import ArtemisClient
from artemis_client.errors import (
ApiError,
ArtemisClientError,
AuthenticationError,
ConflictError,
NetworkError,
NotFoundError,
ProtocolError,
TaskRejectedError,
TaskTimeoutError,
)
from artemis_client.models import Capabilities, Device, TaskHandle, TaskResult
from artemis_client.transport import JsonTransport
__version__ = "0.1.0"
__all__ = [
"ApiError",
"ArtemisClient",
"ArtemisClientError",
"AuthenticationError",
"Capabilities",
"ConflictError",
"Device",
"JsonTransport",
"NetworkError",
"NotFoundError",
"ProtocolError",
"TaskHandle",
"TaskRejectedError",
"TaskResult",
"TaskTimeoutError",
"__version__",
]
@@ -0,0 +1,365 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
"""Asynchronous, remote-only Artemis client."""
from __future__ import annotations
import asyncio
import os
import time
import uuid
from collections.abc import Mapping
from typing import Any, Literal, Protocol
from artemis_client.errors import NotFoundError, ProtocolError, TaskRejectedError, TaskTimeoutError
from artemis_client.models import Capabilities, Device, TaskHandle, TaskResult
from artemis_client.transport import JsonTransport
class _Transport(Protocol):
def request(
self,
method: str,
path: str,
*,
json_body: Mapping[str, Any] | None = None,
) -> Any: ...
class ArtemisClient:
"""Thin client for an Artemis daemon running on another host.
This class never imports ADB, an Artemis agent, an LLM provider, or a
device driver. All execution is delegated to ``base_url``.
"""
_LEGACY_FEATURES = frozenset(
{
"tasks.submit",
"tasks.get",
"tasks.stop",
"devices.list",
"system.readiness",
}
)
def __init__(
self,
base_url: str | None = None,
*,
token: str | None = None,
request_timeout: float = 30.0,
poll_interval: float = 1.0,
headers: Mapping[str, str] | None = None,
transport: _Transport | None = None,
device_id: str | None = None,
device_serial: str | None = None,
default_profile: Literal["flash", "pro"] = "flash",
concurrency_mode: str = "per_device",
max_concurrency: int | None = None,
standalone: bool = False,
) -> None:
if poll_interval <= 0:
raise ValueError("poll_interval must be greater than zero")
resolved_base_url = base_url or os.environ.get("ARTEMIS_BASE_URL")
if not resolved_base_url:
daemon_host = os.environ.get("ARTEMIS_DAEMON_HOST", "127.0.0.1")
daemon_port = os.environ.get("ARTEMIS_DAEMON_PORT", "8000")
resolved_base_url = f"http://{daemon_host}:{daemon_port}"
self.base_url = resolved_base_url.rstrip("/")
self.poll_interval = float(poll_interval)
self._device_serial = device_serial or device_id
self.default_profile = default_profile
self.concurrency_mode = str(concurrency_mode).strip().lower()
self.max_concurrency = max_concurrency
self.standalone = standalone
self._transport = transport or JsonTransport(
resolved_base_url,
token=token,
timeout=request_timeout,
headers=headers,
)
@property
def device_id(self) -> str | None:
"""Default remote device serial (legacy-compatible alias)."""
return self._device_serial
@device_id.setter
def device_id(self, value: str | None) -> None:
self._device_serial = value
@property
def device_serial(self) -> str | None:
return self._device_serial
@device_serial.setter
def device_serial(self, value: str | None) -> None:
self._device_serial = value
def set_device(self, device_serial: str) -> ArtemisClient:
"""Set the default remote device and return this client."""
self._device_serial = device_serial
return self
def set_concurrency_mode(self, mode: str) -> ArtemisClient:
"""Retain the legacy setting for callers migrating to server-side queues."""
self.concurrency_mode = str(mode).strip().lower()
return self
async def health(self) -> Mapping[str, Any]:
"""Perform a fast liveness check against the scheduler status API."""
payload = await self._request("GET", "/api/status")
return self._mapping(payload, endpoint="/api/status")
async def readiness(self) -> Mapping[str, Any]:
"""Run the host's full device and toolchain readiness diagnostics.
This endpoint can take substantially longer than :meth:`health`; set
a larger ``request_timeout`` when constructing the client if needed.
"""
payload = await self._request("GET", "/api/system/readiness")
return self._mapping(payload, endpoint="/api/system/readiness")
async def capabilities(self) -> Capabilities:
"""Discover server features, with a baseline for today's legacy API."""
try:
payload = await self._request("GET", "/api/v1/capabilities")
except NotFoundError:
return Capabilities(api_version="legacy", features=self._LEGACY_FEATURES)
return Capabilities.from_payload(self._mapping(payload, endpoint="/api/v1/capabilities"))
async def list_devices(self) -> tuple[Device, ...]:
"""List Android devices visible to the remote Artemis host."""
payload = await self._request("GET", "/api/devices")
if isinstance(payload, Mapping):
raw_devices = payload.get("devices", [])
else:
raw_devices = payload
if not isinstance(raw_devices, list):
raise ProtocolError("/api/devices response must contain a device list")
devices: list[Device] = []
for item in raw_devices:
if not isinstance(item, Mapping):
raise ProtocolError("/api/devices contained a non-object device entry")
devices.append(Device.from_payload(item))
return tuple(devices)
async def submit(
self,
goal: str,
*,
profile: Literal["flash", "pro"] | None = None,
device_serial: str | None = None,
expected_output: str | None = None,
enable_outputter: bool | None = None,
locked_app_package: str | None = None,
app_path: str | None = None,
conversation_id: str | None = None,
task_id: str | None = None,
options: Mapping[str, Any] | None = None,
) -> TaskHandle:
"""Submit one task and return immediately after scheduler admission.
``task_id`` is generated client-side and sent as the legacy
``session_id``. Reusing it makes submission retries idempotent.
Experimental, forward-compatible fields belong in ``options``.
"""
normalized_goal = goal.strip()
if not normalized_goal:
raise ValueError("goal must not be empty")
if task_id is None:
resolved_task_id = str(uuid.uuid4())
else:
try:
resolved_task_id = str(uuid.UUID(task_id))
except (AttributeError, TypeError, ValueError) as exc:
raise ValueError("task_id must be a valid UUID string") from exc
resolved_profile = profile or self.default_profile
resolved_device = device_serial or self.device_serial
payload: dict[str, Any] = {
"goal": normalized_goal,
"profile": resolved_profile,
"session_id": resolved_task_id,
"ingress": "python_sdk",
}
optional_values = {
"device_serial": resolved_device,
"expected_output": expected_output,
"enable_outputter": enable_outputter,
"locked_app_package": locked_app_package,
"app_path": app_path,
"conversation_id": conversation_id,
"options": dict(options) if options is not None else None,
}
payload.update({key: value for key, value in optional_values.items() if value is not None})
response = self._mapping(
await self._request("POST", "/api/run", json_body=payload),
endpoint="/api/run",
)
status = str(response.get("status") or "unknown").lower()
tasks = response.get("tasks")
if status == "rejected" or (isinstance(tasks, list) and not tasks):
detail = str(response.get("error") or "Artemis host rejected the task")
raise TaskRejectedError(detail)
if not isinstance(tasks, list) or not tasks or not isinstance(tasks[0], Mapping):
raise ProtocolError("/api/run response did not contain an admitted task")
task_payload = dict(tasks[0])
task_payload.setdefault("session_id", resolved_task_id)
task_payload.setdefault("status", status)
return TaskHandle.from_payload(task_payload)
async def get_task(self, task_id: str) -> TaskResult:
"""Get a task from session storage or the live scheduler queue."""
try:
payload = await self._request("GET", f"/api/sessions/{task_id}")
return TaskResult.from_payload(
self._mapping(payload, endpoint=f"/api/sessions/{task_id}"),
task_id=task_id,
)
except NotFoundError:
status_payload = self._mapping(
await self._request("GET", "/api/status"),
endpoint="/api/status",
)
live_task = self._find_live_task(status_payload, task_id)
if live_task is None:
return TaskResult(task_id=task_id, status="launching")
return TaskResult.from_payload(live_task, task_id=task_id)
async def wait_for_task(
self,
task_id: str,
*,
timeout: float = 1800.0,
poll_interval: float | None = None,
) -> TaskResult:
"""Wait until a task reaches a terminal state."""
if timeout <= 0:
raise ValueError("timeout must be greater than zero")
interval = self.poll_interval if poll_interval is None else float(poll_interval)
if interval <= 0:
raise ValueError("poll_interval must be greater than zero")
started = time.monotonic()
while True:
result = await self.get_task(task_id)
if result.done:
return result
elapsed = time.monotonic() - started
if elapsed >= timeout:
raise TaskTimeoutError(task_id, timeout)
await asyncio.sleep(min(interval, max(0.0, timeout - elapsed)))
async def run(
self,
goal: str,
*,
profile: Literal["flash", "pro"] | None = None,
device_serial: str | None = None,
expected_output: str | None = None,
enable_outputter: bool | None = None,
locked_app_package: str | None = None,
app_path: str | None = None,
conversation_id: str | None = None,
task_id: str | None = None,
options: Mapping[str, Any] | None = None,
timeout: float = 1800.0,
poll_interval: float | None = None,
) -> TaskResult:
"""Submit a task and wait for its terminal result."""
handle = await self.submit(
goal,
profile=profile,
device_serial=device_serial,
expected_output=expected_output,
enable_outputter=enable_outputter,
locked_app_package=locked_app_package,
app_path=app_path,
conversation_id=conversation_id,
task_id=task_id,
options=options,
)
return await self.wait_for_task(
handle.task_id,
timeout=timeout,
poll_interval=poll_interval,
)
async def run_task(self, task: Any, **overrides: Any) -> TaskResult:
"""Run a task-like object while delegating all execution remotely."""
goal = getattr(task, "goal", None)
if not isinstance(goal, str) or not goal.strip():
raise ValueError("task must expose a non-empty goal attribute")
values: dict[str, Any] = {
"profile": getattr(task, "profile", None),
"device_serial": getattr(task, "device_serial", None)
or getattr(task, "device_id", None),
"locked_app_package": getattr(task, "locked_package", None),
}
values.update(overrides)
return await self.run(goal, **values)
async def stop(self, task_id: str) -> bool:
"""Request cancellation of one remote task."""
payload = self._mapping(
await self._request(
"POST",
"/api/stop",
json_body={"session_id": task_id},
),
endpoint="/api/stop",
)
return str(payload.get("status") or "").lower() == "stopped"
async def _request(
self,
method: str,
path: str,
*,
json_body: Mapping[str, Any] | None = None,
) -> Any:
return await asyncio.to_thread(
self._transport.request,
method,
path,
json_body=json_body,
)
@staticmethod
def _mapping(payload: Any, *, endpoint: str) -> Mapping[str, Any]:
if not isinstance(payload, Mapping):
raise ProtocolError(f"{endpoint} response must be a JSON object")
return payload
@staticmethod
def _find_live_task(
scheduler: Mapping[str, Any],
task_id: str,
) -> Mapping[str, Any] | None:
for collection_name in ("queue", "active_tasks"):
collection = scheduler.get(collection_name) or []
if not isinstance(collection, list):
continue
for item in collection:
if not isinstance(item, Mapping):
continue
item_id = item.get("task_id") or item.get("session_id")
if str(item_id) == task_id:
return item
active_id = scheduler.get("task_id") or scheduler.get("session_id")
if active_id is not None and str(active_id) == task_id:
synthesized = dict(scheduler)
synthesized.setdefault("session_id", task_id)
return synthesized
return None
@@ -0,0 +1,66 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
"""Exceptions raised by the Artemis remote client."""
from __future__ import annotations
from typing import Any
class ArtemisClientError(Exception):
"""Base class for all errors raised by this package."""
class NetworkError(ArtemisClientError):
"""The Artemis host could not be reached or the connection failed."""
class ProtocolError(ArtemisClientError):
"""The server returned a response that does not satisfy the API contract."""
class ApiError(ArtemisClientError):
"""The Artemis host returned an unsuccessful HTTP response."""
def __init__(
self,
status_code: int,
detail: str,
*,
payload: Any = None,
) -> None:
self.status_code = status_code
self.detail = detail
self.payload = payload
super().__init__(f"Artemis API returned HTTP {status_code}: {detail}")
class AuthenticationError(ApiError):
"""Authentication or authorization failed."""
class NotFoundError(ApiError):
"""The requested Artemis resource does not exist."""
class ConflictError(ApiError):
"""The request conflicts with the current server or device state."""
class TaskRejectedError(ArtemisClientError):
"""The server accepted the request but rejected task admission."""
class TaskTimeoutError(ArtemisClientError):
"""Waiting for an Artemis task exceeded the caller's deadline."""
def __init__(self, task_id: str, timeout: float) -> None:
self.task_id = task_id
self.timeout = timeout
super().__init__(f"Task {task_id} did not finish within {timeout:g} seconds")
@@ -0,0 +1,212 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
"""Dependency-free data models for the Artemis remote API."""
from __future__ import annotations
import json
from dataclasses import dataclass, field
from typing import Any, Mapping
TERMINAL_TASK_STATUSES = frozenset(
{"completed", "success", "failed", "cancelled", "canceled", "rejected"}
)
SUCCESS_TASK_STATUSES = frozenset({"completed", "success"})
def _string(value: Any) -> str | None:
if value is None:
return None
text = str(value).strip()
return text or None
def _integer(value: Any) -> int | None:
if value is None:
return None
try:
return int(value)
except (TypeError, ValueError):
return None
def _device_from_payload(payload: Mapping[str, Any]) -> str | None:
direct = _string(payload.get("device_serial") or payload.get("device_id"))
if direct:
return direct
device_info = payload.get("device_info")
if isinstance(device_info, str):
try:
device_info = json.loads(device_info)
except (TypeError, ValueError, json.JSONDecodeError):
return None
if isinstance(device_info, Mapping):
return _string(device_info.get("device_serial") or device_info.get("device_id"))
return None
@dataclass(frozen=True, slots=True)
class TaskHandle:
"""A task accepted by the remote Artemis scheduler."""
task_id: str
status: str
device_serial: str | None = None
raw: Mapping[str, Any] = field(default_factory=dict, repr=False, compare=False)
@property
def session_id(self) -> str:
"""Compatibility alias for servers that call a task a session."""
return self.task_id
@classmethod
def from_payload(cls, payload: Mapping[str, Any]) -> TaskHandle:
task_id = _string(payload.get("task_id") or payload.get("session_id") or payload.get("id"))
if not task_id:
from artemis_client.errors import ProtocolError
raise ProtocolError("Task admission response did not contain a task/session ID")
return cls(
task_id=task_id,
status=(_string(payload.get("status")) or "queued").lower(),
device_serial=_device_from_payload(payload),
raw=dict(payload),
)
@dataclass(frozen=True, slots=True)
class TaskResult:
"""Current or terminal state of a remote Artemis task."""
task_id: str
status: str
goal: str | None = None
profile: str | None = None
device_serial: str | None = None
output: Any = None
error: str | None = None
turns: int | None = None
raw: Mapping[str, Any] = field(default_factory=dict, repr=False, compare=False)
@property
def session_id(self) -> str:
return self.task_id
@property
def trace_id(self) -> str:
return self.task_id
@property
def done(self) -> bool:
return self.status in TERMINAL_TASK_STATUSES
@property
def succeeded(self) -> bool:
return self.status in SUCCESS_TASK_STATUSES
@classmethod
def from_payload(
cls,
payload: Mapping[str, Any],
*,
task_id: str | None = None,
) -> TaskResult:
resolved_id = _string(
payload.get("task_id")
or payload.get("session_id")
or payload.get("trace_id")
or payload.get("id")
or task_id
)
if not resolved_id:
from artemis_client.errors import ProtocolError
raise ProtocolError("Task response did not contain a task/session ID")
output = payload.get("output")
if output is None:
output = payload.get("result")
if output is None:
output = payload.get("summary")
return cls(
task_id=resolved_id,
status=(_string(payload.get("status")) or "unknown").lower(),
goal=_string(payload.get("goal") or payload.get("initial_goal")),
profile=_string(payload.get("profile")),
device_serial=_device_from_payload(payload),
output=output,
error=_string(payload.get("error") or payload.get("error_message")),
turns=_integer(payload.get("turns") or payload.get("current_turn")),
raw=dict(payload),
)
@dataclass(frozen=True, slots=True)
class Device:
"""A device reported by the remote Artemis host."""
serial: str
state: str
model: str | None = None
product: str | None = None
busy: bool = False
raw: Mapping[str, Any] = field(default_factory=dict, repr=False, compare=False)
@classmethod
def from_payload(cls, payload: Mapping[str, Any]) -> Device:
serial = _string(
payload.get("serial") or payload.get("device_serial") or payload.get("device_id")
)
if not serial:
from artemis_client.errors import ProtocolError
raise ProtocolError("Device response did not contain a serial number")
state = (_string(payload.get("state") or payload.get("status")) or "unknown").lower()
busy_value = payload.get("busy")
if busy_value is None:
busy_value = payload.get("is_busy")
return cls(
serial=serial,
state=state,
model=_string(payload.get("model")),
product=_string(payload.get("product")),
busy=bool(busy_value) or state in {"busy", "running", "locked"},
raw=dict(payload),
)
@dataclass(frozen=True, slots=True)
class Capabilities:
"""Features advertised by an Artemis host."""
api_version: str
features: frozenset[str]
server_version: str | None = None
raw: Mapping[str, Any] = field(default_factory=dict, repr=False, compare=False)
def supports(self, feature: str) -> bool:
return feature in self.features
@classmethod
def from_payload(cls, payload: Mapping[str, Any]) -> Capabilities:
raw_features = payload.get("features") or []
if isinstance(raw_features, Mapping):
features = frozenset(str(key) for key, enabled in raw_features.items() if enabled)
elif isinstance(raw_features, (list, tuple, set, frozenset)):
features = frozenset(str(item) for item in raw_features)
else:
features = frozenset()
return cls(
api_version=_string(payload.get("api_version")) or "unknown",
server_version=_string(payload.get("server_version") or payload.get("version")),
features=features,
raw=dict(payload),
)
@@ -0,0 +1 @@
@@ -0,0 +1,134 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
"""Small JSON-over-HTTP transport implemented with the Python standard library."""
from __future__ import annotations
import json
import ssl
import urllib.error
import urllib.request
from typing import Any, Mapping
from urllib.parse import urlsplit
from artemis_client.errors import (
ApiError,
AuthenticationError,
ConflictError,
NetworkError,
NotFoundError,
ProtocolError,
)
class JsonTransport:
"""Synchronous transport used internally by the async client.
Calls are moved to worker threads by :class:`ArtemisClient`, keeping the
public API asynchronous without adding an HTTP dependency.
"""
def __init__(
self,
base_url: str,
*,
token: str | None = None,
timeout: float = 30.0,
headers: Mapping[str, str] | None = None,
ssl_context: ssl.SSLContext | None = None,
) -> None:
normalized = base_url.rstrip("/")
parsed = urlsplit(normalized)
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
raise ValueError("base_url must be an absolute http:// or https:// URL")
if timeout <= 0:
raise ValueError("timeout must be greater than zero")
self.base_url = normalized
self.timeout = float(timeout)
self.ssl_context = ssl_context
self.headers = {
"Accept": "application/json",
"User-Agent": "artemis-client/0.1",
**dict(headers or {}),
}
if token:
self.headers["Authorization"] = f"Bearer {token}"
def request(
self,
method: str,
path: str,
*,
json_body: Mapping[str, Any] | None = None,
) -> Any:
if not path.startswith("/"):
path = f"/{path}"
url = f"{self.base_url}{path}"
body = None
headers = dict(self.headers)
if json_body is not None:
body = json.dumps(json_body, separators=(",", ":")).encode("utf-8")
headers["Content-Type"] = "application/json"
request = urllib.request.Request(
url,
data=body,
headers=headers,
method=method.upper(),
)
try:
with urllib.request.urlopen(
request,
timeout=self.timeout,
context=self.ssl_context,
) as response:
data = response.read()
if not data:
return None
try:
return json.loads(data.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise ProtocolError(f"Artemis host returned invalid JSON from {path}") from exc
except urllib.error.HTTPError as exc:
payload = self._decode_error_payload(exc)
detail = self._error_detail(payload, exc.reason)
error_type = {
401: AuthenticationError,
403: AuthenticationError,
404: NotFoundError,
409: ConflictError,
}.get(exc.code, ApiError)
raise error_type(exc.code, detail, payload=payload) from exc
except (urllib.error.URLError, TimeoutError, OSError) as exc:
reason = getattr(exc, "reason", exc)
raise NetworkError(f"Could not reach Artemis host at {url}: {reason}") from exc
@staticmethod
def _decode_error_payload(error: urllib.error.HTTPError) -> Any:
try:
data = error.read()
except OSError:
return None
if not data:
return None
try:
return json.loads(data.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError):
return data.decode("utf-8", errors="replace")
@staticmethod
def _error_detail(payload: Any, fallback: Any) -> str:
if isinstance(payload, Mapping):
detail = payload.get("detail") or payload.get("error") or payload.get("message")
if detail is not None:
return str(detail)
if isinstance(payload, str) and payload.strip():
return payload.strip()
return str(fallback or "request failed")
@@ -0,0 +1,246 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
from __future__ import annotations
import unittest
from collections import defaultdict
from collections.abc import Mapping
from typing import Any
from artemis_client import (
ArtemisClient,
NotFoundError,
ProtocolError,
TaskRejectedError,
TaskTimeoutError,
)
class FakeTransport:
def __init__(self) -> None:
self.responses: dict[tuple[str, str], list[Any]] = defaultdict(list)
self.calls: list[tuple[str, str, Mapping[str, Any] | None]] = []
def add(self, method: str, path: str, *responses: Any) -> None:
self.responses[(method, path)].extend(responses)
def request(
self,
method: str,
path: str,
*,
json_body: Mapping[str, Any] | None = None,
) -> Any:
self.calls.append((method, path, json_body))
queued = self.responses[(method, path)]
if not queued:
raise AssertionError(f"Unexpected request: {method} {path}")
response = queued.pop(0)
if isinstance(response, BaseException):
raise response
return response
class ArtemisClientTests(unittest.IsolatedAsyncioTestCase):
def setUp(self) -> None:
self.transport = FakeTransport()
self.client = ArtemisClient(
"https://artemis.example.test",
poll_interval=0.001,
transport=self.transport,
)
async def test_submit_sends_idempotent_session_id(self) -> None:
task_id = "00000000-0000-4000-8000-000000000123"
self.transport.add(
"POST",
"/api/run",
{
"status": "started",
"tasks": [
{
"session_id": task_id,
"status": "pending",
"device_serial": "pixel-8",
}
],
},
)
handle = await self.client.submit(
"Open Settings",
task_id=task_id,
device_serial="pixel-8",
options={"record_video": True},
)
self.assertEqual(handle.task_id, task_id)
self.assertEqual(handle.device_serial, "pixel-8")
body = self.transport.calls[0][2]
assert body is not None
self.assertEqual(body["session_id"], task_id)
self.assertEqual(body["ingress"], "python_sdk")
self.assertEqual(body["options"], {"record_video": True})
async def test_submit_rejected_task_raises_specific_error(self) -> None:
self.transport.add(
"POST",
"/api/run",
{
"status": "rejected",
"error": "Device is offline",
"tasks": [],
},
)
with self.assertRaisesRegex(TaskRejectedError, "Device is offline"):
await self.client.submit("Open Settings")
async def test_submit_rejects_non_uuid_task_id(self) -> None:
with self.assertRaisesRegex(ValueError, "valid UUID"):
await self.client.submit("Open Settings", task_id="not-a-uuid")
async def test_run_finds_queued_task_then_reads_terminal_session(self) -> None:
task_id = "00000000-0000-4000-8000-000000000124"
self.transport.add(
"POST",
"/api/run",
{
"status": "started",
"tasks": [{"session_id": task_id, "status": "pending"}],
},
)
self.transport.add(
"GET",
f"/api/sessions/{task_id}",
NotFoundError(404, "not created yet"),
{
"session_id": task_id,
"status": "completed",
"goal": "Open Settings",
"current_turn": 4,
"summary": "Battery page opened",
},
)
self.transport.add(
"GET",
"/api/status",
{
"status": "running",
"queue": [{"session_id": task_id, "status": "pending"}],
},
)
result = await self.client.run("Open Settings", task_id=task_id, timeout=1)
self.assertTrue(result.done)
self.assertTrue(result.succeeded)
self.assertEqual(result.turns, 4)
self.assertEqual(result.output, "Battery page opened")
async def test_get_task_returns_launching_when_not_visible_yet(self) -> None:
self.transport.add(
"GET",
"/api/sessions/new-task",
NotFoundError(404, "missing"),
)
self.transport.add("GET", "/api/status", {"status": "idle", "queue": []})
result = await self.client.get_task("new-task")
self.assertEqual(result.status, "launching")
self.assertFalse(result.done)
async def test_wait_for_task_times_out(self) -> None:
class LaunchingClient(ArtemisClient):
async def get_task(self, task_id: str): # type: ignore[override]
from artemis_client import TaskResult
return TaskResult(task_id=task_id, status="running")
client = LaunchingClient(
"https://artemis.example.test",
poll_interval=0.001,
transport=self.transport,
)
with self.assertRaises(TaskTimeoutError):
await client.wait_for_task("slow-task", timeout=0.003)
async def test_list_devices_accepts_legacy_shape(self) -> None:
self.transport.add(
"GET",
"/api/devices",
{
"devices": [
{
"serial": "emulator-5554",
"state": "device",
"model": "Pixel_8",
"busy": True,
}
]
},
)
devices = await self.client.list_devices()
self.assertEqual(len(devices), 1)
self.assertEqual(devices[0].serial, "emulator-5554")
self.assertTrue(devices[0].busy)
async def test_capabilities_falls_back_for_legacy_server(self) -> None:
self.transport.add(
"GET",
"/api/v1/capabilities",
NotFoundError(404, "not implemented"),
)
capabilities = await self.client.capabilities()
self.assertEqual(capabilities.api_version, "legacy")
self.assertTrue(capabilities.supports("tasks.submit"))
async def test_health_uses_fast_scheduler_endpoint(self) -> None:
self.transport.add("GET", "/api/status", {"status": "idle"})
health = await self.client.health()
self.assertEqual(health["status"], "idle")
async def test_readiness_uses_full_diagnostics_endpoint(self) -> None:
self.transport.add(
"GET",
"/api/system/readiness",
{"overall_status": "ready"},
)
readiness = await self.client.readiness()
self.assertEqual(readiness["overall_status"], "ready")
async def test_invalid_devices_payload_is_rejected(self) -> None:
self.transport.add("GET", "/api/devices", {"devices": "not-a-list"})
with self.assertRaises(ProtocolError):
await self.client.list_devices()
async def test_stop_targets_one_session(self) -> None:
self.transport.add("POST", "/api/stop", {"status": "stopped"})
stopped = await self.client.stop("task-stop")
self.assertTrue(stopped)
self.assertEqual(
self.transport.calls[0],
("POST", "/api/stop", {"session_id": "task-stop"}),
)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,85 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
from __future__ import annotations
import io
import json
import unittest
import urllib.error
from email.message import Message
from unittest.mock import patch
from artemis_client import AuthenticationError, JsonTransport, NetworkError, ProtocolError
class FakeResponse:
def __init__(self, payload: bytes) -> None:
self.payload = payload
def __enter__(self) -> FakeResponse:
return self
def __exit__(self, *args: object) -> None:
return None
def read(self) -> bytes:
return self.payload
class JsonTransportTests(unittest.TestCase):
def test_rejects_non_http_base_url(self) -> None:
with self.assertRaises(ValueError):
JsonTransport("file:///tmp/artemis")
@patch("urllib.request.urlopen")
def test_sends_json_and_bearer_token(self, urlopen) -> None:
urlopen.return_value = FakeResponse(json.dumps({"ok": True}).encode())
transport = JsonTransport("https://host.example/", token="secret")
result = transport.request("POST", "/api/run", json_body={"goal": "test"})
self.assertEqual(result, {"ok": True})
request = urlopen.call_args.args[0]
self.assertEqual(request.full_url, "https://host.example/api/run")
self.assertEqual(request.headers["Authorization"], "Bearer secret")
self.assertEqual(json.loads(request.data), {"goal": "test"})
@patch("urllib.request.urlopen")
def test_maps_unauthorized_response(self, urlopen) -> None:
urlopen.side_effect = urllib.error.HTTPError(
"https://host.example/api/run",
401,
"Unauthorized",
Message(),
io.BytesIO(b'{"detail":"bad token"}'),
)
transport = JsonTransport("https://host.example")
with self.assertRaisesRegex(AuthenticationError, "bad token"):
transport.request("GET", "/api/run")
@patch("urllib.request.urlopen")
def test_invalid_json_raises_protocol_error(self, urlopen) -> None:
urlopen.return_value = FakeResponse(b"not-json")
transport = JsonTransport("https://host.example")
with self.assertRaises(ProtocolError):
transport.request("GET", "/api/status")
@patch("urllib.request.urlopen")
def test_network_failure_is_wrapped(self, urlopen) -> None:
urlopen.side_effect = urllib.error.URLError("connection refused")
transport = JsonTransport("https://host.example")
with self.assertRaisesRegex(NetworkError, "connection refused"):
transport.request("GET", "/api/status")
if __name__ == "__main__":
unittest.main()
+8
View File
@@ -24,6 +24,8 @@ authors = [
requires-python = ">=3.12"
dependencies = [
# Remote Python SDK (kept as a zero-dependency workspace package)
"artemis-client>=0.1.0",
# Orchestration & Framework
"langgraph>=1.0.2,<2.0.0",
"langchain>=1.0.0",
@@ -100,6 +102,12 @@ build-backend = "setuptools.build_meta"
module-root = ""
module-name = "artemis"
[tool.uv.workspace]
members = ["packages/artemis-client"]
[tool.uv.sources]
artemis-client = { workspace = true }
[tool.ruff]
# Exclude a variety of commonly ignored directories.
exclude = [
+5 -5
View File
@@ -163,14 +163,14 @@ async def run_part_3():
async def task_3a():
print(f"🚀 [Task 3A - Phone] Global mode starting: Executing on {PHONE_SERIAL}...")
res = await client_phone_g.run("Use manage_app to launch Settings app and report task status completed", profile="flash")
print(f"✅ [Task 3A - Phone] Finished")
print("✅ [Task 3A - Phone] Finished")
return res
async def task_3b():
await asyncio.sleep(0.4)
print(f"🚀 [Task 3B - Emulator] Global mode starting: Targeting {EMULATOR_SERIAL} (should queue due to global limit)...")
res = await client_emu_g.run("Use manage_app to launch Settings app and report task status completed", profile="flash")
print(f"✅ [Task 3B - Emulator] Finished")
print("✅ [Task 3B - Emulator] Finished")
return res
t3a = asyncio.create_task(task_3a())
@@ -199,16 +199,16 @@ async def run_part_4():
client_emu = ArtemisClient(device_serial=EMULATOR_SERIAL, concurrency_mode="per_device")
async def task_4a():
print(f"🚀 [Task 4A - Phone] Started...")
print("🚀 [Task 4A - Phone] Started...")
return await client_phone.run("Use manage_app to launch Settings app and report task status completed", profile="flash")
async def task_4b():
await asyncio.sleep(0.4)
print(f"🚀 [Task 4B - Phone] Started (Queued behind 4A)...")
print("🚀 [Task 4B - Phone] Started (Queued behind 4A)...")
return await client_phone.run("Use manage_app to launch Clock app and report task status completed", profile="flash")
async def task_4c():
print(f"🚀 [Task 4C - Emulator] Started concurrently with 4A...")
print("🚀 [Task 4C - Emulator] Started concurrently with 4A...")
return await client_emu.run("Use manage_app to launch Settings app and report task status completed", profile="flash")
t4a = asyncio.create_task(task_4a())
+10 -2
View File
@@ -72,7 +72,13 @@ def main() -> None:
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("-p", "--port", type=int, default=8000, help="Port to run UI on")
parser.add_argument("-H", "--host", type=str, default="0.0.0.0", help="Host address to bind")
parser.add_argument(
"-H",
"--host",
type=str,
default="127.0.0.1",
help="Host address to bind (loopback by default; use a tunnel for remote access)",
)
parser.add_argument(
"-r",
"--restart",
@@ -181,7 +187,9 @@ def main() -> None:
cmd = []
if has_uv and (ROOT_DIR / "pyproject.toml").exists():
cmd = ["uv", "run", "artemis", "ui", "--port", str(args.port), "--host", args.host]
# Launch via `python -m artemis` (not the `artemis` console-script shim) so the
# long-running server never locks .venv/Scripts/artemis.exe against reinstalls.
cmd = ["uv", "run", "python", "-m", "artemis", "ui", "--port", str(args.port), "--host", args.host]
if args.no_open:
cmd.append("--no-open")
if args.reload:
+4 -2
View File
@@ -159,9 +159,11 @@ if ($installMcp -match "^[Yy]") {
Write-Host ""
# 8. Launch unified Showcase UI & open browser
# Launch via `python -m artemis` (not the `artemis.exe` console-script shim) so the
# long-running server never locks .venv\Scripts\artemis.exe against `uv sync` reinstalls.
Write-Host " [INFO] Launching Artemis Showcase UI & Admin Console..." -ForegroundColor Green
if ($NoOpen) {
uv run artemis ui --port $Port --no-open
uv run python -m artemis ui --port $Port --no-open
} else {
uv run artemis ui --port $Port --open
uv run python -m artemis ui --port $Port --open
}
+5 -5
View File
@@ -156,14 +156,14 @@ async def stage_3_global_concurrency_mode():
async def task_phone_g():
print(f"📱 [Task 3A - Phone] Acquiring global lock on {PHONE_SERIAL}...")
res = await client_phone_global.run("Press HOME key and open Settings", profile="flash")
print(f"✅ [Task 3A - Phone] Finished.")
print("✅ [Task 3A - Phone] Finished.")
return res
async def task_emu_g():
await asyncio.sleep(0.4)
print(f"💻 [Task 3B - Emulator] Requesting run on {EMULATOR_SERIAL} under global mode (Should queue!)...")
res = await client_emu_global.run("Press HOME key and open Settings", profile="flash")
print(f"✅ [Task 3B - Emulator] Finished.")
print("✅ [Task 3B - Emulator] Finished.")
return res
p_task = asyncio.create_task(task_phone_g())
@@ -189,16 +189,16 @@ async def stage_4_mixed_queue_and_parallel():
client_emu = ArtemisClient(device_serial=EMULATOR_SERIAL, concurrency_mode="per_device")
async def task_4a():
print(f"📱 [Task 4A - Phone] Running...")
print("📱 [Task 4A - Phone] Running...")
return await client_phone.run("Press HOME key and open Settings", profile="flash")
async def task_4b():
await asyncio.sleep(0.4)
print(f"📱 [Task 4B - Phone] Queued behind 4A...")
print("📱 [Task 4B - Phone] Queued behind 4A...")
return await client_phone.run("Press HOME key and open Clock", profile="flash")
async def task_4c():
print(f"💻 [Task 4C - Emulator] Running in parallel with 4A...")
print("💻 [Task 4C - Emulator] Running in parallel with 4A...")
return await client_emu.run("Press HOME key and open Settings", profile="flash")
t4a = asyncio.create_task(task_4a())
+1 -1
View File
@@ -152,7 +152,7 @@ async def run_experiment_2_multi_device_parallel(real_serial: str, emu_serial: s
async def run_experiment_3_global_concurrency_serialization(real_serial: str, emu_serial: str):
"""Experiment 3: Global concurrency serialization across devices."""
print_header("[Experiment 3] Global Concurrency Mode Test")
print(f"Mode description: concurrency_mode='global', entire system allows only 1 task across all devices.")
print("Mode description: concurrency_mode='global', entire system allows only 1 task across all devices.")
client_a = ArtemisClient(device_serial=real_serial, default_profile="flash", concurrency_mode="global")
client_b = ArtemisClient(device_serial=emu_serial, default_profile="flash", concurrency_mode="global")
+3 -1
View File
@@ -179,4 +179,6 @@ echo ""
# 8. Launch unified Showcase UI & auto-open browser
echo -e " ${GREEN}🚀 Launching Artemis Showcase UI & Admin Console...${NC}"
exec uv run artemis ui --open "$@"
# Launch via `python -m artemis` (not the `artemis` console-script shim) so the
# long-running server never pins .venv/Scripts/artemis[.exe] against reinstalls.
exec uv run python -m artemis ui --open "$@"
+128
View File
@@ -0,0 +1,128 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
"""Opt-in live contract test for the remote-only Python SDK.
Run with:
ARTEMIS_TEST_DEVICE_SERIAL=<serial> pytest tests/integration/test_thin_sdk_live.py
"""
from __future__ import annotations
import os
import asyncio
import time
import uuid
import pytest
from artemis_client import ArtemisClient, TaskRejectedError
def _live_client() -> tuple[ArtemisClient, str]:
serial = os.environ.get("ARTEMIS_TEST_DEVICE_SERIAL")
if not serial:
pytest.skip("ARTEMIS_TEST_DEVICE_SERIAL is required for the live SDK test")
return (
ArtemisClient(
os.environ.get("ARTEMIS_BASE_URL", "http://127.0.0.1:8000"),
device_serial=serial,
default_profile="flash",
poll_interval=0.5,
),
serial,
)
@pytest.mark.android
@pytest.mark.asyncio
async def test_thin_sdk_remote_contract_on_real_device():
client, serial = _live_client()
readiness = await client.health()
assert readiness
capabilities = await client.capabilities()
assert capabilities.supports("tasks.submit")
devices = await client.list_devices()
assert any(device.serial == serial and device.state == "device" for device in devices)
task_id = str(uuid.uuid4())
handle = await client.submit(
"Open Android Settings, navigate to About phone, and verify that the visible model is Pixel 10 "
"and the Android version is 16. Do not change any setting. Stop after both values are observed.",
task_id=task_id,
locked_app_package="com.android.settings",
)
assert handle.task_id == task_id
duplicate = await client.submit(
"Open Android Settings, navigate to About phone, and verify that the visible model is Pixel 10 "
"and the Android version is 16. Do not change any setting. Stop after both values are observed.",
task_id=task_id,
locked_app_package="com.android.settings",
)
assert duplicate.task_id == task_id
result = await client.wait_for_task(task_id, timeout=600)
assert result.succeeded, result.error or result.raw
cleanup_deadline = time.monotonic() + 30
while time.monotonic() < cleanup_deadline:
scheduler = await client.health()
if scheduler.get("status") == "idle" and not scheduler.get("active_tasks"):
break
await asyncio.sleep(0.5)
else:
pytest.fail("Artemis scheduler did not release the device after task completion")
@pytest.mark.android
@pytest.mark.asyncio
async def test_thin_sdk_rejects_unknown_real_device_serial():
client, _ = _live_client()
with pytest.raises(TaskRejectedError):
await client.submit(
"This task must never reach a device",
device_serial="artemis-nonexistent-device",
)
@pytest.mark.android
@pytest.mark.asyncio
async def test_thin_sdk_can_cancel_remote_task_and_release_device():
client, _ = _live_client()
task_id = str(uuid.uuid4())
await client.submit(
"Open Android Settings and inspect the top-level categories without changing anything.",
task_id=task_id,
locked_app_package="com.android.settings",
)
assert await client.stop(task_id)
cleanup_deadline = time.monotonic() + 30
while time.monotonic() < cleanup_deadline:
scheduler = await client.health()
active_ids = {
str(item.get("session_id"))
for item in scheduler.get("active_tasks", [])
if isinstance(item, dict)
}
queued_ids = {
str(item.get("session_id"))
for item in scheduler.get("queue", [])
if isinstance(item, dict)
}
if task_id not in active_ids | queued_ids:
break
await asyncio.sleep(0.5)
else:
pytest.fail("Cancelled task still owns a scheduler slot or device lock")
@@ -0,0 +1,212 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Security regression tests for the Artemis console boundary.
Covers the invariants the no-auth security model depends on:
- secrets never appear in HTTP responses,
- media endpoints cannot read files outside the media allowlist,
- cross-origin browser traffic and DNS-rebinding Hosts are rejected,
- no CORS grants exist,
- lifecycle controls stay loopback-only.
"""
import secrets as py_secrets
from unittest.mock import MagicMock, patch
import pytest
from httpx import ASGITransport, AsyncClient
from pydantic import SecretStr
from apps.admin_console.server import app
from artemis.config import TRACES_PATH, WORKSPACE_ROOT
def _client(**transport_kwargs) -> AsyncClient:
return AsyncClient(
transport=ASGITransport(app=app, **transport_kwargs), base_url="http://localhost"
)
# ---------------------------------------------------------------------------
# Secret exposure
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_credentials_endpoint_never_returns_key_material(monkeypatch):
from artemis.config import settings
honeytoken = f"sk-honeytoken-{py_secrets.token_hex(16)}"
monkeypatch.setattr(
type(settings), "get_api_key", lambda self, provider: SecretStr(honeytoken)
)
async with _client() as ac:
res = await ac.get("/api/system/credentials")
assert res.status_code == 200
assert honeytoken not in res.text
providers = {entry["name"]: entry["configured"] for entry in res.json()["providers"]}
assert providers.get("google") is True
@pytest.mark.asyncio
async def test_server_status_omits_lifecycle_token_and_metadata():
async with _client() as ac:
res = await ac.get("/api/system/server-status")
assert res.status_code == 200
data = res.json()
assert "metadata" not in data
assert "lifecycle_token" not in res.text
assert "cmdline" not in res.text
assert "current_pid" in data
# ---------------------------------------------------------------------------
# File access boundaries
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_videos_endpoint_refuses_dotenv_and_non_video_files():
async with _client() as ac:
for target in (".env", "pyproject.toml", "artemis/__init__.py"):
res = await ac.get(f"/videos/{target}")
assert res.status_code in (403, 404), target
assert "API_KEY" not in res.text
@pytest.mark.asyncio
async def test_videos_endpoint_refuses_encoded_traversal():
async with _client() as ac:
for target in (
"%2e%2e/%2e%2e/etc/passwd",
"..%5c..%5cwindows%5cwin.ini",
"%2e%2e%2f.env",
):
res = await ac.get(f"/videos/{target}")
assert res.status_code in (403, 404), target
@pytest.mark.asyncio
async def test_videos_endpoint_still_serves_real_recordings():
TRACES_PATH.mkdir(parents=True, exist_ok=True)
probe = TRACES_PATH / f"security-probe-{py_secrets.token_hex(4)}.mp4"
probe.write_bytes(b"\x00\x00\x00\x18ftypmp42")
try:
async with _client() as ac:
res = await ac.get(f"/videos/{probe.name}")
assert res.status_code == 200
assert res.headers["content-type"].startswith("video/mp4")
finally:
probe.unlink(missing_ok=True)
@pytest.mark.asyncio
async def test_local_file_endpoint_refuses_non_media_workspace_files():
async with _client() as ac:
for target in (
str(WORKSPACE_ROOT / ".env"),
str(WORKSPACE_ROOT / "pyproject.toml"),
"file://" + str(WORKSPACE_ROOT / ".env"),
):
res = await ac.get("/local_file", params={"path": target})
assert res.status_code in (403, 404), target
assert "API_KEY" not in res.text
@pytest.mark.asyncio
async def test_spa_route_does_not_leak_files_outside_static_roots():
async with _client() as ac:
res = await ac.get("/%2e%2e/%2e%2e/pyproject.toml")
# The catch-all SPA route must fall back to HTML, never the file content.
assert "requires-python" not in res.text
assert res.headers["content-type"].startswith("text/html")
# ---------------------------------------------------------------------------
# Browser boundary: Host, Origin, CORS
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_dns_rebinding_host_is_rejected():
async with AsyncClient(
transport=ASGITransport(app=app), base_url="http://attacker.example"
) as ac:
res = await ac.get("/api/system/emulator/status")
assert res.status_code == 403
@pytest.mark.asyncio
async def test_cross_origin_browser_request_is_rejected_without_cors_grant():
async with _client() as ac:
res = await ac.post(
"/api/system/emulator/dismiss",
headers={"Origin": "https://attacker.example"},
)
assert res.status_code == 403
assert "access-control-allow-origin" not in {k.lower() for k in res.headers}
@pytest.mark.asyncio
async def test_same_origin_browser_request_passes():
async with _client() as ac:
res = await ac.get(
"/api/system/emulator/status", headers={"Origin": "http://localhost"}
)
assert res.status_code == 200
@pytest.mark.asyncio
async def test_null_origin_is_rejected():
async with _client() as ac:
res = await ac.get("/api/system/emulator/status", headers={"Origin": "null"})
assert res.status_code == 403
@pytest.mark.asyncio
async def test_security_headers_present_and_api_responses_uncacheable():
async with _client() as ac:
res = await ac.get("/api/system/emulator/status")
assert res.headers.get("x-content-type-options") == "nosniff"
assert res.headers.get("referrer-policy") == "no-referrer"
assert res.headers.get("cache-control") == "no-store"
# ---------------------------------------------------------------------------
# Lifecycle controls
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_restart_is_loopback_only():
with patch("threading.Thread") as mock_thread:
mock_thread.return_value = MagicMock()
async with AsyncClient(
transport=ASGITransport(app=app, client=("203.0.113.9", 51000)),
base_url="http://localhost",
) as ac:
res = await ac.post("/api/system/restart")
assert res.status_code == 403
mock_thread.assert_not_called()
@@ -0,0 +1,60 @@
"""Legacy "success" session rows must surface as the canonical "completed".
The sessions table historically received "success" from the DataEngine while
the queue reconcile wrote "completed", so API consumers saw a mix of both
terminal strings. Reads through the repository now normalize the legacy alias;
get_session_status stays raw so the queue reconcile can persist the rewrite.
"""
from apps.admin_console.database.connection import db_session
from apps.admin_console.database.repositories.session_repository import SessionRepository
def _insert_session(db_path, session_id, status):
with db_session(db_path) as conn:
conn.execute(
"INSERT INTO sessions "
"(session_id, initial_goal, start_time, end_time, status, device_info, pid) "
"VALUES (?, ?, ?, ?, ?, ?, ?)",
(session_id, session_id, 1.0, 2.0, status, "{}", None),
)
conn.commit()
def test_get_all_sessions_normalizes_legacy_success(tmp_path):
db_path = tmp_path / "sessions.db"
repository = SessionRepository(db_path)
_insert_session(db_path, "legacy", "success")
_insert_session(db_path, "modern", "completed")
_insert_session(db_path, "broken", "failed")
statuses = {r["session_id"]: r["status"] for r in repository.get_all_sessions()}
assert statuses == {"legacy": "completed", "modern": "completed", "broken": "failed"}
def test_get_session_by_id_normalizes_legacy_success(tmp_path):
db_path = tmp_path / "sessions.db"
repository = SessionRepository(db_path)
_insert_session(db_path, "legacy", "success")
row = repository.get_session_by_id("legacy")
assert row is not None
assert row["status"] == "completed"
def test_get_latest_session_normalizes_legacy_success(tmp_path):
db_path = tmp_path / "sessions.db"
repository = SessionRepository(db_path)
_insert_session(db_path, "legacy", "success")
row = repository.get_latest_session()
assert row is not None
assert row["status"] == "completed"
def test_get_session_status_stays_raw_for_queue_reconcile(tmp_path):
db_path = tmp_path / "sessions.db"
repository = SessionRepository(db_path)
_insert_session(db_path, "legacy", "success")
assert repository.get_session_status("legacy") == "success"
@@ -43,7 +43,7 @@ def test_emulator_uses_windows_console_isolation():
async def test_system_emulator_status_endpoint():
"""Verify GET /api/system/emulator/status returns valid schema."""
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as ac:
async with AsyncClient(transport=transport, base_url="http://localhost") as ac:
res = await ac.get("/api/system/emulator/status")
assert res.status_code == 200
data = res.json()
@@ -57,7 +57,7 @@ async def test_system_emulator_status_endpoint():
async def test_system_emulator_dismiss_endpoint():
"""Verify POST /api/system/emulator/dismiss resets tracker state."""
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as ac:
async with AsyncClient(transport=transport, base_url="http://localhost") as ac:
res = await ac.post("/api/system/emulator/dismiss")
assert res.status_code == 200
data = res.json()
@@ -68,6 +68,34 @@ async def test_system_emulator_dismiss_endpoint():
async def test_system_emulator_launch_empty_validation():
"""Verify POST /api/system/emulator/launch validates empty avd_name."""
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as ac:
async with AsyncClient(transport=transport, base_url="http://localhost") as ac:
res = await ac.post("/api/system/emulator/launch", json={"avd_name": " "})
assert res.status_code == 400
@pytest.mark.asyncio
async def test_adb_endpoint_mutation_rejects_cross_origin_browser_requests():
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://localhost") as ac:
res = await ac.post(
"/api/system/adb/server/probe",
json={"host": "127.0.0.1", "port": 5038, "persist": False},
headers={"Origin": "https://untrusted.example"},
)
assert res.status_code == 403
assert "Artemis console" in res.json()["detail"]
@pytest.mark.asyncio
async def test_adb_endpoint_mutation_is_local_only_by_default(monkeypatch):
monkeypatch.delenv("ARTEMIS_ALLOW_REMOTE_ADB_CONFIGURATION", raising=False)
transport = ASGITransport(app=app, client=("192.168.1.20", 42000))
async with AsyncClient(transport=transport, base_url="http://localhost") as ac:
res = await ac.post(
"/api/system/adb/server/probe",
json={"host": "127.0.0.1", "port": 5038, "persist": False},
)
assert res.status_code == 403
assert "local-only" in res.json()["detail"]
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from types import SimpleNamespace
from unittest.mock import AsyncMock
from fastapi import HTTPException
@@ -112,3 +113,131 @@ async def test_run_task_switches_to_unlocked_device_and_syncs_engine(monkeypatch
assert kwargs.get("device_serial") == "emulator-5556"
set_active_mock.assert_called_once_with("emulator-5556")
@pytest.mark.asyncio
async def test_idempotent_retry_skips_device_probe_for_active_session(monkeypatch):
run_probe = AsyncMock()
monkeypatch.setattr(
tasks.readiness_engine, "run_device_submission_probe", run_probe
)
monkeypatch.setattr(tasks.state, "active_session_id", "sdk-task-1")
monkeypatch.setattr(tasks.state, "active_connections", {})
monkeypatch.setattr(
tasks.session_repo,
"get_session_by_id",
lambda session_id: None,
)
result = await tasks.run_task(
RunRequest(
goal="Open Settings",
session_id="sdk-task-1",
device_serial="pixel-10",
ingress="python_sdk",
)
)
assert result["status"] == "running"
assert result["enqueued_count"] == 0
assert result["tasks"][0]["session_id"] == "sdk-task-1"
run_probe.assert_not_awaited()
@pytest.mark.asyncio
async def test_unknown_explicit_device_is_rejected_before_readiness_probe(monkeypatch):
run_probe = AsyncMock()
monkeypatch.setattr(tasks.readiness_engine, "run_device_submission_probe", run_probe)
monkeypatch.setattr(tasks.state, "active_session_id", None)
monkeypatch.setattr(tasks.state, "active_connections", {})
monkeypatch.setattr(tasks.state, "queue_items", [])
monkeypatch.setattr(tasks.session_repo, "get_session_by_id", lambda session_id: None)
monkeypatch.setattr(
tasks.device_pool,
"try_list_devices_async",
AsyncMock(
return_value=[
SimpleNamespace(serial="pixel-10", state="device"),
]
),
)
result = await tasks.run_task(
RunRequest(
goal="Must not run",
device_serial="missing-device",
session_id="00000000-0000-4000-8000-000000000099",
)
)
assert result["status"] == "rejected"
assert result["tasks"] == []
assert "not connected" in result["error"]
run_probe.assert_not_awaited()
@pytest.mark.asyncio
async def test_explicit_device_is_rejected_when_attached_but_not_ready(monkeypatch):
run_probe = AsyncMock()
monkeypatch.setattr(tasks.readiness_engine, "run_device_submission_probe", run_probe)
monkeypatch.setattr(tasks.state, "active_session_id", None)
monkeypatch.setattr(tasks.state, "active_connections", {})
monkeypatch.setattr(tasks.state, "queue_items", [])
monkeypatch.setattr(tasks.session_repo, "get_session_by_id", lambda session_id: None)
monkeypatch.setattr(
tasks.device_pool,
"try_list_devices_async",
AsyncMock(
return_value=[
SimpleNamespace(serial="pixel-10", state="unauthorized"),
]
),
)
result = await tasks.run_task(
RunRequest(
goal="Must not run",
device_serial="pixel-10",
session_id="00000000-0000-4000-8000-000000000100",
)
)
assert result["status"] == "rejected"
assert "unauthorized" in result["error"]
run_probe.assert_not_awaited()
@pytest.mark.parametrize("enumeration", [None, []])
@pytest.mark.asyncio
async def test_explicit_device_proceeds_when_enumeration_is_indeterminate(
monkeypatch, enumeration
):
"""A failed (None) or empty enumeration must never hard-reject an explicit
serial: the submission queues and fails downstream if the device is truly
absent. This is the startup-storm regression guard."""
run_probe = AsyncMock(return_value=None)
enqueue_tasks = AsyncMock(return_value={"status": "queued", "tasks": []})
monkeypatch.setattr(tasks.readiness_engine, "run_device_submission_probe", run_probe)
monkeypatch.setattr(tasks.task_queue_service, "enqueue_tasks", enqueue_tasks)
monkeypatch.setattr(tasks.state, "active_session_id", None)
monkeypatch.setattr(tasks.state, "active_connections", {})
monkeypatch.setattr(tasks.state, "queue_items", [])
monkeypatch.setattr(tasks.session_repo, "get_session_by_id", lambda session_id: None)
monkeypatch.setattr(
tasks.device_pool,
"try_list_devices_async",
AsyncMock(return_value=enumeration),
)
result = await tasks.run_task(
RunRequest(
goal="Queue through the startup storm",
device_serial="pixel-10",
session_id="00000000-0000-4000-8000-000000000101",
)
)
assert result["status"] == "queued"
enqueue_tasks.assert_awaited_once()
_, kwargs = enqueue_tasks.call_args
assert kwargs.get("device_serial") == "pixel-10"
@@ -25,12 +25,22 @@ from apps.admin_console.core.state import state
from apps.admin_console.routers.tasks import get_status
from apps.admin_console.services.task_queue_service import TaskQueueService, task_queue_service
from artemis.runtime.device_lock import DeviceLockOwner
from artemis.runtime.adb_endpoint import AdbEndpoint
@pytest.fixture(autouse=True)
def clean_state(tmp_path, monkeypatch):
"""Reset global state between tests."""
isolated_pause_file = tmp_path / ".artemis_paused"
# Redirect DeviceExecutionLock's lock/queue directory into tmp_path so that
# enqueue reservations never touch the real %TEMP%/artemis/device-locks dir
# (a live daemon merges those tickets into its /api/status queue view).
isolated_lock_dir = tmp_path / "device-locks"
isolated_lock_dir.mkdir(parents=True, exist_ok=True)
monkeypatch.setattr(
"artemis.runtime.device_lock.get_temp_dir",
lambda _subfolder=None: isolated_lock_dir,
)
state_module = importlib.import_module("apps.admin_console.core.state")
queue_module = importlib.import_module("apps.admin_console.services.task_queue_service")
monkeypatch.setattr(state_module, "PAUSE_FILE", isolated_pause_file)
@@ -62,16 +72,6 @@ def clean_state(tmp_path, monkeypatch):
if state.worker_task and not state.worker_task.done():
state.worker_task.cancel()
state.worker_task = None
from artemis.config.paths import get_temp_dir
from artemis.runtime.device_lock import DeviceExecutionLock
DeviceExecutionLock.cleanup_stale_locks()
queue_dir = get_temp_dir("device-locks") / "artemis-global-device.queue"
if queue_dir.exists():
for p in queue_dir.glob("*.wait"):
try:
p.unlink(missing_ok=True)
except OSError:
pass
def test_paused_error_reads_persisted_reason(tmp_path):
@@ -94,6 +94,37 @@ async def test_get_next_pending_task():
assert next_task["goal"] == "Goal A"
@pytest.mark.asyncio
async def test_enqueued_task_keeps_its_adb_endpoint_snapshot():
original = AdbEndpoint.create("127.0.0.1", 5038)
changed = AdbEndpoint.local()
with (
patch.object(TaskQueueService, "ensure_worker_running"),
patch(
"apps.admin_console.services.task_queue_service.current_adb_endpoint",
return_value=original,
),
patch(
"artemis.runtime.device_pool.device_pool.select_device",
return_value="emulator-5554",
),
):
result = await TaskQueueService.enqueue_tasks(["Keep endpoint"])
task_item = result["tasks"][0]
assert task_item["adb_endpoint"]["identity"] == original.identity
with patch(
"apps.admin_console.services.task_queue_service.current_adb_endpoint",
return_value=changed,
):
target = TaskQueueService._task_target(task_item)
assert target.endpoint == original
assert target.serial == "emulator-5554"
assert target.lock_key == f"{original.identity}/emulator-5554"
@pytest.mark.parametrize(
("current_status", "returncode", "stopped", "expected"),
[
@@ -190,8 +221,18 @@ async def test_queue_worker_execution_lifecycle():
with (
patch("asyncio.create_subprocess_exec", side_effect=fake_subprocess_exec),
patch("apps.admin_console.services.task_queue_service.session_repo") as mock_repo,
patch("apps.admin_console.services.task_queue_service.media_service"),
patch(
"artemis.core.diagnostics.readiness_engine.get_active_device_serial",
return_value=None,
),
patch(
"artemis.runtime.device_pool.device_pool.select_device",
return_value="emulator-5554",
),
):
mock_repo.get_running_session_id.return_value = None
mock_repo.get_video_recording_for_session.return_value = {"status": "ready"}
# Enqueue two tasks
res = await task_queue_service.enqueue_tasks(["First Task", "Second Task"])
assert res["enqueued_count"] == 2
@@ -230,8 +271,18 @@ async def test_queue_worker_cmd_construction():
with (
patch("asyncio.create_subprocess_exec", side_effect=fake_subprocess_exec),
patch("apps.admin_console.services.task_queue_service.session_repo") as mock_repo,
patch("apps.admin_console.services.task_queue_service.media_service"),
patch(
"artemis.core.diagnostics.readiness_engine.get_active_device_serial",
return_value=None,
),
patch(
"artemis.runtime.device_pool.device_pool.select_device",
return_value="emulator-5554",
),
):
mock_repo.get_running_session_id.return_value = None
mock_repo.get_video_recording_for_session.return_value = {"status": "ready"}
enqueue_result = await task_queue_service.enqueue_tasks(
["Test Goal with Outputter"],
@@ -262,6 +313,11 @@ async def test_queue_worker_cmd_construction():
executed_kwargs[0]["env"]["ARTEMIS_DEVICE_QUEUE_TICKET"]
== (enqueue_result["tasks"][0]["queue_ticket"])
)
endpoint = enqueue_result["tasks"][0]["adb_endpoint"]
assert executed_kwargs[0]["env"]["ADB_HOST"] == endpoint["host"]
assert executed_kwargs[0]["env"]["ADB_PORT"] == str(endpoint["port"])
assert executed_kwargs[0]["env"]["ADB_SERVER_SOCKET"] == endpoint["socket"]
assert executed_kwargs[0]["env"]["ARTEMIS_ADB_ENDPOINT_ID"] == endpoint["identity"]
if sys.platform == "win32":
assert executed_kwargs[0]["creationflags"] == (
subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.CREATE_NO_WINDOW
@@ -491,8 +547,18 @@ async def test_cancel_task_triggers_next_pending_task():
with (
patch("asyncio.create_subprocess_exec", side_effect=fake_subprocess_exec),
patch("apps.admin_console.services.task_queue_service.session_repo") as mock_repo,
patch("apps.admin_console.services.task_queue_service.media_service"),
patch(
"artemis.core.diagnostics.readiness_engine.get_active_device_serial",
return_value=None,
),
patch(
"artemis.runtime.device_pool.device_pool.select_device",
return_value="emulator-5554",
),
):
mock_repo.get_running_session_id.return_value = None
mock_repo.get_video_recording_for_session.return_value = {"status": "ready"}
await task_queue_service.enqueue_tasks(["Task 1", "Task 2"])
for _ in range(40):
@@ -536,8 +602,18 @@ async def test_immediate_cancel_ignores_stale_ipc_and_runs_next_task():
with (
patch("asyncio.create_subprocess_exec", side_effect=fake_subprocess_exec),
patch("apps.admin_console.services.task_queue_service.session_repo") as mock_repo,
patch("apps.admin_console.services.task_queue_service.media_service"),
patch(
"artemis.core.diagnostics.readiness_engine.get_active_device_serial",
return_value=None,
),
patch(
"artemis.runtime.device_pool.device_pool.select_device",
return_value="emulator-5554",
),
):
mock_repo.get_running_session_id.return_value = None
mock_repo.get_video_recording_for_session.return_value = {"status": "ready"}
# Enqueue Task 1
res1 = await task_queue_service.enqueue_tasks(["Task 1"])
@@ -668,7 +744,7 @@ async def test_queue_worker_notifies_conversation():
patch("mcp_server.notifiers.notify") as mock_notify,
patch("apps.admin_console.services.task_queue_service.session_repo") as mock_repo,
patch(
"artemis.runtime.device_pool.device_pool.list_devices_async",
"artemis.runtime.device_pool.device_pool.try_list_devices_async",
new=AsyncMock(return_value=[]),
),
):
@@ -854,7 +930,7 @@ async def test_enqueue_tasks_debounces_rapid_identical_submissions():
patch("apps.admin_console.services.task_queue_service.session_repo"),
patch("apps.admin_console.services.task_queue_service.DeviceExecutionLock.reserve", return_value="ticket-456"),
patch(
"artemis.runtime.device_pool.device_pool.list_devices_async",
"artemis.runtime.device_pool.device_pool.try_list_devices_async",
new=AsyncMock(return_value=[]),
),
patch.object(TaskQueueService, "ensure_worker_running"),
@@ -0,0 +1,296 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for process-wide local and forwarded ADB server selection."""
import os
import subprocess
import pytest
from artemis.config import settings
from artemis.core.diagnostics.adb_server_connection import (
AdbServerConnectionManager,
AdbServerEndpoint,
InvalidAdbServerEndpoint,
adb_server_connection,
)
from artemis.core.diagnostics.engine import ReadinessEngine
from artemis.runtime.adb_endpoint import ADB_ENDPOINT_ID_ENV, AdbSession
@pytest.mark.parametrize(
("host", "port"),
[
("", 5037),
("http://127.0.0.1", 5037),
("127.0.0.1/path", 5037),
("127.0.0.1", 0),
("127.0.0.1", 65536),
],
)
def test_validate_endpoint_rejects_invalid_addresses(host, port):
with pytest.raises(InvalidAdbServerEndpoint):
AdbServerConnectionManager.validate_endpoint(host, port)
def test_endpoint_formats_ipv6_socket():
endpoint = AdbServerConnectionManager.validate_endpoint("[::1]", 5038)
assert endpoint.socket == "tcp:[::1]:5038"
assert endpoint.mode == "remote"
def test_adb_session_builds_explicit_endpoint_command_and_environment():
endpoint = AdbServerEndpoint("server.example", 5040)
session = AdbSession(endpoint, adb_path="adb")
assert session.command(["devices"]) == [
"adb",
"-H",
"server.example",
"-P",
"5040",
"devices",
]
environment = session.environment({"EXISTING": "value"})
assert environment["EXISTING"] == "value"
assert environment["ADB_SERVER_SOCKET"] == "tcp:server.example:5040"
assert environment[ADB_ENDPOINT_ID_ENV] == endpoint.identity
@pytest.mark.asyncio
async def test_connect_activates_forwarded_server_after_success(monkeypatch):
captured: dict[str, object] = {}
def fake_run(command, **kwargs):
captured["command"] = command
captured["environment"] = kwargs["env"]
return subprocess.CompletedProcess(
command,
0,
stdout=(
"List of devices attached\n"
"R58M1234 device product:husky model:Pixel_8_Pro transport_id:1\n"
),
stderr="",
)
monkeypatch.setattr(subprocess, "run", fake_run)
monkeypatch.setattr(settings, "ADB_HOST", "127.0.0.1")
monkeypatch.setattr(settings, "ADB_PORT", 5037)
monkeypatch.setenv("ADB_SERVER_SOCKET", "tcp:127.0.0.1:5999")
manager = AdbServerConnectionManager(adb_resolver=lambda: "adb", env_files=[])
result = await manager.connect("127.0.0.1", 5038, persist=False)
assert result["success"] is True
assert result["devices"] == [
{
"serial": "R58M1234",
"state": "device",
"model": "Pixel 8 Pro",
"product": "husky",
}
]
assert captured["command"] == [
"adb",
"-H",
"127.0.0.1",
"-P",
"5038",
"devices",
"-l",
]
assert "ADB_SERVER_SOCKET" not in captured["environment"]
assert settings.ADB_HOST == "127.0.0.1"
assert settings.ADB_PORT == 5038
assert os.environ["ADB_SERVER_SOCKET"] == "tcp:127.0.0.1:5038"
@pytest.mark.asyncio
async def test_connect_does_not_activate_unreachable_server(monkeypatch):
def fake_run(command, **kwargs):
return subprocess.CompletedProcess(
command,
1,
stdout="",
stderr="cannot connect to daemon at tcp:127.0.0.1:5038",
)
monkeypatch.setattr(subprocess, "run", fake_run)
monkeypatch.setattr(settings, "ADB_HOST", "127.0.0.1")
monkeypatch.setattr(settings, "ADB_PORT", 5037)
manager = AdbServerConnectionManager(adb_resolver=lambda: "adb", env_files=[])
result = await manager.connect("127.0.0.1", 5038, persist=False)
assert result["success"] is False
assert result["error_code"] == "server_unreachable"
assert settings.ADB_PORT == 5037
@pytest.mark.asyncio
async def test_probe_never_activates_reachable_endpoint(monkeypatch):
monkeypatch.setattr(
subprocess,
"run",
lambda command, **kwargs: subprocess.CompletedProcess(
command,
0,
stdout="List of devices attached\n",
stderr="",
),
)
monkeypatch.setattr(settings, "ADB_HOST", "127.0.0.1")
monkeypatch.setattr(settings, "ADB_PORT", 5037)
manager = AdbServerConnectionManager(adb_resolver=lambda: "adb", env_files=[])
result = await manager.probe("127.0.0.1", 5038)
assert result["success"] is True
assert result["endpoint"]["port"] == 5038
assert settings.ADB_PORT == 5037
@pytest.mark.asyncio
async def test_default_resolver_uses_the_shared_toolchain(monkeypatch):
captured: dict[str, object] = {}
monkeypatch.setattr(
"artemis.core.diagnostics.adb_server_connection.toolchain.resolve",
lambda name: "sdk-platform-tools-adb" if name == "adb" else None,
)
def fake_run(command, **kwargs):
captured["command"] = command
return subprocess.CompletedProcess(
command,
0,
stdout="List of devices attached\n",
stderr="",
)
monkeypatch.setattr(subprocess, "run", fake_run)
manager = AdbServerConnectionManager(env_files=[])
result = await manager.probe("127.0.0.1", 5038)
assert result["success"] is True
assert captured["command"][0] == "sdk-platform-tools-adb"
@pytest.mark.asyncio
async def test_connect_persists_endpoint_without_replacing_other_settings(monkeypatch, tmp_path):
env_file = tmp_path / ".env"
env_file.write_text("EXISTING_VALUE=keep-me\n", encoding="utf-8")
def fake_run(command, **kwargs):
return subprocess.CompletedProcess(
command,
0,
stdout="List of devices attached\n",
stderr="",
)
monkeypatch.setattr(subprocess, "run", fake_run)
manager = AdbServerConnectionManager(adb_resolver=lambda: "adb", env_files=[env_file])
result = await manager.connect("tunnel.local", 5038, persist=True)
content = env_file.read_text(encoding="utf-8")
assert result["persisted"] is True
assert "EXISTING_VALUE=keep-me" in content
assert "ADB_HOST=tunnel.local" in content
assert "ADB_PORT=5038" in content
@pytest.mark.asyncio
async def test_connect_remains_active_when_persistence_fails(monkeypatch):
def fake_run(command, **kwargs):
return subprocess.CompletedProcess(
command,
0,
stdout="List of devices attached\n",
stderr="",
)
def fail_persist(endpoint):
raise OSError("read-only")
monkeypatch.setattr(subprocess, "run", fake_run)
monkeypatch.setattr(settings, "ADB_HOST", "127.0.0.1")
monkeypatch.setattr(settings, "ADB_PORT", 5037)
manager = AdbServerConnectionManager(adb_resolver=lambda: "adb", env_files=[])
monkeypatch.setattr(manager, "_persist", fail_persist)
result = await manager.connect("127.0.0.1", 5038, persist=True)
assert result["success"] is True
assert result["persisted"] is False
assert result["persistence_error"] == "read-only"
assert settings.ADB_PORT == 5038
@pytest.mark.asyncio
async def test_use_local_server_restores_standard_endpoint(monkeypatch):
monkeypatch.setattr(settings, "ADB_HOST", "127.0.0.1")
monkeypatch.setattr(settings, "ADB_PORT", 5038)
manager = AdbServerConnectionManager(env_files=[])
result = await manager.use_local_server(persist=False)
assert result["success"] is True
assert result["endpoint"] == AdbServerEndpoint("127.0.0.1", 5037).to_dict()
assert settings.ADB_PORT == 5037
assert os.environ["ADB_SERVER_SOCKET"] == "tcp:127.0.0.1:5037"
@pytest.mark.asyncio
async def test_restart_skips_kill_server_for_forwarded_endpoint(monkeypatch):
endpoint = AdbServerEndpoint("127.0.0.1", 5038)
monkeypatch.setattr(adb_server_connection, "current_endpoint", lambda: endpoint)
result = await ReadinessEngine().restart_adb_server()
assert result["success"] is True
assert result["skipped"] is True
assert result["endpoint"]["port"] == 5038
@pytest.mark.asyncio
async def test_remote_endpoint_blocks_local_wireless_connection(monkeypatch):
monkeypatch.setattr(
adb_server_connection,
"current_endpoint",
lambda: AdbServerEndpoint("127.0.0.1", 5038),
)
result = await ReadinessEngine().connect_wireless_adb("192.168.1.100", 5555)
assert result["success"] is False
assert "Switch to local ADB" in result["message"]
@pytest.mark.asyncio
async def test_remote_endpoint_blocks_local_emulator_launch(monkeypatch):
monkeypatch.setattr(
adb_server_connection,
"current_endpoint",
lambda: AdbServerEndpoint("127.0.0.1", 5038),
)
result = await ReadinessEngine().launch_emulator("Pixel_8")
assert result["status"] == "failed"
assert "Switch to local ADB" in result["error"]
+17
View File
@@ -327,6 +327,23 @@ def test_resumed_session_monotonic_step_numbering(tmp_path):
assert [s.step_number for s in steps] == [1, 2, 3]
def test_end_session_normalizes_legacy_success_status(tmp_path):
mock_ctx = MagicMock(spec=ArtemisContext)
mock_execution_setup = MagicMock()
mock_execution_setup.traces_path = str(tmp_path)
mock_ctx.execution_setup = mock_execution_setup
mock_ctx.device = None
engine = DataEngine(mock_ctx)
sid = engine.start_session("Terminal status normalization test")
engine.end_session("success")
session = engine.storage.get_session(sid)
assert session is not None
assert session.status == "completed"
assert session.end_time is not None
def test_update_step_summary_includes_step_number_in_sse(tmp_path):
mock_ctx = MagicMock(spec=ArtemisContext)
mock_execution_setup = MagicMock()
Loaded 100 of 110 files, more files were not shown because too many files have changed in this diff. Show more