chenmingxuan/Artemis-Eapil
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 32 Wiki Activity

feat: improve task execution, verification and packaging

Browse Source 
Add execution incidents, action bursts and shared Flash history.

Expose verification and Explorer settings with streamed check results.

Support graceful cancellation, recording recovery and bundled UI resources.

Add CI quality checks and clean up comments and documentation.
This commit is contained in:
somew1nd committed 2026-09-02 21:45:51 -07:00
1 parent 5ccf06d677
commit 67248c887e
289 files changed
+15883 -6919
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

apps/admin_console/core/state.py
+1
View File
@@ -236,6 +236,7 @@ class ServerState:
state = ServerState()
import sys
if __name__ == "admin_console.core.state":
sys.modules["apps.admin_console.core.state"] = sys.modules[__name__]
elif __name__ == "apps.admin_console.core.state":
apps/admin_console/database/repositories/session_repository.py
+55 -8
View File
@@ -13,6 +13,8 @@
# limitations under the License.
import logging
import os
import sqlite3
import time
from typing import Any
@@ -74,10 +76,12 @@ class SessionRepository:
)
if cursor.fetchone() is None:
return {}
columns = {
str(row[1]) for row in cursor.execute("PRAGMA table_info(video_recordings)")
}
ready_clause = "status = 'ready'" if "status" in columns else "end_time IS NOT NULL"
columns = {str(row[1]) for row in cursor.execute("PRAGMA table_info(video_recordings)")}
# Failed recordings may still be recoverable from disk.
# Callers exclude in-progress sessions before resolving paths.
ready_clause = (
"status IN ('ready', 'failed')" if "status" in columns else "end_time IS NOT NULL"
)
cursor.execute(
"SELECT session_id, local_video_path FROM video_recordings "
"WHERE session_id IS NOT NULL AND local_video_path IS NOT NULL "
@@ -104,7 +108,9 @@ class SessionRepository:
return None
result = dict(row)
if "status" not in result:
result["status"] = "ready" if result.get("end_time") is not None else "recording"
result["status"] = (
"ready" if result.get("end_time") is not None else "recording"
)
result.setdefault("error", None)
return result
except Exception:
@@ -147,6 +153,40 @@ class SessionRepository:
except Exception:
return {}
def get_unfinalized_video_recordings(self) -> list[dict[str, Any]]:
"""Recordings whose worker never finalized them, oldest first.
Rows of sessions that are still ``running`` are skipped: their worker
(possibly owned by another daemon) may still be writing the file.
"""
try:
with db_session(self.db_path) as conn:
cursor = conn.cursor()
cursor.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND name='video_recordings'"
)
if cursor.fetchone() is None:
return []
columns = {
str(row[1]) for row in cursor.execute("PRAGMA table_info(video_recordings)")
}
pending_clause = (
"v.status IN ('recording', 'finalizing', 'failed')"
if "status" in columns
else "v.end_time IS NULL"
)
cursor.execute(
"SELECT v.session_id, v.local_video_path, v.start_time, "
"s.status AS session_status, s.start_time AS session_start_time "
"FROM video_recordings v LEFT JOIN sessions s ON s.session_id = v.session_id "
f"WHERE v.session_id IS NOT NULL AND v.local_video_path IS NOT NULL "
f"AND {pending_clause} ORDER BY v.start_time ASC"
)
rows = [dict(r) for r in cursor.fetchall()]
except (OSError, sqlite3.Error):
return []
return [row for row in rows if str(row.get("session_status") or "").lower() != "running"]
def mark_recording_failed_if_pending(self, session_id: str, error: str) -> bool:
"""Close a recording lifecycle when its worker exits before finalization."""
try:
@@ -174,8 +214,16 @@ class SessionRepository:
"WHERE session_id = ?",
(str(local_video_path), time.time(), str(session_id)),
)
updated = cursor.rowcount > 0
if os.path.exists(str(local_video_path)):
# Mirror the worker's own finalization so the session row
# and trace tooling see the recovered file too.
cursor.execute(
"UPDATE sessions SET video_filepath = ? WHERE session_id = ?",
(str(local_video_path), str(session_id)),
)
conn.commit()
return cursor.rowcount > 0
return updated
except Exception:
return False
@@ -324,8 +372,7 @@ class SessionRepository:
# read_status itself never raises; this guards the
# lock/write side of update_trace_status.
logger.warning(
"Could not mark trace %s failed during orphan "
"reconciliation: %s",
"Could not mark trace %s failed during orphan reconciliation: %s",
row["session_id"],
exc,
)
apps/admin_console/routers/media.py
+9 -1
View File
@@ -149,7 +149,9 @@ def _get_session_video_sync(session_id: str):
v_url = media_service.resolve_video_url(row_dict, video_rec_map, video_idx)
video_segments = media_service.resolve_video_segments(v_url)
if v_url:
version = int(float((recording or {}).get("end_time") or row_dict.get("end_time") or 0) * 1000)
version = int(
float((recording or {}).get("end_time") or row_dict.get("end_time") or 0) * 1000
)
separator = "&" if "?" in v_url else "?"
versioned_url = f"{v_url}{separator}v={version}" if version else v_url
for segment in video_segments:
@@ -188,3 +190,9 @@ async def get_task_plan(session_id: str):
@router.get("/api/sessions/{session_id}/notes")
async def get_all_notes(session_id: str):
return {"notes": media_service.get_session_notes_content(session_id)}
@router.get("/api/sessions/{session_id}/checks")
async def get_session_checks(session_id: str):
"""Checker verdict ledger + run outcome (backfill for the Checker panel)."""
return media_service.get_session_checks(session_id)
apps/admin_console/routers/sessions.py
+2 -3
View File
@@ -64,11 +64,10 @@ def _list_sessions_sync():
try:
from artemis.runtime import DeviceExecutionLock
active_owners = DeviceExecutionLock.get_active_owners()
active_owner_sids = {
str(owner.session_id)
for owner in active_owners.values()
if owner.session_id
str(owner.session_id) for owner in active_owners.values() if owner.session_id
}
except Exception:
active_owner_sids = set()
apps/admin_console/routers/system.py
+7 -3
View File
@@ -297,7 +297,11 @@ async def get_credentials():
providers = ("google", "openai", "anthropic", "openrouter", "ocr")
status = {name: bool(settings.get_api_key(name)) for name in providers}
status["gemini"] = status["google"]
return {"providers": [{"name": name, "configured": configured} for name, configured in status.items()]}
return {
"providers": [
{"name": name, "configured": configured} for name, configured in status.items()
]
}
@router.post("/credentials/test")
@@ -370,7 +374,7 @@ async def update_credentials(request: UpdateCredentialsRequest):
async def get_model_config_and_env():
"""Retrieve the current active artemis.jsonc configuration and .env status for custom setup."""
import os
from artemis.config.paths import ROOT_DIR, get_config_path
from artemis.config.paths import get_config_path, get_env_file
from artemis.config import settings
from artemis.utils.file import load_jsonc
@@ -390,7 +394,7 @@ async def get_model_config_and_env():
config_content = f"// Error reading config: {e}"
# 2. Env file resolution
env_path = ROOT_DIR / ".env"
env_path = get_env_file()
# 3. Relevant Env Variables status
def get_real_env_value(key_name: str, provider: str) -> str | None:
apps/admin_console/routers/tasks.py
+45 -25
View File
@@ -90,8 +90,7 @@ async def run_task(request: RunRequest):
(
item
for item in state.queue_items
if isinstance(item, dict)
and str(item.get("session_id")) == requested_sid
if isinstance(item, dict) and str(item.get("session_id")) == requested_sid
),
None,
)
@@ -122,9 +121,7 @@ async def run_task(request: RunRequest):
# fail downstream with a clear error instead.
if request.device_serial:
try:
rejection = await device_pool.validate_explicit_serial_async(
request.device_serial
)
rejection = await device_pool.validate_explicit_serial_async(request.device_serial)
except Exception:
rejection = None
if rejection:
@@ -144,19 +141,19 @@ async def run_task(request: RunRequest):
# prefers the diagnostics target preference, then any unlocked ready
# device); the verified serial is bound below.
target_serial = request.device_serial
device_probe = await readiness_engine.run_device_submission_probe(
target_serial=target_serial
)
device_probe = await readiness_engine.run_device_submission_probe(target_serial=target_serial)
if device_probe and device_probe.summary in {"Device Locked", "Lock State Unknown"}:
locked_serial = (
device_probe.metadata.get("active_device", {}).get("serial")
or target_serial
or ""
device_probe.metadata.get("active_device", {}).get("serial") or target_serial or ""
)
detail = (
f"Android device {locked_serial} is locked. Unlock it and enter the home screen before running a task.".replace(" ", " ").strip()
f"Android device {locked_serial} is locked. Unlock it and enter the home screen before running a task.".replace(
" ", " "
).strip()
if device_probe.summary == "Device Locked"
else f"Android device {locked_serial} lock state could not be verified. Keep it unlocked on the home screen and try again.".replace(" ", " ").strip()
else f"Android device {locked_serial} lock state could not be verified. Keep it unlocked on the home screen and try again.".replace(
" ", " "
).strip()
)
raise HTTPException(status_code=409, detail=detail)
@@ -173,6 +170,8 @@ async def run_task(request: RunRequest):
profile=request.profile or "flash",
expected_output=request.expected_output,
enable_outputter=request.enable_outputter,
verification_level=request.verification_level,
explorer_mode=request.explorer_mode,
locked_app_package=request.locked_app_package,
app_path=request.app_path,
device_serial=target_serial,
@@ -182,6 +181,22 @@ async def run_task(request: RunRequest):
)
@router.get("/api/run/defaults")
async def get_run_defaults():
"""Effective Pro-profile tuning defaults from the agent config.
The launcher's sliders start here so they reflect ``artemis.jsonc`` (and the
``ARTEMIS_EXPLORER_VERSION`` override) instead of a hard-coded guess.
"""
from artemis.config import load_agent_config, verification_level_for_checker
agent_cfg = load_agent_config()
return {
"verification_level": verification_level_for_checker(agent_cfg.checker),
"explorer_mode": agent_cfg.explorer.resolve(profile="pro"),
}
@router.get("/api/devices")
async def list_devices():
"""List all connected Android devices with their busy / idle status."""
@@ -223,7 +238,6 @@ async def stop_task(
return {"status": "no_running_task"}
@router.post("/api/resume")
async def resume_task():
resumed = task_queue_service.resume_task()
@@ -269,12 +283,14 @@ async def get_status():
(global_owner.session_id if global_owner else None)
or state.active_session_id
or (running_task.get("session_id") if running_task else None)
or (active_tasks[0]["session_id"] if active_tasks and active_tasks[0].get("session_id") else None)
or (
active_tasks[0]["session_id"]
if active_tasks and active_tasks[0].get("session_id")
else None
)
or (session_repo.get_running_session_id() if is_running else None)
)
owner_connection = (
state.active_connections.get(str(running_sid), {}) if running_sid else {}
)
owner_connection = state.active_connections.get(str(running_sid), {}) if running_sid else {}
running_goal = (
owner_connection.get("goal")
or (global_owner.description if global_owner else None)
@@ -283,8 +299,10 @@ async def get_status():
or (active_tasks[0]["goal"] if active_tasks else None)
)
active_profile = owner_connection.get("profile") or state.current_profile or (
running_task.get("profile") if running_task and not global_owner else None
active_profile = (
owner_connection.get("profile")
or state.current_profile
or (running_task.get("profile") if running_task and not global_owner else None)
)
if not active_profile and (running_sid or latest_session_id):
check_sid = running_sid or latest_session_id
@@ -407,7 +425,11 @@ async def stream_events(session_id: str = "active", client: str | None = None):
active_sid = state.active_session_id
if not active_sid:
running_item = next(
(t for t in state.queue_items if isinstance(t, dict) and t.get("status") == "running"),
(
t
for t in state.queue_items
if isinstance(t, dict) and t.get("status") == "running"
),
None,
)
if running_item:
@@ -435,16 +457,14 @@ async def stream_events(session_id: str = "active", client: str | None = None):
recorded_steps = step_repo.get_session_steps(str(active_sid))
for step_dict in recorded_steps:
yield (
"event: step_recorded\n"
f"data: {json.dumps(step_dict, default=str)}\n\n"
f"event: step_recorded\ndata: {json.dumps(step_dict, default=str)}\n\n"
)
except Exception as exc:
print(f"[Stream] Could not replay active steps: {exc}")
else:
for progress_event in state.get_startup_progress(session_id):
yield (
"event: startup_progress\n"
f"data: {json.dumps(progress_event, default=str)}\n\n"
f"event: startup_progress\ndata: {json.dumps(progress_event, default=str)}\n\n"
)
shutdown_waiter = asyncio.create_task(state.shutdown_event.wait())
apps/admin_console/schemas/task_schema.py
+5 -2
View File
@@ -21,6 +21,11 @@ class RunRequest(BaseModel):
profile: str | None = "flash"
expected_output: str | None = None
enable_outputter: bool | None = None
# Pro-profile tuning (ignored by the Flash profile): a coarse Checker preset
# ('off' | 'final' | 'checkpoints' | 'strict') and the Explorer perception
# version used by the Operator ('flash' | 'pro' | 'ultra').
verification_level: str | None = None
explorer_mode: str | None = None
locked_app_package: str | None = None
app_path: str | None = None
device_serial: str | None = None
@@ -29,7 +34,6 @@ class RunRequest(BaseModel):
conversation_id: str | None = None
class ReplayRequest(BaseModel):
device_id: str
user_submits: dict
@@ -41,4 +45,3 @@ class StopRequest(BaseModel):
session_id: str | None = None
device_id: str | None = None
all: bool = False
apps/admin_console/server.py
+9 -4
View File
@@ -67,6 +67,7 @@ from artemis.config import (
WORKSPACE_ROOT,
init_ls_address,
)
from artemis.resources import get_bundled_showcase_dist
try:
from admin_console.core.security import SameOriginBoundaryMiddleware
@@ -141,6 +142,9 @@ async def on_startup():
cleaned = session_repo.cleanup_orphans_on_startup()
if cleaned > 0:
print(f"[ServerStartup] Marked {cleaned} orphaned running session(s) as failed.")
# Workers killed together with a previous daemon never remuxed their
# recordings; publish whatever raw files they left behind.
asyncio.create_task(asyncio.to_thread(task_queue_service.recover_orphaned_recordings_on_launch))
await ipc_service.start_server()
state.worker_task = asyncio.create_task(task_queue_service.queue_worker())
@@ -174,9 +178,7 @@ async def on_shutdown():
run_task.cancel()
if run_tasks:
try:
await asyncio.wait_for(
asyncio.gather(*run_tasks, return_exceptions=True), timeout=5.0
)
await asyncio.wait_for(asyncio.gather(*run_tasks, return_exceptions=True), timeout=5.0)
except (TimeoutError, asyncio.CancelledError):
pass
for run in list(state.active_runs.values()):
@@ -231,7 +233,7 @@ except ImportError:
# Showcase UI (Angular 19) Unified Single-Port SPA Static Hosting
# ------------------------------------------------------------------------------
def _get_showcase_dist() -> Path:
"""Return the Showcase UI Angular build directory."""
"""Return the source build, or the immutable build bundled in a wheel."""
base_dist = _workspace_root / "apps" / "showcase_ui" / "dist"
candidates = [
base_dist / "frontend" / "browser",
@@ -242,6 +244,9 @@ def _get_showcase_dist() -> Path:
for candidate in candidates:
if candidate.is_dir() and (candidate / "index.html").exists():
return candidate
bundled_dist = get_bundled_showcase_dist()
if bundled_dist is not None:
return bundled_dist
return base_dist / "frontend" / "browser"
apps/admin_console/services/media_service.py
+179 -7
View File
@@ -13,6 +13,7 @@
# limitations under the License.
import base64
from datetime import datetime
import hashlib
import json
import logging
@@ -20,6 +21,7 @@ import os
from pathlib import Path
import subprocess
import threading
import time
from typing import Any
import urllib.parse
from fastapi import HTTPException
@@ -51,13 +53,23 @@ class MediaService:
if suffix == ".mp4":
return p
if suffix in (".mkv", ".webm"):
if cls.is_live_recording(p):
# scrcpy is still appending to this file: converting now
# would freeze a few-seconds partial as recording.mp4.
return p
src_stat = p.stat()
cache_key = str(p)
cached = cls._playable_cache.get(cache_key)
if cached and cached[0] == src_stat.st_mtime and cached[1] == src_stat.st_size:
return Path(cached[2])
mp4_cand = p.with_suffix(".mp4")
if mp4_cand.exists() and mp4_cand.stat().st_size > 0:
if (
mp4_cand.exists()
and mp4_cand.stat().st_size > 0
and mp4_cand.stat().st_mtime >= src_stat.st_mtime
):
# An MP4 older than its source is a stale partial left by a
# mid-run scan; fall through and convert again.
cls._playable_cache[cache_key] = (
src_stat.st_mtime,
src_stat.st_size,
@@ -88,6 +100,7 @@ class MediaService:
@staticmethod
def _convert_to_mp4(p: Path, mp4_cand: Path) -> Path:
from artemis.utils.video import get_ffmpeg_path
ffmpeg = get_ffmpeg_path()
# 1. Attempt fast copy remux
subprocess.run(
@@ -135,6 +148,115 @@ class MediaService:
return mp4_cand
return p
LIVE_RECORDING_GRACE_SECONDS = 10.0
TRACE_STATUS_SUFFIXES = ("_PASS_", "_FAIL_", "_TESTFAIL_")
@classmethod
def is_live_recording(cls, p: Path) -> bool:
"""Whether ``p`` is a raw .mkv that scrcpy is still appending to.
scrcpy touches the file continuously while recording; a raw file that
has not changed for a few seconds belongs to a finished (or killed)
recording and is safe to convert.
"""
if p.suffix.lower() != ".mkv":
return False
try:
return (time.time() - p.stat().st_mtime) < cls.LIVE_RECORDING_GRACE_SECONDS
except OSError:
return False
@staticmethod
def is_browser_playable(p: Path) -> bool:
return p.suffix.lower() in (".mp4", ".webm")
@classmethod
def strip_trace_status_suffix(cls, folder_name: str) -> str:
for marker in cls.TRACE_STATUS_SUFFIXES:
folder_name = folder_name.split(marker)[0]
return folder_name
@classmethod
def _finalize_trace_folder_name(cls, folder: Path, session_start_time: float | None) -> Path:
"""Give an unfinished temp trace folder the terminal name the worker uses."""
try:
if folder.resolve().parent != Path(TRACES_PATH).resolve():
return folder
except OSError:
return folder
if any(marker in folder.name for marker in cls.TRACE_STATUS_SUFFIXES):
return folder
try:
stamp = datetime.fromtimestamp(float(session_start_time or time.time())).strftime(
"%Y-%m-%dT%H-%M-%S"
)
except (TypeError, ValueError, OSError, OverflowError):
stamp = datetime.now().strftime("%Y-%m-%dT%H-%M-%S")
target = folder.with_name(f"{folder.name}_FAIL_{stamp}")
if target.exists():
return folder
try:
folder.rename(target)
except OSError as exc:
logger.warning("Could not rename orphaned trace folder %s: %s", folder, exc)
return folder
return target
@classmethod
def recover_orphaned_recording(
cls,
local_video_path: str | Path | None,
session_start_time: float | None = None,
) -> Path | None:
"""Finalize the recording of a worker that died before its own stop path ran.
A hard-killed worker leaves the raw scrcpy ``recording.mkv`` behind
(playable, but never remuxed) plus, possibly, a few-seconds partial
``recording.mp4`` that a session-list scan converted mid-run. The raw
file is remuxed over that partial and removed on success, and the temp
trace folder is renamed the way the worker would have done it.
Returns the final browser-playable path, or None when nothing usable
exists (the caller then records a terminal recording failure).
"""
if not local_video_path:
return None
source = Path(str(local_video_path))
folder = source.parent
if not folder.is_dir():
return None
raw = source if source.suffix.lower() == ".mkv" else folder / "recording.mkv"
mp4 = folder / "recording.mp4"
def _non_empty(path: Path) -> bool:
try:
return path.is_file() and path.stat().st_size > 0
except OSError:
return False
if _non_empty(raw):
if cls.is_live_recording(raw):
return None
with cls._playable_cache_lock:
converted = cls._convert_to_mp4(raw, mp4)
cls._playable_cache.pop(str(raw), None)
if converted != mp4 or not _non_empty(mp4):
return None
try:
raw.unlink()
except OSError:
pass
final = mp4
elif _non_empty(mp4):
final = mp4
elif _non_empty(source) and cls.is_browser_playable(source):
final = source
else:
return None
folder = cls._finalize_trace_folder_name(folder, session_start_time)
return folder / final.name
@staticmethod
def path_to_video_url(p: Path) -> str:
resolved = p.resolve()
@@ -168,13 +290,18 @@ class MediaService:
vfile = item / f"recording{ext}"
if vfile.exists():
vfile = cls.ensure_browser_playable_video(vfile)
if not cls.is_browser_playable(vfile):
# Live (still recording) or unconvertible raw file.
break
url = cls.path_to_video_url(vfile)
idx[item.name] = url
prefix = item.name.split("_PASS_")[0].split("_FAIL_")[0]
prefix = cls.strip_trace_status_suffix(item.name)
idx[prefix] = url
break
elif item.suffix in [".mp4", ".mkv", ".webm"]:
item = cls.ensure_browser_playable_video(item)
if not cls.is_browser_playable(item):
continue
url = cls.path_to_video_url(item)
idx[item.stem] = url
idx[item.name] = url
@@ -199,15 +326,17 @@ class MediaService:
v_fp = row_dict.get("video_filepath")
if v_fp and os.path.exists(v_fp):
p = cls.ensure_browser_playable_video(Path(v_fp))
v_url = cls.path_to_video_url(p)
if cls.is_browser_playable(p):
v_url = cls.path_to_video_url(p)
orig_rec = video_rec_map.get(s_id)
if not v_url and orig_rec:
p = Path(orig_rec)
if p.exists():
p = cls.ensure_browser_playable_video(p)
v_url = cls.path_to_video_url(p)
else:
if cls.is_browser_playable(p):
v_url = cls.path_to_video_url(p)
if not v_url:
folder_name = p.parent.name
v_url = video_idx.get(folder_name) or video_idx.get(folder_name.split("_")[0])
@@ -221,13 +350,16 @@ class MediaService:
vfile = sess_dir / f"recording{ext}"
if vfile.exists():
vfile = cls.ensure_browser_playable_video(vfile)
v_url = cls.path_to_video_url(vfile)
if cls.is_browser_playable(vfile):
v_url = cls.path_to_video_url(vfile)
break
if not v_url:
try:
for item in sess_dir.iterdir():
if item.is_file() and item.suffix.lower() in [".mp4", ".webm", ".mkv"]:
item = cls.ensure_browser_playable_video(item)
if not cls.is_browser_playable(item):
continue
v_url = cls.path_to_video_url(item)
break
except OSError:
@@ -260,7 +392,10 @@ class MediaService:
segments = []
for item in payload.get("segments", []):
segment_path = (manifest_path.parent / str(item["file"])).resolve()
if segment_path.parent != manifest_path.parent.resolve() or not segment_path.is_file():
if (
segment_path.parent != manifest_path.parent.resolve()
or not segment_path.is_file()
):
continue
segments.append(
{
@@ -400,6 +535,43 @@ class MediaService:
return f"Error reading task plan: {e}"
return "No task plan created yet."
@staticmethod
def get_session_checks(session_id: str) -> dict[str, Any]:
"""Checker material of one session for UI backfill: the append-only
verdict ledger (``check_ledger.jsonl``) and the machine-readable run
outcome (``run_outcome.json``), both written by
``artemis.graph.checkpoints``. Missing files mean "no checks ran".
"""
session_dir = TRACES_PATH / session_id
records: list[dict[str, Any]] = []
ledger_path = session_dir / "check_ledger.jsonl"
if ledger_path.is_file():
try:
for line in ledger_path.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line:
continue
try:
rec = json.loads(line)
except ValueError:
continue
if isinstance(rec, dict):
records.append(rec)
except (OSError, ValueError) as e:
logger.warning(f"Failed to read check ledger for {session_id}: {e}")
run_outcome: dict[str, Any] | None = None
outcome_path = session_dir / "run_outcome.json"
if outcome_path.is_file():
try:
loaded = json.loads(outcome_path.read_text(encoding="utf-8"))
if isinstance(loaded, dict):
run_outcome = loaded
except (OSError, ValueError) as e:
logger.warning(f"Failed to read run outcome for {session_id}: {e}")
return {"records": records, "run_outcome": run_outcome}
@staticmethod
def get_session_notes_content(session_id: str) -> dict[str, str]:
notes_dir = TRACES_PATH / session_id / "notes"
apps/admin_console/services/task_queue_service.py
+334 -100
View File
@@ -16,8 +16,10 @@ import asyncio
from datetime import datetime
import logging
import os
from pathlib import Path
import shutil
import sys
import threading
import time
from typing import Any
import uuid
@@ -43,8 +45,11 @@ from artemis.runtime import (
AdbEndpoint,
AdbTarget,
DeviceExecutionLock,
clear_cancel_request,
current_adb_endpoint,
pid_is_alive,
process_supervisor,
request_cancel,
trace_store,
)
@@ -59,6 +64,156 @@ class TaskQueueService:
# Strong references to in-flight _execute_task_item tasks (asyncio itself only
# keeps weak references to running tasks).
_run_tasks: set[asyncio.Task] = set()
# Deadline enforcers for graceful stops (see _stop_worker_gracefully).
_forced_stop_tasks: set[asyncio.Task] = set()
DEFAULT_CANCEL_GRACE_SECONDS = 45.0
@classmethod
def _cancel_grace_seconds(cls) -> float:
"""How long a worker may finalize itself before it is killed.
``ARTEMIS_CANCEL_GRACE_SECONDS=0`` restores the legacy immediate kill.
"""
raw = os.getenv("ARTEMIS_CANCEL_GRACE_SECONDS")
if raw is None or not raw.strip():
return cls.DEFAULT_CANCEL_GRACE_SECONDS
try:
return max(0.0, float(raw))
except ValueError:
return cls.DEFAULT_CANCEL_GRACE_SECONDS
@staticmethod
def _hard_kill(pid: int, process_created_at: float = 0.0) -> bool:
if not pid:
return False
if process_created_at and process_created_at > 0:
return process_supervisor.terminate_tree_verified(pid, process_created_at)
try:
return process_supervisor.terminate_tree(pid)
except Exception:
return False
@classmethod
def _stop_worker_gracefully(
cls,
pid: Any,
process_created_at: float = 0.0,
session_id: str | None = None,
reason: str = "Task stopped from the Artemis frontend.",
) -> tuple[bool, bool]:
"""Ask a worker to cancel itself; hard-kill it once the grace period lapses.
Workers are isolated from the daemon's console (and may belong to
another ingress process), so instead of a signal the daemon drops a
cancel marker the worker polls for. Honouring it runs the worker's
normal cancellation path: the screen recording is stopped and remuxed,
the trace folder is compiled, and the device lease is released. A
worker that never picks the marker up is killed after the grace period.
Returns ``(stopped, deferred)``: ``stopped`` mirrors the legacy kill
result, ``deferred`` is True when the kill was handed to the deadline
enforcer instead of happening now.
"""
try:
pid_int = int(pid) if pid else 0
except (TypeError, ValueError):
pid_int = 0
grace = cls._cancel_grace_seconds()
if not pid_int or grace <= 0:
return cls._hard_kill(pid_int, process_created_at), False
created_at = float(process_created_at or 0.0)
if created_at <= 0:
# PID markers carry the creation time so a leftover marker can never
# cancel a future process that reuses this PID.
try:
import psutil
created_at = float(psutil.Process(pid_int).create_time())
except Exception:
created_at = 0.0
if not pid_is_alive(pid_int, created_at or None):
return True, False
written = request_cancel(
session_id=str(session_id) if session_id else None,
pid=pid_int,
process_created_at=created_at,
reason=reason,
)
if not written:
return cls._hard_kill(pid_int, created_at), False
print(
f"[stop_tasks] Cancel requested for worker {pid_int}"
f" (session {session_id or 'n/a'}); forcing termination after {grace:.0f}s"
)
cls._schedule_forced_stop(pid_int, created_at, grace, session_id)
return True, True
@classmethod
def _stop_proc_gracefully(cls, proc: Any, session_id: Any) -> bool:
"""Graceful variant for a locally spawned process; True when deferred."""
pid = getattr(proc, "pid", None)
if not pid or cls._cancel_grace_seconds() <= 0:
return False
try:
_stopped, deferred = cls._stop_worker_gracefully(
pid, session_id=str(session_id) if session_id else None
)
except Exception:
return False
return deferred
@classmethod
def _schedule_forced_stop(
cls, pid: int, process_created_at: float, grace: float, session_id: Any
) -> None:
"""Kill ``pid`` if it is still alive once ``grace`` seconds have passed."""
def _still_alive() -> bool:
return pid_is_alive(pid, process_created_at or None)
def _force() -> None:
print(
f"[stop_tasks] Worker {pid} (session {session_id or 'n/a'}) did not exit"
f" within {grace:.0f}s of the cancel request; forcing termination."
)
cls._hard_kill(pid, process_created_at)
async def _enforce_async() -> None:
deadline = time.monotonic() + grace
while time.monotonic() < deadline:
if not _still_alive():
break
await asyncio.sleep(0.5)
else:
_force()
clear_cancel_request(session_id=str(session_id) if session_id else None, pid=pid)
def _enforce_sync() -> None:
deadline = time.monotonic() + grace
while time.monotonic() < deadline:
if not _still_alive():
break
time.sleep(0.5)
else:
_force()
clear_cancel_request(session_id=str(session_id) if session_id else None, pid=pid)
try:
loop = asyncio.get_running_loop()
except RuntimeError:
loop = None
if loop is not None:
task = loop.create_task(_enforce_async())
cls._forced_stop_tasks.add(task)
task.add_done_callback(cls._forced_stop_tasks.discard)
return
threading.Thread(
target=_enforce_sync, name=f"artemis-forced-stop-{pid}", daemon=True
).start()
@staticmethod
def _task_target(task_item: dict[str, Any]) -> AdbTarget:
@@ -88,9 +243,7 @@ class TaskQueueService:
)
@classmethod
def _broadcast_startup_progress(
cls, session_id: str | None, stage: str, message: str
) -> None:
def _broadcast_startup_progress(cls, session_id: str | None, stage: str, message: str) -> None:
if not session_id:
return
data = {
@@ -223,8 +376,7 @@ class TaskQueueService:
inside each worker process.
"""
print(
"[QueueWorker] Dispatcher initialized "
f"(concurrency mode: {cls._concurrency_mode()})."
f"[QueueWorker] Dispatcher initialized (concurrency mode: {cls._concurrency_mode()})."
)
try:
DeviceExecutionLock.cleanup_stale_locks()
@@ -262,9 +414,7 @@ class TaskQueueService:
# "running" state, so admission must count those too -- active_runs alone
# lags behind by the subprocess startup latency.
in_flight = [
i
for i in state.queue_items
if isinstance(i, dict) and i.get("status") == "running"
i for i in state.queue_items if isinstance(i, dict) and i.get("status") == "running"
]
busy_devices = state.busy_device_ids | {
cls._task_target(i).lock_key for i in in_flight if i.get("device_serial")
@@ -325,9 +475,7 @@ class TaskQueueService:
state.current_profile = profile
state.active_session_id = sess_id
cls._broadcast_startup_progress(
sess_id, "launching", "Starting the execution process"
)
cls._broadcast_startup_progress(sess_id, "launching", "Starting the execution process")
# Broadcast session_started so all connected clients know the task has started
cls._broadcast_event(
@@ -353,6 +501,8 @@ class TaskQueueService:
"""Assemble the worker subprocess command line and environment."""
expected_output = task_item.get("expected_output")
enable_outputter = task_item.get("enable_outputter")
verification_level = task_item.get("verification_level")
explorer_mode = task_item.get("explorer_mode")
locked_app = task_item.get("locked_app_package") or task_item.get("locked_app")
app_path = task_item.get("app_path")
@@ -395,6 +545,10 @@ class TaskQueueService:
cmd.extend(["--output-description", str(expected_output)])
if enable_outputter is not None:
cmd.append("--enable-outputter" if enable_outputter else "--disable-outputter")
if verification_level:
cmd.extend(["--verification-level", str(verification_level)])
if explorer_mode:
cmd.extend(["--explorer-pro-mode", str(explorer_mode)])
if locked_app:
cmd.extend(["--locked-app", str(locked_app)])
if app_path:
@@ -439,9 +593,7 @@ class TaskQueueService:
print(
f"[QueueWorker] Could not record worker pid in status.json for {sess_id}: {exc}"
)
cls._broadcast_startup_progress(
sess_id, "process_ready", "Execution process started"
)
cls._broadcast_startup_progress(sess_id, "process_ready", "Execution process started")
ingress_type = str(task_item.get("ingress", "frontend"))
DeviceExecutionLock.transfer_reservation(
str(task_item.get("queue_ticket")),
@@ -467,9 +619,7 @@ class TaskQueueService:
except Exception:
log_path = None
if isinstance(proc.stdout, asyncio.StreamReader):
return asyncio.create_task(
cls._forward_worker_output(proc.stdout, log_path)
)
return asyncio.create_task(cls._forward_worker_output(proc.stdout, log_path))
return None
@classmethod
@@ -481,9 +631,7 @@ class TaskQueueService:
str(sess_id) in getattr(state, "cancelled_session_ids", set())
or run_key in state.manually_stopped_run_ids
):
print(
f"[QueueWorker] Task [{sess_id}] was cancelled during launch. Terminating."
)
print(f"[QueueWorker] Task [{sess_id}] was cancelled during launch. Terminating.")
await cls._terminate_worker_process(proc)
@classmethod
@@ -505,23 +653,21 @@ class TaskQueueService:
# against when status.json is stale, so a failed DB
# write must not pass silently.
logger.error(
"[QueueWorker] Could not persist terminal DB status "
"'%s' for session %s",
"[QueueWorker] Could not persist terminal DB status '%s' for session %s",
new_status,
sess_id,
)
else:
print(
f"[QueueWorker] Preserved authoritative session {sess_id} "
f"status '{new_status}'"
)
print(f"[QueueWorker] Preserved authoritative session {sess_id} status '{new_status}'")
# A stale status.json would leave MCP pollers seeing "running"
# until their next DB reconcile, so retry transient write
# failures before giving up.
for attempt in range(3):
try:
if trace_store.read_status(str(sess_id)):
canonical_mcp_status = "completed" if new_status in ("completed", "success") else new_status
canonical_mcp_status = (
"completed" if new_status in ("completed", "success") else new_status
)
trace_store.update_trace_status(
str(sess_id),
canonical_mcp_status,
@@ -542,37 +688,68 @@ class TaskQueueService:
@classmethod
async def _recover_or_fail_recording(cls, sess_id: Any) -> None:
"""Attempt recovery of a not-yet-ready recording, else mark it failed."""
"""Finalize a recording whose worker died before it could, else mark it failed.
A worker that stops gracefully finalizes its own recording and this is
a no-op. A hard-killed (or crashed) worker leaves the raw scrcpy file it
registered at recording start; remuxing that file is all that is
needed to publish the full video.
"""
rec_info = session_repo.get_video_recording_for_session(sess_id)
rec_status = (rec_info or {}).get("status")
recovered_video_path = None
if rec_status != "ready":
if rec_status == "ready":
return
recovered_url = None
# 1. Direct recovery from the recording row written at recording start.
try:
local_video_path = (rec_info or {}).get("local_video_path")
if local_video_path:
start_time = (rec_info or {}).get("start_time")
if not start_time:
session_row = session_repo.get_session_by_id(sess_id)
start_time = dict(session_row).get("start_time") if session_row else None
final_path = await asyncio.to_thread(
media_service.recover_orphaned_recording,
local_video_path,
start_time,
)
if final_path:
session_repo.mark_recording_ready(sess_id, str(final_path))
recovered_url = media_service.path_to_video_url(Path(final_path))
except Exception as rec_err:
print(f"[QueueWorker] Error finalizing orphaned recording: {rec_err}")
# 2. Fallback: locate an already finalized file by folder / session naming.
if not recovered_url:
try:
video_rec_map = session_repo.get_video_recordings_map()
video_idx = await asyncio.to_thread(media_service.build_video_index)
recovered_url = await asyncio.to_thread(
fallback_url = await asyncio.to_thread(
media_service.resolve_video_url,
{"session_id": sess_id},
video_rec_map,
video_idx,
)
if recovered_url:
session_repo.mark_recording_ready(sess_id, recovered_url)
cls._broadcast_event(
"recording_ready",
{"session_id": sess_id, "video_url": recovered_url},
)
recovered_video_path = recovered_url
if fallback_url:
session_repo.mark_recording_ready(sess_id, fallback_url)
recovered_url = fallback_url
except Exception as rec_err:
print(f"[QueueWorker] Error attempting recording recovery: {rec_err}")
if not recovered_video_path and rec_status != "ready":
recording_error = "Task worker exited before recording finalization completed"
if session_repo.mark_recording_failed_if_pending(sess_id, recording_error):
cls._broadcast_event(
"recording_failed",
{"session_id": sess_id, "error": recording_error},
)
if recovered_url:
cls._broadcast_event(
"recording_ready",
{"session_id": sess_id, "video_url": recovered_url},
)
return
recording_error = "Task worker exited before recording finalization completed"
if session_repo.mark_recording_failed_if_pending(sess_id, recording_error):
cls._broadcast_event(
"recording_failed",
{"session_id": sess_id, "error": recording_error},
)
@classmethod
def _announce_session_end(
@@ -624,6 +801,15 @@ class TaskQueueService:
state.cancelled_session_ids.discard(str(sess_id))
state.cancelled_session_ids.discard(run_key)
state.manually_stopped_run_ids.discard(run_key)
try:
clear_cancel_request(
session_id=str(sess_id) if sess_id else None,
pid=getattr(proc, "pid", None),
)
except (OSError, TypeError, ValueError):
# Marker cleanup is best-effort: an unwritable temp dir or an odd
# pid value must not block releasing the run slot.
pass
state.active_runs.pop(run_key, None)
if proc is not None and state.current_process is proc:
state.current_process = None
@@ -665,9 +851,7 @@ class TaskQueueService:
env=env,
**cls._subprocess_creation_kwargs(),
)
cls._register_worker_run(
task_item, run_key, sess_id, goal, profile, target, proc
)
cls._register_worker_run(task_item, run_key, sess_id, goal, profile, target, proc)
output_task = cls._start_output_forwarder(sess_id, proc)
await cls._terminate_if_cancelled_during_launch(run_key, sess_id, proc)
@@ -685,9 +869,7 @@ class TaskQueueService:
sess_id, returncode, manual_stop
)
await cls._recover_or_fail_recording(sess_id)
cls._announce_session_end(
task_item, sess_id, goal, new_status, manual_stop
)
cls._announce_session_end(task_item, sess_id, goal, new_status, manual_stop)
except asyncio.CancelledError:
print(f"[QueueWorker] Task [{sess_id}] received cancellation signal.")
@@ -763,9 +945,7 @@ class TaskQueueService:
return None
@classmethod
async def _reject_unavailable_device(
cls, device_serial: str | None
) -> dict[str, Any] | None:
async def _reject_unavailable_device(cls, device_serial: str | None) -> dict[str, Any] | None:
"""Return the rejection response for an unattached explicit serial, if any."""
# Strict device binding: reject an explicitly requested serial that is not
# attached and authorized, instead of silently running on another device.
@@ -804,6 +984,8 @@ class TaskQueueService:
device_serial: str | None,
ingress: str,
conversation_id: str | None,
verification_level: str | None = None,
explorer_mode: str | None = None,
) -> dict[str, Any]:
"""Reserve a device slot and build one pending queue item for a goal."""
sess_id = single_session_id if single_session_id else str(uuid.uuid4())
@@ -829,6 +1011,8 @@ class TaskQueueService:
"profile": profile or "flash",
"expected_output": expected_output,
"enable_outputter": enable_outputter,
"verification_level": verification_level,
"explorer_mode": explorer_mode,
"locked_app_package": locked_app_package,
"app_path": app_path,
"device_serial": assigned_serial,
@@ -854,8 +1038,19 @@ class TaskQueueService:
ingress: str = "frontend",
session_id: str | None = None,
conversation_id: str | None = None,
verification_level: str | None = None,
explorer_mode: str | None = None,
) -> dict[str, Any]:
"""Enqueues one or more goals and wakes up the background worker."""
"""Enqueues one or more goals and wakes up the background worker.
``verification_level`` and ``explorer_mode`` are Pro-profile tuning knobs
forwarded to the worker as ``--verification-level`` / ``--explorer-pro-mode``;
they are normalised here so the queue item and the CLI see one spelling.
"""
verification_level = (
str(verification_level).strip().lower() or None if verification_level else None
)
explorer_mode = str(explorer_mode).strip().lower() or None if explorer_mode else None
cls.ensure_worker_running()
enqueued_tasks = []
@@ -888,6 +1083,8 @@ class TaskQueueService:
device_serial,
ingress,
conversation_id,
verification_level=verification_level,
explorer_mode=explorer_mode,
)
state.queue_items.append(task_item)
enqueued_tasks.append(task_item)
@@ -931,9 +1128,10 @@ class TaskQueueService:
for dev_owner in list(active_owners.values()):
if dev_owner and DeviceExecutionLock.is_active_owner(dev_owner):
process_supervisor.terminate_tree_verified(
cls._stop_worker_gracefully(
dev_owner.pid,
dev_owner.process_created_at,
session_id=dev_owner.session_id,
)
DeviceExecutionLock.cleanup_stale_locks(dev_owner.device_id)
sid = dev_owner.session_id
@@ -973,20 +1171,22 @@ class TaskQueueService:
state.cancelled_session_ids.add(str(run_key))
run_proc = run.get("process")
if run_proc is not None and run_proc.returncode is None:
try:
run_proc.kill()
except (ProcessLookupError, OSError):
# Process already exited between the check and the kill.
pass
if not cls._stop_proc_gracefully(run_proc, run.get("session_id") or run_key):
try:
run_proc.kill()
except (ProcessLookupError, OSError):
# Process already exited between the check and the kill.
pass
# active_runs entries are popped by each run's finalizer once the
# process exit is observed; clearing them here would free the device
# slots before the processes are actually gone.
if state.current_process:
try:
state.current_process.kill()
except (ProcessLookupError, OSError):
# Process already exited between the check and the kill.
pass
if not cls._stop_proc_gracefully(state.current_process, state.active_session_id):
try:
state.current_process.kill()
except (ProcessLookupError, OSError):
# Process already exited between the check and the kill.
pass
state.current_process = None
@classmethod
@@ -1167,19 +1367,24 @@ class TaskQueueService:
stopped = False
reservation_cancelled = False
if owner and DeviceExecutionLock.is_active_owner(owner):
stopped = process_supervisor.terminate_tree_verified(
stopped, _deferred = cls._stop_worker_gracefully(
owner.pid,
owner.process_created_at,
session_id=owner.session_id or target_sid,
)
DeviceExecutionLock.cleanup_stale_locks(owner.device_id)
elif owner is None and owner_record_exists and not target_sid:
# Never fall back to a frontend PID while another process has an
# owner record that is still being published or cannot be parsed.
return None
elif owner is None and local_proc is not None and (
local_run is not None
or not target_sid
or str(state.active_session_id) == target_sid
elif (
owner is None
and local_proc is not None
and (
local_run is not None
or not target_sid
or str(state.active_session_id) == target_sid
)
):
# The locally-managed worker can be stopped during its short
# initialization window before Agent acquires the device lease.
@@ -1187,21 +1392,28 @@ class TaskQueueService:
state.cancelled_session_ids.add(target_sid)
elif local_run_key is not None:
state.manually_stopped_run_ids.add(str(local_run_key))
deferred = False
if local_pid:
try:
stopped = process_supervisor.terminate_tree(local_pid)
stopped, deferred = cls._stop_worker_gracefully(
local_pid,
session_id=target_sid or (local_run or {}).get("session_id"),
)
except Exception:
stopped = False
try:
local_proc.kill()
stopped = True
except (ProcessLookupError, OSError):
# Process already exited; terminate_tree above may have got it.
pass
if not deferred:
try:
local_proc.kill()
stopped = True
except (ProcessLookupError, OSError):
# Process already exited; the stop above may have got it.
pass
stopped = True
is_local_owner = True
elif owner is None and local_item and (
not target_sid or str(local_item.get("session_id")) == target_sid
elif (
owner is None
and local_item
and (not target_sid or str(local_item.get("session_id")) == target_sid)
):
# Cancel a frontend submission before its worker has started. This
# does not touch pending reservations created by other ingresses.
@@ -1226,12 +1438,10 @@ class TaskQueueService:
row_pid = row.get("pid")
if row_pid and session_repo.process_is_alive(row_pid):
try:
process_supervisor.terminate_tree(int(row_pid))
cls._stop_worker_gracefully(int(row_pid), session_id=target_sid)
except Exception as exc:
# Report it: a surviving worker keeps the device busy.
print(
f"[stop_tasks] Could not terminate worker pid {row_pid}: {exc}"
)
print(f"[stop_tasks] Could not terminate worker pid {row_pid}: {exc}")
DeviceExecutionLock.cleanup_stale_locks()
stopped = True
return stopped, is_local_owner, reservation_cancelled
@@ -1264,9 +1474,7 @@ class TaskQueueService:
state.active_connections.pop(sid, None)
if stopped_session_id:
session_repo.update_session_status(
str(stopped_session_id), "cancelled", time.time()
)
session_repo.update_session_status(str(stopped_session_id), "cancelled", time.time())
try:
is_mcp = bool(owner and owner.ingress == "mcp")
if is_mcp or trace_store.read_status(str(stopped_session_id)):
@@ -1287,8 +1495,7 @@ class TaskQueueService:
)
if state.active_session_id and (
not stopped_session_id
or str(state.active_session_id) == str(stopped_session_id)
not stopped_session_id or str(state.active_session_id) == str(stopped_session_id)
):
state.active_session_id = None
state.current_goal = None
@@ -1305,8 +1512,7 @@ class TaskQueueService:
(
item
for item in state.queue_items
if isinstance(item, dict)
and str(item.get("session_id")) == str(stopped_session_id)
if isinstance(item, dict) and str(item.get("session_id")) == str(stopped_session_id)
),
None,
)
@@ -1316,15 +1522,12 @@ class TaskQueueService:
item
for item in state.queue_items
if not (
isinstance(item, dict)
and str(item.get("session_id")) == str(stopped_session_id)
isinstance(item, dict) and str(item.get("session_id")) == str(stopped_session_id)
)
]
@classmethod
def _stop_targeted_task(
cls, target_sid: str | None, target_device: str | None
) -> bool:
def _stop_targeted_task(cls, target_sid: str | None, target_device: str | None) -> bool:
"""Stop a specific task (or default single-device active task)."""
active_owners = {}
try:
@@ -1338,9 +1541,7 @@ class TaskQueueService:
owner_record_exists = DeviceExecutionLock.has_owner_record(target_device)
owner_pid = owner.pid if owner else None
local_run_key, local_run, local_proc = cls._resolve_local_run(
target_sid, target_device
)
local_run_key, local_run, local_proc = cls._resolve_local_run(target_sid, target_device)
local_pid = getattr(local_proc, "pid", None)
is_local_owner = bool(owner_pid and local_pid and owner_pid == local_pid)
@@ -1409,7 +1610,6 @@ class TaskQueueService:
return cls._stop_targeted_task(target_sid, target_device)
@classmethod
def resume_task(cls) -> bool:
if PAUSE_FILE.exists():
@@ -1417,6 +1617,40 @@ class TaskQueueService:
return True
return False
@classmethod
def recover_orphaned_recordings_on_launch(cls) -> int:
"""Finalize recordings left behind by workers that died with the daemon.
The per-run finalizer handles workers that exit while the daemon is up;
this sweep covers everything else (daemon crash, machine reboot, tasks
cancelled before this recovery path existed). Returns the number of
recordings published.
"""
recovered = 0
try:
pending = session_repo.get_unfinalized_video_recordings()
except Exception as exc:
print(f"[ServerStartup] Could not enumerate unfinalized recordings: {exc}")
return 0
for row in pending:
sess_id = row.get("session_id")
try:
final_path = media_service.recover_orphaned_recording(
row.get("local_video_path"),
row.get("start_time") or row.get("session_start_time"),
)
except Exception as exc:
print(f"[ServerStartup] Recording recovery failed for {sess_id}: {exc}")
continue
if not final_path:
continue
if session_repo.mark_recording_ready(str(sess_id), str(final_path)):
recovered += 1
print(f"[ServerStartup] Recovered recording for session {sess_id}: {final_path}")
if recovered:
print(f"[ServerStartup] Recovered {recovered} orphaned recording(s).")
return recovered
@staticmethod
def archive_older_replays_on_launch():
"""Archives all existing step replay output folders on server launch."""
apps/admin_console/services/worker_process_io.py
+1 -3
View File
@@ -47,9 +47,7 @@ def subprocess_creation_kwargs() -> dict[str, Any]:
"stderr": asyncio.subprocess.STDOUT,
}
if sys.platform == "win32":
kwargs["creationflags"] = (
subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.CREATE_NO_WINDOW
)
kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.CREATE_NO_WINDOW
return kwargs