diff --git a/.env.example b/.env.example index 67272f1..4978170 100644 --- a/.env.example +++ b/.env.example @@ -19,6 +19,8 @@ ARTEMIS_VIDEO_ENABLED=false ARTEMIS_RECORDING_ENABLED=false ARTEMIS_OUTPUTTER_ENABLED=true ARTEMIS_STEP_SUMMARIZER_ENABLED=true +# 默认将任务截图长边压至 2048；false 保留设备原始截图。长边值可在 config/artemis.jsonc 的 agent.screenshot 中覆盖。 +ARTEMIS_SCREENSHOT_LIMIT_ENABLED=true # Pro 连续无工具调用或参数校验失败的最大次数；耗尽后明确失败，避免空转。 ARTEMIS_OPERATOR_MAX_NO_PROGRESS_ATTEMPTS=3 # Flash 连续无工具调用的最大次数；无效回复不会重复送回模型。 diff --git a/README.md b/README.md index ac747bd..54abfdb 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,8 @@ cd Artemis-Eapil 也可在项目 `.env` 中设置 `OPENAI_API_KEY`、`OPENAI_BASE_URL` 和 `OPENAI_MODEL`，其他参数见 [.env.example](.env.example)。新安装默认接口地址为 `https://api-n-cd.playones.com/v1`；已保存的地址不变。直接编辑环境文件后需重启相应服务。不要提交自己的密钥。 +“功能开关”默认将任务截图长边限制为 2048 像素；关闭后保留设备原始分辨率。可在 [config/artemis.jsonc](config/artemis.jsonc) 的 `agent.screenshot` 中覆盖 `enabled` 和 `max_long_edge`，重启后对新任务生效。 + 报告会区分“执行完成”和“验证通过”；没有验证证据的结果标为“未验证”。任务步骤与截图可查看，设备动作回放不提供。 ## 让 AI 使用 diff --git a/apps/admin_console/services/model_settings_service.py b/apps/admin_console/services/model_settings_service.py index 95ad6ca..ebcf93a 100644 --- a/apps/admin_console/services/model_settings_service.py +++ b/apps/admin_console/services/model_settings_service.py @@ -9,6 +9,7 @@ from urllib.parse import urlsplit from pydantic import BaseModel, Field, SecretStr, field_validator +from artemis.config.agent import load_agent_config from artemis.config.paths import get_env_file from artemis.config.settings import settings @@ -137,14 +138,19 @@ class CapabilitySettings(BaseModel): recording_enabled: bool = False outputter_enabled: bool = True step_summarizer_enabled: bool = True + screenshot_limit_enabled: bool = True + screenshot_max_long_edge: int = 2048 def get_capability_settings() -> CapabilitySettings: + screenshot = load_agent_config().screenshot return CapabilitySettings( video_enabled=settings.ARTEMIS_VIDEO_ENABLED, recording_enabled=settings.ARTEMIS_RECORDING_ENABLED, outputter_enabled=settings.ARTEMIS_OUTPUTTER_ENABLED, step_summarizer_enabled=settings.ARTEMIS_STEP_SUMMARIZER_ENABLED, + screenshot_limit_enabled=screenshot.enabled, + screenshot_max_long_edge=screenshot.max_long_edge, ) @@ -154,6 +160,7 @@ def save_capability_settings(update: CapabilitySettings) -> CapabilitySettings: "ARTEMIS_RECORDING_ENABLED": update.recording_enabled, "ARTEMIS_OUTPUTTER_ENABLED": update.outputter_enabled, "ARTEMIS_STEP_SUMMARIZER_ENABLED": update.step_summarizer_enabled, + "ARTEMIS_SCREENSHOT_LIMIT_ENABLED": update.screenshot_limit_enabled, } persist_environment({key: str(value).lower() for key, value in values.items()}) for key, value in values.items(): diff --git a/apps/showcase_ui/src/app/components/model-settings/capability-settings.component.ts b/apps/showcase_ui/src/app/components/model-settings/capability-settings.component.ts index d5bede4..e7a3c04 100644 --- a/apps/showcase_ui/src/app/components/model-settings/capability-settings.component.ts +++ b/apps/showcase_ui/src/app/components/model-settings/capability-settings.component.ts @@ -8,6 +8,8 @@ interface Capabilities { recording_enabled: boolean; outputter_enabled: boolean; step_summarizer_enabled: boolean; + screenshot_limit_enabled: boolean; + screenshot_max_long_edge: number; } @Component({ @@ -24,6 +26,8 @@ interface Capabilities { 最终报告生成根据执行记录生成任务结果报告。 + @if (message()) {

{{ message() }}

} @if (error()) { } @@ -39,7 +43,7 @@ interface Capabilities { export class CapabilitySettingsComponent implements OnInit { changed = output(); private http = inject(HttpClient); - config: Capabilities = { video_enabled: false, recording_enabled: false, outputter_enabled: true, step_summarizer_enabled: true }; + config: Capabilities = { video_enabled: false, recording_enabled: false, outputter_enabled: true, step_summarizer_enabled: true, screenshot_limit_enabled: true, screenshot_max_long_edge: 2048 }; busy = signal(false); loading = signal(true); loaded = signal(false); message = signal(''); error = signal(''); ngOnInit(): void { this.http.get('/api/system/capability-settings').pipe(finalize(() => this.loading.set(false))).subscribe({ diff --git a/apps/showcase_ui/src/app/components/model-settings/model-settings.component.spec.ts b/apps/showcase_ui/src/app/components/model-settings/model-settings.component.spec.ts index 2ac8b69..5e37237 100644 --- a/apps/showcase_ui/src/app/components/model-settings/model-settings.component.spec.ts +++ b/apps/showcase_ui/src/app/components/model-settings/model-settings.component.spec.ts @@ -110,17 +110,19 @@ describe('持久化模型与功能设置', () => { }); it('录屏与视频分析独立保存，重新打开后保留各自状态', () => { - const capabilities = { video_enabled: true, recording_enabled: false, outputter_enabled: true, step_summarizer_enabled: false }; + const capabilities = { video_enabled: true, recording_enabled: false, outputter_enabled: true, step_summarizer_enabled: false, screenshot_limit_enabled: true, screenshot_max_long_edge: 2048 }; const fixture = TestBed.createComponent(CapabilitySettingsComponent); fixture.detectChanges(); http.expectOne('/api/system/capability-settings').flush(capabilities); fixture.componentInstance.config.video_enabled = false; fixture.componentInstance.config.recording_enabled = true; + fixture.componentInstance.config.screenshot_limit_enabled = false; fixture.componentInstance.save(); const request = http.expectOne('/api/system/capability-settings'); expect(request.request.body.video_enabled).toBeFalse(); expect(request.request.body.recording_enabled).toBeTrue(); - const saved = { ...capabilities, video_enabled: false, recording_enabled: true }; + expect(request.request.body.screenshot_limit_enabled).toBeFalse(); + const saved = { ...capabilities, video_enabled: false, recording_enabled: true, screenshot_limit_enabled: false }; request.flush(saved); fixture.destroy(); const reopened = TestBed.createComponent(CapabilitySettingsComponent); @@ -128,5 +130,6 @@ describe('持久化模型与功能设置', () => { http.expectOne('/api/system/capability-settings').flush(saved); expect(reopened.componentInstance.config.video_enabled).toBeFalse(); expect(reopened.componentInstance.config.recording_enabled).toBeTrue(); + expect(reopened.componentInstance.config.screenshot_limit_enabled).toBeFalse(); }); }); diff --git a/artemis/config/agent.py b/artemis/config/agent.py index 9cec72a..29a5080 100644 --- a/artemis/config/agent.py +++ b/artemis/config/agent.py @@ -835,6 +835,13 @@ class ProProfileConfig(BaseModel): model_config = {"extra": "allow"} +class ScreenshotConfig(BaseModel): + """Limit screenshots sent to agents while retaining physical device coordinates.""" + + enabled: bool = Field(default_factory=lambda: settings.ARTEMIS_SCREENSHOT_LIMIT_ENABLED) + max_long_edge: int = Field(default=2048, ge=256, le=8192) + + class AgentGlobalConfig(BaseModel): """Global configuration parsed from artemis.jsonc / agent_config.json.""" @@ -889,6 +896,10 @@ class AgentGlobalConfig(BaseModel): default_factory=MemoryConfig, description="Unified step-memory runtime and transcript scrub options.", ) + screenshot: ScreenshotConfig = Field( + default_factory=ScreenshotConfig, + description="Agent screenshot size limit; device/UI coordinates remain native.", + ) model_config = {"extra": "allow"} diff --git a/artemis/config/settings.py b/artemis/config/settings.py index 2fd8ddb..a3bb68c 100644 --- a/artemis/config/settings.py +++ b/artemis/config/settings.py @@ -112,6 +112,7 @@ class Settings(BaseSettings): ARTEMIS_RECORDING_ENABLED: bool = False ARTEMIS_OUTPUTTER_ENABLED: bool = True ARTEMIS_STEP_SUMMARIZER_ENABLED: bool = True + ARTEMIS_SCREENSHOT_LIMIT_ENABLED: bool = True # Consecutive Pro responses without a valid tool call before failing the task. ARTEMIS_OPERATOR_MAX_NO_PROGRESS_ATTEMPTS: int = Field(default=3, ge=1, le=20) # Consecutive Flash responses without any tool call before failing the task. diff --git a/artemis/drivers/android/adb_driver.py b/artemis/drivers/android/adb_driver.py index 4a54381..8934081 100644 --- a/artemis/drivers/android/adb_driver.py +++ b/artemis/drivers/android/adb_driver.py @@ -25,12 +25,14 @@ from artemis.clients.ui_automator_client import ( UIAutomatorClient, _parse_hierarchy_xml_to_elements, ) +from artemis.config.agent import load_agent_config from artemis.config.paths import get_temp_dir from artemis.drivers.base import BaseDeviceDriver, KeyCode, ScreenData, SwipeDirection from artemis.toolchain import find_ffmpeg, find_scrcpy from artemis.utils.video import build_scrcpy_record_command from artemis.utils.ui_filter import filter_ui_hierarchy from artemis.utils.logger import get_logger +from artemis.utils.screenshot_limit import limit_screenshot_long_edge logger = get_logger(__name__) @@ -88,6 +90,7 @@ class AndroidAdbDriver(BaseDeviceDriver): self._device: AdbDevice | None = None self._recording_process: asyncio.subprocess.Process | None = None self._recording_output_path: Path | None = None + self._screenshot_config = load_agent_config().screenshot @property def device_id(self) -> str: @@ -215,6 +218,13 @@ class AndroidAdbDriver(BaseDeviceDriver): if isinstance(screenshot_bytes, bytes) and screenshot_bytes else base64.b64decode(clean_b64) ) + if self._screenshot_config.enabled: + limited_bytes = limit_screenshot_long_edge( + clean_bytes, self._screenshot_config.max_long_edge + ) + if limited_bytes is not clean_bytes: + clean_bytes = limited_bytes + clean_b64 = base64.b64encode(clean_bytes).decode("utf-8") return ScreenData( screenshot_bytes=clean_bytes, diff --git a/artemis/resources/config/artemis.jsonc b/artemis/resources/config/artemis.jsonc index 2a876f2..05d17f9 100644 --- a/artemis/resources/config/artemis.jsonc +++ b/artemis/resources/config/artemis.jsonc @@ -27,6 +27,10 @@ "checker": {} }, "agent": { + "screenshot": { + "enabled": "${env:ARTEMIS_SCREENSHOT_LIMIT_ENABLED:-true}", + "max_long_edge": 2048 + }, "flash": { "max_turns": 0, "explorer_mode": "flash", diff --git a/artemis/utils/screenshot_limit.py b/artemis/utils/screenshot_limit.py new file mode 100644 index 0000000..037d487 --- /dev/null +++ b/artemis/utils/screenshot_limit.py @@ -0,0 +1,23 @@ +"""Prepare Android screenshots for model input without changing device coordinates.""" + +from io import BytesIO + +from PIL import Image + + +def limit_screenshot_long_edge(image_bytes: bytes, max_long_edge: int) -> bytes: + """Downscale oversized screenshots; keep smaller images byte-for-byte unchanged.""" + try: + with Image.open(BytesIO(image_bytes)) as image: + long_edge = max(image.size) + if long_edge <= max_long_edge: + return image_bytes + scale = max_long_edge / long_edge + size = tuple(max(1, round(side * scale)) for side in image.size) + resized = image.convert("RGB").resize(size, Image.Resampling.LANCZOS) + output = BytesIO() + resized.save(output, format="JPEG", quality=80, optimize=True) + return output.getvalue() + except OSError: + # Preserve the capture path's previous behavior for invalid placeholder images. + return image_bytes diff --git a/config/artemis.jsonc b/config/artemis.jsonc index 2a876f2..05d17f9 100644 --- a/config/artemis.jsonc +++ b/config/artemis.jsonc @@ -27,6 +27,10 @@ "checker": {} }, "agent": { + "screenshot": { + "enabled": "${env:ARTEMIS_SCREENSHOT_LIMIT_ENABLED:-true}", + "max_long_edge": 2048 + }, "flash": { "max_turns": 0, "explorer_mode": "flash", diff --git a/tests/unit/admin_console/test_model_settings.py b/tests/unit/admin_console/test_model_settings.py index 3de5387..c8d3f08 100644 --- a/tests/unit/admin_console/test_model_settings.py +++ b/tests/unit/admin_console/test_model_settings.py @@ -27,6 +27,7 @@ def isolated_settings(monkeypatch, tmp_path): "ARTEMIS_VIDEO_ENABLED", "ARTEMIS_OUTPUTTER_ENABLED", "ARTEMIS_STEP_SUMMARIZER_ENABLED", + "ARTEMIS_SCREENSHOT_LIMIT_ENABLED", ] for key in keys: monkeypatch.setattr(settings, key, getattr(settings, key)) @@ -93,21 +94,38 @@ def test_rejects_invalid_base_url(url): def test_capabilities_persist_and_apply_to_new_tasks(isolated_settings): result = service.save_capability_settings( service.CapabilitySettings( - video_enabled=False, outputter_enabled=False, step_summarizer_enabled=False + video_enabled=False, + outputter_enabled=False, + step_summarizer_enabled=False, + screenshot_limit_enabled=False, ) ) assert result.video_enabled is False saved = dotenv_values(isolated_settings) assert saved["ARTEMIS_VIDEO_ENABLED"] == "false" + assert saved["ARTEMIS_SCREENSHOT_LIMIT_ENABLED"] == "false" config = load_agent_config() assert config.video_analyzer.enabled is False assert config.pro.video_analyzer.enabled is False assert config.outputter.enabled is False assert config.flash.step_summarizer.enabled is False + assert config.screenshot.enabled is False service.save_capability_settings(service.CapabilitySettings(video_enabled=True)) assert load_agent_config().video_analyzer.enabled is True +def test_screenshot_json_can_override_environment(monkeypatch, tmp_path): + config_path = tmp_path / "artemis.jsonc" + config_path.write_text( + '{"agent": {"screenshot": {"enabled": true, "max_long_edge": 1024}}}', + encoding="utf-8", + ) + monkeypatch.setenv("ARTEMIS_SCREENSHOT_LIMIT_ENABLED", "false") + screenshot = load_agent_config(config_path).screenshot + assert screenshot.enabled is True + assert screenshot.max_long_edge == 1024 + + def test_failed_write_keeps_running_settings(monkeypatch, isolated_settings): previous = settings.OPENAI_MODEL monkeypatch.setattr( diff --git a/tests/unit/test_drivers.py b/tests/unit/test_drivers.py index 4efba16..592ebdc 100644 --- a/tests/unit/test_drivers.py +++ b/tests/unit/test_drivers.py @@ -15,14 +15,18 @@ """Unit tests for Unified Base Device Driver and implementations.""" import base64 +from io import BytesIO from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock import pytest +from PIL import Image from artemis.drivers.base import KeyCode, ScreenData, SwipeDirection from artemis.drivers.mock.mock_driver import MockDeviceDriver from artemis.drivers.android.adb_driver import AndroidAdbDriver +from artemis.config.agent import ScreenshotConfig +from artemis.utils.screenshot_limit import limit_screenshot_long_edge _ONE_PIXEL_PNG = base64.b64encode( @@ -32,6 +36,13 @@ _ONE_PIXEL_PNG = base64.b64encode( ).decode("ascii") +def test_screenshot_limit_keeps_small_and_invalid_images_unchanged(): + small_image = base64.b64decode(_ONE_PIXEL_PNG) + assert limit_screenshot_long_edge(small_image, 2048) is small_image + invalid_image = b"not an image" + assert limit_screenshot_long_edge(invalid_image, 2048) is invalid_image + + @pytest.mark.asyncio async def test_mock_driver_actions(): """Verify MockDeviceDriver records and performs all core actions.""" @@ -141,6 +152,36 @@ async def test_android_driver_preserves_elements_from_combined_screen_data(): mock_ui_client.get_hierarchy.assert_not_called() +@pytest.mark.asyncio +@pytest.mark.parametrize("enabled", [True, False]) +async def test_android_screenshot_limit_preserves_native_coordinates(enabled): + image = Image.new("RGB", (1200, 2600), "white") + buffer = BytesIO() + image.save(buffer, format="JPEG") + original = buffer.getvalue() + elements = [{"text": "目标", "bounds": "[100,200][300,400]"}] + ui_client = MagicMock() + ui_client.get_screen_data.return_value = SimpleNamespace( + base64=base64.b64encode(original).decode("ascii"), + hierarchy_xml="", + elements=elements, + width=1200, + height=2600, + ) + driver = AndroidAdbDriver("emulator-5554", MagicMock(), ui_adb_client=ui_client) + driver._screenshot_config = ScreenshotConfig(enabled=enabled, max_long_edge=2048) + + screen = await driver.get_screen_data(skip_settling=True) + + with Image.open(BytesIO(screen.screenshot_bytes)) as result: + assert result.size == ((945, 2048) if enabled else (1200, 2600)) + assert screen.width == 1200 and screen.height == 2600 + assert screen.ui_elements == elements + assert screen.screenshot_base64 == base64.b64encode(screen.screenshot_bytes).decode("ascii") + if not enabled: + assert screen.screenshot_bytes == original + + @pytest.mark.asyncio async def test_android_driver_parses_xml_hierarchy_fallback(): """XML-only clients remain supported without discarding their hierarchy."""