From 4d6560dbd47cff63467b82373a023f1a954a9f62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E9=93=AD=E8=BD=A9?= Date: Fri, 9 Oct 2026 16:29:16 +0800 Subject: [PATCH] feat: support batch repetition and selective case import --- .env.example | 1 + apps/admin_console/routers/automation.py | 9 +- apps/admin_console/routers/sessions.py | 8 +- .../services/automation_service.py | 54 +++++++--- apps/admin_console/services/report_export.py | 6 +- apps/admin_console/services/task_import.py | 8 ++ .../automation-panel.component.html | 25 ++++- .../automation-panel.component.scss | 7 ++ .../automation-panel.component.spec.ts | 55 ++++++++++ .../automation-panel.component.ts | 30 +++++- artemis/config/settings.py | 1 + docs/automation-workspace.md | 6 +- mcp_server/tools/automation.py | 10 +- .../artemis-automation/references/batches.md | 5 +- tests/unit/admin_console/test_automation.py | 101 ++++++++++++++++++ tests/unit/mcp/test_automation_tools.py | 7 +- 16 files changed, 296 insertions(+), 37 deletions(-) diff --git a/.env.example b/.env.example index 3888cbc..fbf4cf5 100644 --- a/.env.example +++ b/.env.example @@ -73,4 +73,5 @@ ARTEMIS_DEFAULT_PROFILE=flash # 批量任务与 Excel 导入限制 ARTEMIS_IMPORT_MAX_ROWS=1000 +ARTEMIS_BATCH_MAX_EXECUTIONS=10000 ARTEMIS_IMPORT_MAX_BYTES=10485760 diff --git a/apps/admin_console/routers/automation.py b/apps/admin_console/routers/automation.py index a9e2006..b880be5 100644 --- a/apps/admin_console/routers/automation.py +++ b/apps/admin_console/routers/automation.py @@ -30,6 +30,7 @@ class BatchRequest(ReportPreferences): device_serial: str = Field(min_length=1, max_length=200) profile: Literal["flash", "pro"] = "flash" stop_on_failure: bool = False + repeat_count: int = Field(default=1, ge=1, strict=True) timeout_seconds: int = Field( default_factory=lambda: settings.ARTEMIS_TASK_TIMEOUT_SECONDS, ge=30, @@ -52,6 +53,7 @@ def preferences(): return { **store.preferences(), "batch_timeout_seconds": settings.ARTEMIS_TASK_TIMEOUT_SECONDS, + "batch_max_executions": settings.ARTEMIS_BATCH_MAX_EXECUTIONS, } @@ -256,7 +258,12 @@ def history( [*args, size, (page - 1) * size], ).fetchall() items = [dict(row) for row in rows] + jobs = store.jobs_for_sessions([item["session_id"] for item in items]) for item in items: - if item["case_id"]: + if job := jobs.get(item["session_id"]): + item["name"] = task_import.execution_title(job) + item["iteration"] = job.get("iteration", 1) + item["repeat_count"] = job.get("repeat_count", 1) + elif item["case_id"]: item["name"] = task_import.case_title(item) return {"items": items, "total": total, "page": page, "size": size} diff --git a/apps/admin_console/routers/sessions.py b/apps/admin_console/routers/sessions.py index 675103b..60e7b93 100644 --- a/apps/admin_console/routers/sessions.py +++ b/apps/admin_console/routers/sessions.py @@ -28,7 +28,7 @@ try: from admin_console.services.media_service import media_service from admin_console.services.model_service import model_service from admin_console.services.automation_store import store - from admin_console.services.task_import import case_title + from admin_console.services.task_import import execution_title except ImportError: from apps.admin_console.core.state import state from apps.admin_console.database.repositories.session_repository import session_repo @@ -36,7 +36,7 @@ except ImportError: from apps.admin_console.services.media_service import media_service from apps.admin_console.services.model_service import model_service from apps.admin_console.services.automation_store import store - from apps.admin_console.services.task_import import case_title + from apps.admin_console.services.task_import import execution_title router = APIRouter(tags=["sessions"]) @@ -83,7 +83,9 @@ def _list_sessions_sync(): if job: case = job.get("case") or {} row_dict["case_id"] = case.get("case_id") or "" - row_dict["display_name"] = case_title(case) or row_dict.get("initial_goal") + row_dict["display_name"] = execution_title(job) or row_dict.get("initial_goal") + row_dict["iteration"] = job.get("iteration", 1) + row_dict["repeat_count"] = job.get("repeat_count", 1) row_dict["error"] = job.get("error") or "" if job.get("status") in {"completed", "failed", "cancelled", "interrupted", "skipped"}: row_dict["status"] = job["status"] diff --git a/apps/admin_console/services/automation_service.py b/apps/admin_console/services/automation_service.py index 9f32558..a7ba0b4 100644 --- a/apps/admin_console/services/automation_service.py +++ b/apps/admin_console/services/automation_service.py @@ -9,7 +9,7 @@ from uuid import UUID from apps.admin_console.core.state import state from apps.admin_console.services.automation_store import RECOVERY_ERRORS, store, new_id -from apps.admin_console.services.task_import import case_goal, case_title +from apps.admin_console.services.task_import import case_goal, execution_title from artemis.config.settings import settings from artemis.runtime.task_limits import task_timeout @@ -297,7 +297,7 @@ async def tick_batches(): runtime_overrides=options, session_id=job["session_id"], ingress="batch", - display_name=case_title(job["case"]), + display_name=execution_title(job), ) if response.get("status") == "rejected" or not response.get("tasks"): raise ValueError(response.get("error") or "队列拒绝接收任务") @@ -330,6 +330,15 @@ def has_legacy_replay(batch_id: str) -> bool: def create_batch(payload: dict) -> dict: + repeat_count = payload.get("repeat_count", 1) + if type(repeat_count) is not int or repeat_count < 1: + raise ValueError("循环次数必须为正整数") + cases = payload["cases"] + total = len(cases) * repeat_count + if not cases or total > settings.ARTEMIS_BATCH_MAX_EXECUTIONS: + raise ValueError( + f"请选择用例；用例数 × 循环次数不能超过 {settings.ARTEMIS_BATCH_MAX_EXECUTIONS} 次执行" + ) bid = new_id() batch = { "id": bid, @@ -342,21 +351,28 @@ def create_batch(payload: dict) -> dict: "timeout_seconds": payload.get("timeout_seconds") or settings.ARTEMIS_TASK_TIMEOUT_SECONDS, "stop_on_failure": payload.get("stop_on_failure", False), "request_id": payload.get("request_id"), + "repeat_count": repeat_count, + "case_count": len(cases), + "total_executions": total, } jobs = [] - for number, case in enumerate(payload["cases"], 1): - jobs.append( - { - "session_id": new_id(), - "batch_id": bid, - "position": number, - "case": case, - "status": "pending", - "formats": batch["formats"], - "created_at": time.time(), - "device_serial": batch["device_serial"], - } - ) + for iteration in range(1, repeat_count + 1): + for number, case in enumerate(cases, 1): + jobs.append( + { + "session_id": new_id(), + "batch_id": bid, + "position": len(jobs) + 1, + "iteration": iteration, + "repeat_count": repeat_count, + "case_position": number, + "case": case, + "status": "pending", + "formats": batch["formats"], + "created_at": batch["created_at"], + "device_serial": batch["device_serial"], + } + ) return store.create_batch(batch, jobs, payload) @@ -412,7 +428,13 @@ def batch_action(batch_id: str, action: str) -> dict: if not failed: raise ValueError("没有可重跑的失败或中断任务") revised = create_batch( - {**batch, "name": batch["name"] + " · 失败项重跑", "cases": [j["case"] for j in failed]} + { + **batch, + "name": batch["name"] + " · 失败项重跑", + "cases": [j["case"] for j in failed], + "repeat_count": 1, + "request_id": None, + } ) return revised else: diff --git a/apps/admin_console/services/report_export.py b/apps/admin_console/services/report_export.py index 1267845..ca243ab 100644 --- a/apps/admin_console/services/report_export.py +++ b/apps/admin_console/services/report_export.py @@ -22,7 +22,7 @@ from artemis.config import IMAGES_DIR, TRACES_PATH, WORKSPACE_ROOT from apps.admin_console.database.repositories.session_repository import SessionRepository from apps.admin_console.database.repositories.step_repository import StepRepository from apps.admin_console.services.automation_store import store, identifier -from apps.admin_console.services.task_import import case_title +from apps.admin_console.services.task_import import execution_title EXPORT_ROOT = WORKSPACE_ROOT / "artifacts" / "reports" EXPORT_LOCK = threading.RLock() @@ -190,7 +190,9 @@ def session_report(session_id: str, db_path=None) -> dict: ) return { "id": session_id, - "name": case_title(case) or session.get("initial_goal", session_id)[:120], + "name": execution_title(meta) or session.get("initial_goal", session_id)[:120], + "iteration": meta.get("iteration", 1), + "repeat_count": meta.get("repeat_count", 1), "goal": case.get("goal") or session.get("initial_goal", ""), "expected": case.get("expected", ""), "status": status, diff --git a/apps/admin_console/services/task_import.py b/apps/admin_console/services/task_import.py index eab390c..f1f3eab 100644 --- a/apps/admin_console/services/task_import.py +++ b/apps/admin_console/services/task_import.py @@ -189,3 +189,11 @@ def case_title(case: dict) -> str: case_id = str(case.get("case_id") or "").strip() name = str(case.get("name") or "").strip() return " ".join(part for part in (case_id, name) if part) + + +def execution_title(job: dict) -> str: + """Distinguish repeated executions without altering the original case or prompt.""" + title = case_title(job.get("case") or {}) + if job.get("repeat_count", 1) > 1: + title += f" · 第 {job['iteration']} / {job['repeat_count']} 轮" + return title diff --git a/apps/showcase_ui/src/app/components/automation-panel/automation-panel.component.html b/apps/showcase_ui/src/app/components/automation-panel/automation-panel.component.html index 4f58d49..de13c7a 100644 --- a/apps/showcase_ui/src/app/components/automation-panel/automation-panel.component.html +++ b/apps/showcase_ui/src/app/components/automation-panel/automation-panel.component.html @@ -47,26 +47,41 @@

导入任务

每行一个用例，仅「Artemis提示词」作为任务正文；「任务配置(JSON)」可设置每项模型、模式、超时和录屏。「前置约束提取」仅供查看与搜索。示例在模板的「填写说明」页。

下载 Excel 模板
@if(preview(); as p) { -

有效任务 {{p.tasks.length}} 条 · 错误 {{p.errors.length}} 条

+
已选 {{selectedCount()}} / {{p.tasks.length}} 个用例 · 错误 {{p.errors.length}} 条
@for(e of p.errors; track e.row){

第 {{e.row}} 行：{{e.message}}

} -
@for(t of p.tasks; track t.row){}
行用例提示词任务配置
{{t.row}}{{t.name}}{{t.goal}}{{t.task_config | json}}
+
+ @for(t of p.tasks; track t.row){ +
+ +
{{t.case_id ? t.case_id + ' ' : ''}}{{t.name}}第 {{t.row}} 行 +

{{t.goal}}

+ @if(caseConfig(t.task_config); as config){
自定义参数
{{config}}
}@else{使用批次 / 全局参数} +
+
+ } +
+ +

{{selectedCount()}} 个用例 × {{repeatCount || 0}} 轮，共 {{selectedCount() * (repeatCount || 0)}} 次执行

+

每轮按表格顺序执行所选用例，分别保存结果。未勾选用例不会导入。

+ @if(!validRepetitions()){

循环次数须为正整数，执行总数不超过 {{maxExecutions}}。

} - + }

批次记录

@for(b of batches(); track b.id){
- +
} @if(!batches().length){

暂无批次。

}
@if(batch(); as b){

{{b.name}}

{{label(b.status)}} · {{b.reason}}

+

{{b.case_count || b.jobs?.length || 0}} 个用例 · {{b.repeat_count || 1}} 轮 · 共 {{b.total_executions || b.jobs?.length || 0}} 次执行

@if(b.status === 'running'){} @if(b.status === 'draft' || b.status === 'paused'){} @@ -87,6 +102,6 @@
} @if(b.report_error){

报告生成失败：{{b.report_error}}

} - @for(j of b.jobs; track j.session_id){
{{label(j.status)}}@if(j.error){{{j.error}}}
} + @for(j of b.jobs; track j.session_id){
{{label(j.status)}}@if(j.error){{{j.error}}}
}
} } diff --git a/apps/showcase_ui/src/app/components/automation-panel/automation-panel.component.scss b/apps/showcase_ui/src/app/components/automation-panel/automation-panel.component.scss index 35d8ea6..0d83e66 100644 --- a/apps/showcase_ui/src/app/components/automation-panel/automation-panel.component.scss +++ b/apps/showcase_ui/src/app/components/automation-panel/automation-panel.component.scss @@ -16,6 +16,13 @@ input:not([type=checkbox]):not([type=file]),select,textarea{box-sizing:border-bo input[type=checkbox]{width:15px;height:15px;accent-color:#2563eb;vertical-align:-3px;margin-right:6px;} .upload{position:relative;margin:0;color:#fff;background:#2563eb;border-color:#2563eb;}.upload:hover{background:#1d4ed8;border-color:#1d4ed8;}.upload input{position:absolute;inset:0;width:100%;height:100%;opacity:0;cursor:pointer;} .device{background:#f8fafc;padding:12px 16px;}.device label{margin:0;} +.selection-toolbar{display:flex;align-items:center;justify-content:space-between;gap:8px;margin:12px 0;color:#475569;} +.selection-column{flex:0 0 18px;text-align:center;}.selection-column input{margin:0;} +.case-preview-list{max-height:320px;overflow:auto;border:1px solid #e2e8f0;border-radius:8px;margin-bottom:14px;} +.case-preview{display:flex;align-items:flex-start;gap:10px;padding:12px;border-bottom:1px solid #e2e8f0;&:last-child{border-bottom:0;}&.unselected{background:#f8fafc;color:#64748b;}} +.case-preview-content{min-width:0;flex:1;strong{display:block;overflow-wrap:anywhere;}small{display:block;}details{margin:5px 0 0;}summary{padding:0;font-size:12px;color:#2563eb;}pre{max-height:160px;overflow:auto;white-space:pre-wrap;overflow-wrap:anywhere;background:#f8fafc;border-radius:6px;padding:8px;font-size:11px;}} +.case-goal{display:-webkit-box;-webkit-box-orient:vertical;-webkit-line-clamp:2;overflow:hidden;white-space:pre-wrap;margin:5px 0;font-size:12px;} +.execution-summary{padding:9px 12px;border-radius:8px;background:#eff6ff;color:#1d4ed8;font-weight:500;} .record{display:flex;width:100%;text-align:left;align-items:flex-start;flex-direction:column;gap:4px;margin:8px 0 0;padding:12px;overflow-wrap:anywhere;}.record span{color:#64748b;font-size:12px;font-weight:400;}.record.selected{border-color:#93c5fd;background:#eff6ff;}.record.selected strong{color:#1d4ed8;} .batch-record{display:flex;gap:8px;align-items:stretch;margin-top:8px;}.batch-record .record{flex:1;min-width:0;width:auto;margin:0;}.batch-record .delete-batch{flex:0 0 40px;min-width:40px;padding:0;color:#b91c1c;}.batch-record .delete-batch:hover:not(:disabled){background:#fff1f2;border-color:#fda4af;}.batch-record .delete-batch .material-symbols-outlined{font-size:19px;} .archive-progress{margin:12px 0;padding:11px 13px;border:1px solid #bfdbfe;border-radius:10px;background:#f8fbff;color:#1e40af;} diff --git a/apps/showcase_ui/src/app/components/automation-panel/automation-panel.component.spec.ts b/apps/showcase_ui/src/app/components/automation-panel/automation-panel.component.spec.ts index c96df6f..7e4f550 100644 --- a/apps/showcase_ui/src/app/components/automation-panel/automation-panel.component.spec.ts +++ b/apps/showcase_ui/src/app/components/automation-panel/automation-panel.component.spec.ts @@ -22,6 +22,7 @@ describe('AutomationPanelComponent batch deletion', () => { http.delete.and.returnValue(of({ status: 'deleted' })); http.post.and.returnValue(of({ status: 'ready', stage: '报告包已就绪', completed: 0, total: 0, elapsed_seconds: 0, eta_seconds: null, error: null })); http.delete.calls.reset(); + http.post.calls.reset(); await TestBed.configureTestingModule({ imports: [AutomationPanelComponent], providers: [ @@ -32,6 +33,60 @@ describe('AutomationPanelComponent batch deletion', () => { }).compileComponents(); }); + it('selects all imported cases by default, toggles all and submits only checked cases with rounds', async () => { + const tasks = [ + { row: 2, case_id: '1.1', name: '打开设置', goal: '观察设置', task_config: { model: 'mimo-v2.6-pro' } }, + { row: 3, case_id: '1.2', name: '观察首页', goal: '观察首页', task_config: {} } + ]; + (http.post.and as any).callFake((url: string) => of(url.endsWith('/preview') + ? { valid: true, tasks, errors: [] } : url.endsWith('/batches') + ? { id: 'selected-batch', status: 'draft' } : { id: 'selected-batch', status: 'running' })); + const fixture = TestBed.createComponent(AutomationPanelComponent); + const component = fixture.componentInstance; + component.tab.set('batches'); + component.upload({ target: { files: [new File(['test'], '用例.xlsx')] } } as unknown as Event); + await new Promise(resolve => setTimeout(resolve, 0)); + fixture.detectChanges(); + const checkboxes = () => [...fixture.nativeElement.querySelectorAll('.selection-column input')] as HTMLInputElement[]; + expect(checkboxes().every(c => c.checked)).toBeTrue(); + const toggle = fixture.nativeElement.querySelector('.selection-toolbar button') as HTMLButtonElement; + expect(toggle.textContent).toContain('全不选'); + toggle.click(); fixture.detectChanges(); + expect(component.selectedCount()).toBe(0); + expect((fixture.nativeElement.querySelector('.primary') as HTMLButtonElement).disabled).toBeTrue(); + expect(toggle.textContent).toContain('全选'); + toggle.click(); fixture.detectChanges(); + expect(component.selectedCount()).toBe(2); + checkboxes()[1].click(); + component.repeatCount = 3; component.device = 'test-device'; + fixture.detectChanges(); + expect(fixture.nativeElement.querySelector('.execution-summary').textContent).toContain('共 3 次执行'); + component.startBatch(); + await new Promise(resolve => setTimeout(resolve, 0)); + const request = http.post.calls.allArgs().find(args => args[0] === '/api/automation/batches'); + expect(request?.[1]).toEqual(jasmine.objectContaining({ + cases: [{ case_id: '1.1', name: '打开设置', goal: '观察设置', task_config: { model: 'mimo-v2.6-pro' } }], + repeat_count: 3, device_serial: 'test-device' + })); + }); + + it('rejects empty selections and invalid or excessive loop counts without creating a batch', async () => { + const fixture = TestBed.createComponent(AutomationPanelComponent); + const component = fixture.componentInstance; + component.preview.set({ valid: true, tasks: [{ row: 2, name: '检查', goal: '观察' }], errors: [] }); + component.device = 'test-device'; + component.startBatch(); + await new Promise(resolve => setTimeout(resolve, 0)); + expect(component.error()).toContain('至少勾选'); + component.selectedRows.set(new Set([2])); + for (const count of [0, 1.5, 10001]) { + component.repeatCount = count; component.startBatch(); + await new Promise(resolve => setTimeout(resolve, 0)); + expect(component.error()).toContain('循环次数'); + } + expect(http.post).not.toHaveBeenCalled(); + }); + it('confirms deletion and removes the batch from the visible list', async () => { const fixture = TestBed.createComponent(AutomationPanelComponent); fixture.componentInstance.tab.set('batches'); diff --git a/apps/showcase_ui/src/app/components/automation-panel/automation-panel.component.ts b/apps/showcase_ui/src/app/components/automation-panel/automation-panel.component.ts index 01e1730..fd81ab4 100644 --- a/apps/showcase_ui/src/app/components/automation-panel/automation-panel.component.ts +++ b/apps/showcase_ui/src/app/components/automation-panel/automation-panel.component.ts @@ -25,6 +25,8 @@ export class AutomationPanelComponent { agent = inject(AgentService); system = inject(SystemService); http = inject(HttpClient); tab = signal('reports'); error = signal(''); notice = signal(''); busy = signal(false); batches = signal([]); preview = signal(null); + selectedRows = signal>(new Set()); + selectedCount = computed(() => this.selectedRows().size); batch = signal(null); report = signal(null); archiveProgress = signal(null); archiveBatchId = signal(''); archiveBusy = signal(false); @@ -37,10 +39,11 @@ export class AutomationPanelComponent { }); liveTask = computed(() => ['running', 'paused', 'task_paused', 'resuming'].includes(this.taskStatus())); word = true; excel = true; name = ''; device = ''; profile = 'flash'; stopOnFailure = false; timeout = 1800; + repeatCount = 1; maxExecutions = 10000; private destroyed = false; constructor() { const destroy = inject(DestroyRef); - this.http.get('/api/automation/preferences').subscribe(p => { this.word = p.formats.includes('docx'); this.excel = p.formats.includes('xlsx'); this.timeout = p.batch_timeout_seconds || 1800; }); + this.http.get('/api/automation/preferences').subscribe(p => { this.word = p.formats.includes('docx'); this.excel = p.formats.includes('xlsx'); this.timeout = p.batch_timeout_seconds || 1800; this.maxExecutions = p.batch_max_executions || 10000; }); void this.refresh(); const timer = setInterval(() => void this.refresh(), 5000); destroy.onDestroy(() => { this.destroyed = true; clearInterval(timer); }); @@ -104,13 +107,32 @@ export class AutomationPanelComponent { savePreferences() { void this.action(async () => { if (!this.formats().length) throw Error('请至少选择一种报告格式'); await firstValueFrom(this.http.put('/api/automation/preferences',{formats:this.formats()})); this.notice.set('已保存，后续任务和批次使用此默认格式'); }); } upload(event: Event) { const file = (event.target as HTMLInputElement).files?.[0]; if (!file) return; - void this.action(async () => { this.preview.set(null); const data = new FormData(); data.append('file',file); this.preview.set(await firstValueFrom(this.http.post('/api/automation/import/preview',data))); this.name = file.name.replace(/\.xlsx$/i,''); }); + void this.action(async () => { this.preview.set(null); this.selectedRows.set(new Set()); const data = new FormData(); data.append('file',file); const preview: any = await firstValueFrom(this.http.post('/api/automation/import/preview',data)); this.preview.set(preview); this.selectedRows.set(new Set(preview.tasks.map((t: any) => t.row))); this.repeatCount = 1; this.name = file.name.replace(/\.xlsx$/i,''); }); + } + toggleRow(row: number) { + const selected = new Set(this.selectedRows()); + if (selected.has(row)) selected.delete(row); else selected.add(row); + this.selectedRows.set(selected); + } + toggleAllRows() { + const rows: number[] = (this.preview()?.tasks || []).map((t: any) => t.row); + this.selectedRows.set(this.selectedCount() === rows.length ? new Set() : new Set(rows)); + } + validRepetitions() { + return Number.isInteger(this.repeatCount) && this.repeatCount >= 1 + && this.selectedCount() * this.repeatCount <= this.maxExecutions; + } + caseConfig(config: Record | undefined) { + const overrides = Object.fromEntries(Object.entries(config || {}).filter(([, value]) => value !== null && value !== undefined)); + return Object.keys(overrides).length ? JSON.stringify(overrides, null, 2) : ''; } startBatch() { void this.action(async () => { if (!this.preview()?.valid) throw Error('请先导入并修正所有错误'); + if (!this.selectedCount()) throw Error('请至少勾选一个用例'); + if (!this.validRepetitions()) throw Error(`循环次数必须为正整数，执行总数不能超过 ${this.maxExecutions}`); if (!this.device) throw Error('请选择执行设备'); - const cases = this.preview().tasks.map(({row, ...c}: any) => c); - const created: any = await firstValueFrom(this.http.post('/api/automation/batches',{name:this.name.trim(),cases,device_serial:this.device,profile:this.profile,stop_on_failure:this.stopOnFailure,timeout_seconds:this.timeout,formats:this.formats()})); + const cases = this.preview().tasks.filter((t: any) => this.selectedRows().has(t.row)).map(({row, ...c}: any) => c); + const created: any = await firstValueFrom(this.http.post('/api/automation/batches',{name:this.name.trim(),cases,repeat_count:this.repeatCount,device_serial:this.device,profile:this.profile,stop_on_failure:this.stopOnFailure,timeout_seconds:this.timeout,formats:this.formats()})); this.batch.set(created); await firstValueFrom(this.http.post(`/api/automation/batches/${created.id}/resume`,{})); this.preview.set(null); this.notice.set('批次已开始，可暂停或查看每项结果'); await this.refresh(); diff --git a/artemis/config/settings.py b/artemis/config/settings.py index bca3abb..1720399 100644 --- a/artemis/config/settings.py +++ b/artemis/config/settings.py @@ -124,6 +124,7 @@ class Settings(BaseSettings): # Consecutive Flash responses without any tool call before failing the task. ARTEMIS_FLASH_MAX_NO_PROGRESS_ATTEMPTS: int = Field(default=3, ge=1, le=20) ARTEMIS_TASK_TIMEOUT_SECONDS: int = Field(default=1800, ge=30, le=86400) + ARTEMIS_BATCH_MAX_EXECUTIONS: int = Field(default=10000, ge=1) ARTEMIS_OBSERVATION_TIMEOUT_SECONDS: float = Field(default=60, gt=0, le=300) ARTEMIS_OBSERVATION_MAX_ATTEMPTS: int = Field(default=3, ge=1, le=5) ARTEMIS_DEVICE_COMMAND_TIMEOUT_SECONDS: float = Field(default=10, gt=0, le=60) diff --git a/docs/automation-workspace.md b/docs/automation-workspace.md index 602f9b2..bdf8ba4 100644 --- a/docs/automation-workspace.md +++ b/docs/automation-workspace.md @@ -17,8 +17,10 @@ 1. 点击“下载 Excel 模板”，保留“任务”工作表及表头顺序。 2. 标准列按顺序为“用例编号”“检测项目”“测试子项”“测试用例”“前置约束提取”“Artemis提示词”“任务配置(JSON)”。任务表留空，示例行在“填写说明”页。 3. 每行的“Artemis提示词”必填，且是唯一提交给 AI 的任务正文；其他文本列用于显示、搜索和报告，不自动拼入提示词。任务配置可留空，或填写 JSON 对象，如 `{"profile":"pro","model":"mimo-v2.6-flash","reasoning_effort":"high","record_video":true,"analyze_video":false,"request_timeout_seconds":300}`。支持单项覆盖设备、验证级别、观察模式、目标应用包名、重试及单任务时限；未设置字段继承批次或全局设置。不接受密钥和未知字段。不要填写公式、宏或外部链接。 -4. 上传文件，查看逐行校验结果。所有错误修正后才能提交。 -5. 明确选择设备、执行模式和超时时间，然后点击“确认导入并开始执行”。 +4. 上传文件，查看逐行校验结果。默认勾选全部有效用例，可逐项取消，或用“全选 / 全不选”切换；未勾选用例不会导入。所有错误修正后才能提交。 +5. 明确选择设备、执行模式、超时时间和整批循环次数（默认 1），查看“已选用例数 × 轮数”的执行总数，然后点击“确认导入并开始执行”。 + +循环次数独立保存在每个批次中。每轮按表格顺序执行整批所选用例，然后进入下一轮；各次执行有独立任务编号、状态和报告，队列及报告标题标注轮次。暂停、重启恢复只继续未开始项；“失败即停止”跳过全部后续轮次，“复制失败项为新批次”只复制失败记录各执行一次，不沿用原循环次数。单批默认最多 10000 次执行，可用 `.env` 中 `ARTEMIS_BATCH_MAX_EXECUTIONS` 调整；网页和 MCP 使用同一限制。 同一批次按顺序派发。默认失败后继续且不自动重试，也可以选择失败即停止。当前任务因设备异常或超时失败后，默认继续后续用例；设置失败即停止时才停止后续派发。暂停批次只停止后续派发。单个任务可在安全边界暂停并释放设备；批次下一项可继续，暂停的任务待提交指导或点击继续后在设备队列末尾恢复。运行中提交指导会中断当前模型思考或只读诊断，重新观察画面；已经开始的设备动作会安全结束。网页流程会显示指导的提交与领取状态，仍应检查后续动作是否符合指导。暂停等待和重新排队时间不计入该任务执行时限。 diff --git a/mcp_server/tools/automation.py b/mcp_server/tools/automation.py index e22fc39..743c00d 100644 --- a/mcp_server/tools/automation.py +++ b/mcp_server/tools/automation.py @@ -225,13 +225,19 @@ async def mobile_batch( reasoning_effort?,device_serial?,verification_level?,explorer_mode?, locked_app_package?,record_video?,analyze_video?,request_timeout_seconds?, max_retries?,retry_interval_seconds?,timeout_seconds?}}],profile?:flash|pro, - formats?:[docx,xlsx],stop_on_failure?:false,timeout_seconds?:1800}. Required device + formats?:[docx,xlsx],stop_on_failure?:false,timeout_seconds?:1800,repeat_count?:1}. + repeat_count is a positive integer: execute all selected cases in order for each + round, with independent session IDs, statuses and reports. The default total limit + (cases * repeat_count) is 10000, configurable via ARTEMIS_BATCH_MAX_EXECUTIONS. + Only include user-selected cases; get returns iteration/repeat_count per job. + Required device must be user-selected. Creation validates and saves a DRAFT; start executes it. Reuse the same request_id and exact payload after a connection failure; never invent a new ID for the same submission. lookup(request_id) resolves an uncertain outcome. get returns per-case status. pause stops future dispatch, not the active task; use mobile_manage_task(stop) to stop that task. resume never repeats finished cases. - retry_failed creates a new draft only after an explicit rerun request; it is not + retry_failed copies each failed execution once into a new draft (repeat_count=1) + only after an explicit rerun request; it is not transport-idempotent, so after an uncertain response use list instead of retrying. delete removes a non-active batch and its task sessions, reports and files after an explicit user request. Confirm the batch ID with get first; deletion is irreversible. diff --git a/skills/artemis-automation/references/batches.md b/skills/artemis-automation/references/batches.md index 287cb8d..af42341 100644 --- a/skills/artemis-automation/references/batches.md +++ b/skills/artemis-automation/references/batches.md @@ -17,6 +17,7 @@ "formats": ["docx", "xlsx"], "stop_on_failure": false, "timeout_seconds": 1800, + "repeat_count": 1, "cases": [ { "case_id": "DEMO-001", @@ -42,13 +43,15 @@ `create` 只保存已校验的草稿。检查返回批次编号和范围，用户已经授权执行时接着 `mobile_batch(action="start", batch_id=...)`；用户只要求准备方案时留在草稿，不启动。保存 `batch_id`，用 `get` 跟踪各项状态。 +用户要求压测时，在 `batch.repeat_count` 设置正整数循环次数（默认 1），只向 `batch.cases` 提交用户选择的用例。每轮按原顺序执行整批，再进入下一轮；各次执行独立保存编号和报告，`get` 中的 job 包含 `iteration`、`repeat_count`。总执行数为用例数乘以轮数，默认上限 10000，由服务的 `ARTEMIS_BATCH_MAX_EXECUTIONS` 控制。不要重复调用 `start` 来实现循环，也不要将循环次数写入每项 `task_config`。失败即停止会跳过后续全部轮次；复制失败项时各失败执行只重跑一次，新的批次循环次数为 1。 + 创建请求超时可能已经落库。先 `lookup(request_id=...)`；需要重发时保留**相同 request_id 和相同内容**。服务会返回原批次，内容不同会拒绝。不要通过换编号“修复超时”，否则会形成重复批次。 ## Excel 导入 `mobile_task_file(action="template")` 返回空白模板的本地路径。“任务”页七列依次为：用例编号、检测项目、测试子项、测试用例、前置约束提取、Artemis提示词、任务配置(JSON)。仅提示词必填；说明页有一条示例，不会导入执行。JSON 支持 `profile`、`model`、`reasoning_effort`、`device_serial`、`verification_level`、`explorer_mode`、`locked_app_package`、`record_video`、`analyze_video`、`request_timeout_seconds`、`max_retries`、`retry_interval_seconds`、`timeout_seconds`。未知字段或密钥被拒绝。每行独立，不支持公式、宏、外部链接。 -`preview` 返回 `valid/tasks/errors`，不会执行。所有错误处理完后，将 `tasks` 中每项的预览字段 `row` 去掉，作为 `batch.cases` 提交。不要跳过错误行却声称全部导入成功。 +`preview` 返回 `valid/tasks/errors`，不会执行。网页默认全选，可取消用例；AI 也应按用户指定范围筛选 `tasks`。所有错误处理完后，将所选项的预览字段 `row` 去掉，作为 `batch.cases` 提交，并说明实际选中数量。不要跳过错误行却声称全部导入成功。 ## 状态与异常 diff --git a/tests/unit/admin_console/test_automation.py b/tests/unit/admin_console/test_automation.py index 0dd1675..1674302 100644 --- a/tests/unit/admin_console/test_automation.py +++ b/tests/unit/admin_console/test_automation.py @@ -185,6 +185,107 @@ def test_preferences_and_required_device(client): assert client.post("/api/automation/batches", json=payload(device_serial="")).status_code == 422 +def test_batch_repetitions_are_ordered_independent_and_idempotent(client, isolated): + request = payload(repeat_count=3, request_id="repeat-test") + created = client.post("/api/automation/batches", json=request) + assert created.status_code == 200 + batch = created.json() + assert (batch["case_count"], batch["repeat_count"], batch["total_executions"]) == (2, 3, 6) + jobs = isolated.jobs(batch["id"]) + assert len({j["session_id"] for j in jobs}) == 6 + assert [j["position"] for j in jobs] == list(range(1, 7)) + assert [(j["iteration"], j["case_position"]) for j in jobs] == [ + (1, 1), + (1, 2), + (2, 1), + (2, 2), + (3, 1), + (3, 2), + ] + assert [j["case"]["goal"] for j in jobs] == ["观察设置", "观察首页"] * 3 + assert all(j["status"] == "pending" for j in jobs) + assert client.post("/api/automation/batches", json=request).json()["id"] == batch["id"] + assert len(isolated.jobs(batch["id"])) == 6 + assert ( + client.post("/api/automation/batches", json={**request, "repeat_count": 2}).status_code + == 409 + ) + # Each selected failure is copied once, including failures from different rounds. + for job in (jobs[0], jobs[4]): + isolated.save_job({**job, "status": "failed"}) + retry = service.batch_action(batch["id"], "retry_failed") + assert retry["repeat_count"] == 1 + assert len(isolated.jobs(retry["id"])) == 2 + report = reports.session_report(jobs[4]["session_id"]) + assert "第 3 / 3 轮" in report["name"] + archive = reports.export_batch(batch["id"]) + with zipfile.ZipFile(archive) as z: + assert len([n for n in z.namelist() if n.endswith(".xlsx")]) == 7 + assert len(set(z.namelist())) == len(z.namelist()) + + +@pytest.mark.parametrize("count", [0, -1, 1.5, True, "3", None]) +def test_batch_rejects_invalid_repetitions(client, isolated, count): + assert ( + client.post("/api/automation/batches", json=payload(repeat_count=count)).status_code == 422 + ) + assert isolated.records("batch") == [] + + +def test_batch_limit_is_checked_before_creating_records(client, isolated, monkeypatch): + monkeypatch.setattr(service.settings, "ARTEMIS_BATCH_MAX_EXECUTIONS", 5) + assert client.get("/api/automation/preferences").json()["batch_max_executions"] == 5 + response = client.post("/api/automation/batches", json=payload(repeat_count=3)) + assert response.status_code == 409 + assert "5" in response.json()["detail"] + assert not isolated.records("batch") + assert not isolated.jobs() + + +@pytest.mark.asyncio +async def test_loop_dispatch_continues_after_failure_and_resume_without_repeating( + isolated, monkeypatch +): + from apps.admin_console import server + from artemis.runtime.device_pool import DeviceStatus + + batch = service.create_batch(payload(repeat_count=2)) + service.batch_action(batch["id"], "resume") + monkeypatch.setattr(service, "state", type("State", (), {"is_paused": False})()) + enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]}) + monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue) + monkeypatch.setattr( + "artemis.runtime.device_pool.try_list_devices_async", + AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]), + ) + for index in range(4): + await service.tick_batches() + await service.tick_batches() # Polling must not enqueue the same execution twice. + assert enqueue.await_count == index + 1 + job = isolated.jobs(batch["id"])[index] + assert enqueue.call_args.kwargs["session_id"] == job["session_id"] + assert f"第 {job['iteration']} / 2 轮" in enqueue.call_args.kwargs["display_name"] + isolated.save_job({**job, "status": "failed" if index == 0 else "completed"}) + if index == 1: + service.batch_action(batch["id"], "pause") + await service.tick_batches() + assert enqueue.await_count == 2 + service.batch_action(batch["id"], "resume") + await service.tick_batches() + assert isolated.get("batch", batch["id"])["status"] == "completed" + assert enqueue.await_count == 4 + + +@pytest.mark.asyncio +async def test_repeat_fail_fast_skips_future_rounds(isolated): + batch = service.create_batch(payload(repeat_count=3, stop_on_failure=True)) + first = isolated.jobs(batch["id"])[0] + isolated.save_job({**first, "status": "failed"}) + service.batch_action(batch["id"], "resume") + await service.tick_batches() + assert [j["status"] for j in isolated.jobs(batch["id"])] == ["failed"] + ["skipped"] * 5 + + def test_batch_delete_removes_tasks_and_files_and_prevents_request_replay(client, isolated): created = client.post( "/api/automation/batches", json=payload(request_id="delete-example") diff --git a/tests/unit/mcp/test_automation_tools.py b/tests/unit/mcp/test_automation_tools.py index a41da4c..56a75af 100644 --- a/tests/unit/mcp/test_automation_tools.py +++ b/tests/unit/mcp/test_automation_tools.py @@ -53,9 +53,14 @@ async def test_preview_is_read_only_and_create_is_draft(monkeypatch, tmp_path): with pytest.raises(ValueError): await tools.mobile_batch("create", batch={"name": "x"}) request.assert_not_awaited() - await tools.mobile_batch("create", request_id="stable-id", batch={"name": "x"}) + selected = [{"name": "选中用例", "goal": "观察设置"}] + await tools.mobile_batch( + "create", request_id="stable-id", batch={"name": "x", "cases": selected, "repeat_count": 3} + ) assert request.await_count == 1 assert request.call_args.kwargs["json"]["request_id"] == "stable-id" + assert request.call_args.kwargs["json"]["cases"] == selected + assert request.call_args.kwargs["json"]["repeat_count"] == 3 assert request.call_args.args[1] == "/api/automation/batches"