feat: add multi-device concurrency, server lifecycle CLI, and unified execution stream
- Implement DevicePool and DeviceExecutionLock concurrency modes (per-device FIFO vs global serialization) - Extend ArtemisClient SDK and CLI with explicit device targeting and concurrency strategy configuration - Add server lifecycle management (artemis restart, stop, status) with cross-platform PID discovery - Update MCP server tools to support multi-device dispatch, trace tagging, and per-device execution - Enhance Showcase UI with step-by-step visual replay, playback speed controls, and device serial badges - Add comprehensive unit test coverage and edge-case verification scripts
This commit is contained in:
1 parent
a36a66ce6d
commit
c2c90a6378
84 files changed
+5612
-645
No files matched your search
@@ -85,7 +85,7 @@ describe('action-formatter.util screenshot chaining', () => {
|
||||
expect(act1Pre).toBe('/images/fbdf17fa7a970d2c37aa52ae4336119e76a7d13ea457811cf8aaab19b3f53843');
|
||||
expect(act1Post).toBe('/images/18bd2ac49e64976f1e74e828ddae76e54da1587044b1d3fb01317dd202ff7c67');
|
||||
|
||||
// Action 2: "下一次的决策前是上一次的动作后" -> Pre MUST be Action 1's post (18bd2ac4...), Post should be Action 2's post (484361b1...)
|
||||
// Action 2: "The next pre-action state is the previous post-action state" -> Pre MUST be Action 1's post (18bd2ac4...), Post should be Action 2's post (484361b1...)
|
||||
const act2Pre = getStepPreImageUrl(stepData, action2);
|
||||
const act2Post = getStepPostImageUrl(stepData, action2);
|
||||
expect(act2Pre).toBe('/images/18bd2ac49e64976f1e74e828ddae76e54da1587044b1d3fb01317dd202ff7c67');
|
||||
@@ -154,7 +154,7 @@ describe('extractStepReplayFrames', () => {
|
||||
expect(extractStepReplayFrames(null as any)).toEqual([]);
|
||||
});
|
||||
|
||||
it('should extract ordered StepReplayFrames and deduplicate consecutive identical images', () => {
|
||||
it('should extract 1:1 ordered StepReplayFrames with action objects and pre/post images', () => {
|
||||
const logs = [
|
||||
{
|
||||
type: 'step_updated',
|
||||
@@ -181,13 +181,18 @@ describe('extractStepReplayFrames', () => {
|
||||
];
|
||||
|
||||
const frames = extractStepReplayFrames(logs);
|
||||
expect(frames.length).toBe(3);
|
||||
expect(frames[0].title).toBe('Step 1 (Before Action)');
|
||||
expect(frames.length).toBe(2);
|
||||
expect(frames[0].stepNumber).toBe(1);
|
||||
expect(frames[0].imageUrl).toBe('/images/img_screen_1');
|
||||
expect(frames[1].title).toContain('Step 1');
|
||||
expect(frames[0].preImageUrl).toBe('/images/img_screen_1');
|
||||
expect(frames[0].postImageUrl).toBe('/images/img_screen_2');
|
||||
expect(frames[0].action).toEqual({ action: 'tap', coordinates: [500, 1000] });
|
||||
|
||||
expect(frames[1].stepNumber).toBe(2);
|
||||
expect(frames[1].imageUrl).toBe('/images/img_screen_2');
|
||||
expect(frames[2].title).toContain('Step 2');
|
||||
expect(frames[2].imageUrl).toBe('/images/img_screen_3');
|
||||
expect(frames[1].preImageUrl).toBe('/images/img_screen_2');
|
||||
expect(frames[1].postImageUrl).toBe('/images/img_screen_3');
|
||||
expect(frames[1].action).toEqual({ action: 'input_text', text: 'hello' });
|
||||
});
|
||||
|
||||
it('should handle terminal step with only pre_image (e.g. cancelled / failed task)', () => {
|
||||
|
||||
@@ -706,7 +706,7 @@ export function resolveStepImageMap(stepData: any): Map<any, ResolvedImageEntry>
|
||||
resolvedPost = stepData.post_image_name || stepData.post_screenshot || null;
|
||||
}
|
||||
|
||||
// "下一次的决策前是上一次的动作后"
|
||||
// "The next pre-action state is the previous post-action state"
|
||||
if (resolvedPost) {
|
||||
currentPreImage = resolvedPost;
|
||||
}
|
||||
@@ -778,24 +778,37 @@ export function extractStepReplayFrames(logsOrSteps: any[]): StepReplayFrame[] {
|
||||
const rawSteps: any[] = [];
|
||||
for (const item of logsOrSteps) {
|
||||
if (!item) continue;
|
||||
if (item.type === 'step_updated' && item.data) {
|
||||
rawSteps.push(item.data);
|
||||
} else if (item.type === 'step' && item.data) {
|
||||
if ((item.type === 'step_updated' || item.type === 'step_recorded' || item.type === 'step') && item.data) {
|
||||
rawSteps.push(item.data);
|
||||
} else if (item.step_number !== undefined || item.step_id !== undefined) {
|
||||
rawSteps.push(item);
|
||||
}
|
||||
}
|
||||
|
||||
// Deduplicate by step_id or step_number, keeping latest
|
||||
// Deduplicate and merge by step_id or step_number
|
||||
const stepMap = new Map<any, any>();
|
||||
for (const step of rawSteps) {
|
||||
const key = step.step_id || step.step_number;
|
||||
if (key !== undefined) {
|
||||
stepMap.set(key, step);
|
||||
const existing = stepMap.get(key);
|
||||
if (existing) {
|
||||
stepMap.set(key, {
|
||||
...existing,
|
||||
...step,
|
||||
step_number: step.step_number !== undefined ? step.step_number : existing.step_number,
|
||||
pre_image_name: step.pre_image_name || existing.pre_image_name,
|
||||
post_image_name: step.post_image_name || existing.post_image_name,
|
||||
action_taken: step.action_taken || existing.action_taken,
|
||||
last_execution_result: step.last_execution_result || existing.last_execution_result,
|
||||
generic_tools: step.generic_tools || existing.generic_tools
|
||||
});
|
||||
} else {
|
||||
stepMap.set(key, { ...step });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sort strictly by step_number ascending
|
||||
const steps = Array.from(stepMap.values()).sort((a, b) => {
|
||||
const numA = Number(a.step_number ?? 0);
|
||||
const numB = Number(b.step_number ?? 0);
|
||||
@@ -804,47 +817,45 @@ export function extractStepReplayFrames(logsOrSteps: any[]): StepReplayFrame[] {
|
||||
});
|
||||
|
||||
const frames: StepReplayFrame[] = [];
|
||||
let lastImageUrl: string | null = null;
|
||||
|
||||
for (const stepData of steps) {
|
||||
const stepNum = Number(stepData.step_number ?? (frames.length + 1));
|
||||
for (let i = 0; i < steps.length; i++) {
|
||||
const stepData = steps[i];
|
||||
const stepNum = Number(stepData.step_number ?? (i + 1));
|
||||
const act = stepData.action_taken;
|
||||
const title = act ? (getActionTitle(act) || 'Action') : 'Step';
|
||||
const title = act ? (getActionTitle(act) || 'Action') : `Step ${stepNum}`;
|
||||
const coords = act ? getActionCoords(act) : '';
|
||||
const actionDesc = coords ? `${title} (${coords})` : title;
|
||||
const targetText = act ? getActionTargetText(act) : '';
|
||||
const actionDesc = coords ? `${title} (${coords})` : (targetText ? `${title} (${targetText})` : title);
|
||||
|
||||
const preUrl = getStepPreImageUrl(stepData, act);
|
||||
const postUrl = getStepPostImageUrl(stepData, act);
|
||||
|
||||
// If pre-image exists and is new, add it
|
||||
if (preUrl && preUrl !== lastImageUrl) {
|
||||
frames.push({
|
||||
index: frames.length,
|
||||
stepNumber: stepNum,
|
||||
title: `Step ${stepNum} (Before Action)`,
|
||||
imageUrl: preUrl,
|
||||
actionText: `Preparing: ${actionDesc}`,
|
||||
isPost: false,
|
||||
timestamp: stepData.timestamp
|
||||
});
|
||||
lastImageUrl = preUrl;
|
||||
// Each step gets exactly one primary frame (Pre-action screenshot where the action is taken)
|
||||
const primaryImg = preUrl || postUrl;
|
||||
if (!primaryImg) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// If post-image exists and is different from current screen, add it
|
||||
if (postUrl && postUrl !== lastImageUrl) {
|
||||
frames.push({
|
||||
index: frames.length,
|
||||
stepNumber: stepNum,
|
||||
title: `Step ${stepNum}: ${actionDesc}`,
|
||||
imageUrl: postUrl,
|
||||
actionText: actionDesc,
|
||||
isPost: true,
|
||||
timestamp: stepData.timestamp
|
||||
});
|
||||
lastImageUrl = postUrl;
|
||||
}
|
||||
const failed = isActionFailed(act, stepData);
|
||||
|
||||
frames.push({
|
||||
index: frames.length,
|
||||
stepNumber: stepNum,
|
||||
stepId: String(stepData.step_id || `step-${stepNum}`),
|
||||
title: `Step ${stepNum}: ${title}`,
|
||||
actionText: actionDesc,
|
||||
action: act,
|
||||
actionType: act?.action || act?.name || 'action',
|
||||
targetText,
|
||||
coords,
|
||||
imageUrl: primaryImg,
|
||||
preImageUrl: preUrl,
|
||||
postImageUrl: postUrl,
|
||||
isPost: false,
|
||||
timestamp: stepData.timestamp,
|
||||
status: failed ? 'failed' : 'success'
|
||||
});
|
||||
}
|
||||
|
||||
return frames;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user