feat: add multi-device concurrency, server lifecycle CLI, and unified execution stream

- Implement DevicePool and DeviceExecutionLock concurrency modes (per-device FIFO vs global serialization)
- Extend ArtemisClient SDK and CLI with explicit device targeting and concurrency strategy configuration
- Add server lifecycle management (artemis restart, stop, status) with cross-platform PID discovery
- Update MCP server tools to support multi-device dispatch, trace tagging, and per-device execution
- Enhance Showcase UI with step-by-step visual replay, playback speed controls, and device serial badges
- Add comprehensive unit test coverage and edge-case verification scripts
This commit is contained in:
somew1nd committed 2026-08-26 15:53:14 -07:00
1 parent a36a66ce6d
commit c2c90a6378
84 files changed
+5612 -645

No files matched your search

@@ -85,7 +85,7 @@ describe('action-formatter.util screenshot chaining', () => {
expect(act1Pre).toBe('/images/fbdf17fa7a970d2c37aa52ae4336119e76a7d13ea457811cf8aaab19b3f53843');
expect(act1Post).toBe('/images/18bd2ac49e64976f1e74e828ddae76e54da1587044b1d3fb01317dd202ff7c67');
// Action 2: "下一次的决策前是上一次的动作后" -> Pre MUST be Action 1's post (18bd2ac4...), Post should be Action 2's post (484361b1...)
// Action 2: "The next pre-action state is the previous post-action state" -> Pre MUST be Action 1's post (18bd2ac4...), Post should be Action 2's post (484361b1...)
const act2Pre = getStepPreImageUrl(stepData, action2);
const act2Post = getStepPostImageUrl(stepData, action2);
expect(act2Pre).toBe('/images/18bd2ac49e64976f1e74e828ddae76e54da1587044b1d3fb01317dd202ff7c67');
@@ -154,7 +154,7 @@ describe('extractStepReplayFrames', () => {
expect(extractStepReplayFrames(null as any)).toEqual([]);
});
it('should extract ordered StepReplayFrames and deduplicate consecutive identical images', () => {
it('should extract 1:1 ordered StepReplayFrames with action objects and pre/post images', () => {
const logs = [
{
type: 'step_updated',
@@ -181,13 +181,18 @@ describe('extractStepReplayFrames', () => {
];
const frames = extractStepReplayFrames(logs);
expect(frames.length).toBe(3);
expect(frames[0].title).toBe('Step 1 (Before Action)');
expect(frames.length).toBe(2);
expect(frames[0].stepNumber).toBe(1);
expect(frames[0].imageUrl).toBe('/images/img_screen_1');
expect(frames[1].title).toContain('Step 1');
expect(frames[0].preImageUrl).toBe('/images/img_screen_1');
expect(frames[0].postImageUrl).toBe('/images/img_screen_2');
expect(frames[0].action).toEqual({ action: 'tap', coordinates: [500, 1000] });
expect(frames[1].stepNumber).toBe(2);
expect(frames[1].imageUrl).toBe('/images/img_screen_2');
expect(frames[2].title).toContain('Step 2');
expect(frames[2].imageUrl).toBe('/images/img_screen_3');
expect(frames[1].preImageUrl).toBe('/images/img_screen_2');
expect(frames[1].postImageUrl).toBe('/images/img_screen_3');
expect(frames[1].action).toEqual({ action: 'input_text', text: 'hello' });
});
it('should handle terminal step with only pre_image (e.g. cancelled / failed task)', () => {
@@ -706,7 +706,7 @@ export function resolveStepImageMap(stepData: any): Map<any, ResolvedImageEntry>
resolvedPost = stepData.post_image_name || stepData.post_screenshot || null;
}
// "下一次的决策前是上一次的动作后"
// "The next pre-action state is the previous post-action state"
if (resolvedPost) {
currentPreImage = resolvedPost;
}
@@ -778,24 +778,37 @@ export function extractStepReplayFrames(logsOrSteps: any[]): StepReplayFrame[] {
const rawSteps: any[] = [];
for (const item of logsOrSteps) {
if (!item) continue;
if (item.type === 'step_updated' && item.data) {
rawSteps.push(item.data);
} else if (item.type === 'step' && item.data) {
if ((item.type === 'step_updated' || item.type === 'step_recorded' || item.type === 'step') && item.data) {
rawSteps.push(item.data);
} else if (item.step_number !== undefined || item.step_id !== undefined) {
rawSteps.push(item);
}
}
// Deduplicate by step_id or step_number, keeping latest
// Deduplicate and merge by step_id or step_number
const stepMap = new Map<any, any>();
for (const step of rawSteps) {
const key = step.step_id || step.step_number;
if (key !== undefined) {
stepMap.set(key, step);
const existing = stepMap.get(key);
if (existing) {
stepMap.set(key, {
...existing,
...step,
step_number: step.step_number !== undefined ? step.step_number : existing.step_number,
pre_image_name: step.pre_image_name || existing.pre_image_name,
post_image_name: step.post_image_name || existing.post_image_name,
action_taken: step.action_taken || existing.action_taken,
last_execution_result: step.last_execution_result || existing.last_execution_result,
generic_tools: step.generic_tools || existing.generic_tools
});
} else {
stepMap.set(key, { ...step });
}
}
}
// Sort strictly by step_number ascending
const steps = Array.from(stepMap.values()).sort((a, b) => {
const numA = Number(a.step_number ?? 0);
const numB = Number(b.step_number ?? 0);
@@ -804,47 +817,45 @@ export function extractStepReplayFrames(logsOrSteps: any[]): StepReplayFrame[] {
});
const frames: StepReplayFrame[] = [];
let lastImageUrl: string | null = null;
for (const stepData of steps) {
const stepNum = Number(stepData.step_number ?? (frames.length + 1));
for (let i = 0; i < steps.length; i++) {
const stepData = steps[i];
const stepNum = Number(stepData.step_number ?? (i + 1));
const act = stepData.action_taken;
const title = act ? (getActionTitle(act) || 'Action') : 'Step';
const title = act ? (getActionTitle(act) || 'Action') : `Step ${stepNum}`;
const coords = act ? getActionCoords(act) : '';
const actionDesc = coords ? `${title} (${coords})` : title;
const targetText = act ? getActionTargetText(act) : '';
const actionDesc = coords ? `${title} (${coords})` : (targetText ? `${title} (${targetText})` : title);
const preUrl = getStepPreImageUrl(stepData, act);
const postUrl = getStepPostImageUrl(stepData, act);
// If pre-image exists and is new, add it
if (preUrl && preUrl !== lastImageUrl) {
frames.push({
index: frames.length,
stepNumber: stepNum,
title: `Step ${stepNum} (Before Action)`,
imageUrl: preUrl,
actionText: `Preparing: ${actionDesc}`,
isPost: false,
timestamp: stepData.timestamp
});
lastImageUrl = preUrl;
// Each step gets exactly one primary frame (Pre-action screenshot where the action is taken)
const primaryImg = preUrl || postUrl;
if (!primaryImg) {
continue;
}
// If post-image exists and is different from current screen, add it
if (postUrl && postUrl !== lastImageUrl) {
frames.push({
index: frames.length,
stepNumber: stepNum,
title: `Step ${stepNum}: ${actionDesc}`,
imageUrl: postUrl,
actionText: actionDesc,
isPost: true,
timestamp: stepData.timestamp
});
lastImageUrl = postUrl;
}
const failed = isActionFailed(act, stepData);
frames.push({
index: frames.length,
stepNumber: stepNum,
stepId: String(stepData.step_id || `step-${stepNum}`),
title: `Step ${stepNum}: ${title}`,
actionText: actionDesc,
action: act,
actionType: act?.action || act?.name || 'action',
targetText,
coords,
imageUrl: primaryImg,
preImageUrl: preUrl,
postImageUrl: postUrl,
isPost: false,
timestamp: stepData.timestamp,
status: failed ? 'failed' : 'success'
});
}
return frames;
}