feat: add shared test acceptance and release 1.5.0

artemis/sdk/agent.py

@@ -701,7 +701,16 @@ class Agent:
                             runner = FlashRunner(context, goal=request.goal, max_turns=max_turns)
                             flash_result = await runner.run(state)
 
-                            output = flash_result
+                            from artemis.agents.checker.flash import settle_flash_outcome
+
+                            run_outcome = await settle_flash_outcome(
+                                context,
+                                request.goal,
+                                flash_result,
+                                final_report_received=runner.final_report_received,
+                            )
+                            output = attach_test_summary(flash_result, run_outcome)
+                            state.run_outcome = run_outcome
                             last_state_snapshot = state.model_dump()
 
                             status = flash_result.get("status")
@@ -1545,17 +1554,21 @@ def attach_test_summary(output, run_outcome: dict | None):
         + int(tests.get("inconclusive", 0))
         + int(tests.get("unchecked", 0))
     )
-    if total <= 0:
+    test_result = run_outcome.get("test_result")
+    if total <= 0 and not test_result:
         return output
     summary = {"task_status": run_outcome.get("task_status"), **tests}
+    additions = {"test_summary": summary} if total > 0 else {}
+    if test_result:
+        additions["test_result"] = test_result
     if isinstance(output, dict):
         merged = dict(output)
-        merged.setdefault("test_summary", summary)
+        merged.update(additions)
         return merged
     if output is None:
-        return {"test_summary": summary}
+        return additions
     if isinstance(output, str):
-        return {"result": output, "test_summary": summary}
+        return {"result": output, **additions}
     # Typed/structured outputs keep their shape; the summary stays available in
     # run_outcome.json and the session state.
     return output
@@ -1566,6 +1579,9 @@ def resolve_trace_suffix(task_status: str, run_outcome: dict | None) -> str:
     if task_status != "completed":
         return "_FAIL"
     tests = (run_outcome or {}).get("tests") or {}
-    if int(tests.get("failed", 0)) > 0:
+    if (
+        int(tests.get("failed", 0)) > 0
+        or ((run_outcome or {}).get("test_result") or {}).get("status") == "failed"
+    ):
         return "_TESTFAIL"
     return "_PASS"