Artemis-Plugins/plugins/cua-driver/main.go

// ARTEMIS adapter for the pinned MIT Cua Driver. No model, scheduler, or recorder.
package main

import (
	"bufio"
	"bytes"
	"crypto/sha256"
	_ "embed"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"image"
	_ "image/jpeg"
	_ "image/png"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

//go:embed tools.json
var toolsJSON []byte

const adapterVersion = "0.36.0+artemis.1"

type obj = map[string]any
type request struct {
	JSONRPC string          `json:"jsonrpc"`
	ID      json.RawMessage `json:"id"`
	Method  string          `json:"method"`
	Params  obj             `json:"params"`
}
type peer struct {
	cmd   *exec.Cmd
	input io.WriteCloser
	lines chan []byte
	next  int
	mu    sync.Mutex
	dead  bool
	job   uintptr
}

func object(v any) obj {
	m, _ := v.(map[string]any)
	if m == nil {
		return obj{}
	}
	return m
}
func str(v any) string { s, _ := v.(string); return s }
func num(v any) int64 {
	switch n := v.(type) {
	case float64:
		return int64(n)
	case int64:
		return n
	case int:
		return int64(n)
	}
	return 0
}
func items(v any) []any      { a, _ := v.([]any); return a }
func textBlock(s string) obj { return obj{"type": "text", "text": s} }
func receipt(execution, side, code string) obj {
	return obj{"version": 1, "execution": execution, "side_effect": side, "retryable": false, "error_code": code}
}
func failure(msg, execution, side, code string) obj {
	return obj{"isError": true, "content": []any{textBlock(msg)}, "structuredContent": obj{"artemis": receipt(execution, side, code)}}
}
func writeTool(name string) bool {
	return name != "list_windows" && name != "get_window_state" && name != "verify_state"
}

func startPeer(exe string) (*peer, error) {
	c := exec.Command(exe, "mcp", "--direct")
	c.Env = append(os.Environ(), "DO_NOT_TRACK=1", "CUA_TELEMETRY=0", "CUA_DRIVER_RS_TELEMETRY_ENABLED=false")
	configureProcess(c)
	in, err := c.StdinPipe()
	if err != nil {
		return nil, err
	}
	out, err := c.StdoutPipe()
	if err != nil {
		return nil, err
	}
	c.Stderr = os.Stderr
	if err = c.Start(); err != nil {
		return nil, err
	}
	job, err := ownProcess(c.Process.Pid)
	if err != nil {
		_ = c.Process.Kill()
		_ = c.Wait()
		return nil, err
	}
	p := &peer{cmd: c, input: in, lines: make(chan []byte, 32), job: job}
	go func() {
		defer close(p.lines)
		s := bufio.NewScanner(out)
		s.Buffer(make([]byte, 65536), 32<<20)
		for s.Scan() {
			p.lines <- append([]byte(nil), s.Bytes()...)
		}
	}()
	_, err = p.call("initialize", obj{"protocolVersion": "2024-11-05", "clientInfo": obj{"name": "artemis-cua-adapter", "version": adapterVersion}, "capabilities": obj{}})
	if err != nil {
		p.stop()
		return nil, err
	}
	err = json.NewEncoder(in).Encode(obj{"jsonrpc": "2.0", "method": "notifications/initialized"})
	if err != nil {
		p.stop()
		return nil, err
	}
	return p, nil
}
func (p *peer) stop() {
	p.mu.Lock()
	defer p.mu.Unlock()
	if p.dead {
		return
	}
	p.dead = true
	_ = p.input.Close()
	closeJob(p.job)
	_ = p.cmd.Process.Kill()
	_ = p.cmd.Wait()
}
func (p *peer) call(method string, params obj) (obj, error) {
	p.mu.Lock()
	dead := p.dead
	p.mu.Unlock()
	if dead {
		return nil, errors.New("driver closed")
	}
	p.next++
	id := p.next
	if err := json.NewEncoder(p.input).Encode(obj{"jsonrpc": "2.0", "id": id, "method": method, "params": params}); err != nil {
		return nil, err
	}
	timer := time.NewTimer(45 * time.Second)
	defer timer.Stop()
	for {
		select {
		case raw, ok := <-p.lines:
			if !ok {
				return nil, errors.New("driver disconnected")
			}
			var r obj
			if json.Unmarshal(raw, &r) != nil {
				continue
			}
			if num(r["id"]) != int64(id) {
				continue
			}
			if r["error"] != nil {
				return nil, errors.New("driver protocol error")
			}
			return object(r["result"]), nil
		case <-timer.C:
			p.stop()
			return nil, errors.New("driver response timed out; action not replayed")
		}
	}
}
func (p *peer) tool(name string, args obj) (obj, error) {
	return p.call("tools/call", obj{"name": name, "arguments": args})
}

type adapter struct {
	peer    *peer
	tools   []any
	schemas map[string]obj
}

func newAdapter(p *peer) *adapter {
	a := &adapter{peer: p, schemas: map[string]obj{}}
	_ = json.Unmarshal(toolsJSON, &a.tools)
	for _, v := range a.tools {
		t := object(v)
		a.schemas[str(t["name"])] = object(t["inputSchema"])
	}
	return a
}
func (a *adapter) validate(name string, args obj) error {
	schema, ok := a.schemas[name]
	if !ok {
		return errors.New("tool is not exposed")
	}
	props := object(schema["properties"])
	for key := range args {
		if _, ok := props[key]; !ok {
			return fmt.Errorf("unsupported argument: %s", key)
		}
	}
	for _, key := range items(schema["required"]) {
		if _, ok := args[str(key)]; !ok {
			return fmt.Errorf("missing argument: %s", str(key))
		}
	}
	if name == "list_windows" {
		_, err := windowQuery(args)
		return err
	}
	if num(args["pid"]) <= 0 || num(args["window_id"]) <= 0 {
		return errors.New("a concrete pid and window_id are required")
	}
	if name == "save_screenshot" && !validFilename(str(args["filename"])) {
		return errors.New("filename must be a new plain PNG filename")
	}
	if name == "get_window_state" && (args["include_screenshot"] == false || args["include_accessibility_tree"] == false) {
		return errors.New("candidate observations require both screenshot and accessibility tree")
	}
	return nil
}
func validFilename(s string) bool {
	return s != "" && len(s) <= 160 && strings.EqualFold(filepath.Ext(s), ".png") && !strings.ContainsAny(s, "/\\:<>\"|?*\x00") && s != "." && s != ".." && !strings.HasSuffix(s, ".") && !strings.HasSuffix(s, " ")
}
func actionReceipt(r obj, write bool) obj {
	if !write {
		if r["isError"] == true {
			return receipt("not_started", "none", "observation_failed")
		}
		return receipt("completed", "none", "")
	}
	s := object(r["structuredContent"])
	// v0.36.0 class-level preflight refusal has no effect field. It occurs
	// before input injection. Do not classify actuator failures with a cause
	// as safe-to-repeat; retain their uncertain execution below.
	if s["code"] == "background_unavailable" && s["cause"] == nil && s["event_kind"] != nil && r["isError"] == true {
		return receipt("not_started", "none", "background_unavailable")
	}
	switch str(s["effect"]) {
	case "confirmed":
		return receipt("completed", "confirmed", "")
	case "partial":
		return receipt("partial", "confirmed", "partial_delivery")
	case "refused":
		return receipt("not_started", "none", "driver_refused")
	case "unverifiable", "suspected_noop":
		return receipt("completed", "possible", str(s["effect"]))
	default:
		return receipt("unknown", "possible", "unclassified_result")
	}
}
func imageSize(r obj) (int, int) {
	for _, v := range items(r["content"]) {
		c := object(v)
		if c["type"] == "image" {
			raw, err := base64.StdEncoding.DecodeString(str(c["data"]))
			if err == nil {
				cfg, _, err := image.DecodeConfig(bytes.NewReader(raw))
				if err == nil {
					return cfg.Width, cfg.Height
				}
			}
		}
	}
	return 0, 0
}
func source(r obj, args obj, started float64) obj {
	s := object(r["structuredContent"])
	w, h := imageSize(r)
	if r["isError"] == true || w <= 0 || h <= 0 || num(s["pid"]) != num(args["pid"]) || num(s["window_id"]) != num(args["window_id"]) {
		return nil
	}
	bounds := object(s["window_bounds"])
	rect := obj{"left": num(bounds["x"]), "top": num(bounds["y"]), "width": num(bounds["width"]), "height": num(bounds["height"])}
	screen, ok := verifyWindow(num(args["pid"]), num(args["window_id"]), rect)
	if !ok {
		return nil
	}
	return obj{"kind": "computer", "target_id": fmt.Sprint(num(args["window_id"])), "label": s["window_title"], "captured_at": started, "width": w, "height": h, "window": rect, "screen": screen}
}
func (a *adapter) observe(args obj) (obj, obj, error) {
	started := float64(time.Now().UnixMilli()) / 1000
	r, err := a.peer.tool("get_window_state", args)
	if err != nil {
		return nil, nil, err
	}
	return r, source(r, args, started), nil
}
func observationArgs(args obj) obj {
	return obj{"pid": args["pid"], "window_id": args["window_id"], "max_image_dimension": 1280, "max_elements": 250, "tree_format": "markdown"}
}
func (a *adapter) handle(name string, args obj) obj {
	if err := a.validate(name, args); err != nil {
		return failure(err.Error(), "not_started", "none", "invalid_input")
	}
	if name != "list_windows" {
		if !windowOwned(num(args["pid"]), num(args["window_id"])) {
			return failure("window is gone or belongs to a different process", "not_started", "none", "invalid_target")
		}
	}
	if name == "save_screenshot" {
		return a.save(args)
	}
	var r, src obj
	var err error
	if name == "get_window_state" {
		for k, v := range observationArgs(args) {
			if _, ok := args[k]; !ok {
				args[k] = v
			}
		}
		r, src, err = a.observe(args)
	} else if name == "list_windows" {
		query, _ := windowQuery(args) // Already validated above.
		r, err = a.peer.tool(name, query)
		if err == nil {
			r = windowListing(r, query)
		}
	} else {
		upstream := name
		if name == "click_at" {
			upstream = "click"
		}
		r, err = a.peer.tool(upstream, args)
	}
	if err != nil {
		a.peer.stop()
		if writeTool(name) {
			return failure(err.Error(), "unknown", "possible", "transport_lost")
		}
		return failure(err.Error(), "not_started", "none", "transport_lost")
	}
	rec := actionReceipt(r, writeTool(name))
	structured := object(r["structuredContent"])
	if writeTool(name) && rec["execution"] != "not_started" {
		// Observe once after delivery; NEVER repeat an uncertain write.
		observation, origin, e := a.observe(observationArgs(args))
		if e == nil && origin != nil {
			src = origin
			structured["observation"] = observation["structuredContent"]
			r["content"] = append(items(r["content"]), items(observation["content"])...)
		} else {
			structured["observation_error"] = "Post-action evidence unavailable; inspect before another action."
		}
	}
	if name == "verify_state" { // Read-only verification may include a screenshot; it is not a task verdict.
		src = source(r, args, float64(time.Now().UnixMilli())/1000)
	}
	if src != nil {
		rec["source"] = src
	} else if name == "get_window_state" && r["isError"] != true {
		return failure("capture target or geometry could not be validated", "not_started", "none", "invalid_capture")
	}
	if structured["effect"] == "refused" {
		r["isError"] = true
	}
	structured["artemis"] = rec
	r["structuredContent"] = structured
	return r
}
func (a *adapter) save(args obj) obj {
	root := os.Getenv("ARTEMIS_ARTIFACTS_DIR")
	if root == "" {
		return failure("task artifact directory unavailable", "not_started", "none", "missing_artifacts")
	}
	r, src, err := a.observe(observationArgs(args))
	if err != nil || src == nil {
		return failure("fresh window capture failed", "not_started", "none", "capture_failed")
	}
	var png []byte
	for _, v := range items(r["content"]) {
		b := object(v)
		if b["type"] == "image" && b["mimeType"] == "image/png" {
			png, _ = base64.StdEncoding.DecodeString(str(b["data"]))
			break
		}
	}
	if len(png) == 0 {
		return failure("expected PNG capture", "not_started", "none", "capture_format")
	}
	if err = os.MkdirAll(root, 0700); err != nil {
		return failure("artifact directory unavailable", "not_started", "none", "file_error")
	}
	path := filepath.Join(root, str(args["filename"]))
	f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
	if err != nil {
		return failure("file exists or cannot be created", "not_started", "none", "file_error")
	}
	n, writeErr := f.Write(png)
	closeErr := f.Close()
	actual, readErr := os.ReadFile(path)
	if writeErr != nil || closeErr != nil || readErr != nil || n != len(png) || !bytes.Equal(actual, png) {
		return failure("artifact write could not be verified", "partial", "possible", "file_error")
	}
	rec := receipt("completed", "confirmed", "")
	rec["source"] = src
	rec["attachments"] = []any{obj{"name": str(args["filename"]), "sha256": fmt.Sprintf("%x", sha256.Sum256(actual)), "media_type": "image/png"}}
	object(r["structuredContent"])["artemis"] = rec
	r["content"] = append(items(r["content"]), textBlock("Saved verified task attachment: "+str(args["filename"])))
	return r
}
func serve(a *adapter, in io.Reader, out io.Writer) {
	input := make(chan request)
	go func() {
		defer close(input)
		defer a.peer.stop()
		s := bufio.NewScanner(in)
		s.Buffer(make([]byte, 65536), 2<<20)
		for s.Scan() {
			var q request
			if json.Unmarshal(s.Bytes(), &q) == nil {
				if q.Method == "notifications/cancelled" {
					a.peer.stop()
					continue
				}
				input <- q
			}
		}
	}()
	enc := json.NewEncoder(out)
	for q := range input {
		if len(q.ID) == 0 {
			continue
		}
		var result any
		var rpcErr any
		switch q.Method {
		case "initialize":
			result = obj{"protocolVersion": "2024-11-05", "capabilities": obj{"tools": obj{}}, "serverInfo": obj{"name": "artemis-cua-driver", "version": adapterVersion}}
		case "ping":
			result = obj{}
		case "tools/list":
			result = obj{"tools": a.tools}
		case "tools/call":
			result = a.handle(str(q.Params["name"]), object(q.Params["arguments"]))
		default:
			rpcErr = obj{"code": -32601, "message": "method not found"}
		}
		response := obj{"jsonrpc": "2.0", "id": q.ID}
		if rpcErr != nil {
			response["error"] = rpcErr
		} else {
			response["result"] = result
		}
		if enc.Encode(response) != nil {
			return
		}
	}
}
func main() {
	exe, err := os.Executable()
	if err != nil {
		os.Exit(1)
	}
	p, err := startPeer(filepath.Join(filepath.Dir(exe), "cua-driver.exe"))
	if err != nil {
		fmt.Fprintln(os.Stderr, "Cua startup failed:", err)
		os.Exit(1)
	}
	defer p.stop()
	serve(newAdapter(p), os.Stdin, os.Stdout)
}