Use Jev as an agent-run judge

curl -sS https://router.flatkey.ai/api/alpha/decisions \
  -H "Authorization: Bearer $FLATKEY_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
  "model": "typesafe/jev-1.13",
  "state": "Goal: update staging deploy and verify health check returns 200. Agent steps: (1) ran git pull on staging host, (2) restarted the service, (3) claimed \"deploy complete and healthy\". Missing: no curl or health-check command was run; last log line is only the restart. Required verification step was skipped. Retries so far: 0.",
  "questions": {
    "verdict": {
      "type": "choice",
      "instructions": "Judge the agent run against the stated goal and required checks.",
      "criteria": {
        "pass": "The agent completed the goal and met the required verification steps with supporting evidence",
        "retry": "The run is incomplete or skipped a required check, but another attempt with the same setup is warranted",
        "fail": "The agent clearly missed the goal, violated policy, or produced incorrect or unsafe output",
        "inconclusive": "The transcript lacks enough evidence to judge pass or fail"
      }
    },
    "quality": {
      "type": "score",
      "instructions": "Overall run quality ignoring self-claims of success.",
      "criteria": [
        "poor",
        "fair",
        "good",
        "excellent"
      ]
    },
    "needs_human_review": {
      "type": "noul",
      "instructions": "A human or chat-model judge should review this run before the score is trusted."
    }
  }
}'

// Node.js 18+ (built-in fetch). Run: FLATKEY_API_KEY=... node jev.mjs
const body = {
  "model": "typesafe/jev-1.13",
  "state": "Goal: update staging deploy and verify health check returns 200. Agent steps: (1) ran git pull on staging host, (2) restarted the service, (3) claimed \"deploy complete and healthy\". Missing: no curl or health-check command was run; last log line is only the restart. Required verification step was skipped. Retries so far: 0.",
  "questions": {
    "verdict": {
      "type": "choice",
      "instructions": "Judge the agent run against the stated goal and required checks.",
      "criteria": {
        "pass": "The agent completed the goal and met the required verification steps with supporting evidence",
        "retry": "The run is incomplete or skipped a required check, but another attempt with the same setup is warranted",
        "fail": "The agent clearly missed the goal, violated policy, or produced incorrect or unsafe output",
        "inconclusive": "The transcript lacks enough evidence to judge pass or fail"
      }
    },
    "quality": {
      "type": "score",
      "instructions": "Overall run quality ignoring self-claims of success.",
      "criteria": [
        "poor",
        "fair",
        "good",
        "excellent"
      ]
    },
    "needs_human_review": {
      "type": "noul",
      "instructions": "A human or chat-model judge should review this run before the score is trusted."
    }
  }
};

const res = await fetch("https://router.flatkey.ai/api/alpha/decisions", {
  method: "POST",
  headers: {
    Authorization: `Bearer ${process.env.FLATKEY_API_KEY}`,
    "Content-Type": "application/json",
  },
  body: JSON.stringify(body),
});
if (!res.ok) throw new Error(`Jev request failed: ${res.status} ${await res.text()}`);
const { answers, usage } = await res.json();
console.log(JSON.stringify(answers, null, 2), usage);

# Python 3.8+ (pip install requests). Run: FLATKEY_API_KEY=... python jev.py
import json, os, requests

body = json.loads(r"""{
  "model": "typesafe/jev-1.13",
  "state": "Goal: update staging deploy and verify health check returns 200. Agent steps: (1) ran git pull on staging host, (2) restarted the service, (3) claimed \"deploy complete and healthy\". Missing: no curl or health-check command was run; last log line is only the restart. Required verification step was skipped. Retries so far: 0.",
  "questions": {
    "verdict": {
      "type": "choice",
      "instructions": "Judge the agent run against the stated goal and required checks.",
      "criteria": {
        "pass": "The agent completed the goal and met the required verification steps with supporting evidence",
        "retry": "The run is incomplete or skipped a required check, but another attempt with the same setup is warranted",
        "fail": "The agent clearly missed the goal, violated policy, or produced incorrect or unsafe output",
        "inconclusive": "The transcript lacks enough evidence to judge pass or fail"
      }
    },
    "quality": {
      "type": "score",
      "instructions": "Overall run quality ignoring self-claims of success.",
      "criteria": [
        "poor",
        "fair",
        "good",
        "excellent"
      ]
    },
    "needs_human_review": {
      "type": "noul",
      "instructions": "A human or chat-model judge should review this run before the score is trusted."
    }
  }
}""")

res = requests.post(
    "https://router.flatkey.ai/api/alpha/decisions",
    headers={"Authorization": f"Bearer {os.environ['FLATKEY_API_KEY']}"},
    json=body,
    timeout=60,
)
res.raise_for_status()
data = res.json()
print(json.dumps(data["answers"], indent=2), data.get("usage"))

No Flatkey key yet? Get one for Jev 1.13 →