Use Jev as a grounding / hallucination judge

curl -sS https://router.flatkey.ai/api/alpha/decisions \
  -H "Authorization: Bearer $FLATKEY_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
  "model": "typesafe/jev-1.13",
  "state": "Claim: \"Jev checked 3,157 claims against sources with 78.9% balanced accuracy at roughly 0.2s per verdict and about 14 cents total cost.\"\n\nSource A: Blog excerpt — \"We evaluated mute probability models on 3,157 claim–source pairs. Balanced accuracy: 78.9%. Median latency ~0.2s. Total spend across the run: $0.14.\"\n\nSource B: Same post — \"Judgments collapse when evidence is missing or wrong; calibrate thresholds and cascade pass/block/review.\"\n\nTask: judge grounding of the claim against these sources only.",
  "questions": {
    "judgment": {
      "type": "choice",
      "instructions": "Judge whether the claim is grounded in the provided source evidence. Ignore world knowledge outside the snippets.",
      "criteria": {
        "pass": "The sources support the claim with enough specific evidence; no material contradiction",
        "block": "The sources contradict the claim, or the claim asserts facts the sources do not support",
        "review": "Evidence is partial, ambiguous, or conflicting enough that a human should decide"
      }
    },
    "evidence_strength": {
      "type": "score",
      "instructions": "How strong is the source support (or contradiction) for this claim, ignoring the claim author'\''s confidence.",
      "criteria": [
        "none",
        "weak",
        "moderate",
        "strong"
      ]
    },
    "needs_human": {
      "type": "noul",
      "instructions": "A human should review this claim–source pair before the judgment is trusted in production."
    }
  }
}'

// Node.js 18+ (built-in fetch). Run: FLATKEY_API_KEY=... node jev.mjs
const body = {
  "model": "typesafe/jev-1.13",
  "state": "Claim: \"Jev checked 3,157 claims against sources with 78.9% balanced accuracy at roughly 0.2s per verdict and about 14 cents total cost.\"\n\nSource A: Blog excerpt — \"We evaluated mute probability models on 3,157 claim–source pairs. Balanced accuracy: 78.9%. Median latency ~0.2s. Total spend across the run: $0.14.\"\n\nSource B: Same post — \"Judgments collapse when evidence is missing or wrong; calibrate thresholds and cascade pass/block/review.\"\n\nTask: judge grounding of the claim against these sources only.",
  "questions": {
    "judgment": {
      "type": "choice",
      "instructions": "Judge whether the claim is grounded in the provided source evidence. Ignore world knowledge outside the snippets.",
      "criteria": {
        "pass": "The sources support the claim with enough specific evidence; no material contradiction",
        "block": "The sources contradict the claim, or the claim asserts facts the sources do not support",
        "review": "Evidence is partial, ambiguous, or conflicting enough that a human should decide"
      }
    },
    "evidence_strength": {
      "type": "score",
      "instructions": "How strong is the source support (or contradiction) for this claim, ignoring the claim author's confidence.",
      "criteria": [
        "none",
        "weak",
        "moderate",
        "strong"
      ]
    },
    "needs_human": {
      "type": "noul",
      "instructions": "A human should review this claim–source pair before the judgment is trusted in production."
    }
  }
};

const res = await fetch("https://router.flatkey.ai/api/alpha/decisions", {
  method: "POST",
  headers: {
    Authorization: `Bearer ${process.env.FLATKEY_API_KEY}`,
    "Content-Type": "application/json",
  },
  body: JSON.stringify(body),
});
if (!res.ok) throw new Error(`Jev request failed: ${res.status} ${await res.text()}`);
const { answers, usage } = await res.json();
console.log(JSON.stringify(answers, null, 2), usage);

# Python 3.8+ (pip install requests). Run: FLATKEY_API_KEY=... python jev.py
import json, os, requests

body = json.loads(r"""{
  "model": "typesafe/jev-1.13",
  "state": "Claim: \"Jev checked 3,157 claims against sources with 78.9% balanced accuracy at roughly 0.2s per verdict and about 14 cents total cost.\"\n\nSource A: Blog excerpt — \"We evaluated mute probability models on 3,157 claim–source pairs. Balanced accuracy: 78.9%. Median latency ~0.2s. Total spend across the run: $0.14.\"\n\nSource B: Same post — \"Judgments collapse when evidence is missing or wrong; calibrate thresholds and cascade pass/block/review.\"\n\nTask: judge grounding of the claim against these sources only.",
  "questions": {
    "judgment": {
      "type": "choice",
      "instructions": "Judge whether the claim is grounded in the provided source evidence. Ignore world knowledge outside the snippets.",
      "criteria": {
        "pass": "The sources support the claim with enough specific evidence; no material contradiction",
        "block": "The sources contradict the claim, or the claim asserts facts the sources do not support",
        "review": "Evidence is partial, ambiguous, or conflicting enough that a human should decide"
      }
    },
    "evidence_strength": {
      "type": "score",
      "instructions": "How strong is the source support (or contradiction) for this claim, ignoring the claim author's confidence.",
      "criteria": [
        "none",
        "weak",
        "moderate",
        "strong"
      ]
    },
    "needs_human": {
      "type": "noul",
      "instructions": "A human should review this claim–source pair before the judgment is trusted in production."
    }
  }
}""")

res = requests.post(
    "https://router.flatkey.ai/api/alpha/decisions",
    headers={"Authorization": f"Bearer {os.environ['FLATKEY_API_KEY']}"},
    json=body,
    timeout=60,
)
res.raise_for_status()
data = res.json()
print(json.dumps(data["answers"], indent=2), data.get("usage"))

No Flatkey key yet? Get one for Jev 1.13 →