Check whether a source supports a sentence

Before a generated answer ships, pair each cited sentence with its source passage and ask one Noul: does this passage support this sentence? Anything under your threshold gets flagged or dropped.

async function supported(sentence: string, passage: string) {
  const { answers } = await jev({
    state: JSON.stringify({ sentence, passage }),
    questions: {
      supports: { type: "noul",
        instructions: "Does the passage directly support the sentence, without adding facts?" },
    },
  });
  return answers.supports.noul; // 0..1
}

const flagged = [];
for (const c of citations) {
  if ((await supported(c.sentence, c.passage)) < 0.6) flagged.push(c);
}

def supported(sentence: str, passage: str) -> float:
    a = system_one(state=json.dumps({"sentence": sentence, "passage": passage}), questions={
        "supports": {"type": "noul",
                     "instructions": "Does the passage directly support the sentence, without adding facts?"}
    })
    return a["supports"]["noul"]

flagged = [c for c in citations if supported(c["sentence"], c["passage"]) < 0.6]

Notes

• Run a plain string match first to catch quotes that don't appear in the source at all. It's free.
• Batch many pairs per call where the API allows; it's where the speed advantage shows up.