# metric -> (question, NoulCriteria)
MAIN_QUESTIONS = {
"name_desc_mismatch": (
"Does the `extracted_field` fail to match the field at `path` or the `description` in the "
"`field_spec`? If the `description` is empty, judge against the `path` alone.",
NoulCriteria(
true="the `extracted_field` does not match the field name or its `description`",
false="the `extracted_field` matches the field name and `description`",
),
),
"type_mismatch": (
"Does the `extracted_field` violate the `type` declared in the `field_spec`?",
NoulCriteria(
true="the `extracted_field` violates the declared `type`",
false="the `extracted_field` conforms to the declared `type`",
),
),
"unreasonable": (
"Is the `extracted_field` one that a reasonable person would not have extracted for this "
"`field_spec`?",
NoulCriteria(
true="a reasonable person would not have extracted this value",
false="the extraction is reasonable",
),
),
"hallucinated": (
"Is the `extracted_field` unsupported by, or absent from, the source text?",
NoulCriteria(
true="the `extracted_field` is a hallucination -- not supported by, or absent "
"from, the source text",
false="the `extracted_field` is supported by the source text",
),
),
"off_target": (
"Does the source text fail to genuinely report the thing the `field_spec` describes, so the "
"value was pulled from incidental text?",
NoulCriteria(
true="the source does not genuinely provide this field -- the value was pulled "
"from incidental text",
false="the source genuinely reports this field",
),
),
"incomplete": (
"Does the `extracted_field` fail to capture a value the source supports (note whether the "
"`field_spec` is `required`)?",
NoulCriteria(
true="the field is wrongly empty, null, or missing a value the source supports",
false="the field captures the value the source supports",
),
),
"format_violation": (
"Does the `extracted_field` violate the format or constraints implied by the `description`, "
"the schema `type`, and the extraction instructions (e.g. date format, units, enum membership)?",
NoulCriteria(
true="the `extracted_field` violates the implied format or constraints",
false="the `extracted_field` satisfies the format and constraints",
),
),
}
ABSENCE_QUESTION = (
"The `extracted_field` is empty, null, or an empty collection. Does the source text contain the "
"information the `field_spec` describes, making the empty result wrong?"
)
ABSENCE_CRITERIA = NoulCriteria(
true="a value was wrongly omitted", false="returning nothing is correct"
)
# The pipeline also asks one holistic, whole-record head: "should this be escalated?"
OVERALL_JUDGE = (
"Is this extracted record an incorrect extraction -- some value unsupported by the source or "
"not conforming to the schema, required information missing or wrong, or some field hallucinated -- "
"so it should be escalated to a smarter model?"
)
OVERALL_JUDGE_CRITERIA = NoulCriteria(
true="the record is an incorrect extraction",
false="the record is a correct extraction",
)
def is_empty(v) -> bool:
return v is None or (isinstance(v, (str, list, dict)) and len(v) == 0)
def field_spec(name: str) -> dict:
"""Minimal spec pulled from the schema (unwrapping anyOf/null for optional fields)."""
p = schema["properties"][name]
branches = p.get("anyOf") or []
typ = p.get("type") or next(
(b["type"] for b in branches if b.get("type") != "null"), "unknown"
)
return {
"path": name,
"type": typ,
"description": p.get("description", ""),
"required": name in schema.get("required", []),
}
def build_questions(record: dict) -> dict[str, Noul]:
"""The verify question set: one holistic ``__overall__::judge`` head plus a per-field battery,
keyed ``field::metric`` (mirrors build_verify_prompts)."""
questions: dict[str, Noul] = {
"__overall__::judge": Noul(
instructions=OVERALL_JUDGE, criteria=OVERALL_JUDGE_CRITERIA
),
}
for name, value in record.items():
spec = field_spec(name)
if is_empty(value):
questions[f"{name}::absence_wrong"] = Noul(
instructions={
"field_spec": spec,
"extracted_field": value,
"main_question": ABSENCE_QUESTION,
},
criteria=ABSENCE_CRITERIA,
)
continue
for metric, (question, criteria) in MAIN_QUESTIONS.items():
if metric == "type_mismatch" and spec["type"] == "unknown":
continue
questions[f"{name}::{metric}"] = Noul(
instructions={
"field_spec": spec,
"extracted_field": value,
"main_question": question,
},
criteria=criteria,
)
return questions
@json_cache
def verify(record: dict) -> dict[str, float | str]:
"""Run the whole Noul battery over a record in one TypeSafe call; return ``{field::metric: P(true)}``."""
state = {
"system_message": EXTRACT_SYSTEM,
"instruction": "Extract the structured record from this document",
"source_text": row["content"],
"schema": schema,
"extraction": record,
}
questions = build_questions(record)
answers = ts.system_one(state=state, questions=questions, model=TS_MODEL).answers
return {qid: ans.noul for qid, ans in answers.items()} | {
"playground_link": make_playground_link(state, questions)
}
