{
  "scope": "8 development cases, not held-out or statistically conclusive",
  "source": {
    "knowledge_base": "kbuLoYcmxK7A",
    "entry_paths": [
      "clarifications/premium_conflict_resolution",
      "customer_segmentation/premium_fee",
      "fee_structures/annual_fee_schedules",
      "operational_constraints/prohibited_actions",
      "policy_provenance/source_hashes",
      "policy_rules/applicability_and_conflict"
    ],
    "checked_at": "2026-09-28T23:12:39.697710+00:00",
    "body_sha256": "26d3f432ed654ca453a88f338b8ca1e8468792dfd4c2934ee63464e371e85e5a",
    "record_count": 4,
    "source_url": "https://www.sanity.io/@owmbk81nd/context/knowledge-bases/kbuLoYcmxK7A/entries",
    "adapter_scope": "remote_json_in_six_reviewed_pages_no_local_value_backfill"
  },
  "model": "glm-4.7-flash",
  "cost": "NOT_MEASURED",
  "comparison": "full-context model vs deterministic PolicyTrace; not end-to-end Agent benchmark",
  "rows": [
    {
      "case": {
        "case_id": "historical",
        "synthetic": true,
        "product_type": "Nova Card",
        "event_date": "2026-06-30",
        "intent": "annual_fee",
        "question": "Explain this annual fee.",
        "known_fields": {
          "customer_type": "premium",
          "customer_type_evidence": "synthetic record"
        }
      },
      "clarifications_enabled": true,
      "expected": {
        "decision": "answer",
        "amount": 40,
        "source_ids": [
          "nova-fee-v1"
        ]
      },
      "policytrace": {
        "actual": {
          "decision": "answer",
          "amount": 40,
          "source_ids": [
            "nova-fee-v1"
          ]
        },
        "checks": {
          "decision": true,
          "amount": true,
          "citations": true
        }
      },
      "model": {
        "status": "COMPLETED",
        "actual": {
          "decision": "answer",
          "amount": 20,
          "source_ids": [
            "nova-special-v1"
          ]
        },
        "metadata": {
          "provider": "Zhipu BigModel",
          "requested_model": "glm-4.7-flash",
          "returned_model": "glm-4.7-flash",
          "request_id": "202609290719218856cb1341264d96",
          "usage": {
            "completion_tokens": 29,
            "completion_tokens_details": {
              "reasoning_tokens": 0
            },
            "prompt_tokens": 1329,
            "prompt_tokens_details": {
              "cached_tokens": 42
            },
            "total_tokens": 1358
          },
          "elapsed_ms": 4906
        },
        "checks": {
          "decision": true,
          "amount": false,
          "citations": false
        }
      }
    },
    {
      "case": {
        "case_id": "boundary_standard",
        "synthetic": true,
        "product_type": "Nova Card",
        "event_date": "2026-07-01",
        "intent": "annual_fee",
        "question": "Explain this annual fee.",
        "known_fields": {
          "customer_type": "standard",
          "customer_type_evidence": "synthetic record"
        }
      },
      "clarifications_enabled": true,
      "expected": {
        "decision": "answer",
        "amount": 70,
        "source_ids": [
          "nova-fee-v2"
        ]
      },
      "policytrace": {
        "actual": {
          "decision": "answer",
          "amount": 70,
          "source_ids": [
            "nova-fee-v2"
          ]
        },
        "checks": {
          "decision": true,
          "amount": true,
          "citations": true
        }
      },
      "model": {
        "status": "COMPLETED",
        "actual": {
          "decision": "answer",
          "amount": 70,
          "source_ids": [
            "nova-fee-v2"
          ]
        },
        "metadata": {
          "provider": "Zhipu BigModel",
          "requested_model": "glm-4.7-flash",
          "returned_model": "glm-4.7-flash",
          "request_id": "2026092907192531b9a2e175c14c92",
          "usage": {
            "completion_tokens": 30,
            "completion_tokens_details": {
              "reasoning_tokens": 0
            },
            "prompt_tokens": 1329,
            "prompt_tokens_details": {
              "cached_tokens": 317
            },
            "total_tokens": 1359
          },
          "elapsed_ms": 3109
        },
        "checks": {
          "decision": true,
          "amount": true,
          "citations": true
        }
      }
    },
    {
      "case": {
        "case_id": "premium_clarified",
        "synthetic": true,
        "product_type": "Nova Card",
        "event_date": "2026-09-19",
        "intent": "annual_fee",
        "question": "Explain this annual fee.",
        "known_fields": {
          "customer_type": "premium",
          "customer_type_evidence": "synthetic record"
        }
      },
      "clarifications_enabled": true,
      "expected": {
        "decision": "answer",
        "amount": 20,
        "source_ids": [
          "nova-fee-v2",
          "nova-special-v1"
        ]
      },
      "policytrace": {
        "actual": {
          "decision": "answer",
          "amount": 20,
          "source_ids": [
            "nova-fee-v2",
            "nova-special-v1"
          ]
        },
        "checks": {
          "decision": true,
          "amount": true,
          "citations": true
        }
      },
      "model": {
        "status": "COMPLETED",
        "actual": {
          "decision": "answer",
          "amount": 20,
          "source_ids": [
            "nova-special-v1",
            "nova-fee-v2",
            "nova-fee-v1",
            "clarification-2026-07-premium"
          ]
        },
        "metadata": {
          "provider": "Zhipu BigModel",
          "requested_model": "glm-4.7-flash",
          "returned_model": "glm-4.7-flash",
          "request_id": "20260929071932f41e07ead5cf494a",
          "usage": {
            "completion_tokens": 54,
            "completion_tokens_details": {
              "reasoning_tokens": 0
            },
            "prompt_tokens": 1331,
            "prompt_tokens_details": {
              "cached_tokens": 317
            },
            "total_tokens": 1385
          },
          "elapsed_ms": 12281
        },
        "checks": {
          "decision": true,
          "amount": true,
          "citations": false
        }
      }
    },
    {
      "case": {
        "case_id": "unresolved_conflict",
        "synthetic": true,
        "product_type": "Nova Card",
        "event_date": "2026-09-19",
        "intent": "annual_fee",
        "question": "Explain this annual fee.",
        "known_fields": {
          "customer_type": "premium",
          "customer_type_evidence": "synthetic record"
        }
      },
      "clarifications_enabled": false,
      "expected": {
        "decision": "handoff",
        "amount": null,
        "source_ids": [
          "nova-fee-v2",
          "nova-special-v1"
        ]
      },
      "policytrace": {
        "actual": {
          "decision": "handoff",
          "amount": null,
          "source_ids": [
            "nova-fee-v2",
            "nova-special-v1"
          ]
        },
        "checks": {
          "decision": true,
          "amount": true,
          "citations": true
        }
      },
      "model": {
        "status": "COMPLETED",
        "actual": {
          "decision": "clarify",
          "amount": null,
          "source_ids": [
            "nova-fee-v2",
            "nova-special-v1",
            "nova-fee-v1"
          ]
        },
        "metadata": {
          "provider": "Zhipu BigModel",
          "requested_model": "glm-4.7-flash",
          "returned_model": "glm-4.7-flash",
          "request_id": "202609290719412b479c56a4fc49ce",
          "usage": {
            "completion_tokens": 44,
            "completion_tokens_details": {
              "reasoning_tokens": 0
            },
            "prompt_tokens": 1145,
            "prompt_tokens_details": {
              "cached_tokens": 317
            },
            "total_tokens": 1189
          },
          "elapsed_ms": 2891
        },
        "checks": {
          "decision": false,
          "amount": true,
          "citations": false
        }
      }
    },
    {
      "case": {
        "case_id": "waiver_boundary",
        "synthetic": true,
        "product_type": "Nova Card",
        "event_date": "2026-09-19",
        "intent": "waiver_request",
        "question": "Please waive this fee.",
        "known_fields": {
          "customer_type": "premium",
          "customer_type_evidence": "synthetic record"
        }
      },
      "clarifications_enabled": true,
      "expected": {
        "decision": "handoff",
        "amount": null,
        "source_ids": [
          "nova-fee-v2",
          "nova-special-v1"
        ]
      },
      "policytrace": {
        "actual": {
          "decision": "handoff",
          "amount": null,
          "source_ids": [
            "nova-fee-v2",
            "nova-special-v1"
          ]
        },
        "checks": {
          "decision": true,
          "amount": true,
          "citations": true
        }
      },
      "model": {
        "status": "COMPLETED",
        "actual": {
          "decision": "handoff",
          "amount": null,
          "source_ids": [
            "nova-fee-v2",
            "nova-special-v1",
            "nova-fee-v1",
            "clarification-2026-07-premium"
          ]
        },
        "metadata": {
          "provider": "Zhipu BigModel",
          "requested_model": "glm-4.7-flash",
          "returned_model": "glm-4.7-flash",
          "request_id": "20260929071944979b0b399ea84cf3",
          "usage": {
            "completion_tokens": 55,
            "completion_tokens_details": {
              "reasoning_tokens": 0
            },
            "prompt_tokens": 1330,
            "prompt_tokens_details": {
              "cached_tokens": 317
            },
            "total_tokens": 1385
          },
          "elapsed_ms": 2750
        },
        "checks": {
          "decision": true,
          "amount": true,
          "citations": false
        }
      }
    },
    {
      "case": {
        "case_id": "missing_date",
        "synthetic": true,
        "product_type": "Nova Card",
        "event_date": "",
        "intent": "annual_fee",
        "question": "Explain this annual fee.",
        "known_fields": {
          "customer_type": "premium",
          "customer_type_evidence": "synthetic record"
        }
      },
      "clarifications_enabled": true,
      "expected": {
        "decision": "clarify",
        "amount": null,
        "source_ids": []
      },
      "policytrace": {
        "actual": {
          "decision": "clarify",
          "amount": null,
          "source_ids": []
        },
        "checks": {
          "decision": true,
          "amount": true,
          "citations": true
        }
      },
      "model": {
        "status": "COMPLETED",
        "actual": {
          "decision": "clarify",
          "amount": null,
          "source_ids": [
            "nova-annual-fee",
            "nova-special-fee",
            "nova-annual-fee"
          ]
        },
        "metadata": {
          "provider": "Zhipu BigModel",
          "requested_model": "glm-4.7-flash",
          "returned_model": "glm-4.7-flash",
          "request_id": "20260929071946e6268aede9074f80",
          "usage": {
            "completion_tokens": 44,
            "completion_tokens_details": {
              "reasoning_tokens": 0
            },
            "prompt_tokens": 1322,
            "prompt_tokens_details": {
              "cached_tokens": 317
            },
            "total_tokens": 1366
          },
          "elapsed_ms": 2188
        },
        "checks": {
          "decision": true,
          "amount": true,
          "citations": false
        }
      }
    },
    {
      "case": {
        "case_id": "missing_segment",
        "synthetic": true,
        "product_type": "Nova Card",
        "event_date": "2026-09-19",
        "intent": "annual_fee",
        "question": "Explain this annual fee.",
        "known_fields": {
          "customer_type": "",
          "customer_type_evidence": "synthetic record"
        }
      },
      "clarifications_enabled": true,
      "expected": {
        "decision": "clarify",
        "amount": null,
        "source_ids": []
      },
      "policytrace": {
        "actual": {
          "decision": "clarify",
          "amount": null,
          "source_ids": []
        },
        "checks": {
          "decision": true,
          "amount": true,
          "citations": true
        }
      },
      "model": {
        "status": "COMPLETED",
        "actual": {
          "decision": "clarify",
          "amount": null,
          "source_ids": [
            "nova-fee-v2",
            "nova-special-v1",
            "nova-fee-v1"
          ]
        },
        "metadata": {
          "provider": "Zhipu BigModel",
          "requested_model": "glm-4.7-flash",
          "returned_model": "glm-4.7-flash",
          "request_id": "20260929071948419ce4a82aa94061",
          "usage": {
            "completion_tokens": 44,
            "completion_tokens_details": {
              "reasoning_tokens": 0
            },
            "prompt_tokens": 1327,
            "prompt_tokens_details": {
              "cached_tokens": 318
            },
            "total_tokens": 1371
          },
          "elapsed_ms": 7562
        },
        "checks": {
          "decision": true,
          "amount": true,
          "citations": false
        }
      }
    },
    {
      "case": {
        "case_id": "missing_evidence",
        "synthetic": true,
        "product_type": "Nova Card",
        "event_date": "2026-09-19",
        "intent": "annual_fee",
        "question": "Explain this annual fee.",
        "known_fields": {
          "customer_type": "premium",
          "customer_type_evidence": ""
        }
      },
      "clarifications_enabled": true,
      "expected": {
        "decision": "clarify",
        "amount": null,
        "source_ids": []
      },
      "policytrace": {
        "actual": {
          "decision": "clarify",
          "amount": null,
          "source_ids": []
        },
        "checks": {
          "decision": true,
          "amount": true,
          "citations": true
        }
      },
      "model": {
        "status": "COMPLETED",
        "actual": {
          "decision": "answer",
          "amount": 20,
          "source_ids": [
            "nova-special-v1",
            "nova-fee-v2",
            "nova-fee-v1",
            "clarification-2026-07-premium"
          ]
        },
        "metadata": {
          "provider": "Zhipu BigModel",
          "requested_model": "glm-4.7-flash",
          "returned_model": "glm-4.7-flash",
          "request_id": "20260929071956520c7cf569d74d58",
          "usage": {
            "completion_tokens": 54,
            "completion_tokens_details": {
              "reasoning_tokens": 0
            },
            "prompt_tokens": 1326,
            "prompt_tokens_details": {
              "cached_tokens": 318
            },
            "total_tokens": 1380
          },
          "elapsed_ms": 2265
        },
        "checks": {
          "decision": false,
          "amount": false,
          "citations": false
        }
      }
    }
  ]
}