{
  "scope": "live_v2_development_not_final_benchmark",
  "rows": [
    {
      "case": "premium",
      "status": "PASS",
      "checks": {
        "decision": true,
        "agent": true,
        "amount": true,
        "no_transaction": true
      },
      "response": {
        "decision": "answer",
        "answer_text": "The annual fee is 20 fictional credits for the supplied product and event date. No waiver or refund has been approved or executed.",
        "policy_refs": [
          {
            "policy_id": "nova-annual-fee",
            "revision": "2",
            "source_id": "nova-fee-v2",
            "source_reference": "policytrace-fixture://nova-fee-v2",
            "effective_from": "2026-07-01",
            "effective_to": null,
            "content_hash": "e7602ebede1ea34c60e61f724956e43260c40dff601f7a250c08926fcee57074"
          },
          {
            "policy_id": "nova-special-fee",
            "revision": "1",
            "source_id": "nova-special-v1",
            "source_reference": "policytrace-fixture://nova-special-v1",
            "effective_from": "2026-07-01",
            "effective_to": null,
            "content_hash": "02eeceeafcc6c47ef1b50c750a0127a3e30153ad198c72e7b0536b88ffa095b8"
          }
        ],
        "missing_fields": [],
        "conflicts": [],
        "handoff_summary": null,
        "tool_status": "ok",
        "trace_id": "b89ab6d4-1b37-465f-91bc-447a1b56c2ca",
        "mode": "live",
        "clarification_refs": [
          {
            "source_id": "clarification-2026-07-premium",
            "source_reference": "policytrace-fixture://approved-premium-clarification",
            "approved_by": "demo-policy-owner",
            "reason": "For premium customers, the special fee replaces the standard fee from July 1. This does not grant refund or waiver authority.",
            "content_hash": "16943c921c23576af1b525fe317eab05c3301060d3d88d1d800f587c33c250cc"
          }
        ],
        "audit_context": {
          "customer_fields": {
            "customer_type": "premium",
            "customer_type_evidence": "user_asserted_synthetic_record"
          },
          "original_source_ids": [
            "nova-fee-v2",
            "nova-special-v1"
          ]
        },
        "selected_source_ids": [
          "nova-special-v1"
        ],
        "confirmed_fee": {
          "amount": 20,
          "unit": "fictional credits"
        },
        "audit": {
          "steps": [
            "intake",
            "normalize",
            "retrieve",
            "validate_evidence",
            "resolve_policy",
            "answer",
            "validate_output",
            "respond"
          ],
          "source_snapshot_sha256": "f5c5c4e77cbc75d4256098239ec8e93eb8773d8a5cc3a2d7683596baa50481ea",
          "clarification_snapshot_sha256": "8003811ee9a605779ae8012b4d3e88090ab77f01af1c0f66b4041ff56620e47c",
          "synthetic": true,
          "transaction_executed": false
        },
        "source": {
          "knowledge_base": "kbuLoYcmxK7A",
          "entry_paths": [
            "clarifications/premium_conflict_resolution",
            "customer_segmentation/premium_fee",
            "fee_structures/annual_fee_schedules",
            "operational_constraints/prohibited_actions",
            "policy_provenance/source_hashes",
            "policy_rules/applicability_and_conflict"
          ],
          "checked_at": "2026-09-28T23:32:00.502790+00:00",
          "body_sha256": "26d3f432ed654ca453a88f338b8ca1e8468792dfd4c2934ee63464e371e85e5a",
          "record_count": 4,
          "source_url": "https://www.sanity.io/@owmbk81nd/context/knowledge-bases/kbuLoYcmxK7A/entries",
          "adapter_scope": "remote_json_in_six_reviewed_pages_no_local_value_backfill"
        },
        "agent": {
          "status": "completed",
          "tool": "check_policy",
          "calls": [
            {
              "provider": "Zhipu BigModel",
              "requested_model": "glm-4.7-flash",
              "returned_model": "glm-4.7-flash",
              "request_id": "202609290731452961c6c6acd14f6a",
              "usage": {
                "completion_tokens": 24,
                "completion_tokens_details": {
                  "reasoning_tokens": 0
                },
                "prompt_tokens": 678,
                "prompt_tokens_details": {
                  "cached_tokens": 615
                },
                "total_tokens": 702
              },
              "elapsed_ms": 6062
            },
            {
              "provider": "Zhipu BigModel",
              "requested_model": "glm-4.7-flash",
              "returned_model": "glm-4.7-flash",
              "request_id": "202609290731553f47ce8e73414f6a",
              "usage": {
                "completion_tokens": 31,
                "completion_tokens_details": {
                  "reasoning_tokens": 0
                },
                "prompt_tokens": 656,
                "prompt_tokens_details": {
                  "cached_tokens": 42
                },
                "total_tokens": 687
              },
              "elapsed_ms": 9125
            }
          ]
        },
        "case": {
          "case_id": "364a857fdb2923b1",
          "synthetic": true,
          "intent": "annual_fee",
          "event_date": "2026-08-14",
          "product_type": "Nova Card",
          "question": "Explain the annual fee.",
          "known_fields": {
            "customer_type": "premium",
            "customer_type_evidence": "user_asserted_synthetic_record"
          }
        },
        "source_mode": "live_sanity_json",
        "model_attempted": true,
        "model_called": true,
        "limitations": [
          "Synthetic customer evidence; not verified identity.",
          "Six reviewed pages only. No financial action authority."
        ]
      }
    },
    {
      "case": "old_fee",
      "status": "PASS",
      "checks": {
        "decision": true,
        "agent": true,
        "amount": true,
        "no_transaction": true
      },
      "response": {
        "decision": "answer",
        "answer_text": "The annual fee is 40 fictional credits for the supplied product and event date. No waiver or refund has been approved or executed.",
        "policy_refs": [
          {
            "policy_id": "nova-annual-fee",
            "revision": "1",
            "source_id": "nova-fee-v1",
            "source_reference": "policytrace-fixture://nova-fee-v1",
            "effective_from": "2026-01-01",
            "effective_to": "2026-07-01",
            "content_hash": "5dfe50560d61ab4e886ac4519672a4d309da4b37ed6d7e227ed818338f6c7476"
          }
        ],
        "missing_fields": [],
        "conflicts": [],
        "handoff_summary": null,
        "tool_status": "ok",
        "trace_id": "7eb3f86e-1829-430f-a6af-0d7cbe9a39d3",
        "mode": "live",
        "clarification_refs": [],
        "audit_context": {
          "customer_fields": {
            "customer_type": "premium",
            "customer_type_evidence": "user_asserted_synthetic_record"
          },
          "original_source_ids": [
            "nova-fee-v1"
          ]
        },
        "confirmed_fee": {
          "amount": 40,
          "unit": "fictional credits"
        },
        "audit": {
          "steps": [
            "intake",
            "normalize",
            "retrieve",
            "validate_evidence",
            "resolve_policy",
            "answer",
            "validate_output",
            "respond"
          ],
          "source_snapshot_sha256": "f5c5c4e77cbc75d4256098239ec8e93eb8773d8a5cc3a2d7683596baa50481ea",
          "clarification_snapshot_sha256": "8003811ee9a605779ae8012b4d3e88090ab77f01af1c0f66b4041ff56620e47c",
          "synthetic": true,
          "transaction_executed": false
        },
        "source": {
          "knowledge_base": "kbuLoYcmxK7A",
          "entry_paths": [
            "clarifications/premium_conflict_resolution",
            "customer_segmentation/premium_fee",
            "fee_structures/annual_fee_schedules",
            "operational_constraints/prohibited_actions",
            "policy_provenance/source_hashes",
            "policy_rules/applicability_and_conflict"
          ],
          "checked_at": "2026-09-28T23:32:16.670085+00:00",
          "body_sha256": "26d3f432ed654ca453a88f338b8ca1e8468792dfd4c2934ee63464e371e85e5a",
          "record_count": 4,
          "source_url": "https://www.sanity.io/@owmbk81nd/context/knowledge-bases/kbuLoYcmxK7A/entries",
          "adapter_scope": "remote_json_in_six_reviewed_pages_no_local_value_backfill"
        },
        "agent": {
          "status": "completed",
          "tool": "check_policy",
          "calls": [
            {
              "provider": "Zhipu BigModel",
              "requested_model": "glm-4.7-flash",
              "returned_model": "glm-4.7-flash",
              "request_id": "2026092907320516d56bf61f0343bd",
              "usage": {
                "completion_tokens": 24,
                "completion_tokens_details": {
                  "reasoning_tokens": 0
                },
                "prompt_tokens": 678,
                "prompt_tokens_details": {
                  "cached_tokens": 593
                },
                "total_tokens": 702
              },
              "elapsed_ms": 2125
            },
            {
              "provider": "Zhipu BigModel",
              "requested_model": "glm-4.7-flash",
              "returned_model": "glm-4.7-flash",
              "request_id": "20260929073211d80ce210be204ece",
              "usage": {
                "completion_tokens": 25,
                "completion_tokens_details": {
                  "reasoning_tokens": 0
                },
                "prompt_tokens": 562,
                "prompt_tokens_details": {
                  "cached_tokens": 282
                },
                "total_tokens": 587
              },
              "elapsed_ms": 6750
            }
          ]
        },
        "case": {
          "case_id": "0eb817b6f1651ccc",
          "synthetic": true,
          "intent": "annual_fee",
          "event_date": "2026-06-30",
          "product_type": "Nova Card",
          "question": "Explain the annual fee.",
          "known_fields": {
            "customer_type": "premium",
            "customer_type_evidence": "user_asserted_synthetic_record"
          }
        },
        "source_mode": "live_sanity_json",
        "model_attempted": true,
        "model_called": true,
        "limitations": [
          "Synthetic customer evidence; not verified identity.",
          "Six reviewed pages only. No financial action authority."
        ]
      }
    },
    {
      "case": "waiver",
      "status": "PASS",
      "checks": {
        "decision": true,
        "agent": true,
        "amount": true,
        "no_transaction": true
      },
      "response": {
        "decision": "handoff",
        "answer_text": "This policy permits explanation only. A human must review any waiver request; no waiver or refund has been approved or executed.",
        "policy_refs": [
          {
            "policy_id": "nova-annual-fee",
            "revision": "2",
            "source_id": "nova-fee-v2",
            "source_reference": "policytrace-fixture://nova-fee-v2",
            "effective_from": "2026-07-01",
            "effective_to": null,
            "content_hash": "e7602ebede1ea34c60e61f724956e43260c40dff601f7a250c08926fcee57074"
          },
          {
            "policy_id": "nova-special-fee",
            "revision": "1",
            "source_id": "nova-special-v1",
            "source_reference": "policytrace-fixture://nova-special-v1",
            "effective_from": "2026-07-01",
            "effective_to": null,
            "content_hash": "02eeceeafcc6c47ef1b50c750a0127a3e30153ad198c72e7b0536b88ffa095b8"
          }
        ],
        "missing_fields": [],
        "conflicts": [],
        "handoff_summary": {
          "reason": "waiver_requires_human_authorization",
          "case_id": "e82fbcb8b18a2d6f",
          "draft_only": true,
          "customer_request": "请帮我减免这笔年费。",
          "event_date": "2026-08-14",
          "known_fields": {
            "customer_type": "premium",
            "customer_type_evidence": "user_asserted_synthetic_record"
          },
          "source_ids": [
            "nova-fee-v2",
            "nova-special-v1"
          ],
          "clarification_ids": [
            "clarification-2026-07-premium"
          ],
          "pending_action": "An authorized human must review the waiver; none has been approved."
        },
        "tool_status": "ok",
        "trace_id": "98034306-ddfd-4416-bfdb-4759ccae89b8",
        "mode": "live",
        "clarification_refs": [
          {
            "source_id": "clarification-2026-07-premium",
            "source_reference": "policytrace-fixture://approved-premium-clarification",
            "approved_by": "demo-policy-owner",
            "reason": "For premium customers, the special fee replaces the standard fee from July 1. This does not grant refund or waiver authority.",
            "content_hash": "16943c921c23576af1b525fe317eab05c3301060d3d88d1d800f587c33c250cc"
          }
        ],
        "audit_context": {
          "customer_fields": {
            "customer_type": "premium",
            "customer_type_evidence": "user_asserted_synthetic_record"
          },
          "original_source_ids": [
            "nova-fee-v2",
            "nova-special-v1"
          ]
        },
        "selected_source_ids": [
          "nova-special-v1"
        ],
        "audit": {
          "steps": [
            "intake",
            "normalize",
            "retrieve",
            "validate_evidence",
            "resolve_policy",
            "handoff",
            "validate_output",
            "respond"
          ],
          "source_snapshot_sha256": "f5c5c4e77cbc75d4256098239ec8e93eb8773d8a5cc3a2d7683596baa50481ea",
          "clarification_snapshot_sha256": "8003811ee9a605779ae8012b4d3e88090ab77f01af1c0f66b4041ff56620e47c",
          "synthetic": true,
          "transaction_executed": false
        },
        "source": {
          "knowledge_base": "kbuLoYcmxK7A",
          "entry_paths": [
            "clarifications/premium_conflict_resolution",
            "customer_segmentation/premium_fee",
            "fee_structures/annual_fee_schedules",
            "operational_constraints/prohibited_actions",
            "policy_provenance/source_hashes",
            "policy_rules/applicability_and_conflict"
          ],
          "checked_at": "2026-09-28T23:32:34.318991+00:00",
          "body_sha256": "26d3f432ed654ca453a88f338b8ca1e8468792dfd4c2934ee63464e371e85e5a",
          "record_count": 4,
          "source_url": "https://www.sanity.io/@owmbk81nd/context/knowledge-bases/kbuLoYcmxK7A/entries",
          "adapter_scope": "remote_json_in_six_reviewed_pages_no_local_value_backfill"
        },
        "agent": {
          "status": "completed",
          "tool": "check_policy",
          "calls": [
            {
              "provider": "Zhipu BigModel",
              "requested_model": "glm-4.7-flash",
              "returned_model": "glm-4.7-flash",
              "request_id": "20260929073218ff4d81edb0774e1a",
              "usage": {
                "completion_tokens": 25,
                "completion_tokens_details": {
                  "reasoning_tokens": 0
                },
                "prompt_tokens": 680,
                "prompt_tokens_details": {
                  "cached_tokens": 615
                },
                "total_tokens": 705
              },
              "elapsed_ms": 5750
            },
            {
              "provider": "Zhipu BigModel",
              "requested_model": "glm-4.7-flash",
              "returned_model": "glm-4.7-flash",
              "request_id": "20260929073229d27eb28d88814c69",
              "usage": {
                "completion_tokens": 32,
                "completion_tokens_details": {
                  "reasoning_tokens": 0
                },
                "prompt_tokens": 658,
                "prompt_tokens_details": {
                  "cached_tokens": 263
                },
                "total_tokens": 690
              },
              "elapsed_ms": 1937
            }
          ]
        },
        "case": {
          "case_id": "e82fbcb8b18a2d6f",
          "synthetic": true,
          "intent": "waiver_request",
          "event_date": "2026-08-14",
          "product_type": "Nova Card",
          "question": "请帮我减免这笔年费。",
          "known_fields": {
            "customer_type": "premium",
            "customer_type_evidence": "user_asserted_synthetic_record"
          }
        },
        "source_mode": "live_sanity_json",
        "model_attempted": true,
        "model_called": true,
        "limitations": [
          "Synthetic customer evidence; not verified identity.",
          "Six reviewed pages only. No financial action authority."
        ]
      }
    },
    {
      "case": "missing_evidence",
      "status": "PASS",
      "checks": {
        "decision": true,
        "agent": true,
        "amount": true,
        "no_transaction": true
      },
      "response": {
        "decision": "clarify",
        "answer_text": "Please provide: customer_type_evidence.",
        "policy_refs": [],
        "missing_fields": [
          "customer_type_evidence"
        ],
        "conflicts": [],
        "handoff_summary": null,
        "tool_status": "ok",
        "trace_id": "6297139a-fe5b-40dd-927f-3da9daa3ef99",
        "mode": "live",
        "audit": {
          "steps": [
            "intake",
            "normalize",
            "retrieve",
            "validate_evidence",
            "resolve_policy",
            "clarify",
            "validate_output",
            "respond"
          ],
          "source_snapshot_sha256": "f5c5c4e77cbc75d4256098239ec8e93eb8773d8a5cc3a2d7683596baa50481ea",
          "clarification_snapshot_sha256": "8003811ee9a605779ae8012b4d3e88090ab77f01af1c0f66b4041ff56620e47c",
          "synthetic": true,
          "transaction_executed": false
        },
        "source": {
          "knowledge_base": "kbuLoYcmxK7A",
          "entry_paths": [
            "clarifications/premium_conflict_resolution",
            "customer_segmentation/premium_fee",
            "fee_structures/annual_fee_schedules",
            "operational_constraints/prohibited_actions",
            "policy_provenance/source_hashes",
            "policy_rules/applicability_and_conflict"
          ],
          "checked_at": "2026-09-28T23:32:42.417987+00:00",
          "body_sha256": "26d3f432ed654ca453a88f338b8ca1e8468792dfd4c2934ee63464e371e85e5a",
          "record_count": 4,
          "source_url": "https://www.sanity.io/@owmbk81nd/context/knowledge-bases/kbuLoYcmxK7A/entries",
          "adapter_scope": "remote_json_in_six_reviewed_pages_no_local_value_backfill"
        },
        "agent": {
          "status": "completed",
          "tool": "check_policy",
          "calls": [
            {
              "provider": "Zhipu BigModel",
              "requested_model": "glm-4.7-flash",
              "returned_model": "glm-4.7-flash",
              "request_id": "202609290732313858485843304e18",
              "usage": {
                "completion_tokens": 24,
                "completion_tokens_details": {
                  "reasoning_tokens": 0
                },
                "prompt_tokens": 678,
                "prompt_tokens_details": {
                  "cached_tokens": 676
                },
                "total_tokens": 702
              },
              "elapsed_ms": 2032
            },
            {
              "provider": "Zhipu BigModel",
              "requested_model": "glm-4.7-flash",
              "returned_model": "glm-4.7-flash",
              "request_id": "202609290732376195335e35fa45f4",
              "usage": {
                "completion_tokens": 20,
                "completion_tokens_details": {
                  "reasoning_tokens": 0
                },
                "prompt_tokens": 430,
                "prompt_tokens_details": {
                  "cached_tokens": 354
                },
                "total_tokens": 450
              },
              "elapsed_ms": 1657
            }
          ]
        },
        "case": {
          "case_id": "2bc6db1d070bd074",
          "synthetic": true,
          "intent": "annual_fee",
          "event_date": "2026-08-14",
          "product_type": "Nova Card",
          "question": "Explain the annual fee.",
          "known_fields": {
            "customer_type": "premium",
            "customer_type_evidence": ""
          }
        },
        "source_mode": "live_sanity_json",
        "model_attempted": true,
        "model_called": true,
        "limitations": [
          "Synthetic customer evidence; not verified identity.",
          "Six reviewed pages only. No financial action authority."
        ]
      }
    },
    {
      "case": "missing_date",
      "status": "PASS",
      "checks": {
        "decision": true,
        "agent": true,
        "amount": true,
        "no_transaction": true
      },
      "response": {
        "decision": "clarify",
        "answer_text": "Please provide: event_date.",
        "policy_refs": [],
        "missing_fields": [
          "event_date"
        ],
        "conflicts": [],
        "handoff_summary": null,
        "tool_status": "mock",
        "trace_id": "2f3ebcbb-4a05-419e-b52c-db2695953c03",
        "mode": "live",
        "audit": {
          "steps": [
            "intake",
            "normalize",
            "clarify",
            "validate_output",
            "respond"
          ],
          "source_snapshot_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945",
          "clarification_snapshot_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945",
          "synthetic": true,
          "transaction_executed": false
        },
        "agent": {
          "status": "completed",
          "tool": "request_details",
          "calls": [
            {
              "provider": "Zhipu BigModel",
              "requested_model": "glm-4.7-flash",
              "returned_model": "glm-4.7-flash",
              "request_id": "202609290732386c084021e0c94439",
              "usage": {
                "completion_tokens": 12,
                "completion_tokens_details": {
                  "reasoning_tokens": 0
                },
                "prompt_tokens": 635,
                "prompt_tokens_details": {
                  "cached_tokens": 578
                },
                "total_tokens": 647
              },
              "elapsed_ms": 1391
            }
          ]
        },
        "case": {
          "case_id": "26c905249aa52b2e",
          "synthetic": true,
          "intent": "annual_fee",
          "event_date": "",
          "product_type": "Nova Card",
          "question": "Which annual fee applies?",
          "known_fields": {
            "customer_type": "premium",
            "customer_type_evidence": "user_asserted_synthetic_record"
          }
        },
        "source": null,
        "source_mode": "not_read",
        "model_attempted": true,
        "model_called": true,
        "limitations": [
          "Synthetic customer evidence; not verified identity.",
          "Six reviewed pages only. No financial action authority."
        ]
      }
    }
  ]
}