9258a0739f
The apps branch's documents join the version contract. The answer layer's documents (wai.quantum.answer/1: an answer read, carried out, a seal's request and result, a stored answer checked), described as a draft whose form may change before the extension settles; a correctness record and its corpus (wai.quantum.telemetry/1); the views page's manifest (wai.quantum.views-bundle/1); and the command line's harness report, corpus and token counts (wai.quantum.eval-report/1, eval-answers/1, token-cost/1). The engine's test holds every answer of the layer's conformance set (read, carried out, half arrived, and stored and verified), records, the corpus and the committed manifest to them; the eval tests hold a scripted run's report, a model server's, the corpus and the committed token counts; check_schemas.py holds the committed documents too. The tool and diagnostic schemas take the answer layer's outcomes, dropped and defaulted. The result view's own tool, poll_quantum_job, listed only to a host that draws views, answers to the job's schema, and the strict test holds it. The basis rename reaches the views: a compile's words and its view name the gate set by its basis (both read the certificate's old target), and a test in each holds it. The service's job summary now lives in engine::views (the apps branch moved it there); it carries the durable branch's set summary and the processor packages' energy wording with it. The answer set's stored seal was made at 0.6.16; the values it seals name their engine, so it is written again at 0.7.0 (WAI_ANSWER_BUNDLE_WRITE=1). The views crate's lock moves to wai-quantum 0.7.0. Co-Authored-By: Claude Opus 5.5 <[email protected]>
190 lines
4.4 KiB
JSON
190 lines
4.4 KiB
JSON
{
|
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
"$id": "https://git.transaction.science/Transaction-Science/open-standards/raw/branch/main/wai/wai-quantum-cli/schemas/eval-report.v1.json",
|
|
"title": "wai.quantum.eval-report/1",
|
|
"description": "wai-quantum eval run's report: one model on one task set, its answers scored by the engine. Each task's row, and the summary: how many were valid and solved, first and after repairs, by verdict level, the time, and the joules per solved task where the model server's energy is metered (or why not). A report ranks nothing.",
|
|
"type": "object",
|
|
"required": [
|
|
"format",
|
|
"model",
|
|
"answer_format",
|
|
"tasks",
|
|
"summary"
|
|
],
|
|
"properties": {
|
|
"format": {
|
|
"const": "wai.quantum.eval-report/1"
|
|
},
|
|
"about": {
|
|
"type": "string"
|
|
},
|
|
"model": {
|
|
"type": "object",
|
|
"properties": {
|
|
"scripted": {
|
|
"type": [
|
|
"string",
|
|
"null"
|
|
]
|
|
},
|
|
"server": {
|
|
"type": [
|
|
"string",
|
|
"null"
|
|
]
|
|
},
|
|
"name": {
|
|
"type": [
|
|
"string",
|
|
"null"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"answer_format": {
|
|
"enum": [
|
|
"qasm",
|
|
"views"
|
|
]
|
|
},
|
|
"tasks": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"description": "One task: whether the answer's program reads (valid), the codes that say why not, the verdict's level, whether the intent holds, whether it is solved (holds by proof or evidence), the answer, the tokens each call took as the server counted them, the time, the repair rounds, the first try, and the joules where the server is metered.",
|
|
"required": [
|
|
"name",
|
|
"valid",
|
|
"solved"
|
|
],
|
|
"properties": {
|
|
"name": {
|
|
"type": "string"
|
|
},
|
|
"valid": {
|
|
"type": "boolean"
|
|
},
|
|
"codes": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"level": {
|
|
"type": [
|
|
"string",
|
|
"null"
|
|
]
|
|
},
|
|
"holds": {
|
|
"type": [
|
|
"boolean",
|
|
"null"
|
|
]
|
|
},
|
|
"solved": {
|
|
"type": "boolean"
|
|
},
|
|
"program": {
|
|
"type": [
|
|
"string",
|
|
"null"
|
|
]
|
|
},
|
|
"why": {},
|
|
"statement": {
|
|
"type": "string"
|
|
},
|
|
"answer": {
|
|
"type": "string"
|
|
},
|
|
"tokens": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": [
|
|
"object",
|
|
"null"
|
|
]
|
|
}
|
|
},
|
|
"seconds": {
|
|
"type": "number"
|
|
},
|
|
"repair_rounds": {
|
|
"type": "integer"
|
|
},
|
|
"first_try": {
|
|
"type": [
|
|
"object",
|
|
"null"
|
|
]
|
|
},
|
|
"joules": {}
|
|
}
|
|
}
|
|
},
|
|
"summary": {
|
|
"type": "object",
|
|
"required": [
|
|
"tasks",
|
|
"valid",
|
|
"solved"
|
|
],
|
|
"properties": {
|
|
"tasks": {
|
|
"type": "integer"
|
|
},
|
|
"valid": {
|
|
"type": "integer"
|
|
},
|
|
"solved": {
|
|
"type": "integer"
|
|
},
|
|
"first_try": {
|
|
"type": "object",
|
|
"properties": {
|
|
"valid": {
|
|
"type": "integer"
|
|
},
|
|
"solved": {
|
|
"type": "integer"
|
|
}
|
|
}
|
|
},
|
|
"repair_rounds": {
|
|
"type": "integer"
|
|
},
|
|
"by_level": {
|
|
"type": "object",
|
|
"additionalProperties": {
|
|
"type": "integer"
|
|
}
|
|
},
|
|
"seconds": {
|
|
"type": "number"
|
|
},
|
|
"joules": {
|
|
"type": "object",
|
|
"properties": {
|
|
"per_solved": {
|
|
"type": [
|
|
"number",
|
|
"null"
|
|
]
|
|
},
|
|
"total": {
|
|
"type": "number"
|
|
},
|
|
"covers": {
|
|
"type": "string"
|
|
},
|
|
"unmetered": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"additionalProperties": true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|