{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://git.transaction.science/Transaction-Science/open-standards/raw/branch/main/wai/wai-quantum-cli/schemas/eval-report.v1.json",
  "title": "wai.quantum.eval-report/1",
  "description": "wai-quantum eval run's report: one model on one task set, its answers scored by the engine. Each task's row, and the summary: how many were valid and solved, first and after repairs, by verdict level, the time, and the joules per solved task where the model server's energy is metered (or why not). A report ranks nothing.",
  "type": "object",
  "required": [
    "format",
    "model",
    "answer_format",
    "tasks",
    "summary"
  ],
  "properties": {
    "format": {
      "const": "wai.quantum.eval-report/1"
    },
    "about": {
      "type": "string"
    },
    "model": {
      "type": "object",
      "properties": {
        "scripted": {
          "type": [
            "string",
            "null"
          ]
        },
        "server": {
          "type": [
            "string",
            "null"
          ]
        },
        "name": {
          "type": [
            "string",
            "null"
          ]
        }
      }
    },
    "answer_format": {
      "enum": [
        "qasm",
        "views"
      ]
    },
    "tasks": {
      "type": "array",
      "items": {
        "type": "object",
        "description": "One task: whether the answer's program reads (valid), the codes that say why not, the verdict's level, whether the intent holds, whether it is solved (holds by proof or evidence), the answer, the tokens each call took as the server counted them, the time, the repair rounds, the first try, and the joules where the server is metered.",
        "required": [
          "name",
          "valid",
          "solved"
        ],
        "properties": {
          "name": {
            "type": "string"
          },
          "valid": {
            "type": "boolean"
          },
          "codes": {
            "type": "array",
            "items": {
              "type": "string"
            }
          },
          "level": {
            "type": [
              "string",
              "null"
            ]
          },
          "holds": {
            "type": [
              "boolean",
              "null"
            ]
          },
          "solved": {
            "type": "boolean"
          },
          "program": {
            "type": [
              "string",
              "null"
            ]
          },
          "why": {},
          "statement": {
            "type": "string"
          },
          "answer": {
            "type": "string"
          },
          "tokens": {
            "type": "array",
            "items": {
              "type": [
                "object",
                "null"
              ]
            }
          },
          "seconds": {
            "type": "number"
          },
          "repair_rounds": {
            "type": "integer"
          },
          "first_try": {
            "type": [
              "object",
              "null"
            ]
          },
          "joules": {}
        }
      }
    },
    "summary": {
      "type": "object",
      "required": [
        "tasks",
        "valid",
        "solved"
      ],
      "properties": {
        "tasks": {
          "type": "integer"
        },
        "valid": {
          "type": "integer"
        },
        "solved": {
          "type": "integer"
        },
        "first_try": {
          "type": "object",
          "properties": {
            "valid": {
              "type": "integer"
            },
            "solved": {
              "type": "integer"
            }
          }
        },
        "repair_rounds": {
          "type": "integer"
        },
        "by_level": {
          "type": "object",
          "additionalProperties": {
            "type": "integer"
          }
        },
        "seconds": {
          "type": "number"
        },
        "joules": {
          "type": "object",
          "properties": {
            "per_solved": {
              "type": [
                "number",
                "null"
              ]
            },
            "total": {
              "type": "number"
            },
            "covers": {
              "type": "string"
            },
            "unmetered": {
              "type": "string"
            }
          },
          "additionalProperties": true
        }
      }
    }
  }
}