{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://docs.cognite.com/assets/schemas/agent-eval-cases.schema.json",
  "title": "Cognite CLI agent eval suite",
  "description": "JSON Schema for eval/eval.yaml and included fragment files used by cognite agents eval. Source of truth for validation at runtime remains the Valibot schema in @cognite/cli.",
  "type": "object",
  "additionalProperties": false,
  "properties": {
    "include": {
      "type": "array",
      "description": "Fragment paths relative to the eval/ directory.",
      "items": { "type": "string", "minLength": 1 }
    },
    "cases": {
      "type": "array",
      "items": { "$ref": "#/$defs/evalCase" }
    },
    "dataModels": { "$ref": "#/$defs/dataModels" },
    "instanceSpaces": { "$ref": "#/$defs/instanceSpaces" },
    "appContext": { "$ref": "#/$defs/appContext" },
    "tools": { "$ref": "#/$defs/tools" },
    "tags": {
      "type": "array",
      "description": "File-level tags (included fragments only); unioned into each case.",
      "items": { "type": "string", "minLength": 1 }
    }
  },
  "$defs": {
    "appContext": {
      "type": "string",
      "minLength": 1,
      "description": "Free-text context passed to the agent."
    },
    "instanceSpaces": {
      "type": "array",
      "minItems": 1,
      "items": { "type": "string", "minLength": 1 }
    },
    "dataModels": {
      "type": "array",
      "minItems": 1,
      "items": {
        "type": "object",
        "additionalProperties": false,
        "required": ["space", "externalId", "version"],
        "properties": {
          "space": { "type": "string", "minLength": 1 },
          "externalId": { "type": "string", "minLength": 1 },
          "version": { "type": "string", "minLength": 1 }
        }
      }
    },
    "tools": {
      "type": "array",
      "minItems": 1,
      "description": "Tool list for toolSelection / toolInvocation judges (runtime names and descriptions). Most-specific-wins cascade: root → included file → case → scorer. Falls back to <name>.agent.yaml tools when no eval tools are set and local <name>.agent.yaml is loaded.",
      "items": {
        "type": "object",
        "additionalProperties": false,
        "required": ["name", "description"],
        "properties": {
          "name": {
            "type": "string",
            "minLength": 1,
            "description": "Runtime tool name the judge expects (e.g. find_assets, execute)."
          },
          "description": {
            "type": "string",
            "minLength": 1,
            "description": "What the tool does; include parameter details for toolInvocation."
          }
        }
      }
    },
    "evalCase": {
      "type": "object",
      "additionalProperties": false,
      "required": ["id", "turns"],
      "properties": {
        "id": { "type": "string", "minLength": 1 },
        "tags": {
          "type": "array",
          "items": { "type": "string", "minLength": 1 }
        },
        "expectPass": {
          "type": "boolean",
          "default": true,
          "description": "When false, scorer failure is the expected outcome (negative test)."
        },
        "turns": {
          "type": "array",
          "minItems": 1,
          "items": { "$ref": "#/$defs/evalTurn" }
        },
        "dataModels": { "$ref": "#/$defs/dataModels" },
        "instanceSpaces": { "$ref": "#/$defs/instanceSpaces" },
        "appContext": { "$ref": "#/$defs/appContext" },
        "tools": { "$ref": "#/$defs/tools" }
      }
    },
    "evalTurn": {
      "type": "object",
      "additionalProperties": false,
      "required": ["input"],
      "properties": {
        "input": { "type": "string", "minLength": 1 },
        "scorers": {
          "type": "array",
          "minItems": 1,
          "items": { "$ref": "#/$defs/scorer" }
        }
      }
    },
    "scorer": {
      "oneOf": [
        {
          "type": "object",
          "additionalProperties": false,
          "required": ["type", "criteria"],
          "properties": {
            "type": { "const": "correctness" },
            "criteria": {
              "type": "string",
              "minLength": 1,
              "description": "What a good answer looks like."
            }
          }
        },
        {
          "type": "object",
          "additionalProperties": false,
          "required": ["type", "groundTruth"],
          "properties": {
            "type": { "const": "faithfulness" },
            "groundTruth": {
              "type": "string",
              "minLength": 1,
              "description": "Facts the answer must be grounded in (judge-only)."
            }
          }
        },
        {
          "type": "object",
          "additionalProperties": false,
          "required": ["type"],
          "properties": {
            "type": { "const": "toolSelection" },
            "tools": { "$ref": "#/$defs/tools" }
          }
        },
        {
          "type": "object",
          "additionalProperties": false,
          "required": ["type"],
          "properties": {
            "type": { "const": "toolInvocation" },
            "tools": { "$ref": "#/$defs/tools" }
          }
        }
      ]
    }
  }
}