Groundedness

Available

rag.groundedness

Measures how well the answer's claims are supported by the retrieved evidence rather than outside knowledge or speculation.

Contract

Field	Value
version	1.0.0
metric_type	pointwise
scorer_contract	per_row
direction	higher_is_better
entrypoint	aip_metrics_rag.unanswered.groundedness
target_kind	None

Required columns

• input_id
• prompt
• sut_response
• retrieved_context

Accepted schemas

[
  {
    "name": "gdi_text_v1",
    "task_types": [
      "multi_turn_rag",
      "single_turn_rag"
    ]
  }
]

Methodology

1. An LLM judge (openevals, custom prompt, temperature 0) receives the answer and the joined retrieved context.
2. The judge is instructed to ignore outside world knowledge and assess support from the context alone.
3. It returns a graded 0-1 score with reasoning.
4. The score is normalised to 0-1 (a NaN verdict is surfaced as an error).

Score semantics

Scores range 0-1. A high score means the answer stays within what the context supports; a low score means it makes unsupported, speculative, or contradictory claims. Higher is better.

Worked example

Context states 'Paris is the capital of France'; the answer adds 'Paris, population 12 billion', an unsupported claim, so the judge scores ~ 0.3.

Configuration schema

{
  "properties": {
    "concurrency_limit": {
      "default": 10,
      "title": "Concurrency Limit",
      "type": "integer",
      "x-aip-param-role": "operational"
    },
    "create_prompt_function": {
      "default": "create_prompt",
      "title": "Create Prompt Function",
      "type": "string",
      "x-aip-param-role": "judge_wiring"
    },
    "inverted": {
      "default": false,
      "title": "Inverted",
      "type": "boolean",
      "x-aip-param-role": "scoring_metadata"
    },
    "judge_temperature": {
      "const": 0.0,
      "default": 0.0,
      "maximum": 0.0,
      "minimum": 0.0,
      "title": "Judge Temperature",
      "type": "number"
    },
    "lang": {
      "default": "en",
      "enum": [
        "en",
        "de",
        "fr"
      ],
      "title": "Lang",
      "type": "string"
    },
    "max": {
      "default": 1.0,
      "title": "Max",
      "type": "number",
      "x-aip-param-role": "scoring_metadata"
    },
    "metric_name": {
      "default": "groundedness",
      "title": "Metric Name",
      "type": "string",
      "x-aip-param-role": "scoring_metadata"
    },
    "min": {
      "default": 0.0,
      "title": "Min",
      "type": "number",
      "x-aip-param-role": "scoring_metadata"
    },
    "model": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Model"
    },
    "prompt_composition": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": "compose_llm_fewshot_prompt",
      "title": "Prompt Composition",
      "x-aip-param-role": "judge_wiring"
    },
    "system_type": {
      "const": "rag",
      "default": "rag",
      "title": "System Type",
      "type": "string",
      "x-aip-param-role": "judge_wiring"
    },
    "timeout_seconds": {
      "default": 60.0,
      "exclusiveMinimum": 0,
      "title": "Timeout Seconds",
      "type": "number",
      "x-aip-param-role": "operational"
    },
    "weight": {
      "default": 1.0,
      "title": "Weight",
      "type": "number",
      "x-aip-param-role": "scoring_metadata"
    }
  },
  "title": "InitializationConfig",
  "type": "object"
}

Execution

{
  "emits_metric_family": false,
  "function_name": null,
  "max_concurrency": 8,
  "processing_kind": "network",
  "stream_batch_size": null,
  "timeout_seconds": 900
}

Complete manifest

accepts:
- name: gdi_text_v1
  task_types:
  - multi_turn_rag
  - single_turn_rag
config_schema:
  properties:
    concurrency_limit:
      default: 10
      title: Concurrency Limit
      type: integer
      x-aip-param-role: operational
    create_prompt_function:
      default: create_prompt
      title: Create Prompt Function
      type: string
      x-aip-param-role: judge_wiring
    inverted:
      default: false
      title: Inverted
      type: boolean
      x-aip-param-role: scoring_metadata
    judge_temperature:
      const: 0.0
      default: 0.0
      maximum: 0.0
      minimum: 0.0
      title: Judge Temperature
      type: number
    lang:
      default: en
      enum:
      - en
      - de
      - fr
      title: Lang
      type: string
    max:
      default: 1.0
      title: Max
      type: number
      x-aip-param-role: scoring_metadata
    metric_name:
      default: groundedness
      title: Metric Name
      type: string
      x-aip-param-role: scoring_metadata
    min:
      default: 0.0
      title: Min
      type: number
      x-aip-param-role: scoring_metadata
    model:
      anyOf:
      - type: string
      - type: 'null'
      default: null
      title: Model
    prompt_composition:
      anyOf:
      - type: string
      - type: 'null'
      default: compose_llm_fewshot_prompt
      title: Prompt Composition
      x-aip-param-role: judge_wiring
    system_type:
      const: rag
      default: rag
      title: System Type
      type: string
      x-aip-param-role: judge_wiring
    timeout_seconds:
      default: 60.0
      exclusiveMinimum: 0
      title: Timeout Seconds
      type: number
      x-aip-param-role: operational
    weight:
      default: 1.0
      title: Weight
      type: number
      x-aip-param-role: scoring_metadata
  title: InitializationConfig
  type: object
dependencies: []
description: How well is the answer supported by the retrieved evidence rather than
  assumptions?
direction: higher_is_better
display_name: Groundedness
entrypoint: aip_metrics_rag.unanswered.groundedness
execution:
  emits_metric_family: false
  function_name: null
  max_concurrency: 8
  processing_kind: network
  stream_batch_size: null
  timeout_seconds: 900
kind: metric
manifest_version: '1'
max_prompt_slots: 0
metric_metadata:
  methodology:
  - An LLM judge (openevals, custom prompt, temperature 0) receives the answer and
    the joined retrieved context.
  - The judge is instructed to ignore outside world knowledge and assess support from
    the context alone.
  - It returns a graded 0-1 score with reasoning.
  - The score is normalised to 0-1 (a NaN verdict is surfaced as an error).
  score_semantics: Scores range 0-1. A high score means the answer stays within what
    the context supports; a low score means it makes unsupported, speculative, or
    contradictory claims. Higher is better.
  summary: Measures how well the answer's claims are supported by the retrieved evidence
    rather than outside knowledge or speculation.
  worked_example: Context states 'Paris is the capital of France'; the answer adds
    'Paris, population 12 billion', an unsupported claim, so the judge scores ~ 0.3.
metric_type: pointwise
name: rag.groundedness
partition_types: []
required_columns:
- input_id
- prompt
- sut_response
- retrieved_context
required_kinds: []
scorer_contract: per_row
target_kind: null
unsupported_trace_shapes: []
version: 1.0.0