Retrieval Relevance

Available

rag.retrieval_relevance

Measures how relevant the retrieved material is to the question, judged independently of the answer.

Contract

Field	Value
version	1.0.0
metric_type	pointwise
scorer_contract	per_row
direction	higher_is_better
entrypoint	aip_metrics_rag.unanswered.retrieval_relevance
target_kind	None

Required columns

• input_id
• prompt
• sut_response
• retrieved_context

Accepted schemas

[
  {
    "name": "gdi_text_v1",
    "task_types": [
      "multi_turn_rag",
      "single_turn_rag"
    ]
  }
]

Methodology

1. An LLM judge (openevals, custom prompt, temperature 0) receives the question and the joined retrieved context.
2. The judge enumerates the information the question needs and checks which parts the context addresses.
3. It returns a graded 0-1 score (partial credit allowed) with reasoning.
4. The score is normalised to 0-1 (a NaN verdict is surfaced as an error).

Score semantics

Scores range 0-1. A high score means the retrieved context contains information useful for answering the question; a low score means it is irrelevant or tangential. Higher is better.

Worked example

Question 'What is the boiling point of water?' with chunks about water density plus one stating 'water boils at 100 degrees Celsius' -> partially relevant -> score ~ 0.7.

Configuration schema

{
  "properties": {
    "concurrency_limit": {
      "default": 10,
      "title": "Concurrency Limit",
      "type": "integer",
      "x-aip-param-role": "operational"
    },
    "create_prompt_function": {
      "default": "create_prompt",
      "title": "Create Prompt Function",
      "type": "string",
      "x-aip-param-role": "judge_wiring"
    },
    "inverted": {
      "default": false,
      "title": "Inverted",
      "type": "boolean",
      "x-aip-param-role": "scoring_metadata"
    },
    "judge_temperature": {
      "const": 0.0,
      "default": 0.0,
      "maximum": 0.0,
      "minimum": 0.0,
      "title": "Judge Temperature",
      "type": "number"
    },
    "lang": {
      "default": "en",
      "enum": [
        "en",
        "de",
        "fr"
      ],
      "title": "Lang",
      "type": "string"
    },
    "max": {
      "default": 1.0,
      "title": "Max",
      "type": "number",
      "x-aip-param-role": "scoring_metadata"
    },
    "metric_name": {
      "default": "retrieval_relevance",
      "title": "Metric Name",
      "type": "string",
      "x-aip-param-role": "scoring_metadata"
    },
    "min": {
      "default": 0.0,
      "title": "Min",
      "type": "number",
      "x-aip-param-role": "scoring_metadata"
    },
    "model": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Model"
    },
    "prompt_composition": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": "compose_llm_fewshot_prompt",
      "title": "Prompt Composition",
      "x-aip-param-role": "judge_wiring"
    },
    "system_type": {
      "const": "rag",
      "default": "rag",
      "title": "System Type",
      "type": "string",
      "x-aip-param-role": "judge_wiring"
    },
    "timeout_seconds": {
      "default": 60.0,
      "exclusiveMinimum": 0,
      "title": "Timeout Seconds",
      "type": "number",
      "x-aip-param-role": "operational"
    },
    "weight": {
      "default": 1.0,
      "title": "Weight",
      "type": "number",
      "x-aip-param-role": "scoring_metadata"
    }
  },
  "title": "InitializationConfig",
  "type": "object"
}

Execution

{
  "emits_metric_family": false,
  "function_name": null,
  "max_concurrency": 8,
  "processing_kind": "network",
  "stream_batch_size": null,
  "timeout_seconds": 900
}

Complete manifest

accepts:
- name: gdi_text_v1
  task_types:
  - multi_turn_rag
  - single_turn_rag
config_schema:
  properties:
    concurrency_limit:
      default: 10
      title: Concurrency Limit
      type: integer
      x-aip-param-role: operational
    create_prompt_function:
      default: create_prompt
      title: Create Prompt Function
      type: string
      x-aip-param-role: judge_wiring
    inverted:
      default: false
      title: Inverted
      type: boolean
      x-aip-param-role: scoring_metadata
    judge_temperature:
      const: 0.0
      default: 0.0
      maximum: 0.0
      minimum: 0.0
      title: Judge Temperature
      type: number
    lang:
      default: en
      enum:
      - en
      - de
      - fr
      title: Lang
      type: string
    max:
      default: 1.0
      title: Max
      type: number
      x-aip-param-role: scoring_metadata
    metric_name:
      default: retrieval_relevance
      title: Metric Name
      type: string
      x-aip-param-role: scoring_metadata
    min:
      default: 0.0
      title: Min
      type: number
      x-aip-param-role: scoring_metadata
    model:
      anyOf:
      - type: string
      - type: 'null'
      default: null
      title: Model
    prompt_composition:
      anyOf:
      - type: string
      - type: 'null'
      default: compose_llm_fewshot_prompt
      title: Prompt Composition
      x-aip-param-role: judge_wiring
    system_type:
      const: rag
      default: rag
      title: System Type
      type: string
      x-aip-param-role: judge_wiring
    timeout_seconds:
      default: 60.0
      exclusiveMinimum: 0
      title: Timeout Seconds
      type: number
      x-aip-param-role: operational
    weight:
      default: 1.0
      title: Weight
      type: number
      x-aip-param-role: scoring_metadata
  title: InitializationConfig
  type: object
dependencies: []
description: How relevant is the retrieved material to the question being asked?
direction: higher_is_better
display_name: Retrieval Relevance
entrypoint: aip_metrics_rag.unanswered.retrieval_relevance
execution:
  emits_metric_family: false
  function_name: null
  max_concurrency: 8
  processing_kind: network
  stream_batch_size: null
  timeout_seconds: 900
kind: metric
manifest_version: '1'
max_prompt_slots: 0
metric_metadata:
  methodology:
  - An LLM judge (openevals, custom prompt, temperature 0) receives the question and
    the joined retrieved context.
  - The judge enumerates the information the question needs and checks which parts
    the context addresses.
  - It returns a graded 0-1 score (partial credit allowed) with reasoning.
  - The score is normalised to 0-1 (a NaN verdict is surfaced as an error).
  score_semantics: Scores range 0-1. A high score means the retrieved context contains
    information useful for answering the question; a low score means it is irrelevant
    or tangential. Higher is better.
  summary: Measures how relevant the retrieved material is to the question, judged
    independently of the answer.
  worked_example: Question 'What is the boiling point of water?' with chunks about
    water density plus one stating 'water boils at 100 degrees Celsius' -> partially
    relevant -> score ~ 0.7.
metric_type: pointwise
name: rag.retrieval_relevance
partition_types: []
required_columns:
- input_id
- prompt
- sut_response
- retrieved_context
required_kinds: []
scorer_contract: per_row
target_kind: null
unsupported_trace_shapes: []
version: 1.0.0