Answer Relevance
llm.answer_relevance
Judges whether the response stays on-topic and actually addresses what the question asked.
Contract
| Field | Value |
|---|---|
version | 1.0.0 |
metric_type | pointwise |
scorer_contract | per_row |
direction | higher_is_better |
entrypoint | aip_metrics_llm.unanswered.answer_relevance |
target_kind | None |
Required columns
input_idpromptsut_response
Accepted schemas
[
{
"name": "gdi_text_v1",
"task_types": [
"multi_turn_llm",
"multi_turn_rag",
"single_turn_llm",
"single_turn_rag"
]
}
]
Methodology
- An LLM judge (openevals) scores the response against a relevance rubric.
- The judge receives the query and the response (no reference answer).
- It decides whether the response addresses what was asked.
- The verdict is normalised to a 0-1 score.
Score semantics
Scores range 0-1. A high score means the response directly answers the question; a low score means it is off-topic or tangential. Higher is better.
Worked example
Query 'What is the capital of France?' with response 'Paris is the capital' scores 1.0; 'France has a rich culinary history' scores 0.0.
Configuration schema
{
"properties": {
"concurrency_limit": {
"default": 10,
"title": "Concurrency Limit",
"type": "integer",
"x-aip-param-role": "operational"
},
"create_prompt_function": {
"default": "create_prompt",
"title": "Create Prompt Function",
"type": "string",
"x-aip-param-role": "judge_wiring"
},
"inverted": {
"default": false,
"title": "Inverted",
"type": "boolean",
"x-aip-param-role": "scoring_metadata"
},
"judge_temperature": {
"const": 0.0,
"default": 0.0,
"maximum": 0.0,
"minimum": 0.0,
"title": "Judge Temperature",
"type": "number"
},
"lang": {
"default": "en",
"enum": [
"en",
"de",
"fr"
],
"title": "Lang",
"type": "string"
},
"max": {
"default": 1.0,
"title": "Max",
"type": "number",
"x-aip-param-role": "scoring_metadata"
},
"metric_name": {
"default": "answer_relevance",
"title": "Metric Name",
"type": "string",
"x-aip-param-role": "scoring_metadata"
},
"min": {
"default": 0.0,
"title": "Min",
"type": "number",
"x-aip-param-role": "scoring_metadata"
},
"model": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"title": "Model"
},
"prompt_composition": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": "compose_llm_fewshot_prompt",
"title": "Prompt Composition",
"x-aip-param-role": "judge_wiring"
},
"system_type": {
"default": "llm",
"enum": [
"llm",
"vlm",
"rag"
],
"title": "System Type",
"type": "string",
"x-aip-param-role": "judge_wiring"
},
"timeout_seconds": {
"default": 60.0,
"exclusiveMinimum": 0,
"title": "Timeout Seconds",
"type": "number",
"x-aip-param-role": "operational"
},
"weight": {
"default": 1.0,
"title": "Weight",
"type": "number",
"x-aip-param-role": "scoring_metadata"
}
},
"title": "InitializationConfig",
"type": "object"
}
Execution
{
"emits_metric_family": false,
"function_name": null,
"max_concurrency": 8,
"processing_kind": "network",
"stream_batch_size": 5000,
"timeout_seconds": 21600
}
Complete manifest
accepts:
- name: gdi_text_v1
task_types:
- multi_turn_llm
- multi_turn_rag
- single_turn_llm
- single_turn_rag
config_schema:
properties:
concurrency_limit:
default: 10
title: Concurrency Limit
type: integer
x-aip-param-role: operational
create_prompt_function:
default: create_prompt
title: Create Prompt Function
type: string
x-aip-param-role: judge_wiring
inverted:
default: false
title: Inverted
type: boolean
x-aip-param-role: scoring_metadata
judge_temperature:
const: 0.0
default: 0.0
maximum: 0.0
minimum: 0.0
title: Judge Temperature
type: number
lang:
default: en
enum:
- en
- de
- fr
title: Lang
type: string
max:
default: 1.0
title: Max
type: number
x-aip-param-role: scoring_metadata
metric_name:
default: answer_relevance
title: Metric Name
type: string
x-aip-param-role: scoring_metadata
min:
default: 0.0
title: Min
type: number
x-aip-param-role: scoring_metadata
model:
anyOf:
- type: string
- type: 'null'
default: null
title: Model
prompt_composition:
anyOf:
- type: string
- type: 'null'
default: compose_llm_fewshot_prompt
title: Prompt Composition
x-aip-param-role: judge_wiring
system_type:
default: llm
enum:
- llm
- vlm
- rag
title: System Type
type: string
x-aip-param-role: judge_wiring
timeout_seconds:
default: 60.0
exclusiveMinimum: 0
title: Timeout Seconds
type: number
x-aip-param-role: operational
weight:
default: 1.0
title: Weight
type: number
x-aip-param-role: scoring_metadata
title: InitializationConfig
type: object
dependencies: []
description: How well does the response stay on-topic and address what was actually
asked?
direction: higher_is_better
display_name: Answer Relevance
entrypoint: aip_metrics_llm.unanswered.answer_relevance
execution:
emits_metric_family: false
function_name: null
max_concurrency: 8
processing_kind: network
stream_batch_size: 5000
timeout_seconds: 21600
kind: metric
manifest_version: '1'
max_prompt_slots: 0
metric_metadata:
methodology:
- An LLM judge (openevals) scores the response against a relevance rubric.
- The judge receives the query and the response (no reference answer).
- It decides whether the response addresses what was asked.
- The verdict is normalised to a 0-1 score.
score_semantics: Scores range 0-1. A high score means the response directly answers
the question; a low score means it is off-topic or tangential. Higher is better.
summary: Judges whether the response stays on-topic and actually addresses what
the question asked.
worked_example: Query 'What is the capital of France?' with response 'Paris is the
capital' scores 1.0; 'France has a rich culinary history' scores 0.0.
metric_type: pointwise
name: llm.answer_relevance
partition_types: []
required_columns:
- input_id
- prompt
- sut_response
required_kinds: []
scorer_contract: per_row
target_kind: null
unsupported_trace_shapes: []
version: 1.0.0