Perturbation Robustness

Available

llm.perturbation_robustness

Measures how consistently the system stays correct across meaning-preserving variants of a question.

Contract

Field	Value
version	1.0.0
metric_type	pointwise
scorer_contract	per_row
direction	higher_is_better
entrypoint	aip_metrics_llm.unanswered.perturbation_robustness
target_kind	None

Required columns

• input_id
• prompt
• sut_response

Accepted schemas

[
  {
    "name": "gdi_text_v1",
    "task_types": [
      "multi_turn_llm",
      "multi_turn_rag",
      "single_turn_llm",
      "single_turn_rag"
    ]
  }
]

Methodology

1. An LLM judge scores answers; a generator synthesises perturbed variants (typos, paraphrase, style, format, noise) and a reference.
2. The system under test answers the base query and each perturbed variant.
3. The judge scores base correctness, perturbed correctness, and a per-pair robustness score.
4. Combine as 0.5mean-robustness + 0.5max(0, 1 - mean-correctness-drop).

Score semantics

Scores range 0-1, blending consistency across variants with the drop in correctness. A high score means answers stay correct and stable under rewording; a low score means correctness degrades. Higher is better.

Worked example

Mean robustness 0.9 with a small correctness drop of 0.1 -> 0.50.9 + 0.50.9 = 0.9.

Configuration schema

{
  "properties": {
    "groundtruth_context": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Groundtruth Context"
    },
    "inverted": {
      "default": false,
      "title": "Inverted",
      "type": "boolean",
      "x-aip-param-role": "scoring_metadata"
    },
    "max": {
      "default": 1.0,
      "title": "Max",
      "type": "number",
      "x-aip-param-role": "scoring_metadata"
    },
    "metric_name": {
      "default": "perturbation_robustness_llm",
      "title": "Metric Name",
      "type": "string",
      "x-aip-param-role": "scoring_metadata"
    },
    "min": {
      "default": 0.0,
      "title": "Min",
      "type": "number",
      "x-aip-param-role": "scoring_metadata"
    },
    "n_sut_retries": {
      "default": 0,
      "maximum": 5,
      "minimum": 0,
      "title": "N Sut Retries",
      "type": "integer"
    },
    "num_pairs": {
      "default": 3,
      "maximum": 32,
      "minimum": 1,
      "title": "Num Pairs",
      "type": "integer"
    },
    "reference_answer": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Reference Answer"
    },
    "sub_call_concurrency_limit": {
      "default": 5,
      "maximum": 64,
      "minimum": 1,
      "title": "Sub Call Concurrency Limit",
      "type": "integer",
      "x-aip-param-role": "operational"
    },
    "sut_temperature": {
      "default": 0.7,
      "maximum": 2.0,
      "minimum": 0.0,
      "title": "Sut Temperature",
      "type": "number"
    },
    "synthesis_temperature": {
      "default": 0.0,
      "maximum": 2.0,
      "minimum": 0.0,
      "title": "Synthesis Temperature",
      "type": "number"
    },
    "timeout_seconds": {
      "default": 60.0,
      "exclusiveMinimum": 0,
      "title": "Timeout Seconds",
      "type": "number",
      "x-aip-param-role": "operational"
    },
    "weight": {
      "default": 1.0,
      "title": "Weight",
      "type": "number",
      "x-aip-param-role": "scoring_metadata"
    }
  },
  "title": "PerturbationRobustnessLLMConfig",
  "type": "object"
}

Execution

{
  "emits_metric_family": false,
  "function_name": null,
  "max_concurrency": 8,
  "processing_kind": "network",
  "stream_batch_size": 5000,
  "timeout_seconds": 21600
}

Complete manifest

accepts:
- name: gdi_text_v1
  task_types:
  - multi_turn_llm
  - multi_turn_rag
  - single_turn_llm
  - single_turn_rag
config_schema:
  properties:
    groundtruth_context:
      anyOf:
      - type: string
      - type: 'null'
      default: null
      title: Groundtruth Context
    inverted:
      default: false
      title: Inverted
      type: boolean
      x-aip-param-role: scoring_metadata
    max:
      default: 1.0
      title: Max
      type: number
      x-aip-param-role: scoring_metadata
    metric_name:
      default: perturbation_robustness_llm
      title: Metric Name
      type: string
      x-aip-param-role: scoring_metadata
    min:
      default: 0.0
      title: Min
      type: number
      x-aip-param-role: scoring_metadata
    n_sut_retries:
      default: 0
      maximum: 5
      minimum: 0
      title: N Sut Retries
      type: integer
    num_pairs:
      default: 3
      maximum: 32
      minimum: 1
      title: Num Pairs
      type: integer
    reference_answer:
      anyOf:
      - type: string
      - type: 'null'
      default: null
      title: Reference Answer
    sub_call_concurrency_limit:
      default: 5
      maximum: 64
      minimum: 1
      title: Sub Call Concurrency Limit
      type: integer
      x-aip-param-role: operational
    sut_temperature:
      default: 0.7
      maximum: 2.0
      minimum: 0.0
      title: Sut Temperature
      type: number
    synthesis_temperature:
      default: 0.0
      maximum: 2.0
      minimum: 0.0
      title: Synthesis Temperature
      type: number
    timeout_seconds:
      default: 60.0
      exclusiveMinimum: 0
      title: Timeout Seconds
      type: number
      x-aip-param-role: operational
    weight:
      default: 1.0
      title: Weight
      type: number
      x-aip-param-role: scoring_metadata
  title: PerturbationRobustnessLLMConfig
  type: object
dependencies: []
description: How consistently does the system stay correct across equivalent question
  variants?
direction: higher_is_better
display_name: Perturbation Robustness
entrypoint: aip_metrics_llm.unanswered.perturbation_robustness
execution:
  emits_metric_family: false
  function_name: null
  max_concurrency: 8
  processing_kind: network
  stream_batch_size: 5000
  timeout_seconds: 21600
kind: metric
manifest_version: '1'
max_prompt_slots: 0
metric_metadata:
  methodology:
  - An LLM judge scores answers; a generator synthesises perturbed variants (typos,
    paraphrase, style, format, noise) and a reference.
  - The system under test answers the base query and each perturbed variant.
  - The judge scores base correctness, perturbed correctness, and a per-pair robustness
    score.
  - Combine as 0.5*mean-robustness + 0.5*max(0, 1 - mean-correctness-drop).
  score_semantics: Scores range 0-1, blending consistency across variants with the
    drop in correctness. A high score means answers stay correct and stable under
    rewording; a low score means correctness degrades. Higher is better.
  summary: Measures how consistently the system stays correct across meaning-preserving
    variants of a question.
  worked_example: Mean robustness 0.9 with a small correctness drop of 0.1 -> 0.5*0.9
    + 0.5*0.9 = 0.9.
metric_type: pointwise
name: llm.perturbation_robustness
partition_types: []
required_columns:
- input_id
- prompt
- sut_response
required_kinds: []
scorer_contract: per_row
target_kind: null
unsupported_trace_shapes: []
version: 1.0.0