SacreBLEU

Available

llm.sacrebleu

Measures response-vs-reference wording match on the standardised sacreBLEU scale.

Contract

Field	Value
version	1.0.0
metric_type	pointwise
scorer_contract	per_row
direction	higher_is_better
entrypoint	aip_metrics_llm.answered.sacrebleu
target_kind	None

Required columns

• input_id
• prompt
• expected_output
• sut_response

Accepted schemas

[
  {
    "name": "gdi_text_v1",
    "task_types": [
      "multi_turn_llm",
      "multi_turn_rag",
      "single_turn_llm",
      "single_turn_rag"
    ]
  }
]

Methodology

1. Deterministic - no LLM. Uses the Hugging Face evaluate 'sacrebleu' metric.
2. Compare the response against the reference answer with sacreBLEU's standardised tokenisation.
3. Take the 0-100 sacreBLEU score.
4. Normalise to 0-1 by dividing by 100.

Score semantics

Scores range 0-1 (raw sacreBLEU 0-100 divided by 100). A high score means a strong, reproducible match with the reference; a low score means little overlap. Higher is better.

Worked example

A raw sacreBLEU of 100 (exact match) normalises to 1.0; a raw 45 normalises to 0.45.

Configuration schema

{
  "properties": {
    "concurrency_limit": {
      "default": 10,
      "title": "Concurrency Limit",
      "type": "integer",
      "x-aip-param-role": "operational"
    },
    "huggingface_config_name": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Huggingface Config Name"
    },
    "huggingface_path": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Huggingface Path"
    },
    "inverted": {
      "default": false,
      "title": "Inverted",
      "type": "boolean",
      "x-aip-param-role": "scoring_metadata"
    },
    "max": {
      "default": 100.0,
      "title": "Max",
      "type": "number",
      "x-aip-param-role": "scoring_metadata"
    },
    "metric_name": {
      "default": "sacrebleu",
      "title": "Metric Name",
      "type": "string",
      "x-aip-param-role": "scoring_metadata"
    },
    "min": {
      "default": 0.0,
      "title": "Min",
      "type": "number",
      "x-aip-param-role": "scoring_metadata"
    },
    "primary_key": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": "score",
      "title": "Primary Key"
    },
    "system_type": {
      "default": "llm",
      "enum": [
        "llm",
        "vlm",
        "rag"
      ],
      "title": "System Type",
      "type": "string",
      "x-aip-param-role": "judge_wiring"
    },
    "weight": {
      "default": 1.0,
      "title": "Weight",
      "type": "number",
      "x-aip-param-role": "scoring_metadata"
    }
  },
  "title": "InitializationConfig",
  "type": "object"
}

Execution

{
  "emits_metric_family": false,
  "function_name": null,
  "max_concurrency": null,
  "processing_kind": "cpu",
  "stream_batch_size": null,
  "timeout_seconds": 60
}

Complete manifest

accepts:
- name: gdi_text_v1
  task_types:
  - multi_turn_llm
  - multi_turn_rag
  - single_turn_llm
  - single_turn_rag
config_schema:
  properties:
    concurrency_limit:
      default: 10
      title: Concurrency Limit
      type: integer
      x-aip-param-role: operational
    huggingface_config_name:
      anyOf:
      - type: string
      - type: 'null'
      default: null
      title: Huggingface Config Name
    huggingface_path:
      anyOf:
      - type: string
      - type: 'null'
      default: null
      title: Huggingface Path
    inverted:
      default: false
      title: Inverted
      type: boolean
      x-aip-param-role: scoring_metadata
    max:
      default: 100.0
      title: Max
      type: number
      x-aip-param-role: scoring_metadata
    metric_name:
      default: sacrebleu
      title: Metric Name
      type: string
      x-aip-param-role: scoring_metadata
    min:
      default: 0.0
      title: Min
      type: number
      x-aip-param-role: scoring_metadata
    primary_key:
      anyOf:
      - type: string
      - type: 'null'
      default: score
      title: Primary Key
    system_type:
      default: llm
      enum:
      - llm
      - vlm
      - rag
      title: System Type
      type: string
      x-aip-param-role: judge_wiring
    weight:
      default: 1.0
      title: Weight
      type: number
      x-aip-param-role: scoring_metadata
  title: InitializationConfig
  type: object
dependencies: []
description: How closely does the response match the reference on a standardised wording
  scale?
direction: higher_is_better
display_name: SacreBLEU
entrypoint: aip_metrics_llm.answered.sacrebleu
execution:
  emits_metric_family: false
  function_name: null
  max_concurrency: null
  processing_kind: cpu
  stream_batch_size: null
  timeout_seconds: 60
kind: metric
manifest_version: '1'
max_prompt_slots: 0
metric_metadata:
  methodology:
  - Deterministic - no LLM. Uses the Hugging Face evaluate 'sacrebleu' metric.
  - Compare the response against the reference answer with sacreBLEU's standardised
    tokenisation.
  - Take the 0-100 sacreBLEU score.
  - Normalise to 0-1 by dividing by 100.
  score_range:
    max: 1.0
    min: 0.0
  score_semantics: Scores range 0-1 (raw sacreBLEU 0-100 divided by 100). A high score
    means a strong, reproducible match with the reference; a low score means little
    overlap. Higher is better.
  summary: Measures response-vs-reference wording match on the standardised sacreBLEU
    scale.
  worked_example: A raw sacreBLEU of 100 (exact match) normalises to 1.0; a raw 45
    normalises to 0.45.
metric_type: pointwise
name: llm.sacrebleu
partition_types: []
required_columns:
- input_id
- prompt
- expected_output
- sut_response
required_kinds: []
scorer_contract: per_row
target_kind: null
unsupported_trace_shapes: []
version: 1.0.0