Hit@K

Available

rag.hit_at_k

Checks whether the correct supporting evidence appears anywhere in the top-k retrieved results.

Contract

Field	Value
version	1.0.0
metric_type	pointwise
scorer_contract	per_row
direction	higher_is_better
entrypoint	aip_metrics_rag.answered.hit_at_k
target_kind	None

Required columns

• input_id
• prompt
• reference_contexts
• sut_response
• retrieved_context

Accepted schemas

[
  {
    "name": "gdi_text_v1",
    "task_types": [
      "multi_turn_rag",
      "single_turn_rag"
    ]
  }
]

Methodology

1. Deterministic - no LLM. Uses rapidfuzz string matching against the reference chunks.
2. Take the top-k retrieved chunks (default k=5).
3. Compare each retrieved chunk to each reference chunk with a two-stage gate (token-set ratio >= 0.70, then partial ratio >= 0.65 or token overlap >= 0.50).
4. Score 1.0 if any chunk passes the gate, else 0.0.

Score semantics

The score is binary - 1.0 if at least one of the top-k retrieved chunks matches a reference chunk, otherwise 0.0. Higher is better.

Worked example

Among the top-5 retrieved chunks the 2nd matches a reference chunk (token-set ratio 0.82) -> hit -> score 1.0.

Configuration schema

{
  "properties": {
    "coverage": {
      "default": 0.5,
      "description": "token_overlap stage-2 threshold",
      "maximum": 1.0,
      "minimum": 0.0,
      "title": "Coverage",
      "type": "number"
    },
    "inverted": {
      "default": false,
      "title": "Inverted",
      "type": "boolean",
      "x-aip-param-role": "scoring_metadata"
    },
    "k": {
      "default": 5,
      "minimum": 1,
      "title": "K",
      "type": "integer"
    },
    "language": {
      "default": "en",
      "description": "Language for metric output",
      "enum": [
        "en",
        "de",
        "fr"
      ],
      "title": "Language",
      "type": "string"
    },
    "max": {
      "default": 1.0,
      "title": "Max",
      "type": "number",
      "x-aip-param-role": "scoring_metadata"
    },
    "metric_name": {
      "default": "hit_at_k",
      "title": "Metric Name",
      "type": "string",
      "x-aip-param-role": "scoring_metadata"
    },
    "min": {
      "default": 0.0,
      "title": "Min",
      "type": "number",
      "x-aip-param-role": "scoring_metadata"
    },
    "t1": {
      "default": 0.7,
      "description": "token_set_ratio stage-1 threshold",
      "maximum": 1.0,
      "minimum": 0.0,
      "title": "T1",
      "type": "number"
    },
    "t2": {
      "default": 0.65,
      "description": "partial_ratio stage-2 threshold",
      "maximum": 1.0,
      "minimum": 0.0,
      "title": "T2",
      "type": "number"
    },
    "weight": {
      "default": 1.0,
      "title": "Weight",
      "type": "number",
      "x-aip-param-role": "scoring_metadata"
    }
  },
  "title": "HitAtKConfig",
  "type": "object"
}

Execution

{
  "emits_metric_family": false,
  "function_name": null,
  "max_concurrency": null,
  "processing_kind": "cpu",
  "stream_batch_size": null,
  "timeout_seconds": 60
}

Complete manifest

accepts:
- name: gdi_text_v1
  task_types:
  - multi_turn_rag
  - single_turn_rag
config_schema:
  properties:
    coverage:
      default: 0.5
      description: token_overlap stage-2 threshold
      maximum: 1.0
      minimum: 0.0
      title: Coverage
      type: number
    inverted:
      default: false
      title: Inverted
      type: boolean
      x-aip-param-role: scoring_metadata
    k:
      default: 5
      minimum: 1
      title: K
      type: integer
    language:
      default: en
      description: Language for metric output
      enum:
      - en
      - de
      - fr
      title: Language
      type: string
    max:
      default: 1.0
      title: Max
      type: number
      x-aip-param-role: scoring_metadata
    metric_name:
      default: hit_at_k
      title: Metric Name
      type: string
      x-aip-param-role: scoring_metadata
    min:
      default: 0.0
      title: Min
      type: number
      x-aip-param-role: scoring_metadata
    t1:
      default: 0.7
      description: token_set_ratio stage-1 threshold
      maximum: 1.0
      minimum: 0.0
      title: T1
      type: number
    t2:
      default: 0.65
      description: partial_ratio stage-2 threshold
      maximum: 1.0
      minimum: 0.0
      title: T2
      type: number
    weight:
      default: 1.0
      title: Weight
      type: number
      x-aip-param-role: scoring_metadata
  title: HitAtKConfig
  type: object
dependencies: []
description: Does the right supporting evidence appear among the top retrieved results?
direction: higher_is_better
display_name: Hit@K
entrypoint: aip_metrics_rag.answered.hit_at_k
execution:
  emits_metric_family: false
  function_name: null
  max_concurrency: null
  processing_kind: cpu
  stream_batch_size: null
  timeout_seconds: 60
kind: metric
manifest_version: '1'
max_prompt_slots: 0
metric_metadata:
  methodology:
  - Deterministic - no LLM. Uses rapidfuzz string matching against the reference chunks.
  - Take the top-k retrieved chunks (default k=5).
  - Compare each retrieved chunk to each reference chunk with a two-stage gate (token-set
    ratio >= 0.70, then partial ratio >= 0.65 or token overlap >= 0.50).
  - Score 1.0 if any chunk passes the gate, else 0.0.
  score_semantics: The score is binary - 1.0 if at least one of the top-k retrieved
    chunks matches a reference chunk, otherwise 0.0. Higher is better.
  summary: Checks whether the correct supporting evidence appears anywhere in the
    top-k retrieved results.
  worked_example: Among the top-5 retrieved chunks the 2nd matches a reference chunk
    (token-set ratio 0.82) -> hit -> score 1.0.
metric_type: pointwise
name: rag.hit_at_k
partition_types: []
required_columns:
- input_id
- prompt
- reference_contexts
- sut_response
- retrieved_context
required_kinds: []
scorer_contract: per_row
target_kind: null
unsupported_trace_shapes: []
version: 1.0.0