Runs in your browser — no install. (Works once the repo is public.)

In [ ]:

# --- Colab setup (skipped when running locally) ---
import os, sys
if 'google.colab' in sys.modules and not os.path.exists('data/sample_corpus.json'):
    !git clone -q https://github.com/mdhabibi/llm-search-handbook.git
    %cd llm-search-handbook
    !pip -q install -r requirements.txt

# --- Colab setup (skipped when running locally) --- import os, sys if 'google.colab' in sys.modules and not os.path.exists('data/sample_corpus.json'): !git clone -q https://github.com/mdhabibi/llm-search-handbook.git %cd llm-search-handbook !pip -q install -r requirements.txt

Chapter 9 — Evaluating Search¶

Turn 'this feels better' into numbers. We implement precision@k, recall@k, MRR, nDCG, and MAP from scratch, then build a leaderboard of our retrievers.

BM25 rows run offline; dense/hybrid rows use the embedding model (your machine).

Setup¶

pip install sentence-transformers rank-bm25

In [ ]:

# Bootstrap: locate repo root and import shared helpers
import sys, os, json
d = os.getcwd()
while d != os.path.dirname(d) and not os.path.exists(os.path.join(d, 'data', 'sample_corpus.json')):
    d = os.path.dirname(d)
ROOT = d; sys.path.insert(0, os.path.join(ROOT, 'src'))
import numpy as np
from corpus import load_corpus, tokenize

# Bootstrap: locate repo root and import shared helpers import sys, os, json d = os.getcwd() while d != os.path.dirname(d) and not os.path.exists(os.path.join(d, 'data', 'sample_corpus.json')): d = os.path.dirname(d) ROOT = d; sys.path.insert(0, os.path.join(ROOT, 'src')) import numpy as np from corpus import load_corpus, tokenize

1. Verify the metric implementations¶

Before trusting any number, we check src/metrics.py against hand-computed values.

In [ ]:

import metrics as M
ranked=[0,1,2,3]; rel={1,3}     # relevant at ranks 2 and 4
print('precision@2:', M.precision_at_k(ranked,rel,2))
print('recall@4   :', M.recall_at_k(ranked,rel,4))
print('MRR        :', M.reciprocal_rank(ranked,rel))
print('nDCG@4     :', round(M.ndcg_at_k(ranked,rel,4),4))
assert abs(M.ndcg_at_k(ranked,rel,4) - 0.6509) < 1e-3
assert M.reciprocal_rank(ranked,rel) == 0.5
print('Metrics match the worked example. ✓')

import metrics as M ranked=[0,1,2,3]; rel={1,3} # relevant at ranks 2 and 4 print('precision@2:', M.precision_at_k(ranked,rel,2)) print('recall@4 :', M.recall_at_k(ranked,rel,4)) print('MRR :', M.reciprocal_rank(ranked,rel)) print('nDCG@4 :', round(M.ndcg_at_k(ranked,rel,4),4)) assert abs(M.ndcg_at_k(ranked,rel,4) - 0.6509) < 1e-3 assert M.reciprocal_rank(ranked,rel) == 0.5 print('Metrics match the worked example. ✓')

2. Load the labeled evaluation set¶

Six queries over our corpus, each with judged-relevant document ids.

In [ ]:

eval_path = os.path.join(ROOT,'data','eval_queries.json')
eval_set = json.load(open(eval_path))['queries']
for q in eval_set: print(f"  {q['relevant_ids']}  <-  {q['query']}")

eval_path = os.path.join(ROOT,'data','eval_queries.json') eval_set = json.load(open(eval_path))['queries'] for q in eval_set: print(f" {q['relevant_ids']} <- {q['query']}")

3. Retrievers to compare¶

BM25 (above), dense (Chapter 5), and hybrid via RRF (Chapter 8).

In [ ]:

from semantic_search import SemanticSearch
from sentence_transformers import SentenceTransformer
bi = SentenceTransformer('all-MiniLM-L6-v2')
engine = SemanticSearch(bi.encode).index(docs)
def dense_rank(q,k=10): return [i for i,_,_ in engine.search(q,k=k)]
def rrf(rankings,k=60):
    sc={}
    for r in rankings:
        for rank,i in enumerate(r,1): sc[i]=sc.get(i,0)+1/(k+rank)
    return sorted(sc,key=lambda i:-sc[i])
def hybrid_rank(q,k=10): return rrf([bm25_rank(q,k), dense_rank(q,k)])
retrievers = {'BM25':bm25_rank, 'Dense':dense_rank, 'Hybrid':hybrid_rank}

from semantic_search import SemanticSearch from sentence_transformers import SentenceTransformer bi = SentenceTransformer('all-MiniLM-L6-v2') engine = SemanticSearch(bi.encode).index(docs) def dense_rank(q,k=10): return [i for i,_,_ in engine.search(q,k=k)] def rrf(rankings,k=60): sc={} for r in rankings: for rank,i in enumerate(r,1): sc[i]=sc.get(i,0)+1/(k+rank) return sorted(sc,key=lambda i:-sc[i]) def hybrid_rank(q,k=10): return rrf([bm25_rank(q,k), dense_rank(q,k)]) retrievers = {'BM25':bm25_rank, 'Dense':dense_rank, 'Hybrid':hybrid_rank}

4. The leaderboard¶

Average each metric over all queries, per retriever.

In [ ]:

def evaluate(rank_fn, k=5):
    rankings=[rank_fn(q['query'], k=10) for q in eval_set]
    rels=[set(q['relevant_ids']) for q in eval_set]
    return {
        'nDCG@5' : M.mean_ndcg_at_k(rankings,rels,k),
        'MRR'    : M.mean_reciprocal_rank(rankings,rels),
        'MAP'    : M.mean_average_precision(rankings,rels),
        'P@5'    : M.mean_precision_at_k(rankings,rels,k),
        'R@5'    : M.mean_recall_at_k(rankings,rels,k),
    }

print(f"{'retriever':12}{'nDCG@5':>8}{'MRR':>7}{'MAP':>7}{'P@5':>7}{'R@5':>7}")
for name, fn in retrievers.items():
    m = evaluate(fn)
    print(f"{name:12}{m['nDCG@5']:>8.3f}{m['MRR']:>7.3f}{m['MAP']:>7.3f}{m['P@5']:>7.3f}{m['R@5']:>7.3f}")

def evaluate(rank_fn, k=5): rankings=[rank_fn(q['query'], k=10) for q in eval_set] rels=[set(q['relevant_ids']) for q in eval_set] return { 'nDCG@5' : M.mean_ndcg_at_k(rankings,rels,k), 'MRR' : M.mean_reciprocal_rank(rankings,rels), 'MAP' : M.mean_average_precision(rankings,rels), 'P@5' : M.mean_precision_at_k(rankings,rels,k), 'R@5' : M.mean_recall_at_k(rankings,rels,k), } print(f"{'retriever':12}{'nDCG@5':>8}{'MRR':>7}{'MAP':>7}{'P@5':>7}{'R@5':>7}") for name, fn in retrievers.items(): m = evaluate(fn) print(f"{name:12}{m['nDCG@5']:>8.3f}{m['MRR']:>7.3f}{m['MAP']:>7.3f}{m['P@5']:>7.3f}{m['R@5']:>7.3f}")

5. Interpret¶

The story of the course is now evidence, not assertion. But note: with only six queries the numbers are noisy — differences this small wouldn't be statistically significant on a real (hundreds+) evaluation set. That caveat is itself a core lesson.

Exercises

1. Add 10 more labeled queries. Do the rankings between retrievers stabilize?
2. Add a 'Dense+Rerank' row using the Chapter 7 cross-encoder. Does nDCG improve?
3. Tune BM25's k1/b on a separate dev split, then report on the test split. Why separate them?