Runs in your browser — no install. (Works once the repo is public.)

In [ ]:

# --- Colab setup (skipped when running locally) ---
import os, sys
if 'google.colab' in sys.modules and not os.path.exists('data/sample_corpus.json'):
    !git clone -q https://github.com/mdhabibi/llm-search-handbook.git
    %cd llm-search-handbook
    !pip -q install -r requirements.txt

# --- Colab setup (skipped when running locally) --- import os, sys if 'google.colab' in sys.modules and not os.path.exists('data/sample_corpus.json'): !git clone -q https://github.com/mdhabibi/llm-search-handbook.git %cd llm-search-handbook !pip -q install -r requirements.txt

Chapter 8 — Hybrid Search (fuse keyword + semantic)¶

BM25 and dense retrieval fail differently. We fuse their ranked lists with Reciprocal Rank Fusion (RRF) so the result is strong on both exact terms and meaning.

Setup¶

pip install sentence-transformers rank-bm25

In [ ]:

# Bootstrap: locate repo root and import shared helpers
import sys, os
d = os.getcwd()
while d != os.path.dirname(d) and not os.path.exists(os.path.join(d, 'data', 'sample_corpus.json')):
    d = os.path.dirname(d)
ROOT = d; sys.path.insert(0, os.path.join(ROOT, 'src'))
import numpy as np
from corpus import load_corpus, tokenize

# Bootstrap: locate repo root and import shared helpers import sys, os d = os.getcwd() while d != os.path.dirname(d) and not os.path.exists(os.path.join(d, 'data', 'sample_corpus.json')): d = os.path.dirname(d) ROOT = d; sys.path.insert(0, os.path.join(ROOT, 'src')) import numpy as np from corpus import load_corpus, tokenize

In [ ]:

# BM25 (from Chapter 2) as our keyword retriever
from collections import Counter, defaultdict
import math
docs = load_corpus()
tok = [tokenize(x['text']) for x in docs]
N = len(tok); avgdl = sum(len(t) for t in tok)/N
df = defaultdict(int)
for t in tok:
    for term in set(t): df[term]+=1
def idf(term): return math.log(1 + (N-df[term]+0.5)/(df[term]+0.5))
def bm25_rank(query, k=10, k1=1.5, b=0.75):
    q = tokenize(query); scores=[]
    for i,t in enumerate(tok):
        f=Counter(t); s=0.0
        for term in q:
            if term in f: s += idf(term)*(f[term]*(k1+1))/(f[term]+k1*(1-b+b*len(t)/avgdl))
        scores.append((i,s))
    return [i for i,s in sorted(scores,key=lambda x:-x[1]) if s>0][:k]

# BM25 (from Chapter 2) as our keyword retriever from collections import Counter, defaultdict import math docs = load_corpus() tok = [tokenize(x['text']) for x in docs] N = len(tok); avgdl = sum(len(t) for t in tok)/N df = defaultdict(int) for t in tok: for term in set(t): df[term]+=1 def idf(term): return math.log(1 + (N-df[term]+0.5)/(df[term]+0.5)) def bm25_rank(query, k=10, k1=1.5, b=0.75): q = tokenize(query); scores=[] for i,t in enumerate(tok): f=Counter(t); s=0.0 for term in q: if term in f: s += idf(term)*(f[term]*(k1+1))/(f[term]+k1*(1-b+b*len(t)/avgdl)) scores.append((i,s)) return [i for i,s in sorted(scores,key=lambda x:-x[1]) if s>0][:k]

1. Reciprocal Rank Fusion, in a few lines¶

RRF ignores raw scores and uses only each document's rank in each list: score(d) = Σ 1/(k + rank). Scale-free and robust.

In [ ]:

def rrf(rankings, k=60):
    # rankings: list of ranked lists (each a list of doc ids, best first)
    scores = {}
    for ranking in rankings:
        for rank, doc_id in enumerate(ranking, start=1):
            scores[doc_id] = scores.get(doc_id, 0.0) + 1.0/(k + rank)
    return sorted(scores, key=lambda d: -scores[d]), scores

# Verify against the worked example from the chapter README
bm25_list  = [10, 4, 2]
dense_list = [2, 4, 7]
order, sc = rrf([bm25_list, dense_list])
print('Fused order:', order)
for d in order: print(f'  D{d}: {sc[d]:.5f}')
assert order[0] == 2, 'D2 should win (found #1 by dense, #3 by bm25)'
print('RRF matches the worked example (D2 on top). ✓')

def rrf(rankings, k=60): # rankings: list of ranked lists (each a list of doc ids, best first) scores = {} for ranking in rankings: for rank, doc_id in enumerate(ranking, start=1): scores[doc_id] = scores.get(doc_id, 0.0) + 1.0/(k + rank) return sorted(scores, key=lambda d: -scores[d]), scores # Verify against the worked example from the chapter README bm25_list = [10, 4, 2] dense_list = [2, 4, 7] order, sc = rrf([bm25_list, dense_list]) print('Fused order:', order) for d in order: print(f' D{d}: {sc[d]:.5f}') assert order[0] == 2, 'D2 should win (found #1 by dense, #3 by bm25)' print('RRF matches the worked example (D2 on top). ✓')

2. Build all three retrievers over our corpus¶

BM25 (above) + dense (Chapter 5). We compare BM25-only, dense-only, and hybrid.

In [ ]:

from semantic_search import SemanticSearch
from sentence_transformers import SentenceTransformer
bi = SentenceTransformer('all-MiniLM-L6-v2')
engine = SemanticSearch(bi.encode).index(docs)

def dense_rank(query, k=10):
    return [i for i,_,_ in engine.search(query, k=k)]

def hybrid_rank(query, k=10):
    order, _ = rrf([bm25_rank(query, k=k), dense_rank(query, k=k)])
    return order

def titles(ids): return [docs[i]['title'] for i in ids]

from semantic_search import SemanticSearch from sentence_transformers import SentenceTransformer bi = SentenceTransformer('all-MiniLM-L6-v2') engine = SemanticSearch(bi.encode).index(docs) def dense_rank(query, k=10): return [i for i,_,_ in engine.search(query, k=k)] def hybrid_rank(query, k=10): order, _ = rrf([bm25_rank(query, k=k), dense_rank(query, k=k)]) return order def titles(ids): return [docs[i]['title'] for i in ids]

3. Where each method wins — and hybrid wins both¶

A synonym query (keyword fails) and an exact-token query (semantic may drift).

In [ ]:

for q in ['strong pain in the side of the head', 'Lucene']:
    print('QUERY:', q)
    print('  BM25   :', titles(bm25_rank(q))[:3])
    print('  Dense  :', titles(dense_rank(q))[:3])
    print('  Hybrid :', titles(hybrid_rank(q))[:3])
    print()

for q in ['strong pain in the side of the head', 'Lucene']: print('QUERY:', q) print(' BM25 :', titles(bm25_rank(q))[:3]) print(' Dense :', titles(dense_rank(q))[:3]) print(' Hybrid :', titles(hybrid_rank(q))[:3]) print()

4. (Optional) the full modern stack: fuse, then re-rank¶

Hybrid gives a strong candidate set; a cross-encoder (Chapter 7) can re-rank it for the final order.

In [ ]:

# from sentence_transformers import CrossEncoder
# reranker = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')
# def full_stack(query, cand_k=10, final_k=3):
#     cand_ids = hybrid_rank(query, k=cand_k)
#     scores = reranker.predict([(query, docs[i]['text']) for i in cand_ids])
#     ranked = sorted(zip(cand_ids, scores), key=lambda x: -x[1])[:final_k]
#     return [docs[i]['title'] for i,_ in ranked]
# print(full_stack('a large animal that lives in the sea'))
print('Uncomment to run: BM25 ∪ dense -> RRF -> cross-encoder re-rank -> top results.')

# from sentence_transformers import CrossEncoder # reranker = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2') # def full_stack(query, cand_k=10, final_k=3): # cand_ids = hybrid_rank(query, k=cand_k) # scores = reranker.predict([(query, docs[i]['text']) for i in cand_ids]) # ranked = sorted(zip(cand_ids, scores), key=lambda x: -x[1])[:final_k] # return [docs[i]['title'] for i,_ in ranked] # print(full_stack('a large animal that lives in the sea')) print('Uncomment to run: BM25 ∪ dense -> RRF -> cross-encoder re-rank -> top results.')

Takeaway & exercises¶

Fuse complementary retrievers with RRF to win on both exactness and meaning. Add a re-ranker on top for the full production stack.

Exercises

1. Try weighted RRF: Σ w /(k+rank) with the dense list weighted 2x. When does it help?
2. Sweep RRF's k in {10, 60, 200}. How does it change which documents surface?
3. Construct a query with BOTH an exact code and fuzzy intent; confirm only hybrid handles it.