Runs in your browser — no install. (Works once the repo is public.)

In [ ]:

# --- Colab setup (skipped when running locally) ---
import os, sys
if 'google.colab' in sys.modules and not os.path.exists('data/sample_corpus.json'):
    !git clone -q https://github.com/mdhabibi/llm-search-handbook.git
    %cd llm-search-handbook
    !pip -q install -r requirements.txt

# --- Colab setup (skipped when running locally) --- import os, sys if 'google.colab' in sys.modules and not os.path.exists('data/sample_corpus.json'): !git clone -q https://github.com/mdhabibi/llm-search-handbook.git %cd llm-search-handbook !pip -q install -r requirements.txt

Chapter 4 — Embeddings Deep Dive¶

We use a real, open-source embedding model to turn text into meaning-vectors, and reproduce the lesson's key result: a question's nearest neighbor is its own answer.

First run downloads a small model (~90 MB), so it needs internet once, then works offline.

Setup¶

pip install sentence-transformers

In [ ]:

# Bootstrap: locate repo root and import shared helpers
import sys, os
d = os.getcwd()
while d != os.path.dirname(d) and not os.path.exists(os.path.join(d, 'data', 'sample_corpus.json')):
    d = os.path.dirname(d)
ROOT = d; sys.path.insert(0, os.path.join(ROOT, 'src'))
from corpus import load_corpus, tokenize
import numpy as np

# Bootstrap: locate repo root and import shared helpers import sys, os d = os.getcwd() while d != os.path.dirname(d) and not os.path.exists(os.path.join(d, 'data', 'sample_corpus.json')): d = os.path.dirname(d) ROOT = d; sys.path.insert(0, os.path.join(ROOT, 'src')) from corpus import load_corpus, tokenize import numpy as np

In [ ]:

# Analysis helpers (pure numpy - independent of which model made the vectors)
def l2_normalize(mat):
    mat = np.asarray(mat, float)
    norms = np.linalg.norm(mat, axis=1, keepdims=True)
    return mat / np.clip(norms, 1e-12, None)

def cosine_sim_matrix(mat):
    m = l2_normalize(mat)
    return m @ m.T

def top_k(query_vec, matrix, k=3, exclude=None):
    q = l2_normalize(query_vec[None, :])[0]
    sims = l2_normalize(matrix) @ q
    order = np.argsort(-sims)
    if exclude is not None:
        order = [i for i in order if i != exclude]
    return [(int(i), float(sims[i])) for i in order[:k]]

# Analysis helpers (pure numpy - independent of which model made the vectors) def l2_normalize(mat): mat = np.asarray(mat, float) norms = np.linalg.norm(mat, axis=1, keepdims=True) return mat / np.clip(norms, 1e-12, None) def cosine_sim_matrix(mat): m = l2_normalize(mat) return m @ m.T def top_k(query_vec, matrix, k=3, exclude=None): q = l2_normalize(query_vec[None, :])[0] sims = l2_normalize(matrix) @ q order = np.argsort(-sims) if exclude is not None: order = [i for i in order if i != exclude] return [(int(i), float(sims[i])) for i in order[:k]]

1. Load the model and embed a few sentences¶

all-MiniLM-L6-v2 outputs 384-dimensional vectors and runs on CPU.

In [ ]:

from sentence_transformers import SentenceTransformer
model = SentenceTransformer('all-MiniLM-L6-v2')

vecs = model.encode(['an apple is a fruit', 'a banana is a fruit', 'a car has four wheels'])
print('shape:', vecs.shape, '  (n_sentences, dimensions)')
print('first 8 numbers of sentence 0:', np.round(vecs[0][:8], 3))

from sentence_transformers import SentenceTransformer model = SentenceTransformer('all-MiniLM-L6-v2') vecs = model.encode(['an apple is a fruit', 'a banana is a fruit', 'a car has four wheels']) print('shape:', vecs.shape, ' (n_sentences, dimensions)') print('first 8 numbers of sentence 0:', np.round(vecs[0][:8], 3))

2. Paraphrases are close; unrelated text is not¶

Different words, same meaning -> high cosine similarity.

In [ ]:

pair = model.encode(['hello, how are you?', "hi, how's it going?"])
unrel = model.encode(['hello, how are you?', 'the capital of Canada is Ottawa'])
def cos(a,b): return float(l2_normalize(a[None,:])[0] @ l2_normalize(b[None,:])[0])
print('paraphrase similarity :', round(cos(pair[0], pair[1]), 3))
print('unrelated similarity  :', round(cos(unrel[0], unrel[1]), 3))

pair = model.encode(['hello, how are you?', "hi, how's it going?"]) unrel = model.encode(['hello, how are you?', 'the capital of Canada is Ottawa']) def cos(a,b): return float(l2_normalize(a[None,:])[0] @ l2_normalize(b[None,:])[0]) print('paraphrase similarity :', round(cos(pair[0], pair[1]), 3)) print('unrelated similarity :', round(cos(unrel[0], unrel[1]), 3))

3. The 'aha' demo: questions find their answers¶

Embed Q/A sentences together; each question's nearest neighbor (excluding itself) is its answer -- despite few shared words.

In [ ]:

qa = [
    'what color is the sky?', 'the sky is blue.',
    'what is an apple?', 'an apple is a fruit.',
    'where does the bear live?', 'the bear lives in the woods.',
    'where is the world cup?', 'the world cup is in qatar.',
]
emb = model.encode(qa)
questions = [i for i in range(len(qa)) if qa[i].endswith('?')]
for qi in questions:
    best_i, score = top_k(emb[qi], emb, k=1, exclude=qi)[0]
    print(f'Q: {qa[qi]:32s} -> nearest: {qa[best_i]!r}  (cos={score:.2f})')

qa = [ 'what color is the sky?', 'the sky is blue.', 'what is an apple?', 'an apple is a fruit.', 'where does the bear live?', 'the bear lives in the woods.', 'where is the world cup?', 'the world cup is in qatar.', ] emb = model.encode(qa) questions = [i for i in range(len(qa)) if qa[i].endswith('?')] for qi in questions: best_i, score = top_k(emb[qi], emb, k=1, exclude=qi)[0] print(f'Q: {qa[qi]:32s} -> nearest: {qa[best_i]!r} (cos={score:.2f})')

4. The migraine query keyword search missed (Chapter 2)¶

Embed our course corpus and query it by meaning. The migraine passage should now rise to the top.

In [ ]:

docs = load_corpus()
corpus_emb = model.encode([d['text'] for d in docs])
q = model.encode(['strong pain in the side of the head'])[0]
print('Query: strong pain in the side of the head\n')
for i, s in top_k(q, corpus_emb, k=3):
    print(f'  {s:.3f}  D{i}: {docs[i]["title"]}')
print('\n(Compare with Chapter 2, where BM25 buried the migraine passage.)')

docs = load_corpus() corpus_emb = model.encode([d['text'] for d in docs]) q = model.encode(['strong pain in the side of the head'])[0] print('Query: strong pain in the side of the head\n') for i, s in top_k(q, corpus_emb, k=3): print(f' {s:.3f} D{i}: {docs[i]["title"]}') print('\n(Compare with Chapter 2, where BM25 buried the migraine passage.)')

5. Cluster by meaning and project to 2-D¶

KMeans groups the corpus into topics; PCA squashes 384-D vectors to 2-D so we can see the structure.

In [ ]:

from sklearn.cluster import KMeans
from sklearn.decomposition import PCA
import matplotlib.pyplot as plt

labels = KMeans(n_clusters=5, n_init=10, random_state=0).fit_predict(corpus_emb)
xy = PCA(n_components=2, random_state=0).fit_transform(corpus_emb)
plt.figure(figsize=(8,6))
plt.scatter(xy[:,0], xy[:,1], c=labels, cmap='tab10')
for i, d in enumerate(docs): plt.annotate(d['title'], (xy[i,0], xy[i,1]), fontsize=8)
plt.title('Course corpus embedded, clustered, and projected to 2-D'); plt.show()

from sklearn.cluster import KMeans from sklearn.decomposition import PCA import matplotlib.pyplot as plt labels = KMeans(n_clusters=5, n_init=10, random_state=0).fit_predict(corpus_emb) xy = PCA(n_components=2, random_state=0).fit_transform(corpus_emb) plt.figure(figsize=(8,6)) plt.scatter(xy[:,0], xy[:,1], c=labels, cmap='tab10') for i, d in enumerate(docs): plt.annotate(d['title'], (xy[i,0], xy[i,1]), fontsize=8) plt.title('Course corpus embedded, clustered, and projected to 2-D'); plt.show()

Takeaway & exercises¶

Nearest embedding to a query == search by meaning. That's dense retrieval -- built for real in Chapter 5.

Exercises

1. Add your own Q/A pairs to the demo. Does every question still find its answer?
2. Swap the model for all-mpnet-base-v2 (768-dim). Re-embed everything. Do neighbors improve?
3. Try a query with an exact rare term (a name/code). Where might keyword search still win? (Foreshadows Chapter 8.)