Runs in your browser — no install. (Works once the repo is public.)

In [ ]:

# --- Colab setup (skipped when running locally) ---
import os, sys
if 'google.colab' in sys.modules and not os.path.exists('data/sample_corpus.json'):
    !git clone -q https://github.com/mdhabibi/llm-search-handbook.git
    %cd llm-search-handbook
    !pip -q install -r requirements.txt

# --- Colab setup (skipped when running locally) --- import os, sys if 'google.colab' in sys.modules and not os.path.exists('data/sample_corpus.json'): !git clone -q https://github.com/mdhabibi/llm-search-handbook.git %cd llm-search-handbook !pip -q install -r requirements.txt

Chapter 3 — From Text to Vectors¶

Turn text into numbers, then measure 'closeness'. Pure numpy + matplotlib, so it runs instantly.

We climb three rungs: one-hot → bag-of-words → dense vectors, then learn cosine similarity.

In [ ]:

# Bootstrap: locate repo root and import shared helpers
import sys, os
d = os.getcwd()
while d != os.path.dirname(d) and not os.path.exists(os.path.join(d, 'data', 'sample_corpus.json')):
    d = os.path.dirname(d)
ROOT = d; sys.path.insert(0, os.path.join(ROOT, 'src'))
from corpus import load_corpus, tokenize
import numpy as np

# Bootstrap: locate repo root and import shared helpers import sys, os d = os.getcwd() while d != os.path.dirname(d) and not os.path.exists(os.path.join(d, 'data', 'sample_corpus.json')): d = os.path.dirname(d) ROOT = d; sys.path.insert(0, os.path.join(ROOT, 'src')) from corpus import load_corpus, tokenize import numpy as np

In [ ]:

import matplotlib.pyplot as plt

import matplotlib.pyplot as plt

Rung 1 — One-hot encoding¶

Each word gets its own slot. Note the fatal flaw: every distinct word is equally far from every other.

In [ ]:

vocab = ['apple', 'banana', 'car', 'castle', 'joy']
onehot = {w: np.eye(len(vocab))[i] for i, w in enumerate(vocab)}
print('apple  ->', onehot['apple'])
print('banana ->', onehot['banana'])

def euclid(a, b): return float(np.linalg.norm(a - b))
print('\ndistance(apple, banana) =', euclid(onehot['apple'], onehot['banana']))
print('distance(apple, car)    =', euclid(onehot['apple'], onehot['car']))
print('-> identical! one-hot encodes identity, not meaning.')

vocab = ['apple', 'banana', 'car', 'castle', 'joy'] onehot = {w: np.eye(len(vocab))[i] for i, w in enumerate(vocab)} print('apple ->', onehot['apple']) print('banana ->', onehot['banana']) def euclid(a, b): return float(np.linalg.norm(a - b)) print('\ndistance(apple, banana) =', euclid(onehot['apple'], onehot['banana'])) print('distance(apple, car) =', euclid(onehot['apple'], onehot['car'])) print('-> identical! one-hot encodes identity, not meaning.')

Rung 2 — Bag-of-words¶

Represent a document by its word counts (order ignored). Watch two sentences that mean the same thing with different words share almost nothing.

In [ ]:

def bag_of_words(sentences):
    vocab = sorted({t for s in sentences for t in tokenize(s)})
    idx = {w: i for i, w in enumerate(vocab)}
    mat = np.zeros((len(sentences), len(vocab)))
    for r, s in enumerate(sentences):
        for t in tokenize(s):
            mat[r, idx[t]] += 1
    return mat, vocab

sents = ['an apple is a fruit', 'a banana is a fruit', 'a migraine is a headache']
mat, vocab = bag_of_words(sents)
print('vocab:', vocab)
print(mat.astype(int))
print('\nRows 0 and 1 share "fruit"; row 2 (same *type* of statement) shares only stopwords.')

def bag_of_words(sentences): vocab = sorted({t for s in sentences for t in tokenize(s)}) idx = {w: i for i, w in enumerate(vocab)} mat = np.zeros((len(sentences), len(vocab))) for r, s in enumerate(sentences): for t in tokenize(s): mat[r, idx[t]] += 1 return mat, vocab sents = ['an apple is a fruit', 'a banana is a fruit', 'a migraine is a headache'] mat, vocab = bag_of_words(sents) print('vocab:', vocab) print(mat.astype(int)) print('\nRows 0 and 1 share "fruit"; row 2 (same *type* of statement) shares only stopwords.')

Measuring closeness: cosine similarity¶

Cosine measures the angle (direction), ignoring length. 1 = same direction, 0 = unrelated.

In [ ]:

def cosine(a, b):
    a, b = np.asarray(a, float), np.asarray(b, float)
    denom = np.linalg.norm(a) * np.linalg.norm(b)
    return float(a @ b / denom) if denom else 0.0

a = np.array([1.0, 0.0])   # 'apple' along the fruit axis
b = np.array([2.0, 0.0])   # 'banana', same direction but longer
c = np.array([0.0, 1.0])   # 'car' along the vehicle axis
print('dot(a, b)    =', float(a @ b))
print('cosine(a, b) =', cosine(a, b), '  <- same direction => 1.0, length ignored')
print('cosine(a, c) =', cosine(a, c), '  <- perpendicular => 0.0')

def cosine(a, b): a, b = np.asarray(a, float), np.asarray(b, float) denom = np.linalg.norm(a) * np.linalg.norm(b) return float(a @ b / denom) if denom else 0.0 a = np.array([1.0, 0.0]) # 'apple' along the fruit axis b = np.array([2.0, 0.0]) # 'banana', same direction but longer c = np.array([0.0, 1.0]) # 'car' along the vehicle axis print('dot(a, b) =', float(a @ b)) print('cosine(a, b) =', cosine(a, b), ' <- same direction => 1.0, length ignored') print('cosine(a, c) =', cosine(a, c), ' <- perpendicular => 0.0')

Rung 3 — A toy 2-D 'meaning map'¶

Hand-placed 2-D vectors where location = meaning (real embeddings do this in hundreds of dimensions, learned automatically). Then we drop in a new word and find its neighbors by cosine.

In [ ]:

word_map = {
    'apple': (5.2, 5.0), 'banana': (5.6, 4.7), 'orange': (4.8, 5.3),   # fruits (top-right)
    'car': (5.0, 1.0), 'bike': (5.5, 1.3), 'bus': (4.6, 0.7),          # vehicles (bottom-right)
    'house': (1.0, 1.0), 'castle': (0.7, 1.4), 'tower': (1.3, 0.7),    # buildings (bottom-left)
    'soccer': (1.0, 5.0), 'tennis': (1.4, 5.3), 'boxing': (0.7, 4.6),  # sports (top-left)
}

pts = np.array(list(word_map.values()))
plt.figure(figsize=(6,6))
plt.scatter(pts[:,0], pts[:,1])
for w,(x,y) in word_map.items(): plt.annotate(w, (x,y), fontsize=9)
plt.title('Toy 2-D meaning map'); plt.xlabel('dim 1'); plt.ylabel('dim 2'); plt.show()

word_map = { 'apple': (5.2, 5.0), 'banana': (5.6, 4.7), 'orange': (4.8, 5.3), # fruits (top-right) 'car': (5.0, 1.0), 'bike': (5.5, 1.3), 'bus': (4.6, 0.7), # vehicles (bottom-right) 'house': (1.0, 1.0), 'castle': (0.7, 1.4), 'tower': (1.3, 0.7), # buildings (bottom-left) 'soccer': (1.0, 5.0), 'tennis': (1.4, 5.3), 'boxing': (0.7, 4.6), # sports (top-left) } pts = np.array(list(word_map.values())) plt.figure(figsize=(6,6)) plt.scatter(pts[:,0], pts[:,1]) for w,(x,y) in word_map.items(): plt.annotate(w, (x,y), fontsize=9) plt.title('Toy 2-D meaning map'); plt.xlabel('dim 1'); plt.ylabel('dim 2'); plt.show()

In [ ]:

# Where does a NEW word 'mango' belong? Suppose meaning places it near fruits.
mango = np.array([5.0, 5.1])
def cos2(a,b): return cosine(a,b)
ranked = sorted(word_map.items(), key=lambda kv: -cos2(mango, np.array(kv[1])))
print('Nearest words to mango by cosine:')
for w, v in ranked[:4]:
    print(f'  {cos2(mango, np.array(v)):.3f}  {w}')
print('-> lands among the fruits, exactly as meaning would predict.')

# Where does a NEW word 'mango' belong? Suppose meaning places it near fruits. mango = np.array([5.0, 5.1]) def cos2(a,b): return cosine(a,b) ranked = sorted(word_map.items(), key=lambda kv: -cos2(mango, np.array(kv[1]))) print('Nearest words to mango by cosine:') for w, v in ranked[:4]: print(f' {cos2(mango, np.array(v)):.3f} {w}') print('-> lands among the fruits, exactly as meaning would predict.')

Normalizing makes cosine and Euclidean agree¶

Scale every vector to length 1; then nearest-by-cosine == nearest-by-Euclidean.

In [ ]:

def normalize(v): v = np.asarray(v, float); n = np.linalg.norm(v); return v/n if n else v
words = list(word_map); vecs = [normalize(word_map[w]) for w in words]
m = normalize(mango)
by_cos = sorted(words, key=lambda w: -cosine(m, dict(zip(words,vecs))[w]))
by_euc = sorted(words, key=lambda w: euclid(m, dict(zip(words,vecs))[w]))
print('by cosine:', by_cos[:4])
print('by euclid:', by_euc[:4])
print('same order?', by_cos[:4] == by_euc[:4])

def normalize(v): v = np.asarray(v, float); n = np.linalg.norm(v); return v/n if n else v words = list(word_map); vecs = [normalize(word_map[w]) for w in words] m = normalize(mango) by_cos = sorted(words, key=lambda w: -cosine(m, dict(zip(words,vecs))[w])) by_euc = sorted(words, key=lambda w: euclid(m, dict(zip(words,vecs))[w])) print('by cosine:', by_cos[:4]) print('by euclid:', by_euc[:4]) print('same order?', by_cos[:4] == by_euc[:4])

Takeaway & exercises¶

Meaning-as-location + cosine similarity is the whole engine of semantic search. In Chapter 4 a real model learns these vectors for actual text.

Exercises

1. Add your own word to word_map and predict its neighbors before running.
2. Build bag-of-words for two synonymous sentences and compute their cosine. Why is it low?
3. Prove to yourself that dot product (not cosine) can rank a longer, less-relevant vector higher.