Use BM25 for exact words

Word search still wins for names, invoice numbers, and error codes. Dense search misses those when the embedder smooths them away. BM25 is the lexical lane.

There is no BM25Index in Foundation Models. Implement a compact version: tokenize to lowercase words, store document frequency, and score on query.

struct BM25Index {
    var chunks: [NoteChunk]
    var df: [String: Int] = [:]
    var lengths: [Int] = []
    var avgLength: Double = 1
    let k1 = 1.5
    let b = 0.75

    static func tokenize(_ text: String) -> [String] {
        text.lowercased().split { !$0.isLetter && !$0.isNumber }.map(String.init)
    }

    mutating func build(_ chunks: [NoteChunk]) {
        self.chunks = chunks
        df = [:]
        lengths = chunks.map { Self.tokenize($0.text).count }
        avgLength = Double(lengths.reduce(0, +)) / Double(max(lengths.count, 1))
        for chunk in chunks {
            for word in Set(Self.tokenize(chunk.text)) {
                df[word, default: 0] += 1
            }
        }
    }

    func score(query: String, limit: Int = 8) -> [(NoteChunk, Double)] {
        let terms = Self.tokenize(query)
        let n = Double(chunks.count)
        return chunks.enumerated().map { i, chunk in
            let tfMap = Dictionary(Self.tokenize(chunk.text).map { ($0, 1) }, uniquingKeysWith: +)
            let len = Double(lengths[i])
            let s = terms.reduce(0.0) { acc, term in
                let tf = Double(tfMap[term] ?? 0)
                let dfi = Double(df[term] ?? 0)
                let idf = log((n - dfi + 0.5) / (dfi + 0.5) + 1)
                let denom = tf + k1 * (1 - b + b * len / avgLength)
                return acc + idf * (tf * (k1 + 1)) / max(denom, 0.001)
            }
            return (chunk, s)
        }
        .filter { $0.1 > 0 }
        .sorted { $0.1 > $1.1 }
        .prefix(limit)
        .map { $0 }
    }
}

Query-time scoring stays in Swift on the phone. Building document frequency for a large corpus is easier on a Mac. Write the stats once in Python, then load them in the app. There is still no Apple BM25Index type.

import json
from collections import Counter
from pathlib import Path

def tokenize(text):
    buf = []
    word = []
    for ch in text.lower():
        if ch.isalnum():
            word.append(ch)
        elif word:
            buf.append("".join(word))
            word = []
    if word:
        buf.append("".join(word))
    return buf

chunks = [json.loads(line) for line in Path("artifacts/chunks.jsonl").read_text(encoding="utf-8").splitlines() if line.strip()]
df = Counter()
lengths = []
for chunk in chunks:
    tokens = tokenize(chunk["text"])
    lengths.append(len(tokens))
    df.update(set(tokens))
avg_len = sum(lengths) / max(len(lengths), 1)
index = {
    "k1": 1.5,
    "b": 0.75,
    "avgLength": avg_len,
    "df": dict(df),
    "lengths": lengths,
    "n": len(chunks),
}
Path("artifacts/bm25-index.json").write_text(json.dumps(index), encoding="utf-8")
print("docs", index["n"], "avgLength", round(avg_len, 1))

This is enough for a few thousand chunks. If you later need Spotlight's lexical engine, donate the same text and let Path A run. Do not rename that engine BM25.

Next, add meaning for questions that do not share words.