Chunk notes you can retrieve

A chunk is a passage you can return. Aim for 200 to 400 tokens of text, with a 40 token overlap so a sentence is not cut on a heading. Store docId, chunkId, start offset, and the raw text.

struct NoteChunk: Identifiable, Sendable {
    var id: String { "\(docId)#\(chunkId)" }
    let docId: String
    let chunkId: Int
    let text: String
    let start: Int
}

func chunk(_ text: String, docId: String, size: Int = 320, overlap: Int = 40) -> [NoteChunk] {
    let words = text.split(whereSeparator: \.isWhitespace).map(String.init)
    guard !words.isEmpty else { return [] }
    var chunks: [NoteChunk] = []
    var start = 0
    var index = 0
    while start < words.count {
        let end = min(words.count, start + size)
        let piece = words[start..<end].joined(separator: " ")
        chunks.append(NoteChunk(docId: docId, chunkId: index, text: piece, start: start))
        if end == words.count { break }
        start = max(end - overlap, start + 1)
        index += 1
    }
    return chunks
}

Chunk on the phone when a note is edited. That is the Swift above. For a first index of a large notes folder, run the same split on a Mac in Python, then copy chunks.jsonl into the app. The ids must match.

import json
from pathlib import Path

def chunk(text, doc_id, size=320, overlap=40):
    words = text.split()
    chunks = []
    start = 0
    index = 0
    while start < len(words):
        end = min(len(words), start + size)
        piece = " ".join(words[start:end])
        chunks.append({
            "docId": doc_id,
            "chunkId": index,
            "text": piece,
            "start": start,
            "id": f"{doc_id}#{index}",
        })
        if end == len(words):
            break
        start = max(end - overlap, start + 1)
        index += 1
    return chunks

notes = Path("notes")
out = Path("artifacts/chunks.jsonl")
out.parent.mkdir(parents=True, exist_ok=True)
with out.open("w", encoding="utf-8") as handle:
    for path in sorted(notes.glob("*.md")):
        for row in chunk(path.read_text(encoding="utf-8"), path.stem):
            handle.write(json.dumps(row, ensure_ascii=False) + "\n")
print("wrote", out)

If you also donate to Spotlight, use the same id as CSSearchableItem's unique identifier. Then Path A and Path B can point at the same passage.

Next, score those chunks with words the user actually typed.