feat: add SstData with DictionaryFile and word knowledge

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

Package.swift

@@ -9,6 +9,7 @@ let package = Package(
     ],
     targets: [
         .target(name: "SstCore"),
+        .target(name: "SstData", dependencies: ["SstCore"]),
         .target(
             name: "SstWhisper",
             dependencies: [
@@ -19,6 +20,7 @@ let package = Package(
         .executableTarget(name: "Sst", dependencies: ["SstCore", "SstWhisper"]),
         .executableTarget(name: "SstProbe", dependencies: ["SstCore", "SstWhisper"]),
         .testTarget(name: "SstCoreTests", dependencies: ["SstCore"]),
+        .testTarget(name: "SstDataTests", dependencies: ["SstData", "SstCore"]),
         .testTarget(name: "SstWhisperTests", dependencies: ["SstWhisper"]),
     ],
     swiftLanguageModes: [.v5]

Sources/SstData/DictionaryFile.swift

@@ -0,0 +1,110 @@
+import Foundation
+
+public struct DictionaryEntry: Identifiable, Equatable, Sendable {
+    public enum Kind: Equatable, Sendable {
+        case term(String)
+        case correction(heard: String, wrote: String)
+    }
+
+    public let id: UUID
+    public var kind: Kind
+
+    public init(id: UUID = UUID(), kind: Kind) {
+        self.id = id
+        self.kind = kind
+    }
+
+    /// Equality ignores `id`: two entries with the same content are the same entry.
+    public static func == (lhs: DictionaryEntry, rhs: DictionaryEntry) -> Bool { lhs.kind == rhs.kind }
+
+    /// The text the correction pass listens for (a term listens for itself).
+    public var source: String {
+        switch kind {
+        case .term(let term): term
+        case .correction(let heard, _): heard
+        }
+    }
+
+    /// The text that ends up in the transcript.
+    public var wrote: String {
+        switch kind {
+        case .term(let term): term
+        case .correction(_, let wrote): wrote
+        }
+    }
+}
+
+public enum DictionaryLine: Equatable, Sendable {
+    case entry(DictionaryEntry)
+    case comment(String)
+    case blank
+    case invalid(String)
+}
+
+/// The dictionary as a list of lines, so comments, blank lines and order survive an edit.
+public struct DictionaryFile: Equatable, Sendable {
+    public var lines: [DictionaryLine]
+
+    public init(lines: [DictionaryLine] = []) {
+        self.lines = lines
+    }
+
+    public init(parsing text: String) {
+        var raw = text.replacingOccurrences(of: "\r\n", with: "\n")
+            .split(separator: "\n", omittingEmptySubsequences: false)
+            .map(String.init)
+        if raw.last == "" { raw.removeLast() }
+        lines = raw.map(Self.parseLine)
+    }
+
+    static func parseLine(_ line: String) -> DictionaryLine {
+        let trimmed = line.trimmingCharacters(in: .whitespaces)
+        if trimmed.isEmpty { return .blank }
+        if trimmed.hasPrefix("#") { return .comment(line) }
+        if let arrow = trimmed.range(of: "=>") {
+            let heard = trimmed[..<arrow.lowerBound].trimmingCharacters(in: .whitespaces)
+            let wrote = trimmed[arrow.upperBound...].trimmingCharacters(in: .whitespaces)
+            guard !heard.isEmpty, !wrote.isEmpty else { return .invalid(line) }
+            return .entry(DictionaryEntry(kind: .correction(heard: heard, wrote: wrote)))
+        }
+        return .entry(DictionaryEntry(kind: .term(trimmed)))
+    }
+
+    public func serialized() -> String {
+        let out = lines.map { line -> String in
+            switch line {
+            case .entry(let entry):
+                switch entry.kind {
+                case .term(let term): return term
+                case .correction(let heard, let wrote): return "\(heard) => \(wrote)"
+                }
+            case .comment(let text), .invalid(let text): return text
+            case .blank: return ""
+            }
+        }
+        return out.isEmpty ? "" : out.joined(separator: "\n") + "\n"
+    }
+
+    public var entries: [DictionaryEntry] {
+        lines.compactMap { line in
+            if case .entry(let entry) = line { return entry }
+            return nil
+        }
+    }
+
+    @discardableResult
+    public mutating func append(_ kind: DictionaryEntry.Kind) -> UUID {
+        let entry = DictionaryEntry(kind: kind)
+        lines.append(.entry(entry))
+        return entry.id
+    }
+
+    public mutating func update(id: UUID, kind: DictionaryEntry.Kind) {
+        guard let index = lines.firstIndex(where: { if case .entry(let e) = $0 { return e.id == id } else { return false } }) else { return }
+        lines[index] = .entry(DictionaryEntry(id: id, kind: kind))
+    }
+
+    public mutating func remove(id: UUID) {
+        lines.removeAll { if case .entry(let e) = $0 { return e.id == id } else { return false } }
+    }
+}

Sources/SstData/WordKnowledge.swift

@@ -0,0 +1,72 @@
+import AppKit
+import Foundation
+import NaturalLanguage
+
+public protocol WordKnowledge {
+    /// A common word in Dutch or English (case-insensitive). Rare words, names and product names are not.
+    func isWord(_ word: String) -> Bool
+    /// A very frequent function word ("in", "de", "the", "of").
+    func isFunctionWord(_ word: String) -> Bool
+}
+
+/// Splitting and normalising the left-hand side of an entry, shared by the engine and the warnings.
+enum EntryKey {
+    static func parts(_ source: String) -> [String] {
+        source.split(whereSeparator: { $0.isWhitespace || "-‐‑–".contains($0) }).map(String.init)
+    }
+
+    /// Spacing, hyphens and case do not matter: "Cloud-Code", "cloud code" and "cloudcode" are the same source.
+    static func key(_ source: String) -> String {
+        parts(source).joined().lowercased()
+    }
+}
+
+/// The most frequent Dutch and English function words.
+enum CommonWords {
+    static let english: Set<String> = [
+        "a", "about", "after", "all", "also", "an", "and", "any", "are", "as", "at", "be", "because", "been", "but",
+        "by", "can", "could", "did", "do", "does", "for", "from", "had", "has", "have", "he", "her", "here", "him",
+        "his", "how", "i", "if", "in", "into", "is", "it", "its", "just", "me", "more", "most", "my", "no", "not",
+        "now", "of", "on", "one", "only", "or", "other", "our", "out", "over", "she", "so", "some", "such", "than",
+        "that", "the", "their", "them", "then", "there", "these", "they", "this", "those", "to", "up", "us", "very",
+        "was", "we", "were", "what", "when", "which", "who", "will", "with", "would", "you", "your",
+    ]
+    static let dutch: Set<String> = [
+        "aan", "al", "als", "ben", "bij", "dan", "dat", "de", "den", "der", "deze", "die", "dit", "doen", "door",
+        "dus", "een", "en", "er", "ga", "had", "heb", "heeft", "hem", "het", "hier", "hij", "hoe", "ik", "in", "is",
+        "ja", "je", "jij", "kan", "kon", "maar", "me", "mij", "mijn", "moet", "na", "naar", "niet", "nog", "nu",
+        "of", "om", "ons", "onze", "ook", "op", "over", "te", "ten", "ter", "toch", "toen", "tot", "u", "uit", "van",
+        "veel", "voor", "was", "wat", "we", "wel", "werd", "wij", "wil", "worden", "wordt", "zal", "ze", "zei",
+        "zich", "zij", "zijn", "zo", "zou",
+    ]
+    static let all: Set<String> = english.union(dutch)
+}
+
+/// English: the on-device word list of `NaturalLanguage` (common words only, so "cloud" and "reef" are in, "anthropic" and "vercel" are not).
+/// Dutch has no such list, so the spell checker decides. Function words come from the built-in list.
+public final class SystemWordKnowledge: WordKnowledge, @unchecked Sendable {
+    private let englishWords = NLEmbedding.wordEmbedding(for: .english)
+
+    public init() {}
+
+    public func isWord(_ word: String) -> Bool {
+        let lowered = word.lowercased()
+        guard !lowered.isEmpty else { return false }
+        if let englishWords {
+            if englishWords.contains(lowered) { return true }
+        } else if spellChecks(lowered, language: "en") {
+            return true
+        }
+        return spellChecks(lowered, language: "nl")
+    }
+
+    public func isFunctionWord(_ word: String) -> Bool {
+        CommonWords.all.contains(word.lowercased())
+    }
+
+    private func spellChecks(_ word: String, language: String) -> Bool {
+        NSSpellChecker.shared.checkSpelling(
+            of: word, startingAt: 0, language: language, wrap: false,
+            inSpellDocumentWithTag: 0, wordCount: nil).location == NSNotFound
+    }
+}

Tests/SstDataTests/DictionaryFileTests.swift

@@ -0,0 +1,52 @@
+import Testing
+@testable import SstData
+
+struct DictionaryFileTests {
+    let sample = "# Opmerking\nAnthropic\n\ncloud code => Claude Code\n"
+
+    @Test func parsesTermsPairsCommentsAndBlanks() {
+        let file = DictionaryFile(parsing: sample)
+        #expect(file.lines.count == 4)
+        #expect(file.entries == [term("Anthropic"), pair("cloud code", "Claude Code")])
+        #expect(file.lines[0] == .comment("# Opmerking"))
+        #expect(file.lines[2] == .blank)
+    }
+
+    @Test func roundTripKeepsCommentsBlankLinesAndOrder() {
+        #expect(DictionaryFile(parsing: sample).serialized() == sample)
+    }
+
+    @Test func normalisesSpacingAroundTheArrow() {
+        #expect(DictionaryFile(parsing: "a=>b\n").serialized() == "a => b\n")
+        #expect(DictionaryFile(parsing: "  cloud code   =>   Claude Code  \n").entries == [pair("cloud code", "Claude Code")])
+    }
+
+    @Test func unreadableLinesAreKeptAsInvalid() {
+        let file = DictionaryFile(parsing: "oops =>\n=> nothing\nok\n")
+        #expect(file.lines[0] == .invalid("oops =>"))
+        #expect(file.lines[1] == .invalid("=> nothing"))
+        #expect(file.serialized() == "oops =>\n=> nothing\nok\n")
+    }
+
+    @Test func handlesWindowsLineEndingsAndMissingFinalNewline() {
+        let file = DictionaryFile(parsing: "a\r\nb => c")
+        #expect(file.entries == [term("a"), pair("b", "c")])
+        #expect(file.serialized() == "a\nb => c\n")
+    }
+
+    @Test func emptyTextIsAValidEmptyFile() {
+        #expect(DictionaryFile(parsing: "").entries.isEmpty)
+        #expect(DictionaryFile(parsing: "").serialized() == "")
+    }
+
+    @Test func appendUpdateAndRemove() {
+        var file = DictionaryFile(parsing: "# c\n")
+        let id = file.append(.term("Vercel"))
+        #expect(file.entries == [term("Vercel")])
+        file.update(id: id, kind: .correction(heard: "versel", wrote: "Vercel"))
+        #expect(file.entries == [pair("versel", "Vercel")])
+        file.remove(id: id)
+        #expect(file.entries.isEmpty)
+        #expect(file.serialized() == "# c\n")
+    }
+}

Tests/SstDataTests/Fakes.swift

@@ -0,0 +1,13 @@
+import Foundation
+@testable import SstData
+
+/// Deterministic stand-in for the macOS spell checker.
+struct FakeKnowledge: WordKnowledge {
+    var words: Set<String> = ["cloud", "code", "some", "thing", "something", "reef", "slack", "in", "de", "the", "of"]
+    var functionWords: Set<String> = ["in", "de", "the", "of"]
+    func isWord(_ word: String) -> Bool { words.contains(word.lowercased()) }
+    func isFunctionWord(_ word: String) -> Bool { functionWords.contains(word.lowercased()) }
+}
+
+func term(_ text: String) -> DictionaryEntry { DictionaryEntry(kind: .term(text)) }
+func pair(_ heard: String, _ wrote: String) -> DictionaryEntry { DictionaryEntry(kind: .correction(heard: heard, wrote: wrote)) }

Tests/SstDataTests/SystemWordKnowledgeTests.swift

@@ -0,0 +1,24 @@
+import Testing
+@testable import SstData
+
+struct SystemWordKnowledgeTests {
+    let knowledge = SystemWordKnowledge()
+
+    @Test func recognisesEnglishAndDutchWords() {
+        #expect(knowledge.isWord("cloud"))
+        #expect(knowledge.isWord("Cloud"))
+        #expect(knowledge.isWord("vandaag"))
+    }
+
+    @Test func doesNotTreatProductNamesAsWords() {
+        #expect(!knowledge.isWord("vercel"))
+        #expect(!knowledge.isWord("supabase"))
+        #expect(!knowledge.isWord("anthropic"))
+    }
+
+    @Test func functionWordsComeFromTheBuiltInList() {
+        #expect(knowledge.isFunctionWord("de"))
+        #expect(knowledge.isFunctionWord("The"))
+        #expect(!knowledge.isFunctionWord("vercel"))
+    }
+}