From a5fa0686f06b4b837813660a2f25e06a5d57d81f Mon Sep 17 00:00:00 2001 From: Theo Weijmer Date: Thu, 1 Oct 2026 15:06:22 +0200 Subject: [PATCH] feat: add SstData with DictionaryFile and word knowledge Co-Authored-By: Claude Sonnet 5.5 --- Package.swift | 2 + Sources/SstData/DictionaryFile.swift | 110 ++++++++++++++++++ Sources/SstData/WordKnowledge.swift | 72 ++++++++++++ Tests/SstDataTests/DictionaryFileTests.swift | 52 +++++++++ Tests/SstDataTests/Fakes.swift | 13 +++ .../SystemWordKnowledgeTests.swift | 24 ++++ 6 files changed, 273 insertions(+) create mode 100644 Sources/SstData/DictionaryFile.swift create mode 100644 Sources/SstData/WordKnowledge.swift create mode 100644 Tests/SstDataTests/DictionaryFileTests.swift create mode 100644 Tests/SstDataTests/Fakes.swift create mode 100644 Tests/SstDataTests/SystemWordKnowledgeTests.swift diff --git a/Package.swift b/Package.swift index c8c391c..3ef7517 100644 --- a/Package.swift +++ b/Package.swift @@ -9,6 +9,7 @@ let package = Package( ], targets: [ .target(name: "SstCore"), + .target(name: "SstData", dependencies: ["SstCore"]), .target( name: "SstWhisper", dependencies: [ @@ -19,6 +20,7 @@ let package = Package( .executableTarget(name: "Sst", dependencies: ["SstCore", "SstWhisper"]), .executableTarget(name: "SstProbe", dependencies: ["SstCore", "SstWhisper"]), .testTarget(name: "SstCoreTests", dependencies: ["SstCore"]), + .testTarget(name: "SstDataTests", dependencies: ["SstData", "SstCore"]), .testTarget(name: "SstWhisperTests", dependencies: ["SstWhisper"]), ], swiftLanguageModes: [.v5] diff --git a/Sources/SstData/DictionaryFile.swift b/Sources/SstData/DictionaryFile.swift new file mode 100644 index 0000000..2185841 --- /dev/null +++ b/Sources/SstData/DictionaryFile.swift @@ -0,0 +1,110 @@ +import Foundation + +public struct DictionaryEntry: Identifiable, Equatable, Sendable { + public enum Kind: Equatable, Sendable { + case term(String) + case correction(heard: String, wrote: String) + } + + public let id: UUID + public var kind: Kind + + public init(id: UUID = UUID(), kind: Kind) { + self.id = id + self.kind = kind + } + + /// Equality ignores `id`: two entries with the same content are the same entry. + public static func == (lhs: DictionaryEntry, rhs: DictionaryEntry) -> Bool { lhs.kind == rhs.kind } + + /// The text the correction pass listens for (a term listens for itself). + public var source: String { + switch kind { + case .term(let term): term + case .correction(let heard, _): heard + } + } + + /// The text that ends up in the transcript. + public var wrote: String { + switch kind { + case .term(let term): term + case .correction(_, let wrote): wrote + } + } +} + +public enum DictionaryLine: Equatable, Sendable { + case entry(DictionaryEntry) + case comment(String) + case blank + case invalid(String) +} + +/// The dictionary as a list of lines, so comments, blank lines and order survive an edit. +public struct DictionaryFile: Equatable, Sendable { + public var lines: [DictionaryLine] + + public init(lines: [DictionaryLine] = []) { + self.lines = lines + } + + public init(parsing text: String) { + var raw = text.replacingOccurrences(of: "\r\n", with: "\n") + .split(separator: "\n", omittingEmptySubsequences: false) + .map(String.init) + if raw.last == "" { raw.removeLast() } + lines = raw.map(Self.parseLine) + } + + static func parseLine(_ line: String) -> DictionaryLine { + let trimmed = line.trimmingCharacters(in: .whitespaces) + if trimmed.isEmpty { return .blank } + if trimmed.hasPrefix("#") { return .comment(line) } + if let arrow = trimmed.range(of: "=>") { + let heard = trimmed[.. String { + let out = lines.map { line -> String in + switch line { + case .entry(let entry): + switch entry.kind { + case .term(let term): return term + case .correction(let heard, let wrote): return "\(heard) => \(wrote)" + } + case .comment(let text), .invalid(let text): return text + case .blank: return "" + } + } + return out.isEmpty ? "" : out.joined(separator: "\n") + "\n" + } + + public var entries: [DictionaryEntry] { + lines.compactMap { line in + if case .entry(let entry) = line { return entry } + return nil + } + } + + @discardableResult + public mutating func append(_ kind: DictionaryEntry.Kind) -> UUID { + let entry = DictionaryEntry(kind: kind) + lines.append(.entry(entry)) + return entry.id + } + + public mutating func update(id: UUID, kind: DictionaryEntry.Kind) { + guard let index = lines.firstIndex(where: { if case .entry(let e) = $0 { return e.id == id } else { return false } }) else { return } + lines[index] = .entry(DictionaryEntry(id: id, kind: kind)) + } + + public mutating func remove(id: UUID) { + lines.removeAll { if case .entry(let e) = $0 { return e.id == id } else { return false } } + } +} diff --git a/Sources/SstData/WordKnowledge.swift b/Sources/SstData/WordKnowledge.swift new file mode 100644 index 0000000..b2e0e62 --- /dev/null +++ b/Sources/SstData/WordKnowledge.swift @@ -0,0 +1,72 @@ +import AppKit +import Foundation +import NaturalLanguage + +public protocol WordKnowledge { + /// A common word in Dutch or English (case-insensitive). Rare words, names and product names are not. + func isWord(_ word: String) -> Bool + /// A very frequent function word ("in", "de", "the", "of"). + func isFunctionWord(_ word: String) -> Bool +} + +/// Splitting and normalising the left-hand side of an entry, shared by the engine and the warnings. +enum EntryKey { + static func parts(_ source: String) -> [String] { + source.split(whereSeparator: { $0.isWhitespace || "-‐‑–".contains($0) }).map(String.init) + } + + /// Spacing, hyphens and case do not matter: "Cloud-Code", "cloud code" and "cloudcode" are the same source. + static func key(_ source: String) -> String { + parts(source).joined().lowercased() + } +} + +/// The most frequent Dutch and English function words. +enum CommonWords { + static let english: Set = [ + "a", "about", "after", "all", "also", "an", "and", "any", "are", "as", "at", "be", "because", "been", "but", + "by", "can", "could", "did", "do", "does", "for", "from", "had", "has", "have", "he", "her", "here", "him", + "his", "how", "i", "if", "in", "into", "is", "it", "its", "just", "me", "more", "most", "my", "no", "not", + "now", "of", "on", "one", "only", "or", "other", "our", "out", "over", "she", "so", "some", "such", "than", + "that", "the", "their", "them", "then", "there", "these", "they", "this", "those", "to", "up", "us", "very", + "was", "we", "were", "what", "when", "which", "who", "will", "with", "would", "you", "your", + ] + static let dutch: Set = [ + "aan", "al", "als", "ben", "bij", "dan", "dat", "de", "den", "der", "deze", "die", "dit", "doen", "door", + "dus", "een", "en", "er", "ga", "had", "heb", "heeft", "hem", "het", "hier", "hij", "hoe", "ik", "in", "is", + "ja", "je", "jij", "kan", "kon", "maar", "me", "mij", "mijn", "moet", "na", "naar", "niet", "nog", "nu", + "of", "om", "ons", "onze", "ook", "op", "over", "te", "ten", "ter", "toch", "toen", "tot", "u", "uit", "van", + "veel", "voor", "was", "wat", "we", "wel", "werd", "wij", "wil", "worden", "wordt", "zal", "ze", "zei", + "zich", "zij", "zijn", "zo", "zou", + ] + static let all: Set = english.union(dutch) +} + +/// English: the on-device word list of `NaturalLanguage` (common words only, so "cloud" and "reef" are in, "anthropic" and "vercel" are not). +/// Dutch has no such list, so the spell checker decides. Function words come from the built-in list. +public final class SystemWordKnowledge: WordKnowledge, @unchecked Sendable { + private let englishWords = NLEmbedding.wordEmbedding(for: .english) + + public init() {} + + public func isWord(_ word: String) -> Bool { + let lowered = word.lowercased() + guard !lowered.isEmpty else { return false } + if let englishWords { + if englishWords.contains(lowered) { return true } + } else if spellChecks(lowered, language: "en") { + return true + } + return spellChecks(lowered, language: "nl") + } + + public func isFunctionWord(_ word: String) -> Bool { + CommonWords.all.contains(word.lowercased()) + } + + private func spellChecks(_ word: String, language: String) -> Bool { + NSSpellChecker.shared.checkSpelling( + of: word, startingAt: 0, language: language, wrap: false, + inSpellDocumentWithTag: 0, wordCount: nil).location == NSNotFound + } +} diff --git a/Tests/SstDataTests/DictionaryFileTests.swift b/Tests/SstDataTests/DictionaryFileTests.swift new file mode 100644 index 0000000..6d0fcd6 --- /dev/null +++ b/Tests/SstDataTests/DictionaryFileTests.swift @@ -0,0 +1,52 @@ +import Testing +@testable import SstData + +struct DictionaryFileTests { + let sample = "# Opmerking\nAnthropic\n\ncloud code => Claude Code\n" + + @Test func parsesTermsPairsCommentsAndBlanks() { + let file = DictionaryFile(parsing: sample) + #expect(file.lines.count == 4) + #expect(file.entries == [term("Anthropic"), pair("cloud code", "Claude Code")]) + #expect(file.lines[0] == .comment("# Opmerking")) + #expect(file.lines[2] == .blank) + } + + @Test func roundTripKeepsCommentsBlankLinesAndOrder() { + #expect(DictionaryFile(parsing: sample).serialized() == sample) + } + + @Test func normalisesSpacingAroundTheArrow() { + #expect(DictionaryFile(parsing: "a=>b\n").serialized() == "a => b\n") + #expect(DictionaryFile(parsing: " cloud code => Claude Code \n").entries == [pair("cloud code", "Claude Code")]) + } + + @Test func unreadableLinesAreKeptAsInvalid() { + let file = DictionaryFile(parsing: "oops =>\n=> nothing\nok\n") + #expect(file.lines[0] == .invalid("oops =>")) + #expect(file.lines[1] == .invalid("=> nothing")) + #expect(file.serialized() == "oops =>\n=> nothing\nok\n") + } + + @Test func handlesWindowsLineEndingsAndMissingFinalNewline() { + let file = DictionaryFile(parsing: "a\r\nb => c") + #expect(file.entries == [term("a"), pair("b", "c")]) + #expect(file.serialized() == "a\nb => c\n") + } + + @Test func emptyTextIsAValidEmptyFile() { + #expect(DictionaryFile(parsing: "").entries.isEmpty) + #expect(DictionaryFile(parsing: "").serialized() == "") + } + + @Test func appendUpdateAndRemove() { + var file = DictionaryFile(parsing: "# c\n") + let id = file.append(.term("Vercel")) + #expect(file.entries == [term("Vercel")]) + file.update(id: id, kind: .correction(heard: "versel", wrote: "Vercel")) + #expect(file.entries == [pair("versel", "Vercel")]) + file.remove(id: id) + #expect(file.entries.isEmpty) + #expect(file.serialized() == "# c\n") + } +} diff --git a/Tests/SstDataTests/Fakes.swift b/Tests/SstDataTests/Fakes.swift new file mode 100644 index 0000000..b93dedd --- /dev/null +++ b/Tests/SstDataTests/Fakes.swift @@ -0,0 +1,13 @@ +import Foundation +@testable import SstData + +/// Deterministic stand-in for the macOS spell checker. +struct FakeKnowledge: WordKnowledge { + var words: Set = ["cloud", "code", "some", "thing", "something", "reef", "slack", "in", "de", "the", "of"] + var functionWords: Set = ["in", "de", "the", "of"] + func isWord(_ word: String) -> Bool { words.contains(word.lowercased()) } + func isFunctionWord(_ word: String) -> Bool { functionWords.contains(word.lowercased()) } +} + +func term(_ text: String) -> DictionaryEntry { DictionaryEntry(kind: .term(text)) } +func pair(_ heard: String, _ wrote: String) -> DictionaryEntry { DictionaryEntry(kind: .correction(heard: heard, wrote: wrote)) } diff --git a/Tests/SstDataTests/SystemWordKnowledgeTests.swift b/Tests/SstDataTests/SystemWordKnowledgeTests.swift new file mode 100644 index 0000000..4a42015 --- /dev/null +++ b/Tests/SstDataTests/SystemWordKnowledgeTests.swift @@ -0,0 +1,24 @@ +import Testing +@testable import SstData + +struct SystemWordKnowledgeTests { + let knowledge = SystemWordKnowledge() + + @Test func recognisesEnglishAndDutchWords() { + #expect(knowledge.isWord("cloud")) + #expect(knowledge.isWord("Cloud")) + #expect(knowledge.isWord("vandaag")) + } + + @Test func doesNotTreatProductNamesAsWords() { + #expect(!knowledge.isWord("vercel")) + #expect(!knowledge.isWord("supabase")) + #expect(!knowledge.isWord("anthropic")) + } + + @Test func functionWordsComeFromTheBuiltInList() { + #expect(knowledge.isFunctionWord("de")) + #expect(knowledge.isFunctionWord("The")) + #expect(!knowledge.isFunctionWord("vercel")) + } +}