feat: add SstData with DictionaryFile and word knowledge

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-01 15:06:55 +02:00
parent af1acdc0fa
commit a5fa0686f0
6 changed files with 273 additions and 0 deletions
@@ -0,0 +1,72 @@
+import AppKit
+import Foundation
+import NaturalLanguage
+
+public protocol WordKnowledge {
+    /// A common word in Dutch or English (case-insensitive). Rare words, names and product names are not.
+    func isWord(_ word: String) -> Bool
+    /// A very frequent function word ("in", "de", "the", "of").
+    func isFunctionWord(_ word: String) -> Bool
+}
+
+/// Splitting and normalising the left-hand side of an entry, shared by the engine and the warnings.
+enum EntryKey {
+    static func parts(_ source: String) -> [String] {
+        source.split(whereSeparator: { $0.isWhitespace || "-‐‑–".contains($0) }).map(String.init)
+    }
+
+    /// Spacing, hyphens and case do not matter: "Cloud-Code", "cloud code" and "cloudcode" are the same source.
+    static func key(_ source: String) -> String {
+        parts(source).joined().lowercased()
+    }
+}
+
+/// The most frequent Dutch and English function words.
+enum CommonWords {
+    static let english: Set<String> = [
+        "a", "about", "after", "all", "also", "an", "and", "any", "are", "as", "at", "be", "because", "been", "but",
+        "by", "can", "could", "did", "do", "does", "for", "from", "had", "has", "have", "he", "her", "here", "him",
+        "his", "how", "i", "if", "in", "into", "is", "it", "its", "just", "me", "more", "most", "my", "no", "not",
+        "now", "of", "on", "one", "only", "or", "other", "our", "out", "over", "she", "so", "some", "such", "than",
+        "that", "the", "their", "them", "then", "there", "these", "they", "this", "those", "to", "up", "us", "very",
+        "was", "we", "were", "what", "when", "which", "who", "will", "with", "would", "you", "your",
+    ]
+    static let dutch: Set<String> = [
+        "aan", "al", "als", "ben", "bij", "dan", "dat", "de", "den", "der", "deze", "die", "dit", "doen", "door",
+        "dus", "een", "en", "er", "ga", "had", "heb", "heeft", "hem", "het", "hier", "hij", "hoe", "ik", "in", "is",
+        "ja", "je", "jij", "kan", "kon", "maar", "me", "mij", "mijn", "moet", "na", "naar", "niet", "nog", "nu",
+        "of", "om", "ons", "onze", "ook", "op", "over", "te", "ten", "ter", "toch", "toen", "tot", "u", "uit", "van",
+        "veel", "voor", "was", "wat", "we", "wel", "werd", "wij", "wil", "worden", "wordt", "zal", "ze", "zei",
+        "zich", "zij", "zijn", "zo", "zou",
+    ]
+    static let all: Set<String> = english.union(dutch)
+}
+
+/// English: the on-device word list of `NaturalLanguage` (common words only, so "cloud" and "reef" are in, "anthropic" and "vercel" are not).
+/// Dutch has no such list, so the spell checker decides. Function words come from the built-in list.
+public final class SystemWordKnowledge: WordKnowledge, @unchecked Sendable {
+    private let englishWords = NLEmbedding.wordEmbedding(for: .english)
+
+    public init() {}
+
+    public func isWord(_ word: String) -> Bool {
+        let lowered = word.lowercased()
+        guard !lowered.isEmpty else { return false }
+        if let englishWords {
+            if englishWords.contains(lowered) { return true }
+        } else if spellChecks(lowered, language: "en") {
+            return true
+        }
+        return spellChecks(lowered, language: "nl")
+    }
+
+    public func isFunctionWord(_ word: String) -> Bool {
+        CommonWords.all.contains(word.lowercased())
+    }
+
+    private func spellChecks(_ word: String, language: String) -> Bool {
+        NSSpellChecker.shared.checkSpelling(
+            of: word, startingAt: 0, language: language, wrap: false,
+            inSpellDocumentWithTag: 0, wordCount: nil).location == NSNotFound
+    }
+}