diff --git a/Sources/SstData/EntryWarnings.swift b/Sources/SstData/EntryWarnings.swift new file mode 100644 index 0000000..b8b559f --- /dev/null +++ b/Sources/SstData/EntryWarnings.swift @@ -0,0 +1,67 @@ +import Foundation + +public struct EntryWarning: Equatable, Sendable { + public enum Level: Int, Comparable, Sendable { + case note = 0, warning, error + public static func < (a: Level, b: Level) -> Bool { a.rawValue < b.rawValue } + } + + public let level: Level + public let message: String + + public init(level: Level, message: String) { + self.level = level + self.message = message + } +} + +/// Flags dictionary entries that are likely to corrupt normal text. +public enum EntryWarnings { + /// - Parameter others: all entries except the one being checked. + public static func check(_ kind: DictionaryEntry.Kind, among others: [DictionaryEntry], knowledge: WordKnowledge) -> [EntryWarning] { + let source: String + let isCorrection: Bool + switch kind { + case .term(let term): (source, isCorrection) = (term, false) + case .correction(let heard, _): (source, isCorrection) = (heard, true) + } + + let parts = EntryKey.parts(source) + guard !parts.isEmpty else { return [EntryWarning(level: .error, message: "Leeg: vul een woord in.")] } + + var result: [EntryWarning] = [] + if others.contains(where: { EntryKey.key($0.source) == EntryKey.key(source) }) { + result.append(EntryWarning(level: .error, message: "Dubbele invoer: deze bron staat er al, de eerste telt.")) + } + + if !isCorrection { + if parts.count == 1, knowledge.isWord(source) { + result.append(EntryWarning(level: .note, message: "Gewoon woord: krijgt alleen voorrang bij het model, geen automatische hoofdletterregel.")) + } + return result.sorted { $0.level > $1.level } + } + + if parts.count == 1, knowledge.isWord(parts[0]) { + result.append(EntryWarning(level: .warning, message: "'\(parts[0])' is een gewoon woord en wordt overal in je tekst vervangen.")) + } + if source.filter(\.isLetter).count < 3 { + result.append(EntryWarning(level: .warning, message: "Erg kort (minder dan 3 letters): veel kans op ongewenste vervangingen.")) + } + if parts.count > 1 { + var flagged = false + if let common = parts.first(where: { knowledge.isFunctionWord($0) }) { + result.append(EntryWarning(level: .warning, message: "Bevat het veelvoorkomende woord '\(common)'; dit kan gewone zinnen raken.")) + flagged = true + } + let glued = parts.joined() + if knowledge.isWord(glued) { + result.append(EntryWarning(level: .warning, message: "Aaneengeschreven ('\(glued)') is een bestaand woord en wordt ook vervangen.")) + flagged = true + } + if !flagged, parts.allSatisfy({ knowledge.isWord($0) }) { + result.append(EntryWarning(level: .note, message: "Alle delen zijn gewone woorden: alleen de hele combinatie wordt vervangen.")) + } + } + return result.sorted { $0.level > $1.level } + } +} diff --git a/Sources/SstData/VocabularyBuilder.swift b/Sources/SstData/VocabularyBuilder.swift new file mode 100644 index 0000000..43a2dc0 --- /dev/null +++ b/Sources/SstData/VocabularyBuilder.swift @@ -0,0 +1,19 @@ +import Foundation + +/// The short word list handed to the speech engine as context. +public enum VocabularyBuilder { + /// Terms first (file order), then the unique right-hand sides of correction pairs; at most `maxTerms` words. + public static func build(from entries: [DictionaryEntry], maxTerms: Int = 24) -> [String] { + var out: [String] = [] + var seen = Set() + func add(_ word: String) { + let key = word.lowercased() + guard !word.isEmpty, !seen.contains(key) else { return } + seen.insert(key) + out.append(word) + } + for entry in entries { if case .term(let term) = entry.kind { add(term) } } + for entry in entries { if case .correction(_, let wrote) = entry.kind { add(wrote) } } + return Array(out.prefix(maxTerms)) + } +} diff --git a/Tests/SstDataTests/EntryWarningsTests.swift b/Tests/SstDataTests/EntryWarningsTests.swift new file mode 100644 index 0000000..d37338a --- /dev/null +++ b/Tests/SstDataTests/EntryWarningsTests.swift @@ -0,0 +1,60 @@ +import Testing +@testable import SstData + +struct EntryWarningsTests { + let knowledge = FakeKnowledge() + + func check(_ kind: DictionaryEntry.Kind, others: [DictionaryEntry] = []) -> [EntryWarning] { + EntryWarnings.check(kind, among: others, knowledge: knowledge) + } + + @Test func singleCommonWordIsAWarning() { + let w = check(.correction(heard: "cloud", wrote: "Claude")) + #expect(w.contains { $0.level == .warning && $0.message.contains("cloud") }) + } + + @Test func veryShortSourceIsAWarning() { + #expect(check(.correction(heard: "ab", wrote: "AB")).contains { $0.level == .warning }) + } + + @Test func functionWordInsideAPhraseIsAWarning() { + #expect(check(.correction(heard: "in de", wrote: "indeed")).contains { $0.level == .warning }) + } + + @Test func gluedFormThatIsARealWordIsAWarning() { + let w = check(.correction(heard: "some thing", wrote: "Something")) + #expect(w.contains { $0.level == .warning && $0.message.contains("something") }) + } + + @Test func phraseOfCommonWordsOnlyGetsANote() { + let w = check(.correction(heard: "cloud code", wrote: "Claude Code")) + #expect(w.count == 1) + #expect(w[0].level == .note) + } + + @Test func distinctiveEntriesGetNoWarnings() { + #expect(check(.correction(heard: "versel", wrote: "Vercel")).isEmpty) + #expect(check(.term("Supabase")).isEmpty) + } + + @Test func duplicateSourceIsAnErrorEvenWithDifferentSpelling() { + let others = [pair("cloud code", "Claude Code")] + let w = check(.correction(heard: "Cloud-Code", wrote: "X"), others: others) + #expect(w.first?.level == .error) + } + + @Test func termThatIsARealWordGetsANoteAboutBiasOnly() { + let w = check(.term("Reef")) + #expect(w.count == 1) + #expect(w[0].level == .note) + } + + @Test func emptySourceIsAnError() { + #expect(check(.term(" ")).first?.level == .error) + } + + @Test func warningsAreSortedMostSevereFirst() { + let w = check(.correction(heard: "cloud", wrote: "Claude"), others: [pair("cloud", "x")]) + #expect(w.map(\.level) == w.map(\.level).sorted(by: >)) + } +} diff --git a/Tests/SstDataTests/VocabularyBuilderTests.swift b/Tests/SstDataTests/VocabularyBuilderTests.swift new file mode 100644 index 0000000..f9cd1ca --- /dev/null +++ b/Tests/SstDataTests/VocabularyBuilderTests.swift @@ -0,0 +1,19 @@ +import Testing +@testable import SstData + +struct VocabularyBuilderTests { + @Test func termsComeFirstThenCorrectionTargetsWithoutDuplicates() { + let entries = [pair("cloud code", "Claude Code"), term("Anthropic"), term("anthropic"), pair("versel", "Vercel"), term("Supabase")] + #expect(VocabularyBuilder.build(from: entries) == ["Anthropic", "Supabase", "Claude Code", "Vercel"]) + } + + @Test func listIsCapped() { + let entries = (1...40).map { term("Woord\($0)") } + #expect(VocabularyBuilder.build(from: entries).count == 24) + #expect(VocabularyBuilder.build(from: entries, maxTerms: 5).count == 5) + } + + @Test func emptyDictionaryGivesNoVocabulary() { + #expect(VocabularyBuilder.build(from: []).isEmpty) + } +}