From 93bf59e090a40269ea1f167dd2a0be57b32c3aea Mon Sep 17 00:00:00 2001 From: Theo Weijmer Date: Thu, 1 Oct 2026 15:09:27 +0200 Subject: [PATCH] feat: add CorrectionEngine (whole-word, longest-first, single pass) Co-Authored-By: Claude Sonnet 5.5 --- Sources/SstData/CorrectionEngine.swift | 79 +++++++++++++++ .../SstDataTests/CorrectionEngineTests.swift | 98 +++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100644 Sources/SstData/CorrectionEngine.swift create mode 100644 Tests/SstDataTests/CorrectionEngineTests.swift diff --git a/Sources/SstData/CorrectionEngine.swift b/Sources/SstData/CorrectionEngine.swift new file mode 100644 index 0000000..9e11266 --- /dev/null +++ b/Sources/SstData/CorrectionEngine.swift @@ -0,0 +1,79 @@ +import Foundation +import SstCore + +/// Whole-word, case-insensitive, single-pass replacement. Longest match wins, replaced text is never rescanned. +public struct CorrectionEngine: TextCorrecting { + private struct Rule { + let source: String + let wrote: String + } + + private let rules: [Rule] + private let regex: NSRegularExpression? + + public init(entries: [DictionaryEntry], knowledge: WordKnowledge) { + var seen = Set() + var built: [(rule: Rule, order: Int)] = [] + for (order, entry) in entries.enumerated() { + let source = entry.source.trimmingCharacters(in: .whitespacesAndNewlines) + let key = EntryKey.key(source) + guard !key.isEmpty, !seen.contains(key) else { continue } + // A single-word term that is also a real word only biases the model; capitalising it everywhere would corrupt normal text. + if case .term(let term) = entry.kind, EntryKey.parts(term).count == 1, knowledge.isWord(term) { continue } + seen.insert(key) + built.append((Rule(source: source, wrote: entry.wrote), order)) + } + let ordered = built.sorted { a, b in + let la = a.rule.source.filter(\.isLetter).count + let lb = b.rule.source.filter(\.isLetter).count + return la != lb ? la > lb : a.order < b.order + }.map(\.rule) + rules = ordered + if ordered.isEmpty { + regex = nil + } else { + let alternatives = ordered.map { "(\(Self.pattern($0.source)))" }.joined(separator: "|") + regex = try? NSRegularExpression( + pattern: "(? String { + EntryKey.parts(source) + .map { NSRegularExpression.escapedPattern(for: $0) } + .joined(separator: "[\\s\\-‐‑–]*") + } + + public func correct(_ text: String) -> CorrectionResult { + guard let regex, !text.isEmpty else { return CorrectionResult(text: text, corrections: []) } + let ns = text as NSString + var output = "" + var outputLength = 0 + var last = 0 + var applied: [AppliedCorrection] = [] + + for match in regex.matches(in: text, range: NSRange(location: 0, length: ns.length)) { + let gap = ns.substring(with: NSRange(location: last, length: match.range.location - last)) + output += gap + outputLength += gap.utf16.count + + let heard = ns.substring(with: match.range) + let group = (1.. CorrectionEngine { + CorrectionEngine(entries: entries, knowledge: knowledge) + } + + @Test func replacesAPairAndReportsIt() { + let result = engine([pair("cloud code", "Claude Code")]).correct("ik gebruik cloud code vandaag") + #expect(result.text == "ik gebruik Claude Code vandaag") + #expect(result.corrections.count == 1) + let c = result.corrections[0] + #expect(c.heard == "cloud code" && c.wrote == "Claude Code" && c.rule == "cloud code") + #expect((result.text as NSString).substring(with: NSRange(location: c.start, length: c.length)) == "Claude Code") + } + + @Test func findsGluedHyphenatedAndSpacedVariants() { + let e = engine([pair("cloud code", "Claude Code")]) + for variant in ["CloudCode", "Cloud-Code", "CLOUD CODE", "cloud\u{2011}code", "cloud code"] { + #expect(e.correct(variant).text == "Claude Code", "variant \(variant)") + } + } + + @Test func neverTouchesRealWordsThatMerelyStartTheSame() { + let e = engine([pair("cloud code", "Claude Code")]) + #expect(e.correct("Cloudflare is handig").text == "Cloudflare is handig") + #expect(e.correct("cloud codex").text == "cloud codex") + #expect(e.correct("de cloud en de code").text == "de cloud en de code") + #expect(e.correct("the cloud").corrections.isEmpty) + } + + @Test func respectsPunctuationBoundaries() { + let e = engine([pair("cloud code", "Claude Code")]) + #expect(e.correct("(cloud code)").text == "(Claude Code)") + #expect(e.correct("cloud code.").text == "Claude Code.") + #expect(e.correct("Cloud Code, Cloud Code").text == "Claude Code, Claude Code") + } + + @Test func longestMatchWins() { + let e = engine([pair("cloud code", "Claude Code"), pair("cloud code review", "Claude Code Review")]) + #expect(e.correct("cloud code review nu").text == "Claude Code Review nu") + #expect(e.correct("cloud-code-review").text == "Claude Code Review") + } + + @Test func replacedTextIsNeverRescanned() { + let e = engine([pair("a", "b"), pair("b", "c")]) + #expect(e.correct("a b").text == "b c") + } + + @Test func termsFixCasingAndSpacing() { + let e = engine([term("Anthropic"), term("Claude Code")]) + #expect(e.correct("anthropic ANTHROPIC").text == "Anthropic Anthropic") + #expect(e.correct("claudecode en claude-code").text == "Claude Code en Claude Code") + } + + @Test func termAlreadyRightIsNotReportedAsACorrection() { + let result = engine([term("Anthropic")]).correct("Anthropic bouwt Claude") + #expect(result.text == "Anthropic bouwt Claude") + #expect(result.corrections.isEmpty) + } + + @Test func singleWordTermThatIsARealWordGetsNoCasingRule() { + let e = engine([term("Reef"), term("Slack")]) + #expect(e.correct("het reef en slack").text == "het reef en slack") + } + + @Test func firstEntryWinsOnDuplicateSources() { + let e = engine([pair("cloud code", "Claude Code"), pair("Cloud-Code", "Other")]) + #expect(e.correct("cloudcode").text == "Claude Code") + } + + @Test func replacementIsLiteral() { + let e = engine([pair("price", "US$1 \\0 $1")]) + #expect(e.correct("the price").text == "the US$1 \\0 $1") + } + + @Test func handlesDiacritics() { + let e = engine([term("Café")]) + #expect(e.correct("een CAFÉ om de hoek").text == "een Café om de hoek") + } + + @Test func offsetsAreRelativeToTheCorrectedText() { + let result = engine([pair("a", "xxx"), pair("b", "y")]).correct("a b a") + #expect(result.text == "xxx y xxx") + #expect(result.corrections.map(\.start) == [0, 4, 6]) + #expect(result.corrections.map(\.length) == [3, 1, 3]) + } + + @Test func emptyDictionaryAndEmptyTextAreNoOps() { + #expect(engine([]).correct("hallo").text == "hallo") + #expect(engine([pair("a", "b")]).correct("").text == "") + } +}