feat: add CorrectionEngine (whole-word, longest-first, single pass)
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 5.5
@@ -0,0 +1,79 @@
|
||||
import Foundation
|
||||
import SstCore
|
||||
|
||||
/// Whole-word, case-insensitive, single-pass replacement. Longest match wins, replaced text is never rescanned.
|
||||
public struct CorrectionEngine: TextCorrecting {
|
||||
private struct Rule {
|
||||
let source: String
|
||||
let wrote: String
|
||||
}
|
||||
|
||||
private let rules: [Rule]
|
||||
private let regex: NSRegularExpression?
|
||||
|
||||
public init(entries: [DictionaryEntry], knowledge: WordKnowledge) {
|
||||
var seen = Set<String>()
|
||||
var built: [(rule: Rule, order: Int)] = []
|
||||
for (order, entry) in entries.enumerated() {
|
||||
let source = entry.source.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let key = EntryKey.key(source)
|
||||
guard !key.isEmpty, !seen.contains(key) else { continue }
|
||||
// A single-word term that is also a real word only biases the model; capitalising it everywhere would corrupt normal text.
|
||||
if case .term(let term) = entry.kind, EntryKey.parts(term).count == 1, knowledge.isWord(term) { continue }
|
||||
seen.insert(key)
|
||||
built.append((Rule(source: source, wrote: entry.wrote), order))
|
||||
}
|
||||
let ordered = built.sorted { a, b in
|
||||
let la = a.rule.source.filter(\.isLetter).count
|
||||
let lb = b.rule.source.filter(\.isLetter).count
|
||||
return la != lb ? la > lb : a.order < b.order
|
||||
}.map(\.rule)
|
||||
rules = ordered
|
||||
if ordered.isEmpty {
|
||||
regex = nil
|
||||
} else {
|
||||
let alternatives = ordered.map { "(\(Self.pattern($0.source)))" }.joined(separator: "|")
|
||||
regex = try? NSRegularExpression(
|
||||
pattern: "(?<![\\p{L}\\p{N}])(?:\(alternatives))(?![\\p{L}\\p{N}])",
|
||||
options: [.caseInsensitive]
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Parts joined by "zero or more spaces/hyphens", so "cloud code" also finds "cloudcode" and "Cloud-Code".
|
||||
private static func pattern(_ source: String) -> String {
|
||||
EntryKey.parts(source)
|
||||
.map { NSRegularExpression.escapedPattern(for: $0) }
|
||||
.joined(separator: "[\\s\\-‐‑–]*")
|
||||
}
|
||||
|
||||
public func correct(_ text: String) -> CorrectionResult {
|
||||
guard let regex, !text.isEmpty else { return CorrectionResult(text: text, corrections: []) }
|
||||
let ns = text as NSString
|
||||
var output = ""
|
||||
var outputLength = 0
|
||||
var last = 0
|
||||
var applied: [AppliedCorrection] = []
|
||||
|
||||
for match in regex.matches(in: text, range: NSRange(location: 0, length: ns.length)) {
|
||||
let gap = ns.substring(with: NSRange(location: last, length: match.range.location - last))
|
||||
output += gap
|
||||
outputLength += gap.utf16.count
|
||||
|
||||
let heard = ns.substring(with: match.range)
|
||||
let group = (1..<match.numberOfRanges).first { match.range(at: $0).location != NSNotFound } ?? 1
|
||||
let rule = rules[group - 1]
|
||||
|
||||
output += rule.wrote
|
||||
if heard != rule.wrote {
|
||||
applied.append(AppliedCorrection(
|
||||
heard: heard, wrote: rule.wrote, rule: rule.source,
|
||||
start: outputLength, length: rule.wrote.utf16.count))
|
||||
}
|
||||
outputLength += rule.wrote.utf16.count
|
||||
last = match.range.location + match.range.length
|
||||
}
|
||||
output += ns.substring(from: last)
|
||||
return CorrectionResult(text: output, corrections: applied)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
import Foundation
|
||||
import Testing
|
||||
import SstCore
|
||||
@testable import SstData
|
||||
|
||||
struct CorrectionEngineTests {
|
||||
let knowledge = FakeKnowledge()
|
||||
|
||||
func engine(_ entries: [DictionaryEntry]) -> CorrectionEngine {
|
||||
CorrectionEngine(entries: entries, knowledge: knowledge)
|
||||
}
|
||||
|
||||
@Test func replacesAPairAndReportsIt() {
|
||||
let result = engine([pair("cloud code", "Claude Code")]).correct("ik gebruik cloud code vandaag")
|
||||
#expect(result.text == "ik gebruik Claude Code vandaag")
|
||||
#expect(result.corrections.count == 1)
|
||||
let c = result.corrections[0]
|
||||
#expect(c.heard == "cloud code" && c.wrote == "Claude Code" && c.rule == "cloud code")
|
||||
#expect((result.text as NSString).substring(with: NSRange(location: c.start, length: c.length)) == "Claude Code")
|
||||
}
|
||||
|
||||
@Test func findsGluedHyphenatedAndSpacedVariants() {
|
||||
let e = engine([pair("cloud code", "Claude Code")])
|
||||
for variant in ["CloudCode", "Cloud-Code", "CLOUD CODE", "cloud\u{2011}code", "cloud code"] {
|
||||
#expect(e.correct(variant).text == "Claude Code", "variant \(variant)")
|
||||
}
|
||||
}
|
||||
|
||||
@Test func neverTouchesRealWordsThatMerelyStartTheSame() {
|
||||
let e = engine([pair("cloud code", "Claude Code")])
|
||||
#expect(e.correct("Cloudflare is handig").text == "Cloudflare is handig")
|
||||
#expect(e.correct("cloud codex").text == "cloud codex")
|
||||
#expect(e.correct("de cloud en de code").text == "de cloud en de code")
|
||||
#expect(e.correct("the cloud").corrections.isEmpty)
|
||||
}
|
||||
|
||||
@Test func respectsPunctuationBoundaries() {
|
||||
let e = engine([pair("cloud code", "Claude Code")])
|
||||
#expect(e.correct("(cloud code)").text == "(Claude Code)")
|
||||
#expect(e.correct("cloud code.").text == "Claude Code.")
|
||||
#expect(e.correct("Cloud Code, Cloud Code").text == "Claude Code, Claude Code")
|
||||
}
|
||||
|
||||
@Test func longestMatchWins() {
|
||||
let e = engine([pair("cloud code", "Claude Code"), pair("cloud code review", "Claude Code Review")])
|
||||
#expect(e.correct("cloud code review nu").text == "Claude Code Review nu")
|
||||
#expect(e.correct("cloud-code-review").text == "Claude Code Review")
|
||||
}
|
||||
|
||||
@Test func replacedTextIsNeverRescanned() {
|
||||
let e = engine([pair("a", "b"), pair("b", "c")])
|
||||
#expect(e.correct("a b").text == "b c")
|
||||
}
|
||||
|
||||
@Test func termsFixCasingAndSpacing() {
|
||||
let e = engine([term("Anthropic"), term("Claude Code")])
|
||||
#expect(e.correct("anthropic ANTHROPIC").text == "Anthropic Anthropic")
|
||||
#expect(e.correct("claudecode en claude-code").text == "Claude Code en Claude Code")
|
||||
}
|
||||
|
||||
@Test func termAlreadyRightIsNotReportedAsACorrection() {
|
||||
let result = engine([term("Anthropic")]).correct("Anthropic bouwt Claude")
|
||||
#expect(result.text == "Anthropic bouwt Claude")
|
||||
#expect(result.corrections.isEmpty)
|
||||
}
|
||||
|
||||
@Test func singleWordTermThatIsARealWordGetsNoCasingRule() {
|
||||
let e = engine([term("Reef"), term("Slack")])
|
||||
#expect(e.correct("het reef en slack").text == "het reef en slack")
|
||||
}
|
||||
|
||||
@Test func firstEntryWinsOnDuplicateSources() {
|
||||
let e = engine([pair("cloud code", "Claude Code"), pair("Cloud-Code", "Other")])
|
||||
#expect(e.correct("cloudcode").text == "Claude Code")
|
||||
}
|
||||
|
||||
@Test func replacementIsLiteral() {
|
||||
let e = engine([pair("price", "US$1 \\0 $1")])
|
||||
#expect(e.correct("the price").text == "the US$1 \\0 $1")
|
||||
}
|
||||
|
||||
@Test func handlesDiacritics() {
|
||||
let e = engine([term("Café")])
|
||||
#expect(e.correct("een CAFÉ om de hoek").text == "een Café om de hoek")
|
||||
}
|
||||
|
||||
@Test func offsetsAreRelativeToTheCorrectedText() {
|
||||
let result = engine([pair("a", "xxx"), pair("b", "y")]).correct("a b a")
|
||||
#expect(result.text == "xxx y xxx")
|
||||
#expect(result.corrections.map(\.start) == [0, 4, 6])
|
||||
#expect(result.corrections.map(\.length) == [3, 1, 3])
|
||||
}
|
||||
|
||||
@Test func emptyDictionaryAndEmptyTextAreNoOps() {
|
||||
#expect(engine([]).correct("hallo").text == "hallo")
|
||||
#expect(engine([pair("a", "b")]).correct("").text == "")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user