diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..84f6603 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.build/ +build/ +.DS_Store +*.xcodeproj diff --git a/Package.resolved b/Package.resolved new file mode 100644 index 0000000..20256d6 --- /dev/null +++ b/Package.resolved @@ -0,0 +1,78 @@ +{ + "originHash" : "e6c9264e0494528c9e107c3e12b51524d9f3fe957d63a07601ae76077888c441", + "pins" : [ + { + "identity" : "swift-argument-parser", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-argument-parser.git", + "state" : { + "revision" : "6a52f3251125d74daf04fcbd5e6f08a75d074382", + "version" : "1.8.2" + } + }, + { + "identity" : "swift-asn1", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-asn1.git", + "state" : { + "revision" : "3b6410f7dee09eb33cdd26260c5fd47fda19b0e2", + "version" : "1.7.3" + } + }, + { + "identity" : "swift-collections", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-collections.git", + "state" : { + "revision" : "98ef3c98609a1e31b7e157b5b619579001a789d6", + "version" : "1.7.1" + } + }, + { + "identity" : "swift-crypto", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-crypto.git", + "state" : { + "revision" : "da9d28d69ebe3894b18376c8f2395c2f37b8448f", + "version" : "4.5.2" + } + }, + { + "identity" : "swift-jinja", + "kind" : "remoteSourceControl", + "location" : "https://github.com/huggingface/swift-jinja.git", + "state" : { + "revision" : "4588064a20f3fc093c95f2f7d3359999bf30cae5", + "version" : "2.5.1" + } + }, + { + "identity" : "swift-transformers", + "kind" : "remoteSourceControl", + "location" : "https://github.com/huggingface/swift-transformers.git", + "state" : { + "revision" : "150169bfba0889c229a2ce7494cf8949f18e6906", + "version" : "1.1.9" + } + }, + { + "identity" : "whisperkit", + "kind" : "remoteSourceControl", + "location" : "https://github.com/argmaxinc/WhisperKit.git", + "state" : { + "revision" : "e2adabbe7d98dc4d0ab9a5b75424ecc42a9cdbef", + "version" : "0.18.0" + } + }, + { + "identity" : "yyjson", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ibireme/yyjson.git", + "state" : { + "revision" : "8b4a38dc994a110abaec8a400615567bd996105f", + "version" : "0.12.0" + } + } + ], + "version" : 3 +} diff --git a/Package.swift b/Package.swift new file mode 100644 index 0000000..127d1c3 --- /dev/null +++ b/Package.swift @@ -0,0 +1,24 @@ +// swift-tools-version: 6.0 +import PackageDescription + +let package = Package( + name: "Sst", + platforms: [.macOS(.v14)], + dependencies: [ + .package(url: "https://github.com/argmaxinc/WhisperKit.git", from: "0.9.0"), + ], + targets: [ + .target(name: "SstCore"), + .target( + name: "SstWhisper", + dependencies: [ + "SstCore", + .product(name: "WhisperKit", package: "WhisperKit"), + ] + ), + .executableTarget(name: "Sst", dependencies: ["SstCore", "SstWhisper"]), + .executableTarget(name: "SstProbe", dependencies: ["SstCore", "SstWhisper"]), + .testTarget(name: "SstCoreTests", dependencies: ["SstCore"]), + ], + swiftLanguageModes: [.v5] +) diff --git a/Sources/Sst/AppModel.swift b/Sources/Sst/AppModel.swift new file mode 100644 index 0000000..64a56b7 --- /dev/null +++ b/Sources/Sst/AppModel.swift @@ -0,0 +1,103 @@ +import AVFoundation +import ApplicationServices +import SwiftUI +import SstCore +import SstWhisper + +@MainActor +final class AppModel: ObservableObject { + @Published var state: DictationState = .idle + @Published var modelStatus = "Model laden…" + @Published var modelReady = false + @Published var lastError: String? + @Published var micGranted = false + @Published var accessibilityGranted = false + @Published var inputMonitoringGranted = false + + private let transcriber = WhisperKitTranscriber() + private let hotkey = HotkeyMonitor() + private let controller: DictationController + + init() { + controller = DictationController( + recorder: AudioRecorder(), + transcriber: transcriber, + cleaner: PassthroughCleaner(), + inserter: TextInserter(pasteboard: SystemPasteboard(), paster: CGEventPaster()) + ) + controller.onStateChange = { [weak self] in + self?.state = $0 + if $0 == .recording { self?.lastError = nil } + } + controller.onError = { [weak self] in self?.lastError = $0 } + hotkey.onEvent = { [weak self] event in + guard let self, self.modelReady else { return } + if event == .pressed, AVCaptureDevice.authorizationStatus(for: .audio) != .authorized { + self.lastError = "Microfoon-toegang ontbreekt" + return + } + Task { await self.controller.handle(event) } + } + Task { await start() } + } + + var statusLine: String { + guard modelReady else { return modelStatus } + switch state { + case .idle: return "Klaar — houd Fn ingedrukt om te dicteren" + case .recording: return "Luistert…" + case .transcribing: return "Transcriberen…" + case .inserting: return "Invoegen…" + } + } + + var iconName: String { + switch state { + case .idle: + if lastError != nil { return "exclamationmark.triangle" } + return modelReady ? "mic" : "mic.slash" + case .recording: return "mic.fill" + case .transcribing, .inserting: return "waveform" + } + } + + func refreshPermissions() { + micGranted = AVCaptureDevice.authorizationStatus(for: .audio) == .authorized + accessibilityGranted = AXIsProcessTrusted() + inputMonitoringGranted = CGPreflightListenEventAccess() + } + + func retryHotkey() { + refreshPermissions() + do { + try hotkey.start() + lastError = nil + } catch { + lastError = error.localizedDescription + } + } + + private func start() async { + _ = await AVCaptureDevice.requestAccess(for: .audio) + if !CGPreflightListenEventAccess() { _ = CGRequestListenEventAccess() } + if !AXIsProcessTrusted() { + _ = AXIsProcessTrustedWithOptions(["AXTrustedCheckOptionPrompt": true] as CFDictionary) + } + retryHotkey() + transcriber.onProgress = { [weak self] fraction in + Task { @MainActor in + self?.modelStatus = fraction < 1 + ? "Model downloaden… \(Int(fraction * 100))%" + : "Model laden…" + } + } + do { + try await transcriber.load() + modelReady = true + modelStatus = "Klaar" + } catch { + modelStatus = "Model laden mislukt" + lastError = error.localizedDescription + } + } +} diff --git a/Sources/Sst/MenuContent.swift b/Sources/Sst/MenuContent.swift new file mode 100644 index 0000000..5d9b15f --- /dev/null +++ b/Sources/Sst/MenuContent.swift @@ -0,0 +1,34 @@ +import AppKit +import SwiftUI + +struct MenuContent: View { + @ObservedObject var model: AppModel + + var body: some View { + Text(model.statusLine) + if let error = model.lastError { + Text("Fout: \(error)") + } + Divider() + permissionRow("Microfoon", granted: model.micGranted, pane: "Privacy_Microphone") + permissionRow("Accessibility", granted: model.accessibilityGranted, pane: "Privacy_Accessibility") + permissionRow("Input Monitoring", granted: model.inputMonitoringGranted, pane: "Privacy_ListenEvent") + Button("Rechten opnieuw controleren") { model.retryHotkey() } + Divider() + Button("Stop") { NSApplication.shared.terminate(nil) } + .keyboardShortcut("q") + } + + @ViewBuilder + private func permissionRow(_ title: String, granted: Bool, pane: String) -> some View { + if granted { + Text("✓ \(title)") + } else { + Button("⚠︎ \(title) — open Instellingen") { + if let url = URL(string: "x-apple.systempreferences:com.apple.preference.security?\(pane)") { + NSWorkspace.shared.open(url) + } + } + } + } +} diff --git a/Sources/Sst/SstApp.swift b/Sources/Sst/SstApp.swift new file mode 100644 index 0000000..e4d86be --- /dev/null +++ b/Sources/Sst/SstApp.swift @@ -0,0 +1,14 @@ +import SwiftUI + +@main +struct SstApp: App { + @StateObject private var model = AppModel() + + var body: some Scene { + MenuBarExtra { + MenuContent(model: model) + } label: { + Image(systemName: model.iconName) + } + } +} diff --git a/Sources/SstCore/AudioRecorder.swift b/Sources/SstCore/AudioRecorder.swift new file mode 100644 index 0000000..b537465 --- /dev/null +++ b/Sources/SstCore/AudioRecorder.swift @@ -0,0 +1,73 @@ +import AVFoundation + +public final class AudioRecorder: AudioRecording { + public enum RecorderError: Error, LocalizedError { + case converterUnavailable + + public var errorDescription: String? { + "Kon de microfoon niet omzetten naar 16 kHz mono." + } + } + + private let engine = AVAudioEngine() + private let lock = NSLock() + private var buffer: [Float] = [] + private let targetFormat = AVAudioFormat( + commonFormat: .pcmFormatFloat32, + sampleRate: 16_000, + channels: 1, + interleaved: false + )! + + public init() {} + + public func start() throws { + lock.withLock { buffer.removeAll(keepingCapacity: true) } + + let input = engine.inputNode + let inputFormat = input.outputFormat(forBus: 0) + guard let converter = AVAudioConverter(from: inputFormat, to: targetFormat) else { + throw RecorderError.converterUnavailable + } + let ratio = targetFormat.sampleRate / inputFormat.sampleRate + + input.removeTap(onBus: 0) + input.installTap(onBus: 0, bufferSize: 4096, format: inputFormat) { [weak self] pcm, _ in + guard let self else { return } + let capacity = AVAudioFrameCount(Double(pcm.frameLength) * ratio) + 1 + guard let out = AVAudioPCMBuffer(pcmFormat: self.targetFormat, frameCapacity: capacity) else { return } + + var consumed = false + var error: NSError? + converter.convert(to: out, error: &error) { _, status in + if consumed { + status.pointee = .noDataNow + return nil + } + consumed = true + status.pointee = .haveData + return pcm + } + guard error == nil, let channel = out.floatChannelData?[0] else { return } + let samples = Array(UnsafeBufferPointer(start: channel, count: Int(out.frameLength))) + self.lock.withLock { self.buffer.append(contentsOf: samples) } + } + engine.prepare() + do { + try engine.start() + } catch { + input.removeTap(onBus: 0) + throw error + } + } + + public func stop() -> [Float] { + engine.inputNode.removeTap(onBus: 0) + engine.stop() + return lock.withLock { + let samples = buffer + buffer = [] + return samples + } + } +} diff --git a/Sources/SstCore/DictationController.swift b/Sources/SstCore/DictationController.swift new file mode 100644 index 0000000..5db278c --- /dev/null +++ b/Sources/SstCore/DictationController.swift @@ -0,0 +1,76 @@ +import Foundation + +@MainActor +public final class DictationController { + public private(set) var state: DictationState = .idle { + didSet { onStateChange?(state) } + } + public var onStateChange: ((DictationState) -> Void)? + public var onError: ((String) -> Void)? + + private let recorder: AudioRecording + private let transcriber: Transcriber + private let cleaner: Cleaner + private let inserter: TextInserting + private let minimumSamples: Int + private let silenceThreshold: Float + + /// - Parameter minimumSamples: 16 kHz samples; default is 0.3 s. + /// - Parameter silenceThreshold: RMS below which a recording counts as silence and is skipped. + public init( + recorder: AudioRecording, + transcriber: Transcriber, + cleaner: Cleaner, + inserter: TextInserting, + minimumSamples: Int = 4_800, + silenceThreshold: Float = 0.005 + ) { + self.recorder = recorder + self.transcriber = transcriber + self.cleaner = cleaner + self.inserter = inserter + self.minimumSamples = minimumSamples + self.silenceThreshold = silenceThreshold + } + + public func handle(_ event: HotkeyEvent) async { + switch (event, state) { + case (.pressed, .idle): + do { + try recorder.start() + state = .recording + } catch { + onError?(error.localizedDescription) + } + case (.released, .recording): + let samples = recorder.stop() + guard samples.count >= minimumSamples else { + state = .idle + return + } + let energy = samples.reduce(0) { $0 + $1 * $1 } + guard (energy / Float(samples.count)).squareRoot() >= silenceThreshold else { + state = .idle + return + } + await process(samples) + default: + break + } + } + + private func process(_ samples: [Float]) async { + defer { state = .idle } + do { + state = .transcribing + let raw = try await transcriber.transcribe(samples) + let text = raw.trimmingCharacters(in: .whitespacesAndNewlines) + guard !text.isEmpty else { return } + let cleaned = try await cleaner.clean(text, context: CleaningContext()) + state = .inserting + try await inserter.insert(cleaned) + } catch { + onError?(error.localizedDescription) + } + } +} diff --git a/Sources/SstCore/FnKeyTracker.swift b/Sources/SstCore/FnKeyTracker.swift new file mode 100644 index 0000000..2ba1e10 --- /dev/null +++ b/Sources/SstCore/FnKeyTracker.swift @@ -0,0 +1,12 @@ +public struct FnKeyTracker { + private var isDown = false + + public init() {} + + /// Feed the current Fn flag state; returns an event only on a transition. + public mutating func process(fnDown: Bool) -> HotkeyEvent? { + guard fnDown != isDown else { return nil } + isDown = fnDown + return fnDown ? .pressed : .released + } +} diff --git a/Sources/SstCore/HotkeyMonitor.swift b/Sources/SstCore/HotkeyMonitor.swift new file mode 100644 index 0000000..07d4f22 --- /dev/null +++ b/Sources/SstCore/HotkeyMonitor.swift @@ -0,0 +1,69 @@ +import ApplicationServices +import CoreGraphics + +public final class HotkeyMonitor { + public enum HotkeyError: Error, LocalizedError { + case tapUnavailable + + public var errorDescription: String? { + "Kan de Fn-toets niet volgen: geef Input Monitoring-rechten in Systeeminstellingen en probeer opnieuw (soms is een herstart van de app nodig)." + } + } + + public var onEvent: ((HotkeyEvent) -> Void)? + + private var tracker = FnKeyTracker() + private var tap: CFMachPort? + private var source: CFRunLoopSource? + + public init() {} + + public func start() throws { + guard tap == nil else { return } + let mask = CGEventMask(1 << CGEventType.flagsChanged.rawValue) + let callback: CGEventTapCallBack = { _, type, event, refcon in + if let refcon { + let monitor = Unmanaged.fromOpaque(refcon).takeUnretainedValue() + monitor.handle(type: type, event: event) + } + return Unmanaged.passUnretained(event) + } + guard let tap = CGEvent.tapCreate( + tap: .cgSessionEventTap, + place: .headInsertEventTap, + options: .listenOnly, + eventsOfInterest: mask, + callback: callback, + userInfo: Unmanaged.passUnretained(self).toOpaque() + ) else { throw HotkeyError.tapUnavailable } + + let source = CFMachPortCreateRunLoopSource(kCFAllocatorDefault, tap, 0) + CFRunLoopAddSource(CFRunLoopGetMain(), source, .commonModes) + CGEvent.tapEnable(tap: tap, enable: true) + self.tap = tap + self.source = source + } + + public func stop() { + if let source { CFRunLoopRemoveSource(CFRunLoopGetMain(), source, .commonModes) } + if let tap { CGEvent.tapEnable(tap: tap, enable: false) } + tap = nil + source = nil + } + + private func handle(type: CGEventType, event: CGEvent) { + switch type { + case .tapDisabledByTimeout, .tapDisabledByUserInput: + if let tap { CGEvent.tapEnable(tap: tap, enable: true) } + case .flagsChanged: + // .maskSecondaryFn is also set while arrows/F-keys are held; only react to the Fn key itself. + guard event.getIntegerValueField(.keyboardEventKeycode) == 63 else { return } + let fnDown = event.flags.contains(.maskSecondaryFn) + if let hotkeyEvent = tracker.process(fnDown: fnDown) { + onEvent?(hotkeyEvent) + } + default: + break + } + } +} diff --git a/Sources/SstCore/Models.swift b/Sources/SstCore/Models.swift new file mode 100644 index 0000000..36444a2 --- /dev/null +++ b/Sources/SstCore/Models.swift @@ -0,0 +1,35 @@ +import Foundation + +public enum HotkeyEvent: Equatable, Sendable { + case pressed + case released +} + +public enum DictationState: Equatable, Sendable { + case idle + case recording + case transcribing + case inserting +} + +public struct CleaningContext: Sendable { + public init() {} +} + +public protocol AudioRecording { + func start() throws + /// Stops recording and returns all captured 16 kHz mono Float samples. + func stop() -> [Float] +} + +public protocol Transcriber { + func transcribe(_ samples: [Float]) async throws -> String +} + +public protocol Cleaner { + func clean(_ text: String, context: CleaningContext) async throws -> String +} + +public protocol TextInserting { + func insert(_ text: String) async throws +} diff --git a/Sources/SstCore/PassthroughCleaner.swift b/Sources/SstCore/PassthroughCleaner.swift new file mode 100644 index 0000000..46ec88f --- /dev/null +++ b/Sources/SstCore/PassthroughCleaner.swift @@ -0,0 +1,7 @@ +public struct PassthroughCleaner: Cleaner { + public init() {} + + public func clean(_ text: String, context: CleaningContext) async throws -> String { + text + } +} diff --git a/Sources/SstCore/SystemPasteboard.swift b/Sources/SstCore/SystemPasteboard.swift new file mode 100644 index 0000000..629ccbf --- /dev/null +++ b/Sources/SstCore/SystemPasteboard.swift @@ -0,0 +1,65 @@ +import AppKit +import ApplicationServices + +public struct SystemPasteboard: PasteboardProviding { + public init() {} + + public func readItems() -> [PasteboardItem] { + (NSPasteboard.general.pasteboardItems ?? []).map { item in + var dict: PasteboardItem = [:] + for type in item.types { + if let data = item.data(forType: type) { dict[type.rawValue] = data } + } + return dict + } + } + + public func writeString(_ text: String) { + let pasteboard = NSPasteboard.general + pasteboard.clearContents() + pasteboard.setString(text, forType: .string) + } + + public func writeItems(_ items: [PasteboardItem]) { + let pasteboard = NSPasteboard.general + pasteboard.clearContents() + let restored = items.map { dict -> NSPasteboardItem in + let item = NSPasteboardItem() + for (type, data) in dict { item.setData(data, forType: NSPasteboard.PasteboardType(type)) } + return item + } + if !restored.isEmpty { pasteboard.writeObjects(restored) } + } +} + +public struct CGEventPaster: KeystrokePosting { + public enum PasteError: Error, LocalizedError { + case accessibilityDenied + case eventCreationFailed + + public var errorDescription: String? { + switch self { + case .accessibilityDenied: + "Geen Accessibility-rechten: tekst staat op het klembord, plak zelf met ⌘V." + case .eventCreationFailed: + "Kon het plak-event niet aanmaken: tekst staat op het klembord." + } + } + } + + public init() {} + + public func postPaste() throws { + guard AXIsProcessTrusted() else { throw PasteError.accessibilityDenied } + let source = CGEventSource(stateID: .combinedSessionState) + let vKey: CGKeyCode = 0x09 + guard + let down = CGEvent(keyboardEventSource: source, virtualKey: vKey, keyDown: true), + let up = CGEvent(keyboardEventSource: source, virtualKey: vKey, keyDown: false) + else { throw PasteError.eventCreationFailed } + down.flags = .maskCommand + up.flags = .maskCommand + down.post(tap: .cgAnnotatedSessionEventTap) + up.post(tap: .cgAnnotatedSessionEventTap) + } +} diff --git a/Sources/SstCore/TextInserter.swift b/Sources/SstCore/TextInserter.swift new file mode 100644 index 0000000..f852bc8 --- /dev/null +++ b/Sources/SstCore/TextInserter.swift @@ -0,0 +1,39 @@ +import Foundation + +public typealias PasteboardItem = [String: Data] + +public protocol PasteboardProviding { + func readItems() -> [PasteboardItem] + func writeString(_ text: String) + func writeItems(_ items: [PasteboardItem]) +} + +public protocol KeystrokePosting { + func postPaste() throws +} + +public struct TextInserter: TextInserting { + private let pasteboard: PasteboardProviding + private let paster: KeystrokePosting + private let restoreDelay: Duration + + public init( + pasteboard: PasteboardProviding, + paster: KeystrokePosting, + restoreDelay: Duration = .milliseconds(400) + ) { + self.pasteboard = pasteboard + self.paster = paster + self.restoreDelay = restoreDelay + } + + public func insert(_ text: String) async throws { + let saved = pasteboard.readItems() + pasteboard.writeString(text) + // On failure the text deliberately stays on the clipboard. + try paster.postPaste() + // Give the target app time to read the clipboard before restoring it. + try await Task.sleep(for: restoreDelay) + pasteboard.writeItems(saved) + } +} diff --git a/Sources/SstProbe/HotkeyProbe.swift b/Sources/SstProbe/HotkeyProbe.swift new file mode 100644 index 0000000..d052046 --- /dev/null +++ b/Sources/SstProbe/HotkeyProbe.swift @@ -0,0 +1,16 @@ +import Foundation +import CoreGraphics +import SstCore + +func runHotkeyProbe() -> Never { + if !CGPreflightListenEventAccess() { _ = CGRequestListenEventAccess() } + let monitor = HotkeyMonitor() + monitor.onEvent = { print("hotkey:", $0) } + do { try monitor.start() } catch { + print("failed:", error.localizedDescription) + exit(1) + } + print("Houd Fn ingedrukt en laat los. Ctrl-C om te stoppen.") + RunLoop.main.run() + exit(0) +} diff --git a/Sources/SstProbe/RecordProbe.swift b/Sources/SstProbe/RecordProbe.swift new file mode 100644 index 0000000..e8e3c3f --- /dev/null +++ b/Sources/SstProbe/RecordProbe.swift @@ -0,0 +1,24 @@ +import AVFoundation +import SstCore + +/// Records `seconds` of microphone audio; returns nil if access is denied or recording fails. +func recordSamples(seconds: Double) async -> [Float]? { + guard await AVCaptureDevice.requestAccess(for: .audio) else { + print("Microfoon-toegang geweigerd (Systeeminstellingen → Privacy → Microfoon).") + return nil + } + let recorder = AudioRecorder() + do { try recorder.start() } catch { + print("start mislukt:", error.localizedDescription) + return nil + } + print("Neem \(Int(seconds)) s op — praat nu") + try? await Task.sleep(for: .seconds(seconds)) + return recorder.stop() +} + +func runRecordProbe() async { + guard let samples = await recordSamples(seconds: 3) else { return } + let peak = samples.map { abs($0) }.max() ?? 0 + print("samples: \(samples.count) (verwacht ≈ 48000), piek: \(peak)") +} diff --git a/Sources/SstProbe/TranscribeProbe.swift b/Sources/SstProbe/TranscribeProbe.swift new file mode 100644 index 0000000..f678751 --- /dev/null +++ b/Sources/SstProbe/TranscribeProbe.swift @@ -0,0 +1,35 @@ +import Foundation +import SstWhisper + +func runTranscribeProbe() async { + print("Model laden (eerste keer: download ≈1.5 GB)…") + let transcriber = WhisperKitTranscriber() + do { + try await transcriber.load() + } catch { + print("mislukt:", error.localizedDescription) + return + } + print("Model klaar.") + for i in [3, 2, 1] { + print("\(i)...") + try? await Task.sleep(for: .seconds(1)) + } + guard let samples = await recordSamples(seconds: 8) else { return } + let peak = samples.map { abs($0) }.max() ?? 0 + print("samples: \(samples.count), piek: \(peak)") + let perSecond = stride(from: 0, to: samples.count, by: 16_000).map { start -> String in + let chunk = samples[start.. Void)? + + public init(model: String = "large-v3-v20240930_turbo") { + self.modelName = model + } + + /// Hugging Face hub root; WhisperKit stores models in `/models//`. + /// Kept in Application Support so it is neither in ~/Documents (TCC prompt, iCloud) nor re-fetched. + private static var storageBase: URL { + FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask)[0] + .appending(path: "Sst/models", directoryHint: .isDirectory) + } + + private static let repo = "argmaxinc/whisperkit-coreml" + + /// A previously downloaded variant folder, if it contains the compiled Core ML models. + private func localModelFolder(base: URL) -> URL? { + let repoDir = base.appending(path: "models/\(Self.repo)", directoryHint: .isDirectory) + let folders = (try? FileManager.default.contentsOfDirectory(atPath: repoDir.path)) ?? [] + return folders + .filter { $0.hasSuffix(modelName) } + .map { repoDir.appending(path: $0, directoryHint: .isDirectory) } + .first { folder in + ["MelSpectrogram", "AudioEncoder", "TextDecoder"].allSatisfy { + FileManager.default.fileExists( + atPath: folder.appending(path: "\($0).mlmodelc/coremldata.bin").path) + } + } + } + + private func loadPipeline(from folder: URL, base: URL) async throws -> WhisperKit { + try await WhisperKit(WhisperKitConfig( + downloadBase: base, + modelFolder: folder.path, + tokenizerFolder: base, + prewarm: true, + load: true, + download: false + )) + } + + /// Loads the model into memory. Downloads it (≈1.5 GB) only if it is not stored locally yet; + /// once downloaded, launching needs no network. + public func load() async throws { + guard pipeline == nil else { return } + let base = Self.storageBase + try FileManager.default.createDirectory(at: base, withIntermediateDirectories: true) + + if let folder = localModelFolder(base: base), + let loaded = try? await loadPipeline(from: folder, base: base) { + pipeline = loaded + return + } + let folder = try await WhisperKit.download( + variant: modelName, + downloadBase: base, + from: Self.repo + ) { [onProgress] progress in + onProgress?(progress.fractionCompleted) + } + pipeline = try await loadPipeline(from: folder, base: base) + } + + public func transcribe(_ samples: [Float]) async throws -> String { + guard let pipeline else { throw TranscriberError.notLoaded } + // language nil + detectLanguage: Whisper picks NL or EN per utterance. + let options = DecodingOptions( + task: .transcribe, + language: nil, + detectLanguage: true, + skipSpecialTokens: true, + withoutTimestamps: true + ) + let results = try await pipeline.transcribe(audioArray: samples, decodeOptions: options) + return results + .map(\.text) + .joined(separator: " ") + .trimmingCharacters(in: .whitespacesAndNewlines) + } +} diff --git a/Tests/SstCoreTests/DictationControllerTests.swift b/Tests/SstCoreTests/DictationControllerTests.swift new file mode 100644 index 0000000..0fa5329 --- /dev/null +++ b/Tests/SstCoreTests/DictationControllerTests.swift @@ -0,0 +1,138 @@ +import Testing +@testable import SstCore + +@MainActor +struct DictationControllerTests { + let recorder = FakeRecorder() + let transcriber = FakeTranscriber() + let cleaner = FakeCleaner() + let inserter = FakeInserter() + static let oneSecond = [Float](repeating: 0.1, count: 16_000) + + func makeController() -> DictationController { + DictationController( + recorder: recorder, + transcriber: transcriber, + cleaner: cleaner, + inserter: inserter + ) + } + + @Test func pressAndReleaseTranscribesCleansAndInserts() async { + recorder.samples = Self.oneSecond + let controller = makeController() + var states: [DictationState] = [] + controller.onStateChange = { states.append($0) } + + await controller.handle(.pressed) + await controller.handle(.released) + + #expect(states == [.recording, .transcribing, .inserting, .idle]) + #expect(inserter.inserted == ["[hallo wereld]"]) + } + + @Test func shortRecordingIsDiscarded() async { + recorder.samples = [Float](repeating: 0.1, count: 1_000) + let controller = makeController() + var states: [DictationState] = [] + controller.onStateChange = { states.append($0) } + + await controller.handle(.pressed) + await controller.handle(.released) + + #expect(states == [.recording, .idle]) + #expect(transcriber.callCount == 0) + #expect(inserter.inserted.isEmpty) + } + + @Test func silentRecordingIsDiscarded() async { + recorder.samples = [Float](repeating: 0, count: 16_000) + let controller = makeController() + var states: [DictationState] = [] + var errors: [String] = [] + controller.onStateChange = { states.append($0) } + controller.onError = { errors.append($0) } + + await controller.handle(.pressed) + await controller.handle(.released) + + #expect(states == [.recording, .idle]) + #expect(transcriber.callCount == 0) + #expect(inserter.inserted.isEmpty) + #expect(errors.isEmpty) + } + + @Test func emptyTranscriptionIsNotInserted() async { + recorder.samples = Self.oneSecond + transcriber.result = .success(" \n") + let controller = makeController() + var errors: [String] = [] + controller.onError = { errors.append($0) } + + await controller.handle(.pressed) + await controller.handle(.released) + + #expect(inserter.inserted.isEmpty) + #expect(errors.isEmpty) + #expect(controller.state == .idle) + } + + @Test func transcriberErrorReportsAndReturnsToIdle() async { + recorder.samples = Self.oneSecond + transcriber.result = .failure(TestError()) + let controller = makeController() + var errors: [String] = [] + controller.onError = { errors.append($0) } + + await controller.handle(.pressed) + await controller.handle(.released) + + #expect(errors.count == 1) + #expect(inserter.inserted.isEmpty) + #expect(controller.state == .idle) + } + + @Test func inserterErrorReportsAndReturnsToIdle() async { + recorder.samples = Self.oneSecond + inserter.error = TestError() + let controller = makeController() + var errors: [String] = [] + controller.onError = { errors.append($0) } + + await controller.handle(.pressed) + await controller.handle(.released) + + #expect(errors.count == 1) + #expect(controller.state == .idle) + } + + @Test func recorderStartFailureStaysIdle() async { + recorder.startError = TestError() + let controller = makeController() + var errors: [String] = [] + controller.onError = { errors.append($0) } + + await controller.handle(.pressed) + + #expect(errors.count == 1) + #expect(controller.state == .idle) + } + + @Test func secondPressWhileRecordingIsIgnored() async { + let controller = makeController() + await controller.handle(.pressed) + await controller.handle(.pressed) + #expect(recorder.startCount == 1) + } + + @Test func releaseWithoutPressIsIgnored() async { + let controller = makeController() + var states: [DictationState] = [] + controller.onStateChange = { states.append($0) } + + await controller.handle(.released) + + #expect(states.isEmpty) + #expect(transcriber.callCount == 0) + } +} diff --git a/Tests/SstCoreTests/Fakes.swift b/Tests/SstCoreTests/Fakes.swift new file mode 100644 index 0000000..f8b2cf7 --- /dev/null +++ b/Tests/SstCoreTests/Fakes.swift @@ -0,0 +1,43 @@ +import Foundation +@testable import SstCore + +struct TestError: Error {} + +final class FakeRecorder: AudioRecording { + var samples: [Float] = [] + var startError: Error? + private(set) var startCount = 0 + + func start() throws { + startCount += 1 + if let startError { throw startError } + } + + func stop() -> [Float] { samples } +} + +final class FakeTranscriber: Transcriber { + var result: Result = .success("hallo wereld") + private(set) var callCount = 0 + + func transcribe(_ samples: [Float]) async throws -> String { + callCount += 1 + return try result.get() + } +} + +final class FakeCleaner: Cleaner { + func clean(_ text: String, context: CleaningContext) async throws -> String { + "[\(text)]" + } +} + +final class FakeInserter: TextInserting { + var error: Error? + private(set) var inserted: [String] = [] + + func insert(_ text: String) async throws { + if let error { throw error } + inserted.append(text) + } +} diff --git a/Tests/SstCoreTests/FnKeyTrackerTests.swift b/Tests/SstCoreTests/FnKeyTrackerTests.swift new file mode 100644 index 0000000..fdef9eb --- /dev/null +++ b/Tests/SstCoreTests/FnKeyTrackerTests.swift @@ -0,0 +1,13 @@ +import Testing +@testable import SstCore + +struct FnKeyTrackerTests { + @Test func emitsPressedThenReleasedOnlyOnTransitions() { + var tracker = FnKeyTracker() + #expect(tracker.process(fnDown: false) == nil) + #expect(tracker.process(fnDown: true) == .pressed) + #expect(tracker.process(fnDown: true) == nil) + #expect(tracker.process(fnDown: false) == .released) + #expect(tracker.process(fnDown: false) == nil) + } +} diff --git a/Tests/SstCoreTests/TextInserterTests.swift b/Tests/SstCoreTests/TextInserterTests.swift new file mode 100644 index 0000000..9bf95bc --- /dev/null +++ b/Tests/SstCoreTests/TextInserterTests.swift @@ -0,0 +1,63 @@ +import Foundation +import Testing +@testable import SstCore + +final class FakePasteboard: PasteboardProviding { + var items: [PasteboardItem] = [] + + func readItems() -> [PasteboardItem] { items } + func writeString(_ text: String) { + items = [["public.utf8-plain-text": Data(text.utf8)]] + } + func writeItems(_ items: [PasteboardItem]) { self.items = items } +} + +final class FakePaster: KeystrokePosting { + var shouldFail = false + var onPaste: (() -> Void)? + private(set) var pasteCount = 0 + + func postPaste() throws { + pasteCount += 1 + if shouldFail { throw TestError() } + onPaste?() + } +} + +struct TextInserterTests { + let pasteboard = FakePasteboard() + let paster = FakePaster() + + func makeInserter() -> TextInserter { + TextInserter(pasteboard: pasteboard, paster: paster, restoreDelay: .zero) + } + + @Test func pastesTextThenRestoresPreviousClipboard() async throws { + let original: [PasteboardItem] = [["public.utf8-plain-text": Data("oud".utf8)]] + pasteboard.items = original + var seenAtPaste: Data? + paster.onPaste = { seenAtPaste = pasteboard.items.first?["public.utf8-plain-text"] } + + try await makeInserter().insert("hallo") + + #expect(seenAtPaste == Data("hallo".utf8)) + #expect(paster.pasteCount == 1) + #expect(pasteboard.items == original) + } + + @Test func failedPasteLeavesTextOnClipboard() async { + pasteboard.items = [["public.utf8-plain-text": Data("oud".utf8)]] + paster.shouldFail = true + + await #expect(throws: TestError.self) { + try await makeInserter().insert("hallo") + } + + #expect(pasteboard.items.first?["public.utf8-plain-text"] == Data("hallo".utf8)) + } + + @Test func passthroughCleanerReturnsTextUnchanged() async throws { + let result = try await PassthroughCleaner().clean("hoi", context: CleaningContext()) + #expect(result == "hoi") + } +} diff --git a/scripts/make-app.sh b/scripts/make-app.sh new file mode 100755 index 0000000..55b449a --- /dev/null +++ b/scripts/make-app.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." + +swift build -c release --product Sst +BIN=".build/release/Sst" +APP="build/Sst.app" + +rm -rf "$APP" +mkdir -p "$APP/Contents/MacOS" "$APP/Contents/Resources" +cp "$BIN" "$APP/Contents/MacOS/Sst" +# SwiftPM resource bundles (if any dependency ships them) live next to the binary. +for b in .build/release/*.bundle; do + [ -e "$b" ] && cp -R "$b" "$APP/Contents/Resources/" +done + +cat > "$APP/Contents/Info.plist" <<'PLIST' + + + + + CFBundleIdentifierlocal.sst.app + CFBundleNameSst + CFBundleExecutableSst + CFBundlePackageTypeAPPL + CFBundleShortVersionString0.1.0 + CFBundleVersion1 + LSMinimumSystemVersion14.0 + LSUIElement + NSMicrophoneUsageDescription + Sst neemt je stem op zolang je Fn ingedrukt houdt, om die lokaal naar tekst om te zetten. + + +PLIST + +codesign --force --sign - --identifier local.sst.app "$APP" +echo "Gebouwd: $APP" +echo "Na een herbouw: tccutil reset Accessibility local.sst.app; tccutil reset ListenEvent local.sst.app; tccutil reset Microphone local.sst.app (en geef de rechten opnieuw)."