Merge feat/dictation-skeleton: push-to-talk dictation skeleton
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 5.5
@@ -0,0 +1,4 @@
|
||||
.build/
|
||||
build/
|
||||
.DS_Store
|
||||
*.xcodeproj
|
||||
@@ -0,0 +1,78 @@
|
||||
{
|
||||
"originHash" : "e6c9264e0494528c9e107c3e12b51524d9f3fe957d63a07601ae76077888c441",
|
||||
"pins" : [
|
||||
{
|
||||
"identity" : "swift-argument-parser",
|
||||
"kind" : "remoteSourceControl",
|
||||
"location" : "https://github.com/apple/swift-argument-parser.git",
|
||||
"state" : {
|
||||
"revision" : "6a52f3251125d74daf04fcbd5e6f08a75d074382",
|
||||
"version" : "1.8.2"
|
||||
}
|
||||
},
|
||||
{
|
||||
"identity" : "swift-asn1",
|
||||
"kind" : "remoteSourceControl",
|
||||
"location" : "https://github.com/apple/swift-asn1.git",
|
||||
"state" : {
|
||||
"revision" : "3b6410f7dee09eb33cdd26260c5fd47fda19b0e2",
|
||||
"version" : "1.7.3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"identity" : "swift-collections",
|
||||
"kind" : "remoteSourceControl",
|
||||
"location" : "https://github.com/apple/swift-collections.git",
|
||||
"state" : {
|
||||
"revision" : "98ef3c98609a1e31b7e157b5b619579001a789d6",
|
||||
"version" : "1.7.1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"identity" : "swift-crypto",
|
||||
"kind" : "remoteSourceControl",
|
||||
"location" : "https://github.com/apple/swift-crypto.git",
|
||||
"state" : {
|
||||
"revision" : "da9d28d69ebe3894b18376c8f2395c2f37b8448f",
|
||||
"version" : "4.5.2"
|
||||
}
|
||||
},
|
||||
{
|
||||
"identity" : "swift-jinja",
|
||||
"kind" : "remoteSourceControl",
|
||||
"location" : "https://github.com/huggingface/swift-jinja.git",
|
||||
"state" : {
|
||||
"revision" : "4588064a20f3fc093c95f2f7d3359999bf30cae5",
|
||||
"version" : "2.5.1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"identity" : "swift-transformers",
|
||||
"kind" : "remoteSourceControl",
|
||||
"location" : "https://github.com/huggingface/swift-transformers.git",
|
||||
"state" : {
|
||||
"revision" : "150169bfba0889c229a2ce7494cf8949f18e6906",
|
||||
"version" : "1.1.9"
|
||||
}
|
||||
},
|
||||
{
|
||||
"identity" : "whisperkit",
|
||||
"kind" : "remoteSourceControl",
|
||||
"location" : "https://github.com/argmaxinc/WhisperKit.git",
|
||||
"state" : {
|
||||
"revision" : "e2adabbe7d98dc4d0ab9a5b75424ecc42a9cdbef",
|
||||
"version" : "0.18.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"identity" : "yyjson",
|
||||
"kind" : "remoteSourceControl",
|
||||
"location" : "https://github.com/ibireme/yyjson.git",
|
||||
"state" : {
|
||||
"revision" : "8b4a38dc994a110abaec8a400615567bd996105f",
|
||||
"version" : "0.12.0"
|
||||
}
|
||||
}
|
||||
],
|
||||
"version" : 3
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
// swift-tools-version: 6.0
|
||||
import PackageDescription
|
||||
|
||||
let package = Package(
|
||||
name: "Sst",
|
||||
platforms: [.macOS(.v14)],
|
||||
dependencies: [
|
||||
.package(url: "https://github.com/argmaxinc/WhisperKit.git", from: "0.9.0"),
|
||||
],
|
||||
targets: [
|
||||
.target(name: "SstCore"),
|
||||
.target(
|
||||
name: "SstWhisper",
|
||||
dependencies: [
|
||||
"SstCore",
|
||||
.product(name: "WhisperKit", package: "WhisperKit"),
|
||||
]
|
||||
),
|
||||
.executableTarget(name: "Sst", dependencies: ["SstCore", "SstWhisper"]),
|
||||
.executableTarget(name: "SstProbe", dependencies: ["SstCore", "SstWhisper"]),
|
||||
.testTarget(name: "SstCoreTests", dependencies: ["SstCore"]),
|
||||
],
|
||||
swiftLanguageModes: [.v5]
|
||||
)
|
||||
@@ -0,0 +1,103 @@
|
||||
import AVFoundation
|
||||
import ApplicationServices
|
||||
import SwiftUI
|
||||
import SstCore
|
||||
import SstWhisper
|
||||
|
||||
@MainActor
|
||||
final class AppModel: ObservableObject {
|
||||
@Published var state: DictationState = .idle
|
||||
@Published var modelStatus = "Model laden…"
|
||||
@Published var modelReady = false
|
||||
@Published var lastError: String?
|
||||
@Published var micGranted = false
|
||||
@Published var accessibilityGranted = false
|
||||
@Published var inputMonitoringGranted = false
|
||||
|
||||
private let transcriber = WhisperKitTranscriber()
|
||||
private let hotkey = HotkeyMonitor()
|
||||
private let controller: DictationController
|
||||
|
||||
init() {
|
||||
controller = DictationController(
|
||||
recorder: AudioRecorder(),
|
||||
transcriber: transcriber,
|
||||
cleaner: PassthroughCleaner(),
|
||||
inserter: TextInserter(pasteboard: SystemPasteboard(), paster: CGEventPaster())
|
||||
)
|
||||
controller.onStateChange = { [weak self] in
|
||||
self?.state = $0
|
||||
if $0 == .recording { self?.lastError = nil }
|
||||
}
|
||||
controller.onError = { [weak self] in self?.lastError = $0 }
|
||||
hotkey.onEvent = { [weak self] event in
|
||||
guard let self, self.modelReady else { return }
|
||||
if event == .pressed, AVCaptureDevice.authorizationStatus(for: .audio) != .authorized {
|
||||
self.lastError = "Microfoon-toegang ontbreekt"
|
||||
return
|
||||
}
|
||||
Task { await self.controller.handle(event) }
|
||||
}
|
||||
Task { await start() }
|
||||
}
|
||||
|
||||
var statusLine: String {
|
||||
guard modelReady else { return modelStatus }
|
||||
switch state {
|
||||
case .idle: return "Klaar — houd Fn ingedrukt om te dicteren"
|
||||
case .recording: return "Luistert…"
|
||||
case .transcribing: return "Transcriberen…"
|
||||
case .inserting: return "Invoegen…"
|
||||
}
|
||||
}
|
||||
|
||||
var iconName: String {
|
||||
switch state {
|
||||
case .idle:
|
||||
if lastError != nil { return "exclamationmark.triangle" }
|
||||
return modelReady ? "mic" : "mic.slash"
|
||||
case .recording: return "mic.fill"
|
||||
case .transcribing, .inserting: return "waveform"
|
||||
}
|
||||
}
|
||||
|
||||
func refreshPermissions() {
|
||||
micGranted = AVCaptureDevice.authorizationStatus(for: .audio) == .authorized
|
||||
accessibilityGranted = AXIsProcessTrusted()
|
||||
inputMonitoringGranted = CGPreflightListenEventAccess()
|
||||
}
|
||||
|
||||
func retryHotkey() {
|
||||
refreshPermissions()
|
||||
do {
|
||||
try hotkey.start()
|
||||
lastError = nil
|
||||
} catch {
|
||||
lastError = error.localizedDescription
|
||||
}
|
||||
}
|
||||
|
||||
private func start() async {
|
||||
_ = await AVCaptureDevice.requestAccess(for: .audio)
|
||||
if !CGPreflightListenEventAccess() { _ = CGRequestListenEventAccess() }
|
||||
if !AXIsProcessTrusted() {
|
||||
_ = AXIsProcessTrustedWithOptions(["AXTrustedCheckOptionPrompt": true] as CFDictionary)
|
||||
}
|
||||
retryHotkey()
|
||||
transcriber.onProgress = { [weak self] fraction in
|
||||
Task { @MainActor in
|
||||
self?.modelStatus = fraction < 1
|
||||
? "Model downloaden… \(Int(fraction * 100))%"
|
||||
: "Model laden…"
|
||||
}
|
||||
}
|
||||
do {
|
||||
try await transcriber.load()
|
||||
modelReady = true
|
||||
modelStatus = "Klaar"
|
||||
} catch {
|
||||
modelStatus = "Model laden mislukt"
|
||||
lastError = error.localizedDescription
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import AppKit
|
||||
import SwiftUI
|
||||
|
||||
struct MenuContent: View {
|
||||
@ObservedObject var model: AppModel
|
||||
|
||||
var body: some View {
|
||||
Text(model.statusLine)
|
||||
if let error = model.lastError {
|
||||
Text("Fout: \(error)")
|
||||
}
|
||||
Divider()
|
||||
permissionRow("Microfoon", granted: model.micGranted, pane: "Privacy_Microphone")
|
||||
permissionRow("Accessibility", granted: model.accessibilityGranted, pane: "Privacy_Accessibility")
|
||||
permissionRow("Input Monitoring", granted: model.inputMonitoringGranted, pane: "Privacy_ListenEvent")
|
||||
Button("Rechten opnieuw controleren") { model.retryHotkey() }
|
||||
Divider()
|
||||
Button("Stop") { NSApplication.shared.terminate(nil) }
|
||||
.keyboardShortcut("q")
|
||||
}
|
||||
|
||||
@ViewBuilder
|
||||
private func permissionRow(_ title: String, granted: Bool, pane: String) -> some View {
|
||||
if granted {
|
||||
Text("✓ \(title)")
|
||||
} else {
|
||||
Button("⚠︎ \(title) — open Instellingen") {
|
||||
if let url = URL(string: "x-apple.systempreferences:com.apple.preference.security?\(pane)") {
|
||||
NSWorkspace.shared.open(url)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
import SwiftUI
|
||||
|
||||
@main
|
||||
struct SstApp: App {
|
||||
@StateObject private var model = AppModel()
|
||||
|
||||
var body: some Scene {
|
||||
MenuBarExtra {
|
||||
MenuContent(model: model)
|
||||
} label: {
|
||||
Image(systemName: model.iconName)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
import AVFoundation
|
||||
|
||||
public final class AudioRecorder: AudioRecording {
|
||||
public enum RecorderError: Error, LocalizedError {
|
||||
case converterUnavailable
|
||||
|
||||
public var errorDescription: String? {
|
||||
"Kon de microfoon niet omzetten naar 16 kHz mono."
|
||||
}
|
||||
}
|
||||
|
||||
private let engine = AVAudioEngine()
|
||||
private let lock = NSLock()
|
||||
private var buffer: [Float] = []
|
||||
private let targetFormat = AVAudioFormat(
|
||||
commonFormat: .pcmFormatFloat32,
|
||||
sampleRate: 16_000,
|
||||
channels: 1,
|
||||
interleaved: false
|
||||
)!
|
||||
|
||||
public init() {}
|
||||
|
||||
public func start() throws {
|
||||
lock.withLock { buffer.removeAll(keepingCapacity: true) }
|
||||
|
||||
let input = engine.inputNode
|
||||
let inputFormat = input.outputFormat(forBus: 0)
|
||||
guard let converter = AVAudioConverter(from: inputFormat, to: targetFormat) else {
|
||||
throw RecorderError.converterUnavailable
|
||||
}
|
||||
let ratio = targetFormat.sampleRate / inputFormat.sampleRate
|
||||
|
||||
input.removeTap(onBus: 0)
|
||||
input.installTap(onBus: 0, bufferSize: 4096, format: inputFormat) { [weak self] pcm, _ in
|
||||
guard let self else { return }
|
||||
let capacity = AVAudioFrameCount(Double(pcm.frameLength) * ratio) + 1
|
||||
guard let out = AVAudioPCMBuffer(pcmFormat: self.targetFormat, frameCapacity: capacity) else { return }
|
||||
|
||||
var consumed = false
|
||||
var error: NSError?
|
||||
converter.convert(to: out, error: &error) { _, status in
|
||||
if consumed {
|
||||
status.pointee = .noDataNow
|
||||
return nil
|
||||
}
|
||||
consumed = true
|
||||
status.pointee = .haveData
|
||||
return pcm
|
||||
}
|
||||
guard error == nil, let channel = out.floatChannelData?[0] else { return }
|
||||
let samples = Array(UnsafeBufferPointer(start: channel, count: Int(out.frameLength)))
|
||||
self.lock.withLock { self.buffer.append(contentsOf: samples) }
|
||||
}
|
||||
engine.prepare()
|
||||
do {
|
||||
try engine.start()
|
||||
} catch {
|
||||
input.removeTap(onBus: 0)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
public func stop() -> [Float] {
|
||||
engine.inputNode.removeTap(onBus: 0)
|
||||
engine.stop()
|
||||
return lock.withLock {
|
||||
let samples = buffer
|
||||
buffer = []
|
||||
return samples
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
import Foundation
|
||||
|
||||
@MainActor
|
||||
public final class DictationController {
|
||||
public private(set) var state: DictationState = .idle {
|
||||
didSet { onStateChange?(state) }
|
||||
}
|
||||
public var onStateChange: ((DictationState) -> Void)?
|
||||
public var onError: ((String) -> Void)?
|
||||
|
||||
private let recorder: AudioRecording
|
||||
private let transcriber: Transcriber
|
||||
private let cleaner: Cleaner
|
||||
private let inserter: TextInserting
|
||||
private let minimumSamples: Int
|
||||
private let silenceThreshold: Float
|
||||
|
||||
/// - Parameter minimumSamples: 16 kHz samples; default is 0.3 s.
|
||||
/// - Parameter silenceThreshold: RMS below which a recording counts as silence and is skipped.
|
||||
public init(
|
||||
recorder: AudioRecording,
|
||||
transcriber: Transcriber,
|
||||
cleaner: Cleaner,
|
||||
inserter: TextInserting,
|
||||
minimumSamples: Int = 4_800,
|
||||
silenceThreshold: Float = 0.005
|
||||
) {
|
||||
self.recorder = recorder
|
||||
self.transcriber = transcriber
|
||||
self.cleaner = cleaner
|
||||
self.inserter = inserter
|
||||
self.minimumSamples = minimumSamples
|
||||
self.silenceThreshold = silenceThreshold
|
||||
}
|
||||
|
||||
public func handle(_ event: HotkeyEvent) async {
|
||||
switch (event, state) {
|
||||
case (.pressed, .idle):
|
||||
do {
|
||||
try recorder.start()
|
||||
state = .recording
|
||||
} catch {
|
||||
onError?(error.localizedDescription)
|
||||
}
|
||||
case (.released, .recording):
|
||||
let samples = recorder.stop()
|
||||
guard samples.count >= minimumSamples else {
|
||||
state = .idle
|
||||
return
|
||||
}
|
||||
let energy = samples.reduce(0) { $0 + $1 * $1 }
|
||||
guard (energy / Float(samples.count)).squareRoot() >= silenceThreshold else {
|
||||
state = .idle
|
||||
return
|
||||
}
|
||||
await process(samples)
|
||||
default:
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
private func process(_ samples: [Float]) async {
|
||||
defer { state = .idle }
|
||||
do {
|
||||
state = .transcribing
|
||||
let raw = try await transcriber.transcribe(samples)
|
||||
let text = raw.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !text.isEmpty else { return }
|
||||
let cleaned = try await cleaner.clean(text, context: CleaningContext())
|
||||
state = .inserting
|
||||
try await inserter.insert(cleaned)
|
||||
} catch {
|
||||
onError?(error.localizedDescription)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
public struct FnKeyTracker {
|
||||
private var isDown = false
|
||||
|
||||
public init() {}
|
||||
|
||||
/// Feed the current Fn flag state; returns an event only on a transition.
|
||||
public mutating func process(fnDown: Bool) -> HotkeyEvent? {
|
||||
guard fnDown != isDown else { return nil }
|
||||
isDown = fnDown
|
||||
return fnDown ? .pressed : .released
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
import ApplicationServices
|
||||
import CoreGraphics
|
||||
|
||||
public final class HotkeyMonitor {
|
||||
public enum HotkeyError: Error, LocalizedError {
|
||||
case tapUnavailable
|
||||
|
||||
public var errorDescription: String? {
|
||||
"Kan de Fn-toets niet volgen: geef Input Monitoring-rechten in Systeeminstellingen en probeer opnieuw (soms is een herstart van de app nodig)."
|
||||
}
|
||||
}
|
||||
|
||||
public var onEvent: ((HotkeyEvent) -> Void)?
|
||||
|
||||
private var tracker = FnKeyTracker()
|
||||
private var tap: CFMachPort?
|
||||
private var source: CFRunLoopSource?
|
||||
|
||||
public init() {}
|
||||
|
||||
public func start() throws {
|
||||
guard tap == nil else { return }
|
||||
let mask = CGEventMask(1 << CGEventType.flagsChanged.rawValue)
|
||||
let callback: CGEventTapCallBack = { _, type, event, refcon in
|
||||
if let refcon {
|
||||
let monitor = Unmanaged<HotkeyMonitor>.fromOpaque(refcon).takeUnretainedValue()
|
||||
monitor.handle(type: type, event: event)
|
||||
}
|
||||
return Unmanaged.passUnretained(event)
|
||||
}
|
||||
guard let tap = CGEvent.tapCreate(
|
||||
tap: .cgSessionEventTap,
|
||||
place: .headInsertEventTap,
|
||||
options: .listenOnly,
|
||||
eventsOfInterest: mask,
|
||||
callback: callback,
|
||||
userInfo: Unmanaged.passUnretained(self).toOpaque()
|
||||
) else { throw HotkeyError.tapUnavailable }
|
||||
|
||||
let source = CFMachPortCreateRunLoopSource(kCFAllocatorDefault, tap, 0)
|
||||
CFRunLoopAddSource(CFRunLoopGetMain(), source, .commonModes)
|
||||
CGEvent.tapEnable(tap: tap, enable: true)
|
||||
self.tap = tap
|
||||
self.source = source
|
||||
}
|
||||
|
||||
public func stop() {
|
||||
if let source { CFRunLoopRemoveSource(CFRunLoopGetMain(), source, .commonModes) }
|
||||
if let tap { CGEvent.tapEnable(tap: tap, enable: false) }
|
||||
tap = nil
|
||||
source = nil
|
||||
}
|
||||
|
||||
private func handle(type: CGEventType, event: CGEvent) {
|
||||
switch type {
|
||||
case .tapDisabledByTimeout, .tapDisabledByUserInput:
|
||||
if let tap { CGEvent.tapEnable(tap: tap, enable: true) }
|
||||
case .flagsChanged:
|
||||
// .maskSecondaryFn is also set while arrows/F-keys are held; only react to the Fn key itself.
|
||||
guard event.getIntegerValueField(.keyboardEventKeycode) == 63 else { return }
|
||||
let fnDown = event.flags.contains(.maskSecondaryFn)
|
||||
if let hotkeyEvent = tracker.process(fnDown: fnDown) {
|
||||
onEvent?(hotkeyEvent)
|
||||
}
|
||||
default:
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import Foundation
|
||||
|
||||
public enum HotkeyEvent: Equatable, Sendable {
|
||||
case pressed
|
||||
case released
|
||||
}
|
||||
|
||||
public enum DictationState: Equatable, Sendable {
|
||||
case idle
|
||||
case recording
|
||||
case transcribing
|
||||
case inserting
|
||||
}
|
||||
|
||||
public struct CleaningContext: Sendable {
|
||||
public init() {}
|
||||
}
|
||||
|
||||
public protocol AudioRecording {
|
||||
func start() throws
|
||||
/// Stops recording and returns all captured 16 kHz mono Float samples.
|
||||
func stop() -> [Float]
|
||||
}
|
||||
|
||||
public protocol Transcriber {
|
||||
func transcribe(_ samples: [Float]) async throws -> String
|
||||
}
|
||||
|
||||
public protocol Cleaner {
|
||||
func clean(_ text: String, context: CleaningContext) async throws -> String
|
||||
}
|
||||
|
||||
public protocol TextInserting {
|
||||
func insert(_ text: String) async throws
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
public struct PassthroughCleaner: Cleaner {
|
||||
public init() {}
|
||||
|
||||
public func clean(_ text: String, context: CleaningContext) async throws -> String {
|
||||
text
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
import AppKit
|
||||
import ApplicationServices
|
||||
|
||||
public struct SystemPasteboard: PasteboardProviding {
|
||||
public init() {}
|
||||
|
||||
public func readItems() -> [PasteboardItem] {
|
||||
(NSPasteboard.general.pasteboardItems ?? []).map { item in
|
||||
var dict: PasteboardItem = [:]
|
||||
for type in item.types {
|
||||
if let data = item.data(forType: type) { dict[type.rawValue] = data }
|
||||
}
|
||||
return dict
|
||||
}
|
||||
}
|
||||
|
||||
public func writeString(_ text: String) {
|
||||
let pasteboard = NSPasteboard.general
|
||||
pasteboard.clearContents()
|
||||
pasteboard.setString(text, forType: .string)
|
||||
}
|
||||
|
||||
public func writeItems(_ items: [PasteboardItem]) {
|
||||
let pasteboard = NSPasteboard.general
|
||||
pasteboard.clearContents()
|
||||
let restored = items.map { dict -> NSPasteboardItem in
|
||||
let item = NSPasteboardItem()
|
||||
for (type, data) in dict { item.setData(data, forType: NSPasteboard.PasteboardType(type)) }
|
||||
return item
|
||||
}
|
||||
if !restored.isEmpty { pasteboard.writeObjects(restored) }
|
||||
}
|
||||
}
|
||||
|
||||
public struct CGEventPaster: KeystrokePosting {
|
||||
public enum PasteError: Error, LocalizedError {
|
||||
case accessibilityDenied
|
||||
case eventCreationFailed
|
||||
|
||||
public var errorDescription: String? {
|
||||
switch self {
|
||||
case .accessibilityDenied:
|
||||
"Geen Accessibility-rechten: tekst staat op het klembord, plak zelf met ⌘V."
|
||||
case .eventCreationFailed:
|
||||
"Kon het plak-event niet aanmaken: tekst staat op het klembord."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public init() {}
|
||||
|
||||
public func postPaste() throws {
|
||||
guard AXIsProcessTrusted() else { throw PasteError.accessibilityDenied }
|
||||
let source = CGEventSource(stateID: .combinedSessionState)
|
||||
let vKey: CGKeyCode = 0x09
|
||||
guard
|
||||
let down = CGEvent(keyboardEventSource: source, virtualKey: vKey, keyDown: true),
|
||||
let up = CGEvent(keyboardEventSource: source, virtualKey: vKey, keyDown: false)
|
||||
else { throw PasteError.eventCreationFailed }
|
||||
down.flags = .maskCommand
|
||||
up.flags = .maskCommand
|
||||
down.post(tap: .cgAnnotatedSessionEventTap)
|
||||
up.post(tap: .cgAnnotatedSessionEventTap)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
import Foundation
|
||||
|
||||
public typealias PasteboardItem = [String: Data]
|
||||
|
||||
public protocol PasteboardProviding {
|
||||
func readItems() -> [PasteboardItem]
|
||||
func writeString(_ text: String)
|
||||
func writeItems(_ items: [PasteboardItem])
|
||||
}
|
||||
|
||||
public protocol KeystrokePosting {
|
||||
func postPaste() throws
|
||||
}
|
||||
|
||||
public struct TextInserter: TextInserting {
|
||||
private let pasteboard: PasteboardProviding
|
||||
private let paster: KeystrokePosting
|
||||
private let restoreDelay: Duration
|
||||
|
||||
public init(
|
||||
pasteboard: PasteboardProviding,
|
||||
paster: KeystrokePosting,
|
||||
restoreDelay: Duration = .milliseconds(400)
|
||||
) {
|
||||
self.pasteboard = pasteboard
|
||||
self.paster = paster
|
||||
self.restoreDelay = restoreDelay
|
||||
}
|
||||
|
||||
public func insert(_ text: String) async throws {
|
||||
let saved = pasteboard.readItems()
|
||||
pasteboard.writeString(text)
|
||||
// On failure the text deliberately stays on the clipboard.
|
||||
try paster.postPaste()
|
||||
// Give the target app time to read the clipboard before restoring it.
|
||||
try await Task.sleep(for: restoreDelay)
|
||||
pasteboard.writeItems(saved)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import Foundation
|
||||
import CoreGraphics
|
||||
import SstCore
|
||||
|
||||
func runHotkeyProbe() -> Never {
|
||||
if !CGPreflightListenEventAccess() { _ = CGRequestListenEventAccess() }
|
||||
let monitor = HotkeyMonitor()
|
||||
monitor.onEvent = { print("hotkey:", $0) }
|
||||
do { try monitor.start() } catch {
|
||||
print("failed:", error.localizedDescription)
|
||||
exit(1)
|
||||
}
|
||||
print("Houd Fn ingedrukt en laat los. Ctrl-C om te stoppen.")
|
||||
RunLoop.main.run()
|
||||
exit(0)
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import AVFoundation
|
||||
import SstCore
|
||||
|
||||
/// Records `seconds` of microphone audio; returns nil if access is denied or recording fails.
|
||||
func recordSamples(seconds: Double) async -> [Float]? {
|
||||
guard await AVCaptureDevice.requestAccess(for: .audio) else {
|
||||
print("Microfoon-toegang geweigerd (Systeeminstellingen → Privacy → Microfoon).")
|
||||
return nil
|
||||
}
|
||||
let recorder = AudioRecorder()
|
||||
do { try recorder.start() } catch {
|
||||
print("start mislukt:", error.localizedDescription)
|
||||
return nil
|
||||
}
|
||||
print("Neem \(Int(seconds)) s op — praat nu")
|
||||
try? await Task.sleep(for: .seconds(seconds))
|
||||
return recorder.stop()
|
||||
}
|
||||
|
||||
func runRecordProbe() async {
|
||||
guard let samples = await recordSamples(seconds: 3) else { return }
|
||||
let peak = samples.map { abs($0) }.max() ?? 0
|
||||
print("samples: \(samples.count) (verwacht ≈ 48000), piek: \(peak)")
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import Foundation
|
||||
import SstWhisper
|
||||
|
||||
func runTranscribeProbe() async {
|
||||
print("Model laden (eerste keer: download ≈1.5 GB)…")
|
||||
let transcriber = WhisperKitTranscriber()
|
||||
do {
|
||||
try await transcriber.load()
|
||||
} catch {
|
||||
print("mislukt:", error.localizedDescription)
|
||||
return
|
||||
}
|
||||
print("Model klaar.")
|
||||
for i in [3, 2, 1] {
|
||||
print("\(i)...")
|
||||
try? await Task.sleep(for: .seconds(1))
|
||||
}
|
||||
guard let samples = await recordSamples(seconds: 8) else { return }
|
||||
let peak = samples.map { abs($0) }.max() ?? 0
|
||||
print("samples: \(samples.count), piek: \(peak)")
|
||||
let perSecond = stride(from: 0, to: samples.count, by: 16_000).map { start -> String in
|
||||
let chunk = samples[start..<min(start + 16_000, samples.count)]
|
||||
let rms = (chunk.reduce(0) { $0 + $1 * $1 } / Float(chunk.count)).squareRoot()
|
||||
return String(format: "%.3f", rms)
|
||||
}
|
||||
print("RMS per seconde:", perSecond.joined(separator: " "))
|
||||
do {
|
||||
let started = Date.now
|
||||
let text = try await transcriber.transcribe(samples)
|
||||
let elapsed = Date.now.timeIntervalSince(started)
|
||||
print(String(format: "(%.2f s) ", elapsed) + text)
|
||||
} catch {
|
||||
print("mislukt:", error.localizedDescription)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import Foundation
|
||||
|
||||
let command = CommandLine.arguments.dropFirst().first ?? "help"
|
||||
switch command {
|
||||
case "hotkey":
|
||||
runHotkeyProbe()
|
||||
case "record":
|
||||
await runRecordProbe()
|
||||
case "transcribe":
|
||||
await runTranscribeProbe()
|
||||
default:
|
||||
print("usage: sst-probe hotkey|record|transcribe")
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
import Foundation
|
||||
import SstCore
|
||||
import WhisperKit
|
||||
|
||||
public final class WhisperKitTranscriber: Transcriber {
|
||||
public enum TranscriberError: Error, LocalizedError {
|
||||
case notLoaded
|
||||
|
||||
public var errorDescription: String? { "Spraakmodel is nog niet geladen." }
|
||||
}
|
||||
|
||||
private let modelName: String
|
||||
private var pipeline: WhisperKit?
|
||||
|
||||
/// Download progress (0...1) during the one-time model download. May be called from any thread.
|
||||
public var onProgress: ((Double) -> Void)?
|
||||
|
||||
public init(model: String = "large-v3-v20240930_turbo") {
|
||||
self.modelName = model
|
||||
}
|
||||
|
||||
/// Hugging Face hub root; WhisperKit stores models in `<root>/models/<repo>/<variant>`.
|
||||
/// Kept in Application Support so it is neither in ~/Documents (TCC prompt, iCloud) nor re-fetched.
|
||||
private static var storageBase: URL {
|
||||
FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask)[0]
|
||||
.appending(path: "Sst/models", directoryHint: .isDirectory)
|
||||
}
|
||||
|
||||
private static let repo = "argmaxinc/whisperkit-coreml"
|
||||
|
||||
/// A previously downloaded variant folder, if it contains the compiled Core ML models.
|
||||
private func localModelFolder(base: URL) -> URL? {
|
||||
let repoDir = base.appending(path: "models/\(Self.repo)", directoryHint: .isDirectory)
|
||||
let folders = (try? FileManager.default.contentsOfDirectory(atPath: repoDir.path)) ?? []
|
||||
return folders
|
||||
.filter { $0.hasSuffix(modelName) }
|
||||
.map { repoDir.appending(path: $0, directoryHint: .isDirectory) }
|
||||
.first { folder in
|
||||
["MelSpectrogram", "AudioEncoder", "TextDecoder"].allSatisfy {
|
||||
FileManager.default.fileExists(
|
||||
atPath: folder.appending(path: "\($0).mlmodelc/coremldata.bin").path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func loadPipeline(from folder: URL, base: URL) async throws -> WhisperKit {
|
||||
try await WhisperKit(WhisperKitConfig(
|
||||
downloadBase: base,
|
||||
modelFolder: folder.path,
|
||||
tokenizerFolder: base,
|
||||
prewarm: true,
|
||||
load: true,
|
||||
download: false
|
||||
))
|
||||
}
|
||||
|
||||
/// Loads the model into memory. Downloads it (≈1.5 GB) only if it is not stored locally yet;
|
||||
/// once downloaded, launching needs no network.
|
||||
public func load() async throws {
|
||||
guard pipeline == nil else { return }
|
||||
let base = Self.storageBase
|
||||
try FileManager.default.createDirectory(at: base, withIntermediateDirectories: true)
|
||||
|
||||
if let folder = localModelFolder(base: base),
|
||||
let loaded = try? await loadPipeline(from: folder, base: base) {
|
||||
pipeline = loaded
|
||||
return
|
||||
}
|
||||
let folder = try await WhisperKit.download(
|
||||
variant: modelName,
|
||||
downloadBase: base,
|
||||
from: Self.repo
|
||||
) { [onProgress] progress in
|
||||
onProgress?(progress.fractionCompleted)
|
||||
}
|
||||
pipeline = try await loadPipeline(from: folder, base: base)
|
||||
}
|
||||
|
||||
public func transcribe(_ samples: [Float]) async throws -> String {
|
||||
guard let pipeline else { throw TranscriberError.notLoaded }
|
||||
// language nil + detectLanguage: Whisper picks NL or EN per utterance.
|
||||
let options = DecodingOptions(
|
||||
task: .transcribe,
|
||||
language: nil,
|
||||
detectLanguage: true,
|
||||
skipSpecialTokens: true,
|
||||
withoutTimestamps: true
|
||||
)
|
||||
let results = try await pipeline.transcribe(audioArray: samples, decodeOptions: options)
|
||||
return results
|
||||
.map(\.text)
|
||||
.joined(separator: " ")
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
import Testing
|
||||
@testable import SstCore
|
||||
|
||||
@MainActor
|
||||
struct DictationControllerTests {
|
||||
let recorder = FakeRecorder()
|
||||
let transcriber = FakeTranscriber()
|
||||
let cleaner = FakeCleaner()
|
||||
let inserter = FakeInserter()
|
||||
static let oneSecond = [Float](repeating: 0.1, count: 16_000)
|
||||
|
||||
func makeController() -> DictationController {
|
||||
DictationController(
|
||||
recorder: recorder,
|
||||
transcriber: transcriber,
|
||||
cleaner: cleaner,
|
||||
inserter: inserter
|
||||
)
|
||||
}
|
||||
|
||||
@Test func pressAndReleaseTranscribesCleansAndInserts() async {
|
||||
recorder.samples = Self.oneSecond
|
||||
let controller = makeController()
|
||||
var states: [DictationState] = []
|
||||
controller.onStateChange = { states.append($0) }
|
||||
|
||||
await controller.handle(.pressed)
|
||||
await controller.handle(.released)
|
||||
|
||||
#expect(states == [.recording, .transcribing, .inserting, .idle])
|
||||
#expect(inserter.inserted == ["[hallo wereld]"])
|
||||
}
|
||||
|
||||
@Test func shortRecordingIsDiscarded() async {
|
||||
recorder.samples = [Float](repeating: 0.1, count: 1_000)
|
||||
let controller = makeController()
|
||||
var states: [DictationState] = []
|
||||
controller.onStateChange = { states.append($0) }
|
||||
|
||||
await controller.handle(.pressed)
|
||||
await controller.handle(.released)
|
||||
|
||||
#expect(states == [.recording, .idle])
|
||||
#expect(transcriber.callCount == 0)
|
||||
#expect(inserter.inserted.isEmpty)
|
||||
}
|
||||
|
||||
@Test func silentRecordingIsDiscarded() async {
|
||||
recorder.samples = [Float](repeating: 0, count: 16_000)
|
||||
let controller = makeController()
|
||||
var states: [DictationState] = []
|
||||
var errors: [String] = []
|
||||
controller.onStateChange = { states.append($0) }
|
||||
controller.onError = { errors.append($0) }
|
||||
|
||||
await controller.handle(.pressed)
|
||||
await controller.handle(.released)
|
||||
|
||||
#expect(states == [.recording, .idle])
|
||||
#expect(transcriber.callCount == 0)
|
||||
#expect(inserter.inserted.isEmpty)
|
||||
#expect(errors.isEmpty)
|
||||
}
|
||||
|
||||
@Test func emptyTranscriptionIsNotInserted() async {
|
||||
recorder.samples = Self.oneSecond
|
||||
transcriber.result = .success(" \n")
|
||||
let controller = makeController()
|
||||
var errors: [String] = []
|
||||
controller.onError = { errors.append($0) }
|
||||
|
||||
await controller.handle(.pressed)
|
||||
await controller.handle(.released)
|
||||
|
||||
#expect(inserter.inserted.isEmpty)
|
||||
#expect(errors.isEmpty)
|
||||
#expect(controller.state == .idle)
|
||||
}
|
||||
|
||||
@Test func transcriberErrorReportsAndReturnsToIdle() async {
|
||||
recorder.samples = Self.oneSecond
|
||||
transcriber.result = .failure(TestError())
|
||||
let controller = makeController()
|
||||
var errors: [String] = []
|
||||
controller.onError = { errors.append($0) }
|
||||
|
||||
await controller.handle(.pressed)
|
||||
await controller.handle(.released)
|
||||
|
||||
#expect(errors.count == 1)
|
||||
#expect(inserter.inserted.isEmpty)
|
||||
#expect(controller.state == .idle)
|
||||
}
|
||||
|
||||
@Test func inserterErrorReportsAndReturnsToIdle() async {
|
||||
recorder.samples = Self.oneSecond
|
||||
inserter.error = TestError()
|
||||
let controller = makeController()
|
||||
var errors: [String] = []
|
||||
controller.onError = { errors.append($0) }
|
||||
|
||||
await controller.handle(.pressed)
|
||||
await controller.handle(.released)
|
||||
|
||||
#expect(errors.count == 1)
|
||||
#expect(controller.state == .idle)
|
||||
}
|
||||
|
||||
@Test func recorderStartFailureStaysIdle() async {
|
||||
recorder.startError = TestError()
|
||||
let controller = makeController()
|
||||
var errors: [String] = []
|
||||
controller.onError = { errors.append($0) }
|
||||
|
||||
await controller.handle(.pressed)
|
||||
|
||||
#expect(errors.count == 1)
|
||||
#expect(controller.state == .idle)
|
||||
}
|
||||
|
||||
@Test func secondPressWhileRecordingIsIgnored() async {
|
||||
let controller = makeController()
|
||||
await controller.handle(.pressed)
|
||||
await controller.handle(.pressed)
|
||||
#expect(recorder.startCount == 1)
|
||||
}
|
||||
|
||||
@Test func releaseWithoutPressIsIgnored() async {
|
||||
let controller = makeController()
|
||||
var states: [DictationState] = []
|
||||
controller.onStateChange = { states.append($0) }
|
||||
|
||||
await controller.handle(.released)
|
||||
|
||||
#expect(states.isEmpty)
|
||||
#expect(transcriber.callCount == 0)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import Foundation
|
||||
@testable import SstCore
|
||||
|
||||
struct TestError: Error {}
|
||||
|
||||
final class FakeRecorder: AudioRecording {
|
||||
var samples: [Float] = []
|
||||
var startError: Error?
|
||||
private(set) var startCount = 0
|
||||
|
||||
func start() throws {
|
||||
startCount += 1
|
||||
if let startError { throw startError }
|
||||
}
|
||||
|
||||
func stop() -> [Float] { samples }
|
||||
}
|
||||
|
||||
final class FakeTranscriber: Transcriber {
|
||||
var result: Result<String, Error> = .success("hallo wereld")
|
||||
private(set) var callCount = 0
|
||||
|
||||
func transcribe(_ samples: [Float]) async throws -> String {
|
||||
callCount += 1
|
||||
return try result.get()
|
||||
}
|
||||
}
|
||||
|
||||
final class FakeCleaner: Cleaner {
|
||||
func clean(_ text: String, context: CleaningContext) async throws -> String {
|
||||
"[\(text)]"
|
||||
}
|
||||
}
|
||||
|
||||
final class FakeInserter: TextInserting {
|
||||
var error: Error?
|
||||
private(set) var inserted: [String] = []
|
||||
|
||||
func insert(_ text: String) async throws {
|
||||
if let error { throw error }
|
||||
inserted.append(text)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import Testing
|
||||
@testable import SstCore
|
||||
|
||||
struct FnKeyTrackerTests {
|
||||
@Test func emitsPressedThenReleasedOnlyOnTransitions() {
|
||||
var tracker = FnKeyTracker()
|
||||
#expect(tracker.process(fnDown: false) == nil)
|
||||
#expect(tracker.process(fnDown: true) == .pressed)
|
||||
#expect(tracker.process(fnDown: true) == nil)
|
||||
#expect(tracker.process(fnDown: false) == .released)
|
||||
#expect(tracker.process(fnDown: false) == nil)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
import Foundation
|
||||
import Testing
|
||||
@testable import SstCore
|
||||
|
||||
final class FakePasteboard: PasteboardProviding {
|
||||
var items: [PasteboardItem] = []
|
||||
|
||||
func readItems() -> [PasteboardItem] { items }
|
||||
func writeString(_ text: String) {
|
||||
items = [["public.utf8-plain-text": Data(text.utf8)]]
|
||||
}
|
||||
func writeItems(_ items: [PasteboardItem]) { self.items = items }
|
||||
}
|
||||
|
||||
final class FakePaster: KeystrokePosting {
|
||||
var shouldFail = false
|
||||
var onPaste: (() -> Void)?
|
||||
private(set) var pasteCount = 0
|
||||
|
||||
func postPaste() throws {
|
||||
pasteCount += 1
|
||||
if shouldFail { throw TestError() }
|
||||
onPaste?()
|
||||
}
|
||||
}
|
||||
|
||||
struct TextInserterTests {
|
||||
let pasteboard = FakePasteboard()
|
||||
let paster = FakePaster()
|
||||
|
||||
func makeInserter() -> TextInserter {
|
||||
TextInserter(pasteboard: pasteboard, paster: paster, restoreDelay: .zero)
|
||||
}
|
||||
|
||||
@Test func pastesTextThenRestoresPreviousClipboard() async throws {
|
||||
let original: [PasteboardItem] = [["public.utf8-plain-text": Data("oud".utf8)]]
|
||||
pasteboard.items = original
|
||||
var seenAtPaste: Data?
|
||||
paster.onPaste = { seenAtPaste = pasteboard.items.first?["public.utf8-plain-text"] }
|
||||
|
||||
try await makeInserter().insert("hallo")
|
||||
|
||||
#expect(seenAtPaste == Data("hallo".utf8))
|
||||
#expect(paster.pasteCount == 1)
|
||||
#expect(pasteboard.items == original)
|
||||
}
|
||||
|
||||
@Test func failedPasteLeavesTextOnClipboard() async {
|
||||
pasteboard.items = [["public.utf8-plain-text": Data("oud".utf8)]]
|
||||
paster.shouldFail = true
|
||||
|
||||
await #expect(throws: TestError.self) {
|
||||
try await makeInserter().insert("hallo")
|
||||
}
|
||||
|
||||
#expect(pasteboard.items.first?["public.utf8-plain-text"] == Data("hallo".utf8))
|
||||
}
|
||||
|
||||
@Test func passthroughCleanerReturnsTextUnchanged() async throws {
|
||||
let result = try await PassthroughCleaner().clean("hoi", context: CleaningContext())
|
||||
#expect(result == "hoi")
|
||||
}
|
||||
}
|
||||
Executable
+38
@@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
swift build -c release --product Sst
|
||||
BIN=".build/release/Sst"
|
||||
APP="build/Sst.app"
|
||||
|
||||
rm -rf "$APP"
|
||||
mkdir -p "$APP/Contents/MacOS" "$APP/Contents/Resources"
|
||||
cp "$BIN" "$APP/Contents/MacOS/Sst"
|
||||
# SwiftPM resource bundles (if any dependency ships them) live next to the binary.
|
||||
for b in .build/release/*.bundle; do
|
||||
[ -e "$b" ] && cp -R "$b" "$APP/Contents/Resources/"
|
||||
done
|
||||
|
||||
cat > "$APP/Contents/Info.plist" <<'PLIST'
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>CFBundleIdentifier</key><string>local.sst.app</string>
|
||||
<key>CFBundleName</key><string>Sst</string>
|
||||
<key>CFBundleExecutable</key><string>Sst</string>
|
||||
<key>CFBundlePackageType</key><string>APPL</string>
|
||||
<key>CFBundleShortVersionString</key><string>0.1.0</string>
|
||||
<key>CFBundleVersion</key><string>1</string>
|
||||
<key>LSMinimumSystemVersion</key><string>14.0</string>
|
||||
<key>LSUIElement</key><true/>
|
||||
<key>NSMicrophoneUsageDescription</key>
|
||||
<string>Sst neemt je stem op zolang je Fn ingedrukt houdt, om die lokaal naar tekst om te zetten.</string>
|
||||
</dict>
|
||||
</plist>
|
||||
PLIST
|
||||
|
||||
codesign --force --sign - --identifier local.sst.app "$APP"
|
||||
echo "Gebouwd: $APP"
|
||||
echo "Na een herbouw: tccutil reset Accessibility local.sst.app; tccutil reset ListenEvent local.sst.app; tccutil reset Microphone local.sst.app (en geef de rechten opnieuw)."
|
||||
Reference in New Issue
Block a user