yum/FastTextPager

Compressed text paging over OSC.

git clone https://git.yummers.dev/yum/FastTextPager

yumSet target loudness to -16, and enable segment metadata logging by default043a447

master
branchesmasterall refs →
34.1 KiB915 linesraw
1from datetime import datetime
2from faster_whisper import WhisperModel
3import json
4import langcodes
5import numpy as np
6import os
7import noisereduce as nr
8try:
9    from profanity_filter import ProfanityFilter
10    PROFANITY_FILTER_AVAILABLE = True
11except ImportError:
12    PROFANITY_FILTER_AVAILABLE = False
13    print("Warning: profanity_filter module not available", file=sys.stderr)
14import pyaudio
15from pydub import AudioSegment
16from shared_thread_data import SharedThreadData
17from silero_vad import load_silero_vad, get_speech_timestamps
18import sys
19import time
20import typing
21import wave
22
23APP_ROOT = os.path.dirname(os.path.abspath(__file__))
24PROJECT_ROOT = os.path.dirname(APP_ROOT)
25
26class AudioStream():
27    FORMAT = pyaudio.paInt16
28    # Size of each frame (audio sample), in bytes. If you change FORMAT, make
29    # sure this stays up to date!
30    FRAME_SZ = 2
31    # Frames per second.
32    FPS = 16000
33    CHANNELS = 1
34    def __init__(self):
35        pass
36
37    def getSamples(self) -> bytes:
38        raise NotImplementedError("getSamples is not implemented!")
39
40class MicStream(AudioStream):
41    CHUNK_SZ = 1024
42
43    def __init__(self, cfg: typing.Dict):
44        self.p = pyaudio.PyAudio()
45        self.stream = None
46        self.sample_rate = None
47        # Each time pyaudio gives us audio data, it's in the form of a chunk of
48        # samples. We keep these in a list to keep the audio callback as light
49        # as possible. Whenever downstream layers want data, we collapse the
50        # list into a single array of data (a bytes object).
51        self.chunks = []
52        # If set, incoming frames are simply discarded.
53        self.paused = False
54
55        which_mic = cfg["microphone"]
56
57        if cfg["enable_debug_mode"]:
58            print(f"Finding mic {which_mic}", file=sys.stderr)
59            self.dumpMicDevices()
60
61        got_match = False
62        device_index = -1
63        if which_mic == "index":
64            target_str = "Digital Audio Interface"
65        elif which_mic == "focusrite":
66            target_str = "Focusrite"
67        elif which_mic == "motu":
68            target_str = "In 1-2 (MOTU M Series)"
69        elif which_mic == "beyond":
70            target_str = "Microphone (Beyond)"
71        else:
72            if cfg["enable_debug_mode"]:
73                print(f"Mic {which_mic} requested, treating it as a numerical " +
74                        "device ID", file=sys.stderr)
75            device_index = int(which_mic)
76            got_match = True
77        if not got_match:
78            info = self.p.get_host_api_info_by_index(0)
79            numdevices = info.get('deviceCount')
80            for i in range(0, numdevices):
81                if (self.p.get_device_info_by_host_api_device_index(0, i).get('maxInputChannels')) > 0:
82                    device_name = self.p.get_device_info_by_host_api_device_index(0, i).get('name')
83                    if target_str in device_name:
84                        print(f"Got matching mic: {device_name}",
85                                file=sys.stderr)
86                        device_index = i
87                        got_match = True
88                        break
89        if not got_match:
90            raise KeyError(f"Mic {which_mic} not found")
91
92        info = self.p.get_device_info_by_host_api_device_index(0, device_index)
93        if cfg["enable_debug_mode"]:
94            print(f"Found mic {which_mic}: {info['name']}", file=sys.stderr)
95        self.sample_rate = int(info['defaultSampleRate'])
96        if cfg["enable_debug_mode"]:
97            print(f"Mic sample rate: {self.sample_rate}", file=sys.stderr)
98
99        self.stream = self.p.open(
100                rate=self.sample_rate,
101                channels=AudioStream.CHANNELS,
102                format=AudioStream.FORMAT,
103                input=True,
104                frames_per_buffer=MicStream.CHUNK_SZ,
105                input_device_index=device_index,
106                stream_callback=self.onAudioFramesAvailable)
107
108        self.stream.start_stream()
109
110        AudioStream.__init__(self)
111
112    def pause(self, state: bool = True):
113        self.paused = state
114
115    def dumpMicDevices(self):
116        info = self.p.get_host_api_info_by_index(0)
117        numdevices = info.get('deviceCount')
118
119        for i in range(0, numdevices):
120            if (self.p.get_device_info_by_host_api_device_index(0, i).get('maxInputChannels')) > 0:
121                device_name = self.p.get_device_info_by_host_api_device_index(0, i).get('name')
122                print("Input Device id ", i, " - ", device_name)
123
124    def onAudioFramesAvailable(self,
125            frames,
126            frame_count,
127            time_info,
128            status_flags):
129        if self.paused:
130            # Don't literally pause, just start returning silence. This allows
131            # the `min_segment_age_s` check to work while paused.
132            n_frames = int(frame_count * AudioStream.FPS /
133                    float(self.sample_rate))
134            self.chunks.append(np.zeros(n_frames,
135                dtype=np.int16).tobytes())
136            return (frames, pyaudio.paContinue)
137
138        decimated = b''
139        # In pyaudio, a `frame` is a single sample of audio data.
140        frame_len = AudioStream.FRAME_SZ
141        next_frame = 0.0
142        # The mic probably has a higher sample rate than Whisper wants, so
143        # decrease the sample rate by dropping samples. Note that this
144        # algorithm only works if the mic's rate is higher than whisper's
145        # expected rate.
146        keep_every = float(self.sample_rate) / AudioStream.FPS
147        for i in range(frame_count):
148            if i >= next_frame:
149                decimated += frames[i*frame_len:(i+1)*frame_len]
150                next_frame += keep_every
151        self.chunks.append(decimated)
152
153        return (frames, pyaudio.paContinue)
154
155    # Get audio data and the corresponding timestamp.
156    def getSamples(self) -> bytes:
157        chunks = self.chunks
158        self.chunks = []
159        result = b''.join(chunks)
160        return result
161
162class AudioCollector:
163    def __init__(self, stream: AudioStream):
164        self.stream = stream
165        self.frames = b''
166        # Note: by design, this is the only spot where we anchor our timestamps
167        # against the real world. This is done to make it possible to profile
168        # test cases which read from disk (at much faster than real speed) in
169        # the same way that we profile real-time data.
170        self.wall_ts = time.time()
171
172    def getAudio(self) -> bytes:
173        frames = self.stream.getSamples()
174        if frames:
175            self.frames += frames
176        return self.frames
177
178    def dropAudioPrefix(self, dur_s: float) -> bytes:
179        n_bytes = int(dur_s * AudioStream.FPS) * self.stream.FRAME_SZ
180        n_bytes = min(n_bytes, len(self.frames))
181        cut_portion = self.frames[:n_bytes]
182        self.frames = self.frames[n_bytes:]
183        self.wall_ts += float(n_bytes / self.stream.FRAME_SZ) / self.stream.FPS
184        return cut_portion
185
186    def dropAudioPrefixByFrames(self, dur_frames: int) -> bytes:
187        n_bytes = dur_frames * self.stream.FRAME_SZ
188        n_bytes = min(n_bytes, len(self.frames))
189        cut_portion = self.frames[:n_bytes]
190        self.frames = self.frames[n_bytes:]
191        self.wall_ts += float(n_bytes / self.stream.FRAME_SZ) / self.stream.FPS
192        return cut_portion
193
194    def keepLast(self, dur_s: float) -> bytes:
195        drop_len = max(0, self.duration() - dur_s)
196        return self.dropAudioPrefix(drop_len)
197
198    def dropAudio(self):
199        self.wall_ts += self.duration()
200        cut_portion = self.frames
201        self.frames = b''
202        return cut_portion
203
204    def duration(self):
205        return len(self.frames) / (AudioStream.FPS * self.stream.FRAME_SZ)
206
207    def begin(self):
208        return self.wall_ts
209
210    def now(self):
211        return self.begin() + self.duration()
212
213class AudioCollectorFilter:
214    def __init__(self, parent: AudioCollector):
215        self.parent = parent
216
217    def getAudio(self) -> bytes:
218        return self.parent.getAudio()
219    def dropAudioPrefix(self, dur_s: float):
220        return self.parent.dropAudioPrefix(dur_s)
221    def dropAudioPrefixByFrames(self, dur_frames: int):
222        return self.parent.dropAudioPrefixByFrames(dur_frames)
223    def keepLast(self, dur_s):
224        return self.parent.keepLast(dur_s)
225    def dropAudio(self):
226        return self.parent.dropAudio()
227    def duration(self):
228        return self.parent.duration()
229    def begin(self):
230        return self.parent.begin()
231    def now(self):
232        return self.parent.now()
233
234# Audio collector that enforces a minimum length on its audio data.
235class LengthEnforcingAudioCollector(AudioCollectorFilter):
236    def __init__(self, parent: AudioCollector, min_duration_s: float):
237        AudioCollectorFilter.__init__(self, parent)
238        self.min_duration_s = min_duration_s
239
240    def getAudio(self) -> bytes:
241        audio = self.parent.getAudio()
242        min_duration_frames = int(self.min_duration_s * AudioStream.FPS)
243        pad_len_frames = max(0, min_duration_frames - int(len(audio) /
244            AudioStream.FRAME_SZ))
245        pad = np.zeros(pad_len_frames, dtype=np.int16).tobytes()
246        return pad + audio
247
248class NormalizingAudioCollector(AudioCollectorFilter):
249    def __init__(self, parent: AudioCollector):
250        AudioCollectorFilter.__init__(self, parent)
251
252    def getAudio(self) -> bytes:
253        audio = self.parent.getAudio()
254
255        audio = AudioSegment(audio, sample_width=AudioStream.FRAME_SZ,
256                frame_rate=AudioStream.FPS, channels=AudioStream.CHANNELS)
257        audio = audio.normalize()
258
259        frames = np.array(audio.get_array_of_samples())
260        frames = np.int16(frames).tobytes()
261
262        return frames
263
264class BoostingAudioCollector(AudioCollectorFilter):
265    def __init__(self, parent: AudioCollector,
266                 target_dBFS: float,
267                 max_gain_dB: float,
268                 cfg: typing.Dict):
269        AudioCollectorFilter.__init__(self, parent)
270        self.target_dBFS = target_dBFS
271        self.max_gain_dB = max_gain_dB
272        self.cfg = cfg
273
274    def getAudio(self) -> bytes:
275        audio = self.parent.getAudio()
276
277        audio = AudioSegment(audio, sample_width=AudioStream.FRAME_SZ,
278                frame_rate=AudioStream.FPS, channels=AudioStream.CHANNELS)
279        gain = min(self.target_dBFS - audio.dBFS, self.max_gain_dB)
280        if self.cfg["enable_debug_mode"]:
281            print(f"Boosting audio by {gain} dB (from {audio.dBFS} to {audio.dBFS + gain})", flush=True)
282        audio = audio.apply_gain(gain)
283
284        frames = np.array(audio.get_array_of_samples())
285        frames = np.int16(frames).tobytes()
286
287        return frames
288
289class CompressingAudioCollector(AudioCollectorFilter):
290    def __init__(self, parent: AudioCollector):
291        AudioCollectorFilter.__init__(self, parent)
292
293    def getAudio(self) -> bytes:
294        audio = self.parent.getAudio()
295
296        audio = AudioSegment(audio, sample_width=AudioStream.FRAME_SZ,
297                frame_rate=AudioStream.FPS, channels=AudioStream.CHANNELS)
298        # subtle compression has a slight positive effect on my benchmark
299        audio = audio.compress_dynamic_range(threshold=-10, ratio=2.0)
300
301        frames = np.array(audio.get_array_of_samples())
302        frames = np.int16(frames).tobytes()
303
304        return frames
305
306class NoiseReducingAudioCollector(AudioCollectorFilter):
307    def __init__(self, parent: AudioCollector, cfg: typing.Dict):
308        AudioCollectorFilter.__init__(self, parent)
309        self.cfg = cfg
310
311    def getAudio(self) -> bytes:
312        audio = self.parent.getAudio()
313        audio_array = np.frombuffer(audio, dtype=np.int16).astype(np.float32)
314
315        reduced_audio = nr.reduce_noise(
316            y=audio_array,
317            sr=AudioStream.FPS,
318        )
319
320        # Convert back to int16
321        reduced_audio = np.clip(reduced_audio, -32768, 32767)
322        frames = np.int16(reduced_audio).tobytes()
323
324        return frames
325
326class AudioSegmenter:
327    def __init__(self,
328            min_silence_ms=250,
329            max_speech_s=5,
330            min_speech_duration_ms=100):
331        self.min_silence_ms = min_silence_ms
332        self.max_speech_s = max_speech_s
333        self.min_speech_duration_ms = min_speech_duration_ms
334
335        # Load Silero VAD model
336        self.model = load_silero_vad()
337
338        self.vad_threshold = 0.3
339        self.min_silence_duration_ms = min_silence_ms
340        self.max_speech_duration_s = max_speech_s
341        self.min_speech_duration_ms = min_speech_duration_ms
342
343    def segmentAudio(self, audio: bytes):
344        # Convert audio bytes to numpy array expected by silero-vad
345        audio_array = np.frombuffer(audio,
346                dtype=np.int16).flatten().astype(np.float32) / 32768.0
347
348        # Get speech timestamps using silero-vad
349        # Note: silero-vad expects sample rate of 16000 Hz which matches AudioStream.FPS
350        speech_timestamps = get_speech_timestamps(
351            audio_array,
352            self.model,
353            sampling_rate=AudioStream.FPS,
354            threshold=self.vad_threshold,
355            min_silence_duration_ms=self.min_silence_duration_ms,
356            max_speech_duration_s=self.max_speech_duration_s,
357            min_speech_duration_ms=self.min_speech_duration_ms,
358            return_seconds=False  # We want frame indices, not seconds
359        )
360
361        return speech_timestamps
362
363    # Returns the stable cutoff (if any) and whether there are any segments.
364    def getStableCutoff(self, audio: bytes) -> typing.Tuple[int, bool]:
365        min_delta_frames = int((self.min_silence_duration_ms *
366                AudioStream.FPS) / 1000.0)
367        cutoff = None
368
369        last_end = None
370        segments = self.segmentAudio(audio)
371
372        for i in range(len(segments)):
373            s = segments[i]
374            #print(f"s: {s}")
375            #print(f"last_end: {last_end}")
376
377            if last_end:
378                delta_frames = s['start'] - last_end
379                #print(f"delta frames: {delta_frames}")
380                if delta_frames > min_delta_frames:
381                    cutoff = s['start']
382            else:
383                last_end = s['end']
384
385            if i == len(segments) - 1:
386                now = int(len(audio) / AudioStream.FRAME_SZ)
387                #print(f"now: {now}")
388                #print(f"min d: {min_delta_frames}")
389                delta_frames = now - s['end']
390                if delta_frames > min_delta_frames:
391                    cutoff = now - int(min_delta_frames / 2)
392
393        return (cutoff, len(segments) > 0)
394
395# A segment of transcribed audio. `start_ts` and `end_ts` are floating point
396# number of seconds since the beginning of audio data.
397class Segment:
398    def __init__(self,
399            transcript: str,
400            start_ts: float,
401            end_ts: float,
402            wall_ts: float,
403            avg_logprob: float,
404            no_speech_prob: float,
405            compression_ratio: float):
406        self.transcript = transcript
407        # start_ts, end_ts are timestamps in seconds relative to `wall_ts`.
408        self.start_ts = start_ts
409        self.end_ts = end_ts
410        # wall_ts is the time.time() at which the oldest audio sample leading
411        # to this transcript was collected.
412        self.wall_ts = wall_ts
413        self.avg_logprob = avg_logprob
414        self.no_speech_prob = no_speech_prob
415        self.compression_ratio = compression_ratio
416
417    def __str__(self):
418        ts = f"(ts: {self.start_ts}-{self.end_ts}) "
419
420        wall_ts_start = datetime.utcfromtimestamp(self.start_ts + self.wall_ts).strftime('%H:%M:%S')
421        wall_ts_end = datetime.utcfromtimestamp(self.end_ts + self.wall_ts).strftime('%H:%M:%S')
422        wall_ts = f"(wall ts: {wall_ts_start}-{wall_ts_end}) "
423
424        no_speech = f"(no_speech: {self.no_speech_prob}) "
425        avg_logprob = f"(avg_logprob: {self.avg_logprob}) "
426        return f"{self.transcript} " + ts + wall_ts + no_speech + avg_logprob
427
428def join_segments(a, b):
429    if len(a) > 0 and a[-1] != ' ':
430        return a + ' ' + b
431    else:
432        return a + b
433
434class Whisper:
435    def __init__(self,
436            collector: AudioCollector,
437            cfg: typing.Dict):
438        self.collector = collector
439        self.model = None
440        self.cfg = cfg
441
442        model_str = cfg["model"]
443        model_root = os.path.join(PROJECT_ROOT, "Models",
444                os.path.normpath(model_str))
445        if cfg["enable_debug_mode"]:
446            print(f"Model {cfg['model']} will be saved to {model_root}",
447                    file=sys.stderr)
448
449        model_device = "cuda"
450        compute_type = cfg["compute_type"]
451        if cfg["use_cpu"]:
452            model_device = "cpu"
453            compute_type = "int8"
454
455        already_downloaded = os.path.exists(model_root)
456
457        if not already_downloaded:
458            print(f"Model {model_str} not already downloaded, downloading now...", flush=True)
459
460        self.model = WhisperModel(model_str,
461                device = model_device,
462                device_index = cfg["gpu_idx"],
463                compute_type = compute_type,
464                download_root = model_root,
465                local_files_only = already_downloaded)
466
467        self.context_window_chars = 200  # Keep last 200 chars of context
468        self.recent_context = ""  # Store recent committed text
469
470    def update_context(self, committed_text: str):
471        """Update the context with recently committed text."""
472        self.recent_context = join_segments(self.recent_context, committed_text).strip()
473        # Drop half of the context window.
474        if len(self.recent_context) > self.context_window_chars:
475            words = self.recent_context.split()
476            words = words[len(words)//2:]
477            self.recent_context = ' '.join(words)
478
479    def transcribe(self, frames: bytes = None) -> typing.List[Segment]:
480        if frames is None:
481            frames = self.collector.getAudio()
482
483        # Convert audio to float32
484        audio = np.frombuffer(frames,
485                dtype=np.int16).flatten().astype(np.float32) / 32768.0
486
487        # Build context-aware prompt
488        prompt = self._build_prompt()
489
490        if self.cfg["enable_debug_mode"]:
491            print(f"Prompt: {prompt}", flush=True)
492
493        t0 = time.time()
494        segments, info = self.model.transcribe(
495                audio,
496                language = langcodes.find(self.cfg["language"]).language,
497                vad_filter = True,
498                temperature=0.0,
499                without_timestamps = False,
500                initial_prompt=prompt,
501                beam_size=self.cfg.get("beam_size", 5),
502                best_of=self.cfg.get("best_of", 5),
503                condition_on_previous_text=True
504        )
505        res = []
506        for s in segments:
507            # Manual touchup. I see a decent number of hallucinations sneaking
508            # in with high `no_speech_prob` and modest `avg_logprob`.
509            if s.no_speech_prob > 0.6 and s.avg_logprob < -0.5:
510                if self.cfg["enable_debug_mode"]:
511                    print(f"Drop probable hallucination (case 1) " +
512                            f"(text='{s.text}', " +
513                            f"no_speech_prob={s.no_speech_prob}, " +
514                            f"avg_logprob={s.avg_logprob})", file=sys.stderr)
515                continue
516            # Another touchup targeted at the vexatious "thanks for watching!"
517            # hallucination. This triggers a lot when listening to
518            # instrumental/electronic music.
519            if s.no_speech_prob > 0.15 and s.avg_logprob < -0.7:
520                if self.cfg["enable_debug_mode"]:
521                    print(f"Drop probable hallucination (case 2) " +
522                            f"(text='{s.text}', " +
523                            f"no_speech_prob={s.no_speech_prob}, " +
524                            f"avg_logprob={s.avg_logprob})", file=sys.stderr)
525                continue
526            if s.avg_logprob < -0.75:
527                if self.cfg["enable_debug_mode"]:
528                    print(f"Drop probable hallucination (case 3) " +
529                            f"(text='{s.text}', " +
530                            f"no_speech_prob={s.no_speech_prob}, " +
531                            f"avg_logprob={s.avg_logprob})", file=sys.stderr)
532                continue
533            if self.cfg["enable_debug_mode"]:
534                print(f"s get: {s}")
535            if s.avg_logprob < -1.0:
536                continue
537            if s.compression_ratio > 2.4:
538                continue
539            res.append(Segment(s.text, s.start, s.end,
540                self.collector.begin(),
541                s.avg_logprob, s.no_speech_prob, s.compression_ratio))
542        t1 = time.time()
543        if self.cfg["enable_debug_mode"]:
544            print(f"Transcription latency (s): {t1 - t0}")
545        return res
546
547    def _build_prompt(self) -> str:
548        """Build a context-aware prompt for Whisper."""
549        user_prompt = self.cfg["user_prompt"]
550        context_prompt = ""
551        if self.recent_context and len(self.recent_context) > 0:
552            context_prompt = f"Here is the context so far: {self.recent_context}"
553
554        prompts = [user_prompt, context_prompt]
555        prompts = [p for p in prompts if p and len(p) > 0]
556        return " ".join(prompts)
557
558class TranscriptCommit:
559    def __init__(self,
560            delta: str,
561            preview: str,
562            latency_s: float = None,
563            thresh_at_commit: int = None,
564            audio: bytes = None,
565            duration_s: float = None,
566            start_ts: float = None):
567        self.delta = delta
568        self.preview = preview
569        self.latency_s = latency_s
570        self.thresh_at_commit = thresh_at_commit
571        self.audio = audio
572        # Time at which the commit is generated
573        self.ts = time.time()
574        # Time corresponding to the start of the segment
575        self.start_ts = start_ts
576        # The duration of the audio segment, in seconds.
577        self.duration_s = duration_s
578
579
580def saveAudio(audio: bytes, path: str, cfg: typing.Dict):
581    with wave.open(path, 'wb') as wf:
582        if cfg["enable_debug_mode"]:
583            print(f"Saving audio to {path}", file=sys.stderr)
584        wf.setnchannels(AudioStream.CHANNELS)
585        wf.setsampwidth(AudioStream.FRAME_SZ)
586        wf.setframerate(AudioStream.FPS)
587        wf.writeframes(audio)
588
589
590class SegmentLogger:
591    def __init__(self, cfg: typing.Dict):
592        self.cfg = cfg
593        self.enabled = cfg.get("enable_segment_logging", False)
594        self.session_data = []
595        self.log_file = None
596
597        if self.enabled:
598            log_dir = os.path.join(PROJECT_ROOT, "logs")
599            if not os.path.exists(log_dir):
600                os.makedirs(log_dir)
601
602            # Create file
603            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
604            self.log_file = os.path.join(log_dir, f"session_debug_{timestamp}.json")
605            print(f"Segment logging enabled. Logging to: {self.log_file}", file=sys.stderr)
606
607    def log_segment(self, segment: Segment, commit_type: str = "commit"):
608        if not self.enabled:
609            return
610
611        segment_data = {
612            "timestamp": datetime.now().isoformat(),
613            "type": commit_type,
614            "text": segment.transcript,
615            "start_ts": segment.start_ts,
616            "end_ts": segment.end_ts,
617            "wall_ts": segment.wall_ts,
618            "avg_logprob": segment.avg_logprob,
619            "no_speech_prob": segment.no_speech_prob,
620            "compression_ratio": segment.compression_ratio,
621            "duration": segment.end_ts - segment.start_ts
622        }
623
624        self.session_data.append(segment_data)
625
626        # Write to file incrementally
627        try:
628            with open(self.log_file, 'w') as f:
629                json.dump({
630                    "session_start": self.session_data[0]["timestamp"] if self.session_data else None,
631                    "segments": self.session_data
632                }, f, indent=2)
633        except Exception as e:
634            print(f"Error writing segment log: {e}", file=sys.stderr)
635
636    def close(self):
637        if self.enabled and self.session_data:
638            print(f"Session complete. Logged {len(self.session_data)} segments to {self.log_file}", file=sys.stderr)
639
640
641class VadCommitter:
642    def __init__(self,
643            cfg: typing.Dict,
644            collector: AudioCollector,
645            whisper: Whisper,
646            segmenter: AudioSegmenter,
647            segment_logger: SegmentLogger = None):
648        self.cfg = cfg
649        self.collector = collector
650        self.whisper = whisper
651        self.segmenter = segmenter
652        self.segment_logger = segment_logger
653
654    def getDelta(self) -> TranscriptCommit:
655        audio = self.collector.getAudio()
656        stable_cutoff, has_audio = self.segmenter.getStableCutoff(audio)
657
658        delta = ""
659        commit_audio = None
660        latency_s = None
661        duration_s = self.collector.duration()
662        start_ts = self.collector.begin()
663
664        if has_audio and stable_cutoff:
665            latency_s = self.collector.now() - self.collector.begin()
666            duration_s = stable_cutoff / AudioStream.FPS
667            start_ts = self.collector.begin()
668
669            # Get the filtered audio first, then extract the portion we need
670            filtered_audio = self.collector.getAudio()
671            commit_audio = filtered_audio[:stable_cutoff * AudioStream.FRAME_SZ]
672
673            # Now drop the prefix from the collector
674            self.collector.dropAudioPrefixByFrames(stable_cutoff)
675
676            segments = self.whisper.transcribe(commit_audio)
677            delta = ''.join(s.transcript for s in segments)
678
679            # Update whisper's context with the committed text
680            if delta.strip():
681                self.whisper.update_context(delta.strip())
682
683            if self.segment_logger:
684                for s in segments:
685                    self.segment_logger.log_segment(s, "commit")
686
687            audio = self.collector.getAudio()
688            if self.cfg["enable_debug_mode"]:
689                for s in segments:
690                    print(f"commit segment: {s}", file=sys.stderr)
691                if len(delta) > 0:
692                    print(f"delta get: {delta}", file=sys.stderr)
693
694            if self.cfg["save_audio"] and len(delta) > 0:
695                ts = datetime.fromtimestamp(self.collector.now() - latency_s)
696                filename = str(ts.strftime('%Y_%m_%d__%H-%M-%S')) + delta.strip() + ".wav"
697                audio_dir = os.path.join(PROJECT_ROOT, "audio")
698                if not os.path.exists(audio_dir):
699                    os.makedirs(audio_dir)
700                saveAudio(commit_audio, os.path.join(audio_dir, filename), self.cfg)
701
702        preview = ""
703        if self.cfg["enable_previews"] and has_audio:
704            segments = self.whisper.transcribe(audio)
705            preview = "".join(s.transcript for s in segments)
706
707            if self.segment_logger:
708                for s in segments:
709                    self.segment_logger.log_segment(s, "preview")
710
711        if not has_audio:
712            self.collector.keepLast(1.0)
713
714        return TranscriptCommit(
715                delta.strip(),
716                preview.strip(),
717                latency_s,
718                audio=audio,
719                duration_s=duration_s,
720                start_ts=start_ts)
721
722
723class StreamingPlugin:
724    def __init__(self):
725        pass
726
727    def transform(self, commit: TranscriptCommit) -> TranscriptCommit:
728        return commit
729
730    def stop(self):
731        pass
732
733
734class LowercasePlugin(StreamingPlugin):
735    def __init__(self, cfg):
736        self.cfg = cfg
737
738    def transform(self, commit: TranscriptCommit) -> TranscriptCommit:
739        if self.cfg["enable_lowercase_filter"]:
740            commit.delta = commit.delta.lower()
741            commit.preview = commit.preview.lower()
742        return commit
743
744
745class UppercasePlugin(StreamingPlugin):
746    def __init__(self, cfg):
747        self.cfg = cfg
748
749    def transform(self, commit: TranscriptCommit) -> TranscriptCommit:
750        if self.cfg["enable_uppercase_filter"]:
751            commit.delta = commit.delta.upper()
752            commit.preview = commit.preview.upper()
753        return commit
754
755
756class ProfanityPlugin(StreamingPlugin):
757    def __init__(self, cfg):
758        self.cfg = cfg
759        self.filter = None
760        if PROFANITY_FILTER_AVAILABLE and cfg["enable_profanity_filter"]:
761            en_profanity_path = os.path.join(PROJECT_ROOT, "Third_Party/Profanity/en")
762            try:
763                self.filter = ProfanityFilter(en_profanity_path)
764                self.filter.load()
765            except Exception as e:
766                print(f"Warning: Could not load profanity filter: {e}", file=sys.stderr)
767                self.filter = None
768
769    def transform(self, commit: TranscriptCommit) -> TranscriptCommit:
770        if self.cfg["enable_profanity_filter"] and self.filter:
771            commit.delta = self.filter.filter(commit.delta)
772            commit.preview = self.filter.filter(commit.preview)
773        return commit
774
775
776class PresentationFilter:
777    def __init__(self):
778        pass
779
780    def transform(self, transcript: str, preview: str) -> typing.Tuple[str, str]:
781        return transcript, preview
782
783    def stop(self):
784        pass
785
786
787class TrailingPeriodFilter(PresentationFilter):
788    def __init__(self, cfg):
789        self.cfg = cfg
790
791    def transform(self, transcript: str, preview: str) -> typing.Tuple[str, str]:
792        if self.cfg["remove_trailing_period"]:
793            def _remove_trailing_period(s: str) -> str:
794                if len(s) > 0 and s[-1] == '.' and not s.endswith("..."):
795                    s = s[0:len(s)-1]
796                return s
797            if len(preview) == 0:
798                transcript = _remove_trailing_period(transcript)
799            else:
800                preview = _remove_trailing_period(preview)
801        return transcript, preview
802
803
804def transcriptionThread(shared_data: SharedThreadData):
805    last_stable_commit = None
806
807    stream = MicStream(shared_data.cfg)
808    collector = AudioCollector(stream)
809    collector = CompressingAudioCollector(collector)
810    collector = BoostingAudioCollector(collector, -16.0, 24.0,
811                                       shared_data.cfg)
812    collector = NoiseReducingAudioCollector(collector, shared_data.cfg)
813    #collector = NormalizingAudioCollector(collector)
814    whisper = Whisper(collector, shared_data.cfg)
815    segmenter = AudioSegmenter(min_silence_ms=shared_data.cfg["min_silence_duration_ms"],
816            max_speech_s=shared_data.cfg["max_speech_duration_s"],
817            min_speech_duration_ms=shared_data.cfg["min_speech_duration_ms"])
818
819    segment_logger = SegmentLogger(shared_data.cfg)
820    committer = VadCommitter(shared_data.cfg, collector, whisper, segmenter, segment_logger)
821
822    plugins = []
823    # plugins.append(TranslationPlugin(shared_data.cfg))  # Not implemented yet
824    plugins.append(UppercasePlugin(shared_data.cfg))
825    plugins.append(LowercasePlugin(shared_data.cfg))
826    plugins.append(ProfanityPlugin(shared_data.cfg))
827    # plugins.append(UwuPlugin(shared_data.cfg))  # Not implemented yet
828    # plugins.append(BrowserSource(shared_data.cfg))  # Not implemented yet
829
830    filters = []
831    filters.append(TrailingPeriodFilter(shared_data.cfg))
832
833    transcript = ""
834    preview = ""
835
836    with shared_data.word_lock:
837        shared_data.stream = stream
838        shared_data.collector = collector
839
840    print(f"Ready to go!", flush=True)
841
842    while not shared_data.exit_event.is_set():
843        time.sleep(shared_data.cfg["transcription_loop_delay_ms"] / 1000.0);
844
845        op = None
846
847        commit = committer.getDelta()
848
849        with shared_data.word_lock:
850            for plugin in plugins:
851                commit = plugin.transform(commit)
852
853            if len(commit.delta) > 0 or len(commit.preview) > 0:
854                # Avoid re-sending text after long pauses
855                if shared_data.cfg["reset_after_silence_s"] > 0:
856                    silence_duration = 0
857                    if last_stable_commit:
858                        last_commit_end_ts = \
859                                last_stable_commit.start_ts + \
860                                last_stable_commit.duration_s
861                        silence_duration = commit.start_ts - last_commit_end_ts
862                    if silence_duration > shared_data.cfg["reset_after_silence_s"]:
863                        if shared_data.cfg["enable_debug_mode"]:
864                            print(f"Resetting transcript after {silence_duration}-second "
865                                    "silence", file=sys.stderr)
866                        shared_data.transcript = ""
867                        shared_data.preview = ""
868                        whisper.recent_context = ""  # Reset context too
869                    if commit.delta:
870                        last_stable_commit = commit
871
872                # Hard-cap displayed transcript length to prevent
873                # runaway memory use in UI. Keep the full transcript to avoid
874                # breaking OSC pager.
875                if len(shared_data.transcript) >= 1024:
876                    shared_data.transcript = shared_data.transcript[-512:]
877                shared_data.transcript = \
878                        join_segments(shared_data.transcript, commit.delta)
879                shared_data.preview = commit.preview
880
881                for filt in filters:
882                    shared_data.transcript, shared_data.preview = \
883                            filt.transform(shared_data.transcript,
884                                           shared_data.preview)
885
886                try:
887                    print(f"Transcript: {shared_data.transcript}", flush=True)
888                except UnicodeEncodeError:
889                    print("Failed to encode transcript - discarding delta",
890                            file=sys.stderr)
891                    continue
892                try:
893                    print(f"Preview: {shared_data.preview}", flush=True)
894                except UnicodeEncodeError:
895                    print("Failed to encode preview - discarding", file=sys.stderr)
896
897                if shared_data.cfg["enable_debug_mode"]:
898                    print(f"commit latency: {commit.latency_s}", file=sys.stderr)
899                    print(f"commit thresh: {commit.thresh_at_commit}",
900                            file=sys.stderr)
901
902            if len(shared_data.transcript) > 0 and \
903                    (not shared_data.transcript.endswith(' ')) and \
904                    (not commit.delta.startswith(' ')):
905                commit.delta = ' ' + commit.delta
906            if len(commit.delta) > 0 and \
907                    (not commit.delta.endswith(' ')) and \
908                    (not commit.preview.startswith(' ')):
909                commit.preview = ' ' + commit.preview
910    for plugin in plugins:
911        plugin.stop()
912    for filt in filters:
913        filt.stop()
914    segment_logger.close()
915