From 0bda49279ec80187d49a922ff2a47141ffb2fd8f Mon Sep 17 00:00:00 2001 From: yum Date: Tue, 30 May 2023 19:01:56 -0700 Subject: Finish translation for Western European language speakers NLLB needs its input to be split up into sentences. I use the sentence_splitter Python package to do this. It supports ~20 Western European languages, but notably, no Asian languages. * Sort spoken language list. English is still at the top. * Remove 'Translation source' dropdown. Infer this from the spoken language. * Add lang_compat.py to map language codes between the various libraries (whisper, nllb, sentence_splitter). * Fix bug where old text would appear in textbox when you first bring it up. --- GUI/GUI/GUI/Config.cpp | 3 - GUI/GUI/GUI/Config.h | 1 - GUI/GUI/GUI/Frame.cpp | 235 +++++++++++++++++++----------------------- GUI/GUI/GUI/Frame.h | 1 - GUI/GUI/GUI/Logging.h | 3 +- GUI/GUI/GUI/PythonWrapper.cpp | 1 - Scripts/lang_compat.py | 58 +++++++++++ Scripts/requirements.txt | 1 + Scripts/transcribe.py | 78 ++++++++------ 9 files changed, 211 insertions(+), 170 deletions(-) create mode 100644 Scripts/lang_compat.py diff --git a/GUI/GUI/GUI/Config.cpp b/GUI/GUI/GUI/Config.cpp index bf0c1c6..2bf17cf 100644 --- a/GUI/GUI/GUI/Config.cpp +++ b/GUI/GUI/GUI/Config.cpp @@ -64,7 +64,6 @@ AppConfig::AppConfig(wxTextCtrl* out) microphone("index"), language("english"), - language_source("Do not translate"), language_target("Do not translate"), model("base.en"), model_translation("nllb-200-distilled-600M"), @@ -115,7 +114,6 @@ bool AppConfig::Serialize(const std::filesystem::path& path) { cm.Set("microphone", microphone); cm.Set("language", language); - cm.Set("language_source", language_source); cm.Set("language_target", language_target); cm.Set("model", model); cm.Set("model_translation", model_translation); @@ -179,7 +177,6 @@ bool AppConfig::Deserialize(const std::filesystem::path& path) { AppConfig c(out_); cm.Get("microphone", c.microphone); cm.Get("language", c.language); - cm.Get("language_source", c.language_source); cm.Get("language_target", c.language_target); cm.Get("model", c.model); cm.Get("model_translation", c.model_translation); diff --git a/GUI/GUI/GUI/Config.h b/GUI/GUI/GUI/Config.h index bc942d4..bf714bc 100644 --- a/GUI/GUI/GUI/Config.h +++ b/GUI/GUI/GUI/Config.h @@ -50,7 +50,6 @@ public: // Transcription-specific settings. std::string microphone; std::string language; - std::string language_source; std::string language_target; std::string model; std::string model_translation; diff --git a/GUI/GUI/GUI/Frame.cpp b/GUI/GUI/GUI/Frame.cpp index 8cebc2d..a2b128a 100644 --- a/GUI/GUI/GUI/Frame.cpp +++ b/GUI/GUI/GUI/Frame.cpp @@ -30,7 +30,6 @@ namespace { ID_PY_APP_MIC, ID_PY_APP_MIC_PANEL, ID_PY_APP_LANG, - ID_PY_APP_TRANSLATE_SOURCE, ID_PY_APP_TRANSLATE_TARGET, ID_PY_APP_LANG_PANEL, ID_PY_APP_MODEL, @@ -130,107 +129,107 @@ namespace { constexpr int kMicDefault = 0; // index // lifted from whisper/tokenizer.py - const wxString kLangChoices[] = { - "english", - "chinese", - "german", - "spanish", - "russian", - "korean", - "french", - "japanese", - "portuguese", - "turkish", - "polish", - "catalan", - "dutch", - "arabic", - "swedish", - "italian", - "indonesian", - "hindi", - "finnish", - "vietnamese", - "hebrew", - "ukrainian", - "greek", - "malay", - "czech", - "romanian", - "danish", - "hungarian", - "tamil", - "norwegian", - "thai", - "urdu", - "croatian", - "bulgarian", - "lithuanian", - "latin", - "maori", - "malayalam", - "welsh", - "slovak", - "telugu", - "persian", - "latvian", - "bengali", - "serbian", - "azerbaijani", - "slovenian", - "kannada", - "estonian", - "macedonian", - "breton", - "basque", - "icelandic", - "armenian", - "nepali", - "mongolian", - "bosnian", - "kazakh", - "albanian", - "swahili", - "galician", - "marathi", - "punjabi", - "sinhala", - "khmer", - "shona", - "yoruba", - "somali", - "afrikaans", - "occitan", - "georgian", - "belarusian", - "tajik", - "sindhi", - "gujarati", - "amharic", - "yiddish", - "lao", - "uzbek", - "faroese", - "haitian creole", - "pashto", - "turkmen", - "nynorsk", - "maltese", - "sanskrit", - "luxembourgish", - "myanmar", - "tibetan", - "tagalog", - "malagasy", - "assamese", - "tatar", - "hawaiian", - "lingala", - "hausa", - "bashkir", - "javanese", - "sundanese" - }; + const wxString kLangChoices[] = { + "english", + "afrikaans", + "albanian", + "amharic", + "arabic", + "armenian", + "assamese", + "azerbaijani", + "bashkir", + "basque", + "belarusian", + "bengali", + "bosnian", + "breton", + "bulgarian", + "catalan", + "chinese", + "croatian", + "czech", + "danish", + "dutch", + "estonian", + "faroese", + "finnish", + "french", + "galician", + "georgian", + "german", + "greek", + "gujarati", + "haitian creole", + "hausa", + "hawaiian", + "hebrew", + "hindi", + "hungarian", + "icelandic", + "indonesian", + "italian", + "japanese", + "javanese", + "kannada", + "kazakh", + "khmer", + "korean", + "lao", + "latin", + "latvian", + "lingala", + "lithuanian", + "luxembourgish", + "macedonian", + "malagasy", + "malay", + "malayalam", + "maltese", + "maori", + "marathi", + "mongolian", + "myanmar", + "nepali", + "norwegian", + "nynorsk", + "occitan", + "pashto", + "persian", + "polish", + "portuguese", + "punjabi", + "romanian", + "russian", + "sanskrit", + "serbian", + "shona", + "sindhi", + "sinhala", + "slovak", + "slovenian", + "somali", + "spanish", + "sundanese" + "swahili", + "swedish", + "tagalog", + "tajik", + "tamil", + "tatar", + "telugu", + "thai", + "tibetan", + "turkish", + "turkmen", + "ukrainian", + "urdu", + "uzbek", + "vietnamese", + "welsh", + "yiddish", + "yoruba", + }; const size_t kNumLangChoices = sizeof(kLangChoices) / sizeof(kLangChoices[0]); constexpr int kLangDefault = 0; // english @@ -633,17 +632,9 @@ Frame::Frame() ID_PY_APP_LANG, wxDefaultPosition, wxDefaultSize, kNumLangChoices, kLangChoices); py_app_lang->SetToolTip("Select which language you will " - "speak in. If using something other than English, " - "make sure you're not using a *.en model."); + "speak in."); py_app_lang_ = py_app_lang; - auto* py_app_translate_source = new wxChoice(py_app_config_panel_pairs, - ID_PY_APP_TRANSLATE_SOURCE, wxDefaultPosition, wxDefaultSize, - kNumLangTargetChoices, kLangTargetChoices); - py_app_translate_source->SetToolTip("Select which language to " - "translate from, in other words, the language you are transcribing into."); - py_app_translate_source_ = py_app_translate_source; - auto* py_app_translate_target = new wxChoice(py_app_config_panel_pairs, ID_PY_APP_TRANSLATE_TARGET, wxDefaultPosition, wxDefaultSize, kNumLangTargetChoices, kLangTargetChoices); @@ -674,7 +665,8 @@ Frame::Frame() py_app_model_translation->SetToolTip("Select which " "version of the translation model to use. 600M params " "uses 4.1 GB of memory, while 1.3B uses ~7GB of " - "memory."); + "memory. If 'Translate to' is set to 'Do not " + "translate', this does nothing."); py_app_model_translation_ = py_app_model_translation; auto* py_app_chars_per_sync = new wxChoice( @@ -767,8 +759,8 @@ Frame::Frame() /*flags=*/wxEXPAND); sizer->Add(new wxStaticText(py_app_config_panel_pairs, - wxID_ANY, /*label=*/"Translate from:")); - sizer->Add(py_app_translate_source, /*proportion=*/0, + wxID_ANY, /*label=*/"Transcription model:")); + sizer->Add(py_app_model, /*proportion=*/0, /*flags=*/wxEXPAND); sizer->Add(new wxStaticText(py_app_config_panel_pairs, @@ -776,11 +768,6 @@ Frame::Frame() sizer->Add(py_app_translate_target, /*proportion=*/0, /*flags=*/wxEXPAND); - sizer->Add(new wxStaticText(py_app_config_panel_pairs, - wxID_ANY, /*label=*/"Transcription model:")); - sizer->Add(py_app_model, /*proportion=*/0, - /*flags=*/wxEXPAND); - sizer->Add(new wxStaticText(py_app_config_panel_pairs, wxID_ANY, /*label=*/"Translation model:")); sizer->Add(py_app_model_translation, /*proportion=*/0, @@ -1344,11 +1331,6 @@ void Frame::ApplyConfigToInputFields() kNumLangChoices, app_c_->language, kLangDefault); py_app_lang->SetSelection(lang_idx); - auto* py_app_translate_source = static_cast(FindWindowById(ID_PY_APP_TRANSLATE_SOURCE)); - int translate_source_idx = GetDropdownChoiceIndex(kLangTargetChoices, - kNumLangTargetChoices, app_c_->language_source, kLangTargetDefault); - py_app_translate_source->SetSelection(translate_source_idx); - auto* py_app_translate_target = static_cast(FindWindowById(ID_PY_APP_TRANSLATE_TARGET)); int translate_target_idx = GetDropdownChoiceIndex(kLangTargetChoices, kNumLangTargetChoices, app_c_->language_target, kLangTargetDefault); @@ -1968,10 +1950,6 @@ void Frame::OnAppStart(wxCommandEvent& event) { if (which_lang == wxNOT_FOUND) { which_lang = kLangDefault; } - int which_translate_source = py_app_translate_source_->GetSelection(); - if (which_translate_source == wxNOT_FOUND) { - which_translate_source = kLangDefault; - } int which_translate_target = py_app_translate_target_->GetSelection(); if (which_translate_target == wxNOT_FOUND) { which_translate_target = kLangDefault; @@ -2061,7 +2039,6 @@ void Frame::OnAppStart(wxCommandEvent& event) { app_c_->microphone = kMicChoices[which_mic].ToStdString(); app_c_->language = kLangChoices[which_lang].ToStdString(); - app_c_->language_source = kLangTargetChoices[which_translate_source].ToStdString(); app_c_->language_target = kLangTargetChoices[which_translate_target].ToStdString(); app_c_->model = kModelChoices[which_model].ToStdString(); app_c_->model_translation = kModelTranslationChoices[which_model_translation].ToStdString(); diff --git a/GUI/GUI/GUI/Frame.h b/GUI/GUI/GUI/Frame.h index 24a0594..4176e83 100644 --- a/GUI/GUI/GUI/Frame.h +++ b/GUI/GUI/GUI/Frame.h @@ -62,7 +62,6 @@ private: wxChoice* py_app_mic_; wxChoice* py_app_lang_; - wxChoice* py_app_translate_source_; wxChoice* py_app_translate_target_; wxChoice* py_app_model_; wxChoice* py_app_model_translation_; diff --git a/GUI/GUI/GUI/Logging.h b/GUI/GUI/GUI/Logging.h index c85a376..193617a 100644 --- a/GUI/GUI/GUI/Logging.h +++ b/GUI/GUI/GUI/Logging.h @@ -40,8 +40,9 @@ namespace Logging { void Log(wxTextCtrl* frame, std::string_view format, Args&&... args) { const std::string raw = std::vformat(format, std::make_format_args(args...)); const std::string masked = HidePII(std::move(raw)); + const std::string decoded = wxString::FromUTF8(masked).ToStdString(); - kThreadLogger.Append(frame, std::move(masked)); + kThreadLogger.Append(frame, std::move(decoded)); } void DrainAsyncOutput(wxProcess* proc, wxTextCtrl* frame); diff --git a/GUI/GUI/GUI/PythonWrapper.cpp b/GUI/GUI/GUI/PythonWrapper.cpp index b95d6b3..2172ac4 100644 --- a/GUI/GUI/GUI/PythonWrapper.cpp +++ b/GUI/GUI/GUI/PythonWrapper.cpp @@ -463,7 +463,6 @@ std::future PythonWrapper::StartApp( "Resources/Scripts/transcribe.py", "--mic", config.microphone, "--language", config.language, - "--language_source", Quote(config.language_source), "--language_target", Quote(config.language_target), "--model", config.model, "--model_translation", config.model_translation, diff --git a/Scripts/lang_compat.py b/Scripts/lang_compat.py new file mode 100644 index 0000000..af35921 --- /dev/null +++ b/Scripts/lang_compat.py @@ -0,0 +1,58 @@ +# This file provides mappings between language codes used by different +# third-party libraries. + +# Whisper to NLLB. +whisper_to_nllb = { + "catalan": "cat_Ltn", # catalan + "czech": "ces_Latn", # czech + "danish": "dan_Latn", # danish + "dutch": "nld_Latn", # dutch + "english": "eng_Latn", # english + "finnish": "fin_Latn", # finnish + "french": "fra_Latn", # french + "german": "deu_Latn", # german + "greek": "ell_Grek", # greek + "hungarian": "hun_Latn", # hungarian + "icelandic": "isl_Latn", # icelandic + "italian": "ita_Latn", # italian + "latvian": "lvs_Latn", # latvian + "lithuanian": "lit_Latn", # lithuanian + "norwegian": "nob_Latn", # norwegian (bokmal) + "polish": "pol_Latn", # polish + "portugese": "por_Latn", # portugese + "romanian": "ron_Latn", # romanian + "russian": "rus_Cyrl", # russian + "slovak": "slk_Latn", # slovak + "slovene": "slv_Latn", # slovene + "spanish": "spa_Latn", # spanish + "swedish": "swe_Latn", # swedish + "turkish": "tur_Latn", # turkish + } + +# NLLB to sentence_splitter (SS). +nllb_to_ss = { + "cat_Ltn": "ca", # catalan + "ces_Latn": "cs", # czech + "dan_Latn": "da", # danish + "nld_Latn": "nl", # dutch + "eng_Latn": "en", # english + "fin_Latn": "fi", # finnish + "fra_Latn": "fr", # french + "deu_Latn": "de", # german + "ell_Grek": "el", # greek + "hun_Latn": "hu", # hungarian + "isl_Latn": "is", # icelandic + "ita_Latn": "it", # italian + "lvs_Latn": "lv", # latvian + "lit_Latn": "lt", # lithuanian + "nob_Latn": "no", # norwegian (bokmal) + "pol_Latn": "pl", # polish + "por_Latn": "pt", # portugese + "ron_Latn": "ro", # romanian + "rus_Cyrl": "ru", # russian + "slk_Latn": "sk", # slovak + "slv_Latn": "sl", # slovene + "spa_Latn": "es", # spanish + "swe_Latn": "sv", # swedish + "tur_Latn": "tr", # turkish + } diff --git a/Scripts/requirements.txt b/Scripts/requirements.txt index 5500a91..647e942 100644 --- a/Scripts/requirements.txt +++ b/Scripts/requirements.txt @@ -11,4 +11,5 @@ pyaudio python-osc playsound==1.2.2 pyyaml +sentence_splitter transformers>=4.21.0 diff --git a/Scripts/transcribe.py b/Scripts/transcribe.py index e113be1..fe06631 100644 --- a/Scripts/transcribe.py +++ b/Scripts/transcribe.py @@ -5,6 +5,7 @@ from emotes_v2 import EmotesState from faster_whisper import WhisperModel from functools import partial from playsound import playsound +from sentence_splitter import split_text_into_sentences import argparse import copy @@ -12,6 +13,7 @@ import ctranslate2 import generate_utils import keybind_event_machine import keyboard +import lang_compat import langcodes import numpy as np import os @@ -71,7 +73,7 @@ class AudioState: # The language the user is speaking in. Default is English but user may set # this to whatever they want. - self.language = "en" + self.language = "english" self.audio_paused = False @@ -257,6 +259,8 @@ def transcribeAudio(audio_state, if audio_state.drop_transcription: audio_state.drop_transcription = False + audio_state.text = "" + audio_state.filtered_text = "" print("drop transcription ({} seconds)".format(time.time() - last_transcribe_time)) last_transcribe_time = time.time() continue @@ -265,16 +269,30 @@ def transcribeAudio(audio_state, audio_state.text = string_matcher.matchStrings(audio_state.text, text, window_size = 25) + now = time.time() + print("Transcription ({} seconds): {}".format( + now - last_transcribe_time, + audio_state.text)) + last_transcribe_time = now + # Translate if requested. - if audio_state.language_source and audio_state.language_target: - source = audio_state.tokenizer.convert_ids_to_tokens(audio_state.tokenizer.encode(copy.copy(audio_state.text))) - target_prefix = [audio_state.language_target] - results = audio_state.translator.translate_batch([source], target_prefix=[target_prefix]) - target = results[0].hypotheses[0][1:] - translated = audio_state.tokenizer.decode(audio_state.tokenizer.convert_tokens_to_ids(target)) - print(f"Translated text: {translated}") - else: - translated = copy.copy(audio_state.text) + translated = audio_state.text + if audio_state.language_target: + whisper_lang = audio_state.whisper_language + nllb_lang = lang_compat.whisper_to_nllb[whisper_lang] + ss_lang = lang_compat.nllb_to_ss[nllb_lang] + sentences = split_text_into_sentences(translated, language=ss_lang) + + translated_sentences = [] + for sentence in sentences: + source = audio_state.tokenizer.convert_ids_to_tokens(audio_state.tokenizer.encode(sentence)) + target_prefix = [audio_state.language_target] + results = audio_state.translator.translate_batch([source], target_prefix=[target_prefix]) + target = results[0].hypotheses[0][1:] + translated_sentence = audio_state.tokenizer.decode(audio_state.tokenizer.convert_tokens_to_ids(target)) + translated_sentences.append(translated_sentence) + translated = " ".join(translated_sentences) + print(f"Translation: {translated}") # Apply filters to transcription filtered_text = translated @@ -296,12 +314,6 @@ def transcribeAudio(audio_state, filtered_text = filtered_text.lower() audio_state.filtered_text = filtered_text - now = time.time() - print("Transcription ({} seconds): {}".format( - now - last_transcribe_time, - audio_state.text)) - last_transcribe_time = now - if old_text != audio_state.text: # We think the user said something, so reset the amount of # time we sleep between transcriptions to the minimum. @@ -358,10 +370,10 @@ def readKeyboardInput(audio_state, enable_local_beep: bool, osc_ctrl.toggleBoard(audio_state.osc_state.client, False) #playsound(os.path.abspath("../Sounds/Noise_Off_Quiet.wav")) - resetAudioLocked(audio_state) - resetDisplayLocked(audio_state) audio_state.drop_transcription = True audio_state.audio_paused = True + resetAudioLocked(audio_state) + resetDisplayLocked(audio_state) continue # Short hold @@ -383,12 +395,12 @@ def readKeyboardInput(audio_state, enable_local_beep: bool, osc_ctrl.indicateSpeech(audio_state.osc_state.client, True) osc_ctrl.toggleBoard(audio_state.osc_state.client, True) osc_ctrl.lockWorld(audio_state.osc_state.client, False) - resetAudioLocked(audio_state) - resetDisplayLocked(audio_state) - audio_state.drop_transcription = True audio_state.audio_paused = False + resetAudioLocked(audio_state) + resetDisplayLocked(audio_state) + if enable_local_beep == 1: playsound(os.path.abspath("Resources/Sounds/Noise_On_Quiet.wav"), block=False) @@ -506,7 +518,6 @@ def readControllerInput(audio_state, enable_local_beep: bool, # whisper/__init__.py. Examples: tiny, base, small, medium. def transcribeLoop(mic: str, language: str, - language_source: str, language_target: str, model: str, model_translation: str, @@ -523,6 +534,7 @@ def transcribeLoop(mic: str, gpu_idx: int, keyboard_hotkey: str): audio_state = getMicStream(mic) + audio_state.whisper_language = language audio_state.language = langcodes.find(language).language audio_state.MAX_LENGTH_S = window_duration_s @@ -532,14 +544,8 @@ def transcribeLoop(mic: str, audio_state.language_target = lang_code else: audio_state.language_target = None - lang_bits = language_source.split(" | ") - if len(lang_bits) == 2: - lang_code = lang_bits[1] - audio_state.language_source = lang_code - else: - audio_state.language_source = None - if audio_state.language_source and audio_state.language_target: + if audio_state.language_target: print("Translation requested") print("Installing torch and sentencepiece in virtual environment. " @@ -579,9 +585,15 @@ def transcribeLoop(mic: str, print(f"Using model at {output_dir}") audio_state.translator = ctranslate2.Translator(output_dir) + + whisper_lang = audio_state.whisper_language + nllb_lang = lang_compat.whisper_to_nllb[whisper_lang] + audio_state.tokenizer = transformers.AutoTokenizer.from_pretrained( "facebook/" + model_translation, - src_lang=audio_state.language_source) + src_lang=nllb_lang) + + print(f"Translation ready to go") print("Safe to start talking") @@ -661,7 +673,6 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--mic", type=str, help="Which mic to use. Options: index, focusrite. Default: index") parser.add_argument("--language", type=str, help="Which language to use. Ex: english, japanese, chinese, french, german.") - parser.add_argument("--language_source", type=str, help="Which language to translate from. See kLangTargetChoices in Frame.cpp for valid choices") parser.add_argument("--language_target", type=str, help="Which language to translate into. See kLangTargetChoices in Frame.cpp for valid choices") parser.add_argument("--model", type=str, help="Which transcription model to use. " \ "Options: tiny, tiny.en, base, base.en, small, small.en, " \ @@ -692,8 +703,8 @@ if __name__ == "__main__": if not args.language: args.language = "english" - if not args.language_source or not args.language_target: - print("--language_source and --language_target required", file=sys.stderr) + if not args.language_target: + print("--language_target required", file=sys.stderr) if not args.model: args.model = "base" @@ -769,7 +780,6 @@ if __name__ == "__main__": transcribeLoop(args.mic, args.language, - args.language_source, args.language_target, args.model, args.model_translation, -- cgit v1.2.3