From ba7fb6d1085a4981e3dbaa2f1151483051ebc808 Mon Sep 17 00:00:00 2001 From: yum Date: Wed, 22 Feb 2023 23:40:46 -0800 Subject: Add input fields for browser source Not wired up yet. * Add browser source fields to persistent config --- GUI/GUI/GUI/.gitignore | 1 + GUI/GUI/GUI/BrowserSource.cpp | 19 +++++++++++++-- GUI/GUI/GUI/BrowserSource.h | 27 ++++++++++++++------ GUI/GUI/GUI/Config.cpp | 17 ++++++++++++- GUI/GUI/GUI/Config.h | 6 +++++ GUI/GUI/GUI/Frame.cpp | 57 +++++++++++++++++++++++++++++++++++-------- GUI/GUI/GUI/Frame.h | 5 +++- GUI/GUI/GUI/WhisperCPP.cpp | 53 +++++++++++++++++++++++++++++++--------- GUI/GUI/GUI/WhisperCPP.h | 14 ++++++++--- 9 files changed, 162 insertions(+), 37 deletions(-) (limited to 'GUI') diff --git a/GUI/GUI/GUI/.gitignore b/GUI/GUI/GUI/.gitignore index 36df1a2..75f9933 100644 --- a/GUI/GUI/GUI/.gitignore +++ b/GUI/GUI/GUI/.gitignore @@ -6,3 +6,4 @@ GUI.APS # No fetched files ryml.h whisper/ +oatpp/ diff --git a/GUI/GUI/GUI/BrowserSource.cpp b/GUI/GUI/GUI/BrowserSource.cpp index 9df4f81..030b982 100644 --- a/GUI/GUI/GUI/BrowserSource.cpp +++ b/GUI/GUI/GUI/BrowserSource.cpp @@ -1,12 +1,27 @@ #include "BrowserSource.h" +#include "Logging.h" #include "oatpp/network/Server.hpp" -BrowserSource::BrowserSource(uint16_t port) - : port_(port) +BrowserSource::BrowserSource(uint16_t port, wxTextCtrl *out) + : port_(port), out_(out) +{} + +void BrowserSource::Run(volatile bool* run) { AppComponent components; + OATPP_COMPONENT(std::shared_ptr, router); router->addController(std::make_shared()); + OATPP_COMPONENT(std::shared_ptr, connectionHandler); + + OATPP_COMPONENT(std::shared_ptr, connectionProvider); + + oatpp::network::Server server(connectionProvider, connectionHandler); + + OATPP_LOGI("BrowserSource", "Server running on port %s", + connectionProvider->getProperty("port").getData()); + + server.run(std::function([run]() { return *run == true; })); } diff --git a/GUI/GUI/GUI/BrowserSource.h b/GUI/GUI/GUI/BrowserSource.h index efc6142..5b75c21 100644 --- a/GUI/GUI/GUI/BrowserSource.h +++ b/GUI/GUI/GUI/BrowserSource.h @@ -1,5 +1,11 @@ #pragma once +#include + +#ifndef WX_PRECOMP +#include +#endif + #include "oatpp/core/macro/codegen.hpp" #include "oatpp/core/macro/component.hpp" #include "oatpp/core/Types.hpp" @@ -13,20 +19,20 @@ #include OATPP_CODEGEN_BEGIN(DTO) -class AppDto : public oatpp::DTO { - +class AppDto : public oatpp::DTO +{ DTO_INIT(AppDto, DTO) - DTO_FIELD(Int32, statusCode); + DTO_FIELD(Int32, statusCode); DTO_FIELD(String, message); - }; #include OATPP_CODEGEN_END(DTO) #include OATPP_CODEGEN_BEGIN(ApiController) -class AppController : public oatpp::web::server::api::ApiController { +class AppController : public oatpp::web::server::api::ApiController +{ public: AppController(OATPP_COMPONENT(std::shared_ptr, objectMapper)) : oatpp::web::server::api::ApiController(objectMapper) @@ -43,7 +49,8 @@ public: #include OATPP_CODEGEN_END(ApiController) -class AppComponent { +class AppComponent +{ public: // TODO(yum) parameterize port OATPP_CREATE_COMPONENT(std::shared_ptr, serverConnectionProvider)([] { @@ -64,10 +71,14 @@ public: }()); }; -class BrowserSource { +class BrowserSource +{ public: - BrowserSource(uint16_t port); + BrowserSource(uint16_t port, wxTextCtrl *out); + + void Run(volatile bool* run); private: const uint16_t port_; + wxTextCtrl* const out_; }; diff --git a/GUI/GUI/GUI/Config.cpp b/GUI/GUI/GUI/Config.cpp index a877d4a..bcc7375 100644 --- a/GUI/GUI/GUI/Config.cpp +++ b/GUI/GUI/GUI/Config.cpp @@ -90,7 +90,12 @@ AppConfig::AppConfig() clear_osc(false), whisper_model("ggml-base.en.bin"), - whisper_mic(0) + whisper_mic(0), + + browser_src_port(9010), + whisper_enable_builtin(false), + whisper_enable_custom(true), + whisper_enable_browser_src(false) {} bool AppConfig::Serialize(const std::filesystem::path& path) { @@ -121,6 +126,11 @@ bool AppConfig::Serialize(const std::filesystem::path& path) { root["whisper_model"] << whisper_model; root["whisper_mic"] << whisper_mic; + root["browser_src_port"] << browser_src_port; + root["whisper_enable_builtin"] << whisper_enable_builtin; + root["whisper_enable_custom"] << whisper_enable_custom; + root["whisper_enable_browser_src"] << whisper_enable_browser_src; + return Config::Serialize(path, &t); } @@ -163,6 +173,11 @@ bool AppConfig::Deserialize(const std::filesystem::path& path) { root.get_if("whisper_model", &c.whisper_model); root.get_if("whisper_mic", &c.whisper_mic); + root.get_if("browser_src_port", &c.browser_src_port); + root.get_if("whisper_enable_builtin", &c.whisper_enable_builtin); + root.get_if("whisper_enable_custom", &c.whisper_enable_custom); + root.get_if("whisper_enable_browser_src", &c.whisper_enable_browser_src); + *this = std::move(c); return true; } diff --git a/GUI/GUI/GUI/Config.h b/GUI/GUI/GUI/Config.h index b27dcc5..bf5bfb0 100644 --- a/GUI/GUI/GUI/Config.h +++ b/GUI/GUI/GUI/Config.h @@ -64,5 +64,11 @@ public: // WhisperCPP-specific settings. std::string whisper_model; int whisper_mic; + + // Browser source-specific settings. + int browser_src_port; + bool whisper_enable_builtin; + bool whisper_enable_custom; + bool whisper_enable_browser_src; }; diff --git a/GUI/GUI/GUI/Frame.cpp b/GUI/GUI/GUI/Frame.cpp index 0bac3aa..ce3d832 100644 --- a/GUI/GUI/GUI/Frame.cpp +++ b/GUI/GUI/GUI/Frame.cpp @@ -83,9 +83,12 @@ namespace { ID_WHISPER_ROWS, ID_WHISPER_COLS, ID_WHISPER_WINDOW_DURATION, + ID_WHISPER_BROWSER_SRC_PORT, ID_WHISPER_ENABLE_LOCAL_BEEP, ID_WHISPER_USE_CPU, - ID_WHISPER_USE_BUILTIN, + ID_WHISPER_ENABLE_BUILTIN, + ID_WHISPER_ENABLE_CUSTOM, + ID_WHISPER_ENABLE_BROWSER_SRC, ID_WHISPER_START_BUTTON, ID_WHISPER_STOP_BUTTON, }; @@ -949,6 +952,16 @@ Frame::Frame() "but are far more accurate."); whisper_window_duration_ = whisper_window_duration; + auto* whisper_browser_src_port = new wxTextCtrl( + whisper_config_panel_pairs, ID_WHISPER_BROWSER_SRC_PORT, + std::to_string(app_c_.browser_src_port), wxDefaultPosition, + wxDefaultSize, /*style=*/0); + whisper_browser_src_port->SetToolTip( + "This is the port that the browser source is hosted " + "on. If you aren't using TaSTT to stream, you can " + "ignore this option."); + whisper_browser_src_port_ = whisper_browser_src_port; + auto* sizer = new wxFlexGridSizer(/*cols=*/2); whisper_config_panel_pairs->SetSizer(sizer); @@ -996,6 +1009,11 @@ Frame::Frame() wxID_ANY, /*label=*/"Window duration (s):")); sizer->Add(whisper_window_duration, /*proportion=*/0, /*flags=*/wxEXPAND); + + sizer->Add(new wxStaticText(whisper_config_panel_pairs, + wxID_ANY, /*label=*/"Browser source port:")); + sizer->Add(whisper_browser_src_port, /*proportion=*/0, + /*flags=*/wxEXPAND); } auto* whisper_enable_local_beep = new wxCheckBox(whisper_config_panel, @@ -1019,14 +1037,29 @@ Frame::Frame() ); whisper_use_cpu_ = whisper_use_cpu; - auto* whisper_use_builtin = new wxCheckBox(whisper_config_panel, - ID_WHISPER_USE_BUILTIN, "Use built-in chatbox"); - whisper_use_builtin->SetValue(app_c_.use_builtin); - whisper_use_builtin->SetToolTip( - "If checked, text will be sent to the built-in text box " - "instead of one attached to the current avatar." + auto* whisper_enable_builtin = new wxCheckBox(whisper_config_panel, + ID_WHISPER_ENABLE_BUILTIN, "Send to built-in chatbox"); + whisper_enable_builtin->SetValue(app_c_.whisper_enable_builtin); + whisper_enable_builtin->SetToolTip( + "If checked, text will be sent to the built-in text box." + ); + whisper_enable_builtin_ = whisper_enable_builtin; + + auto* whisper_enable_custom = new wxCheckBox(whisper_config_panel, + ID_WHISPER_ENABLE_CUSTOM, "Send to custom chatbox"); + whisper_enable_custom->SetValue(app_c_.whisper_enable_custom); + whisper_enable_custom->SetToolTip( + "If checked, text will be sent to the custom text box." ); - whisper_use_builtin_ = whisper_use_builtin; + whisper_enable_custom_ = whisper_enable_custom; + + auto* whisper_enable_browser_src = new wxCheckBox(whisper_config_panel, + ID_WHISPER_ENABLE_BROWSER_SRC, "Send to browser source"); + whisper_enable_browser_src->SetValue(app_c_.whisper_enable_browser_src); + whisper_enable_browser_src->SetToolTip( + "If checked, text will be sent to a browser source. If " + "you're not using TaSTT to stream, you can ignore this option."); + whisper_enable_browser_src_ = whisper_enable_browser_src; // Hack: Add newlines before and after the button text to make // the buttons bigger, and easier to click from inside VR. @@ -1043,7 +1076,11 @@ Frame::Frame() /*flags=*/wxEXPAND); sizer->Add(whisper_use_cpu, /*proportion=*/0, /*flags=*/wxEXPAND); - sizer->Add(whisper_use_builtin, /*proportion=*/0, + sizer->Add(whisper_enable_builtin, /*proportion=*/0, + /*flags=*/wxEXPAND); + sizer->Add(whisper_enable_custom, /*proportion=*/0, + /*flags=*/wxEXPAND); + sizer->Add(whisper_enable_browser_src, /*proportion=*/0, /*flags=*/wxEXPAND); sizer->Add(whisper_start_button, /*proportion=*/0, /*flags=*/wxEXPAND); @@ -1919,7 +1956,7 @@ void Frame::OnWhisperStart(wxCommandEvent& event) { } const bool enable_local_beep = whisper_enable_local_beep_->GetValue(); const bool use_cpu = whisper_use_cpu_->GetValue(); - const bool use_builtin = whisper_use_builtin_->GetValue(); + const bool use_builtin = whisper_enable_builtin_->GetValue(); std::string rows_str = whisper_rows_->GetValue().ToStdString(); std::string cols_str = whisper_cols_->GetValue().ToStdString(); std::string chars_per_sync_str = diff --git a/GUI/GUI/GUI/Frame.h b/GUI/GUI/GUI/Frame.h index 5d69679..d40ba24 100644 --- a/GUI/GUI/GUI/Frame.h +++ b/GUI/GUI/GUI/Frame.h @@ -44,6 +44,7 @@ private: wxTextCtrl* whisper_rows_; wxTextCtrl* whisper_cols_; wxTextCtrl* whisper_window_duration_; + wxTextCtrl* whisper_browser_src_port_; wxDirPickerCtrl* unity_assets_file_picker_; wxFilePickerCtrl* unity_animator_file_picker_; @@ -72,7 +73,9 @@ private: wxCheckBox* unity_clear_osc_; wxCheckBox* whisper_enable_local_beep_; wxCheckBox* whisper_use_cpu_; - wxCheckBox* whisper_use_builtin_; + wxCheckBox* whisper_enable_builtin_; + wxCheckBox* whisper_enable_custom_; + wxCheckBox* whisper_enable_browser_src_; wxProcess* py_app_; wxProcess* env_proc_; diff --git a/GUI/GUI/GUI/WhisperCPP.cpp b/GUI/GUI/GUI/WhisperCPP.cpp index 2bb5b34..b730236 100644 --- a/GUI/GUI/GUI/WhisperCPP.cpp +++ b/GUI/GUI/GUI/WhisperCPP.cpp @@ -1,3 +1,4 @@ +#include "BrowserSource.h" #include "Logging.h" #include "PythonWrapper.h" #include "ScopeGuard.h" @@ -8,7 +9,7 @@ #include #include -#include "whisperWindows.h" +#include "whisper/whisperWindows.h" #include #include @@ -61,11 +62,20 @@ namespace { }; WhisperCPP::WhisperCPP(wxTextCtrl* out) - : out_(out), f_(nullptr), did_init_(false), proc_(), run_(false) + : out_(out), f_(nullptr), did_init_(false), run_transcription_(false), run_browser_src_(false) { - auto p = std::promise(); - proc_ = p.get_future(); - p.set_value(); + // Initialize futures so that valid() returns true. We use this as a proxy + // to tell whether they're still executing. + { + auto p = std::promise(); + transcription_thd_ = p.get_future(); + p.set_value(); + } + { + auto p = std::promise(); + browser_src_thd_ = p.get_future(); + p.set_value(); + } } WhisperCPP::~WhisperCPP() { @@ -222,14 +232,14 @@ bool WhisperCPP::CreateContext(Whisper::iModel* model, Whisper::iContext*& conte } void WhisperCPP::Start(const AppConfig& c) { - if (!proc_.valid()) { + if (!transcription_thd_.valid()) { Log(out_, "Transcription engine already running\n"); return; } - proc_ = std::async(std::launch::async, [&]() -> void { + transcription_thd_ = std::async(std::launch::async, [&]() -> void { Log(out_, "Transcription thread top\n"); - run_ = true; + run_transcription_ = true; Whisper::iAudioCapture* mic_stream; if (!OpenMic(c.whisper_mic, mic_stream)) { @@ -339,7 +349,7 @@ void WhisperCPP::Start(const AppConfig& c) { callbacks.shouldCancel = [](void* pv) noexcept -> HRESULT __stdcall { WhisperCPP* app = static_cast(pv); - if (!app->run_) { + if (!app->run_transcription_) { Log(app->out_, "Exit transcription loop\n"); return S_FALSE; } @@ -369,8 +379,29 @@ void WhisperCPP::Start(const AppConfig& c) { void WhisperCPP::Stop() { Log(out_, "Stopping transcription engine...\n"); - run_ = false; - proc_.wait(); + run_transcription_ = false; + transcription_thd_.wait(); + Log(out_, "Done!\n"); +} + +void WhisperCPP::StartBrowserSource(const AppConfig& c) { + if (!browser_src_thd_.valid()) { + Log(out_, "Transcription engine already running\n"); + return; + } + + browser_src_thd_ = std::async(std::launch::async, [&]() -> void { +#if 0 + BrowserSource src(c.browser_src_port, out_); + src.Run(&run_browser_src_); +#endif + }); +} + +void WhisperCPP::StopBrowserSource() { + Log(out_, "Stopping browser source engine...\n"); + run_browser_src_ = false; + browser_src_thd_.wait(); Log(out_, "Done!\n"); } diff --git a/GUI/GUI/GUI/WhisperCPP.h b/GUI/GUI/GUI/WhisperCPP.h index ed934be..c6e1011 100644 --- a/GUI/GUI/GUI/WhisperCPP.h +++ b/GUI/GUI/GUI/WhisperCPP.h @@ -1,6 +1,5 @@ #pragma once -#include #include #ifndef WX_PRECOMP @@ -11,7 +10,7 @@ #include #include -#include "whisperWindows.h" +#include "whisper/whisperWindows.h" #include "Config.h" @@ -38,12 +37,19 @@ public: void Start(const AppConfig& c); void Stop(); + void StartBrowserSource(const AppConfig& c); + void StopBrowserSource(); + private: bool GetMicsImpl(std::vector& mics); wxTextCtrl* out_; Whisper::iMediaFoundation* f_; bool did_init_; - std::future proc_; - volatile bool run_; + + std::future transcription_thd_; + volatile bool run_transcription_; + + std::future browser_src_thd_; + volatile bool run_browser_src_; }; -- cgit v1.2.3