Add input fields for browser source

Not wired up yet.

* Add browser source fields to persistent config

9 files changed, 162 insertions, 37 deletions

diff --git a/GUI/GUI/GUI/.gitignore b/GUI/GUI/GUI/.gitignore
index 36df1a2..75f9933 100644
--- a/GUI/GUI/GUI/.gitignore
+++ b/GUI/GUI/GUI/.gitignore
@@ -6,3 +6,4 @@ GUI.APS
 # No fetched files
 ryml.h
 whisper/
+oatpp/
diff --git a/GUI/GUI/GUI/BrowserSource.cpp b/GUI/GUI/GUI/BrowserSource.cpp
index 9df4f81..030b982 100644
--- a/GUI/GUI/GUI/BrowserSource.cpp
+++ b/GUI/GUI/GUI/BrowserSource.cpp
@@ -1,12 +1,27 @@
 #include "BrowserSource.h"

+#include "Logging.h"

 

 #include "oatpp/network/Server.hpp"

 

-BrowserSource::BrowserSource(uint16_t port)

-	: port_(port)

+BrowserSource::BrowserSource(uint16_t port, wxTextCtrl *out)

+	: port_(port), out_(out)

+{}

+

+void BrowserSource::Run(volatile bool* run)

 {

 	AppComponent components;

+

 	OATPP_COMPONENT(std::shared_ptr<oatpp::web::server::HttpRouter>, router);

 	router->addController(std::make_shared<AppController>());

 

+	OATPP_COMPONENT(std::shared_ptr<oatpp::network::ConnectionHandler>, connectionHandler);

+

+	OATPP_COMPONENT(std::shared_ptr<oatpp::network::ServerConnectionProvider>, connectionProvider);

+

+	oatpp::network::Server server(connectionProvider, connectionHandler);

+

+	OATPP_LOGI("BrowserSource", "Server running on port %s",

+		connectionProvider->getProperty("port").getData());

+

+	server.run(std::function<bool()>([run]() { return *run == true; }));

 }

diff --git a/GUI/GUI/GUI/BrowserSource.h b/GUI/GUI/GUI/BrowserSource.h
index efc6142..5b75c21 100644
--- a/GUI/GUI/GUI/BrowserSource.h
+++ b/GUI/GUI/GUI/BrowserSource.h
@@ -1,5 +1,11 @@
 #pragma once

 

+#include <wx/wxprec.h>

+

+#ifndef WX_PRECOMP

+#include <wx/wx.h>

+#endif

+

 #include "oatpp/core/macro/codegen.hpp"

 #include "oatpp/core/macro/component.hpp"

 #include "oatpp/core/Types.hpp"

@@ -13,20 +19,20 @@
 

 #include OATPP_CODEGEN_BEGIN(DTO)

 

-class AppDto : public oatpp::DTO {

-

+class AppDto : public oatpp::DTO

+{

     DTO_INIT(AppDto, DTO)

 

-        DTO_FIELD(Int32, statusCode);

+	DTO_FIELD(Int32, statusCode);

     DTO_FIELD(String, message);

-

 };

 

 #include OATPP_CODEGEN_END(DTO)

 

 #include OATPP_CODEGEN_BEGIN(ApiController)

 

-class AppController : public oatpp::web::server::api::ApiController {

+class AppController : public oatpp::web::server::api::ApiController

+{

 public:

     AppController(OATPP_COMPONENT(std::shared_ptr<ObjectMapper>, objectMapper))

         : oatpp::web::server::api::ApiController(objectMapper)

@@ -43,7 +49,8 @@ public:
 

 #include OATPP_CODEGEN_END(ApiController)

 

-class AppComponent {

+class AppComponent

+{

 public:

     // TODO(yum) parameterize port

 	OATPP_CREATE_COMPONENT(std::shared_ptr<oatpp::network::ServerConnectionProvider>, serverConnectionProvider)([] {

@@ -64,10 +71,14 @@ public:
         }());

 };

 

-class BrowserSource {

+class BrowserSource

+{

 public:

-	BrowserSource(uint16_t port);

+	BrowserSource(uint16_t port, wxTextCtrl *out);

+

+    void Run(volatile bool* run);

 

 private:

 	const uint16_t port_;

+    wxTextCtrl* const out_;

 };

diff --git a/GUI/GUI/GUI/Config.cpp b/GUI/GUI/GUI/Config.cpp
index a877d4a..bcc7375 100644
--- a/GUI/GUI/GUI/Config.cpp
+++ b/GUI/GUI/GUI/Config.cpp
@@ -90,7 +90,12 @@ AppConfig::AppConfig()
 	clear_osc(false),

 

 	whisper_model("ggml-base.en.bin"),

-	whisper_mic(0)

+	whisper_mic(0),

+

+	browser_src_port(9010),

+	whisper_enable_builtin(false),

+	whisper_enable_custom(true),

+	whisper_enable_browser_src(false)

 {}

 

 bool AppConfig::Serialize(const std::filesystem::path& path) {

@@ -121,6 +126,11 @@ bool AppConfig::Serialize(const std::filesystem::path& path) {
 	root["whisper_model"] << whisper_model;

 	root["whisper_mic"] << whisper_mic;

 

+	root["browser_src_port"] << browser_src_port;

+	root["whisper_enable_builtin"] << whisper_enable_builtin;

+	root["whisper_enable_custom"] << whisper_enable_custom;

+	root["whisper_enable_browser_src"] << whisper_enable_browser_src;

+

   return Config::Serialize(path, &t);

 }

 

@@ -163,6 +173,11 @@ bool AppConfig::Deserialize(const std::filesystem::path& path) {
 	root.get_if("whisper_model", &c.whisper_model);

 	root.get_if("whisper_mic", &c.whisper_mic);

 

+	root.get_if("browser_src_port", &c.browser_src_port);

+	root.get_if("whisper_enable_builtin", &c.whisper_enable_builtin);

+	root.get_if("whisper_enable_custom", &c.whisper_enable_custom);

+	root.get_if("whisper_enable_browser_src", &c.whisper_enable_browser_src);

+

 	*this = std::move(c);

 	return true;

 }

diff --git a/GUI/GUI/GUI/Config.h b/GUI/GUI/GUI/Config.h
index b27dcc5..bf5bfb0 100644
--- a/GUI/GUI/GUI/Config.h
+++ b/GUI/GUI/GUI/Config.h
@@ -64,5 +64,11 @@ public:
 	// WhisperCPP-specific settings.

 	std::string whisper_model;

 	int whisper_mic;

+

+	// Browser source-specific settings.

+	int browser_src_port;

+	bool whisper_enable_builtin;

+	bool whisper_enable_custom;

+	bool whisper_enable_browser_src;

 };

 

diff --git a/GUI/GUI/GUI/Frame.cpp b/GUI/GUI/GUI/Frame.cpp
index 0bac3aa..ce3d832 100644
--- a/GUI/GUI/GUI/Frame.cpp
+++ b/GUI/GUI/GUI/Frame.cpp
@@ -83,9 +83,12 @@ namespace {
 		ID_WHISPER_ROWS,

 		ID_WHISPER_COLS,

 		ID_WHISPER_WINDOW_DURATION,

+		ID_WHISPER_BROWSER_SRC_PORT,

 		ID_WHISPER_ENABLE_LOCAL_BEEP,

 		ID_WHISPER_USE_CPU,

-		ID_WHISPER_USE_BUILTIN,

+		ID_WHISPER_ENABLE_BUILTIN,

+		ID_WHISPER_ENABLE_CUSTOM,

+		ID_WHISPER_ENABLE_BROWSER_SRC,

 		ID_WHISPER_START_BUTTON,

 		ID_WHISPER_STOP_BUTTON,

     };

@@ -949,6 +952,16 @@ Frame::Frame()
                         "but are far more accurate.");

                     whisper_window_duration_ = whisper_window_duration;

 

+                    auto* whisper_browser_src_port = new wxTextCtrl(

+                        whisper_config_panel_pairs, ID_WHISPER_BROWSER_SRC_PORT,

+                        std::to_string(app_c_.browser_src_port), wxDefaultPosition,

+                        wxDefaultSize, /*style=*/0);

+                    whisper_browser_src_port->SetToolTip(

+                        "This is the port that the browser source is hosted "

+                        "on. If you aren't using TaSTT to stream, you can "

+                        "ignore this option.");

+                    whisper_browser_src_port_ = whisper_browser_src_port;

+

                     auto* sizer = new wxFlexGridSizer(/*cols=*/2);

                     whisper_config_panel_pairs->SetSizer(sizer);

 

@@ -996,6 +1009,11 @@ Frame::Frame()
                         wxID_ANY, /*label=*/"Window duration (s):"));

                     sizer->Add(whisper_window_duration, /*proportion=*/0,

                         /*flags=*/wxEXPAND);

+

+                    sizer->Add(new wxStaticText(whisper_config_panel_pairs,

+                        wxID_ANY, /*label=*/"Browser source port:"));

+                    sizer->Add(whisper_browser_src_port, /*proportion=*/0,

+                        /*flags=*/wxEXPAND);

                 }

 

                 auto* whisper_enable_local_beep = new wxCheckBox(whisper_config_panel,

@@ -1019,14 +1037,29 @@ Frame::Frame()
                 );

                 whisper_use_cpu_ = whisper_use_cpu;

 

-                auto* whisper_use_builtin = new wxCheckBox(whisper_config_panel,

-                    ID_WHISPER_USE_BUILTIN, "Use built-in chatbox");

-                whisper_use_builtin->SetValue(app_c_.use_builtin);

-                whisper_use_builtin->SetToolTip(

-                    "If checked, text will be sent to the built-in text box "

-                    "instead of one attached to the current avatar."

+                auto* whisper_enable_builtin = new wxCheckBox(whisper_config_panel,

+                    ID_WHISPER_ENABLE_BUILTIN, "Send to built-in chatbox");

+                whisper_enable_builtin->SetValue(app_c_.whisper_enable_builtin);

+                whisper_enable_builtin->SetToolTip(

+                    "If checked, text will be sent to the built-in text box."

+                );

+                whisper_enable_builtin_ = whisper_enable_builtin;

+

+                auto* whisper_enable_custom = new wxCheckBox(whisper_config_panel,

+                    ID_WHISPER_ENABLE_CUSTOM, "Send to custom chatbox");

+                whisper_enable_custom->SetValue(app_c_.whisper_enable_custom);

+                whisper_enable_custom->SetToolTip(

+                    "If checked, text will be sent to the custom text box."

                 );

-                whisper_use_builtin_ = whisper_use_builtin;

+                whisper_enable_custom_ = whisper_enable_custom;

+

+                auto* whisper_enable_browser_src = new wxCheckBox(whisper_config_panel,

+                    ID_WHISPER_ENABLE_BROWSER_SRC, "Send to browser source");

+                whisper_enable_browser_src->SetValue(app_c_.whisper_enable_browser_src);

+                whisper_enable_browser_src->SetToolTip(

+                    "If checked, text will be sent to a browser source. If "

+                    "you're not using TaSTT to stream, you can ignore this option.");

+                whisper_enable_browser_src_ = whisper_enable_browser_src;

 

                 // Hack: Add newlines before and after the button text to make

                 // the buttons bigger, and easier to click from inside VR.

@@ -1043,7 +1076,11 @@ Frame::Frame()
                     /*flags=*/wxEXPAND);

                 sizer->Add(whisper_use_cpu, /*proportion=*/0,

                     /*flags=*/wxEXPAND);

-                sizer->Add(whisper_use_builtin, /*proportion=*/0,

+                sizer->Add(whisper_enable_builtin, /*proportion=*/0,

+                    /*flags=*/wxEXPAND);

+                sizer->Add(whisper_enable_custom, /*proportion=*/0,

+                    /*flags=*/wxEXPAND);

+                sizer->Add(whisper_enable_browser_src, /*proportion=*/0,

                     /*flags=*/wxEXPAND);

                 sizer->Add(whisper_start_button, /*proportion=*/0,

                     /*flags=*/wxEXPAND);

@@ -1919,7 +1956,7 @@ void Frame::OnWhisperStart(wxCommandEvent& event) {
     }

     const bool enable_local_beep = whisper_enable_local_beep_->GetValue();

     const bool use_cpu = whisper_use_cpu_->GetValue();

-    const bool use_builtin = whisper_use_builtin_->GetValue();

+    const bool use_builtin = whisper_enable_builtin_->GetValue();

     std::string rows_str = whisper_rows_->GetValue().ToStdString();

     std::string cols_str = whisper_cols_->GetValue().ToStdString();

     std::string chars_per_sync_str =

diff --git a/GUI/GUI/GUI/Frame.h b/GUI/GUI/GUI/Frame.h
index 5d69679..d40ba24 100644
--- a/GUI/GUI/GUI/Frame.h
+++ b/GUI/GUI/GUI/Frame.h
@@ -44,6 +44,7 @@ private:
     wxTextCtrl* whisper_rows_;

     wxTextCtrl* whisper_cols_;

     wxTextCtrl* whisper_window_duration_;

+    wxTextCtrl* whisper_browser_src_port_;

 

     wxDirPickerCtrl* unity_assets_file_picker_;

     wxFilePickerCtrl* unity_animator_file_picker_;

@@ -72,7 +73,9 @@ private:
     wxCheckBox* unity_clear_osc_;

     wxCheckBox* whisper_enable_local_beep_;

     wxCheckBox* whisper_use_cpu_;

-    wxCheckBox* whisper_use_builtin_;

+    wxCheckBox* whisper_enable_builtin_;

+    wxCheckBox* whisper_enable_custom_;

+    wxCheckBox* whisper_enable_browser_src_;

 

     wxProcess* py_app_;

     wxProcess* env_proc_;

diff --git a/GUI/GUI/GUI/WhisperCPP.cpp b/GUI/GUI/GUI/WhisperCPP.cpp
index 2bb5b34..b730236 100644
--- a/GUI/GUI/GUI/WhisperCPP.cpp
+++ b/GUI/GUI/GUI/WhisperCPP.cpp
@@ -1,3 +1,4 @@
+#include "BrowserSource.h"

 #include "Logging.h"

 #include "PythonWrapper.h"

 #include "ScopeGuard.h"

@@ -8,7 +9,7 @@
 #include <wchar.h>

 #include <winerror.h>

 

-#include "whisperWindows.h"

+#include "whisper/whisperWindows.h"

 

 #include <charconv>

 #include <codecvt>

@@ -61,11 +62,20 @@ namespace {
 };

 

 WhisperCPP::WhisperCPP(wxTextCtrl* out)

-	: out_(out), f_(nullptr), did_init_(false), proc_(), run_(false)

+	: out_(out), f_(nullptr), did_init_(false), run_transcription_(false), run_browser_src_(false)

 {

-	auto p = std::promise<void>();

-	proc_ = p.get_future();

-	p.set_value();

+	// Initialize futures so that valid() returns true. We use this as a proxy

+	// to tell whether they're still executing.

+	{

+		auto p = std::promise<void>();

+		transcription_thd_ = p.get_future();

+		p.set_value();

+	}

+	{

+		auto p = std::promise<void>();

+		browser_src_thd_ = p.get_future();

+		p.set_value();

+	}

 }

 

 WhisperCPP::~WhisperCPP() {

@@ -222,14 +232,14 @@ bool WhisperCPP::CreateContext(Whisper::iModel* model, Whisper::iContext*& conte
 }

 

 void WhisperCPP::Start(const AppConfig& c) {

-	if (!proc_.valid()) {

+	if (!transcription_thd_.valid()) {

 		Log(out_, "Transcription engine already running\n");

 		return;

 	}

 

-	proc_ = std::async(std::launch::async, [&]() -> void {

+	transcription_thd_ = std::async(std::launch::async, [&]() -> void {

 		Log(out_, "Transcription thread top\n");

-		run_ = true;

+		run_transcription_ = true;

 

 		Whisper::iAudioCapture* mic_stream;

 		if (!OpenMic(c.whisper_mic, mic_stream)) {

@@ -339,7 +349,7 @@ void WhisperCPP::Start(const AppConfig& c) {
 

 		callbacks.shouldCancel = [](void* pv) noexcept -> HRESULT __stdcall {

 			WhisperCPP* app = static_cast<WhisperCPP*>(pv);

-			if (!app->run_) {

+			if (!app->run_transcription_) {

 				Log(app->out_, "Exit transcription loop\n");

 				return S_FALSE;

 			}

@@ -369,8 +379,29 @@ void WhisperCPP::Start(const AppConfig& c) {
 

 void WhisperCPP::Stop() {

 	Log(out_, "Stopping transcription engine...\n");

-	run_ = false;

-	proc_.wait();

+	run_transcription_ = false;

+	transcription_thd_.wait();

+	Log(out_, "Done!\n");

+}

+

+void WhisperCPP::StartBrowserSource(const AppConfig& c) {

+	if (!browser_src_thd_.valid()) {

+		Log(out_, "Transcription engine already running\n");

+		return;

+	}

+

+	browser_src_thd_ = std::async(std::launch::async, [&]() -> void {

+#if 0

+		BrowserSource src(c.browser_src_port, out_);

+		src.Run(&run_browser_src_);

+#endif

+	});

+}

+

+void WhisperCPP::StopBrowserSource() {

+	Log(out_, "Stopping browser source engine...\n");

+	run_browser_src_ = false;

+	browser_src_thd_.wait();

 	Log(out_, "Done!\n");

 }

 

diff --git a/GUI/GUI/GUI/WhisperCPP.h b/GUI/GUI/GUI/WhisperCPP.h
index ed934be..c6e1011 100644
--- a/GUI/GUI/GUI/WhisperCPP.h
+++ b/GUI/GUI/GUI/WhisperCPP.h
@@ -1,6 +1,5 @@
 #pragma once

 

-#include <wx/filepicker.h>

 #include <wx/wxprec.h>

 

 #ifndef WX_PRECOMP

@@ -11,7 +10,7 @@
 #include <wchar.h>

 #include <winerror.h>

 

-#include "whisperWindows.h"

+#include "whisper/whisperWindows.h"

 

 #include "Config.h"

 

@@ -38,12 +37,19 @@ public:
 	void Start(const AppConfig& c);

 	void Stop();

 

+	void StartBrowserSource(const AppConfig& c);

+	void StopBrowserSource();

+

 private:

 	bool GetMicsImpl(std::vector<Whisper::sCaptureDevice>& mics);

 

 	wxTextCtrl* out_;

 	Whisper::iMediaFoundation* f_;

 	bool did_init_;

-	std::future<void> proc_;

-	volatile bool run_;

+

+	std::future<void> transcription_thd_;

+	volatile bool run_transcription_;

+

+	std::future<void> browser_src_thd_;

+	volatile bool run_browser_src_;

 };