From 0ed379f2c99ac5c126a6f101965ef1eaa58c017b Mon Sep 17 00:00:00 2001 From: yum Date: Mon, 26 Jun 2023 00:58:58 -0700 Subject: Add browser source, hardcoded to port 8097 Transcription output now streams to localhost:8097. In OBS: * Create a browser source. * url: localhost:8097 * width: 2200 * height: 400 TODO: * Put behind toggle. * Create input field for port. Misc cleanup: * transcribe.py: Drop frames from audio capture thread instead of the transcription thread. Doing it the other way would result in occasional data loss. --- GUI/GUI/GUI/BrowserSource.cpp | 2 ++ GUI/GUI/GUI/Frame.cpp | 38 ++++++++++++++++++++++++++++++++++++-- GUI/GUI/GUI/Frame.h | 3 +++ GUI/GUI/GUI/PythonWrapper.cpp | 3 +++ GUI/GUI/GUI/PythonWrapper.h | 1 + GUI/GUI/GUI/Transcript.cpp | 6 ++++++ GUI/GUI/GUI/Transcript.h | 1 + 7 files changed, 52 insertions(+), 2 deletions(-) (limited to 'GUI') diff --git a/GUI/GUI/GUI/BrowserSource.cpp b/GUI/GUI/GUI/BrowserSource.cpp index c43f1a0..62e3e43 100644 --- a/GUI/GUI/GUI/BrowserSource.cpp +++ b/GUI/GUI/GUI/BrowserSource.cpp @@ -61,6 +61,8 @@ void BrowserSource::Run(volatile bool* run) resp_oss << "}"; payload = resp_oss.str(); type = WebServer::JSON; + + //Log(out_, "Serving transcript to port {}: {}\n", port_, transcript_oss.str()); }); if (!ws.Run(run)) { diff --git a/GUI/GUI/GUI/Frame.cpp b/GUI/GUI/GUI/Frame.cpp index 6bce67e..d39222b 100644 --- a/GUI/GUI/GUI/Frame.cpp +++ b/GUI/GUI/GUI/Frame.cpp @@ -1,3 +1,4 @@ +#include "BrowserSource.h" #include "Frame.h" #include "Logging.h" #include "PythonWrapper.h" @@ -6,6 +7,7 @@ #include #include +#include #include #include #include @@ -551,6 +553,11 @@ Frame::Frame() py_app_ = p.get_future(); p.set_value(true); } + { + auto p = std::promise(); + obs_app_ = p.get_future(); + p.set_value(true); + } { auto p = std::promise(); unity_app_ = p.get_future(); @@ -1930,6 +1937,12 @@ void Frame::OnAppStart(wxCommandEvent& event) { return; } + status = obs_app_.wait_for(std::chrono::seconds(0)); + if (status != std::future_status::ready) { + Log(transcribe_out_, "Transcription engine (OBS server) already running\n"); + return; + } + Log(transcribe_out_, "Launching transcription engine\n"); int which_mic = py_app_mic_->GetSelection(); @@ -2043,6 +2056,16 @@ void Frame::OnAppStart(wxCommandEvent& event) { auto out_cb = [&](const std::string& out, const std::string& err) { Log(transcribe_out_, "{}", out); Log(transcribe_out_, "{}", err); + + std::regex pattern("^Transcription \\(([0-9]*\\.[0-9]+) seconds\\):"); + if (std::regex_search(out, pattern)) { + std::string filtered_transcript = std::regex_replace(out, pattern, ""); + filtered_transcript.erase(std::remove_if(filtered_transcript.begin(), filtered_transcript.end(), [](char c) { + return c == '\n' || c == '\r'; + }), filtered_transcript.end()); + Log(transcribe_out_, "Got transcription line! Transcript: \"{}\"", filtered_transcript); + transcript_.Set(std::move(filtered_transcript)); + } }; auto in_cb = [&](std::string& in) {}; auto run_cb = [&]() { @@ -2052,8 +2075,17 @@ void Frame::OnAppStart(wxCommandEvent& event) { auto prestart_cb = [this]() -> void { EnsureVirtualEnv(/*block=*/true); }; - py_app_ = std::move(PythonWrapper::StartApp(*app_c_, std::move(out_cb), - std::move(in_cb), std::move(run_cb), std::move(prestart_cb))); + + // TODO(yum) parameterize port + obs_app_ = std::async(std::launch::async, + [&]() -> bool { + BrowserSource browser_src(8097, transcribe_out_, &transcript_); + browser_src.Run(&run_py_app_); + return true; + }); + py_app_ = std::move(PythonWrapper::StartApp(*app_c_, transcribe_out_, + std::move(out_cb), std::move(in_cb), std::move(run_cb), + std::move(prestart_cb))); Log(transcribe_out_, "py app valid: {}\n", py_app_.valid()); } @@ -2065,6 +2097,8 @@ void Frame::OnAppStop() { } run_py_app_ = false; py_app_.wait(); + obs_app_.wait(); + transcript_.Clear(); Log(transcribe_out_, "Stopped transcription engine\n"); } diff --git a/GUI/GUI/GUI/Frame.h b/GUI/GUI/GUI/Frame.h index 94f2a68..904df4f 100644 --- a/GUI/GUI/GUI/Frame.h +++ b/GUI/GUI/GUI/Frame.h @@ -8,6 +8,7 @@ #endif #include "Config.h" +#include "Transcript.h" #include #include @@ -93,6 +94,8 @@ private: wxCheckBox* whisper_enable_browser_src_; std::future py_app_; + std::future obs_app_; + Transcript transcript_; bool run_py_app_; std::future unity_app_; std::future dump_mics_; diff --git a/GUI/GUI/GUI/PythonWrapper.cpp b/GUI/GUI/GUI/PythonWrapper.cpp index a061e34..037d961 100644 --- a/GUI/GUI/GUI/PythonWrapper.cpp +++ b/GUI/GUI/GUI/PythonWrapper.cpp @@ -448,16 +448,19 @@ bool PythonWrapper::InstallPip( std::future PythonWrapper::StartApp( const AppConfig& config, + wxTextCtrl *out, const std::function&& out_cb, const std::function&& in_cb, const std::function&& run_cb, const std::function&& prestart_cb) { + return std::move(std::async(std::launch::async, [&]( const std::function&& out_cb, const std::function&& in_cb, const std::function&& run_cb) -> bool { prestart_cb(); + return InvokeWithArgs({ "-u", // Unbuffered output "Resources/Scripts/transcribe.py", diff --git a/GUI/GUI/GUI/PythonWrapper.h b/GUI/GUI/GUI/PythonWrapper.h index 17f5e1d..6366247 100644 --- a/GUI/GUI/GUI/PythonWrapper.h +++ b/GUI/GUI/GUI/PythonWrapper.h @@ -73,6 +73,7 @@ namespace PythonWrapper // app restarts. std::future StartApp( const AppConfig& config, + wxTextCtrl *out, const std::function&& out_cb, const std::function&& in_cb = [](std::string&) {}, const std::function&& run_cb = []() { return true; }, diff --git a/GUI/GUI/GUI/Transcript.cpp b/GUI/GUI/GUI/Transcript.cpp index 30f1f76..9ef607f 100644 --- a/GUI/GUI/GUI/Transcript.cpp +++ b/GUI/GUI/GUI/Transcript.cpp @@ -5,6 +5,12 @@ void Transcript::Append(std::string&& segment) { segments_.push_back(std::move(segment)); } +void Transcript::Set(std::string&& segment) { + std::scoped_lock l(mu_); + segments_.clear(); + segments_.push_back(std::move(segment)); +} + void Transcript::Clear() { std::scoped_lock l(mu_); segments_.clear(); diff --git a/GUI/GUI/GUI/Transcript.h b/GUI/GUI/GUI/Transcript.h index 09858b0..fae2bad 100644 --- a/GUI/GUI/GUI/Transcript.h +++ b/GUI/GUI/GUI/Transcript.h @@ -10,6 +10,7 @@ public: Transcript() = default; void Append(std::string&& segment); + void Set(std::string&& segment); void Clear(); std::vector Get(); -- cgit v1.2.3