From fa59ca6064ce9400adb1765deeb960bfe51c2f6f Mon Sep 17 00:00:00 2001 From: yum Date: Sat, 25 Feb 2023 12:09:53 -0800 Subject: Begin work on custom webserver oatpp was a crashy mess. Begin making a simple web server from scratch. * Add Designs/ folder to document nontrivial things like the webserver design --- GUI/GUI/GUI/BrowserSource.cpp | 43 +++---------------- GUI/GUI/GUI/BrowserSource.h | 58 ------------------------- GUI/GUI/GUI/Config.cpp | 2 +- GUI/GUI/GUI/Frame.cpp | 52 +++++++++++------------ GUI/GUI/GUI/GUI.vcxproj | 9 +++- GUI/GUI/GUI/GUI.vcxproj.filters | 18 ++++++++ GUI/GUI/GUI/HTTPMapper.cpp | 0 GUI/GUI/GUI/HTTPMapper.h | 1 + GUI/GUI/GUI/WebCommon.h | 8 ++++ GUI/GUI/GUI/WebServer.cpp | 94 +++++++++++++++++++++++++++++++++++++++++ GUI/GUI/GUI/WebServer.h | 50 ++++++++++++++++++++++ GUI/GUI/GUI/WhisperCPP.cpp | 35 ++++----------- GUI/Libraries/fetch.ps1 | 40 +----------------- 13 files changed, 220 insertions(+), 190 deletions(-) create mode 100644 GUI/GUI/GUI/HTTPMapper.cpp create mode 100644 GUI/GUI/GUI/HTTPMapper.h create mode 100644 GUI/GUI/GUI/WebCommon.h create mode 100644 GUI/GUI/GUI/WebServer.cpp create mode 100644 GUI/GUI/GUI/WebServer.h (limited to 'GUI') diff --git a/GUI/GUI/GUI/BrowserSource.cpp b/GUI/GUI/GUI/BrowserSource.cpp index 2403411..aa96f46 100644 --- a/GUI/GUI/GUI/BrowserSource.cpp +++ b/GUI/GUI/GUI/BrowserSource.cpp @@ -1,10 +1,10 @@ #include "BrowserSource.h" #include "Logging.h" #include "ScopeGuard.h" - -#include "oatpp/network/Server.hpp" +#include "WebServer.h" using ::Logging::Log; +//using ::WebServer::WebServer; BrowserSource::BrowserSource(uint16_t port, wxTextCtrl *out, Transcript *transcript) : port_(port), out_(out), transcript_(transcript) @@ -12,40 +12,9 @@ BrowserSource::BrowserSource(uint16_t port, wxTextCtrl *out, Transcript *transcr void BrowserSource::Run(volatile bool* run) { - // TODO(yum) oatpp::base::Environment::destroy() accesses invalid memory if - // it's called after serving a connection. Probably a bug in my code. Fix it - // and then use a pattern like - // init(); - // ScopeGuard cleanup([]() { destroy(); }); - static bool did_init = false; - if (!did_init) { - oatpp::base::Environment::init(); + WebServer::WebServer ws(out_, port_); + if (!ws.Run(run)) { + Log(out_, "Failed to launch browser source!\n"); } - //ScopeGuard oatpp_env_cleanup([]() { oatpp::base::Environment::destroy(); }); - - OATPP_CREATE_COMPONENT( - std::shared_ptr, - serverConnectionProvider)([&] { - return oatpp::network::tcp::server::ConnectionProvider::createShared( - { "0.0.0.0", port_, oatpp::network::Address::IP_4 }); - }()); - - OATPP_CREATE_COMPONENT(std::shared_ptr, apiObjectMapper)([] { - return oatpp::parser::json::mapping::ObjectMapper::createShared(); - }()); - - OATPP_CREATE_COMPONENT(std::shared_ptr, httpRouter)([] { - return oatpp::web::server::HttpRouter::createShared(); - }()); - httpRouter.getObject()->addController(std::make_shared(apiObjectMapper.getObject(), transcript_)); - - OATPP_CREATE_COMPONENT(std::shared_ptr, serverConnectionHandler)([&] { - return oatpp::web::server::HttpConnectionHandler::createShared(httpRouter.getObject()); - }()); - - oatpp::network::Server server(serverConnectionProvider.getObject(), serverConnectionHandler.getObject()); - Log(out_, "Server running on port {}\n", - static_cast(serverConnectionProvider.getObject()->getProperty("port").getData())); - - server.run(std::function([run]() { return *run == true; })); + return; } diff --git a/GUI/GUI/GUI/BrowserSource.h b/GUI/GUI/GUI/BrowserSource.h index 25b9aa5..fe732ba 100644 --- a/GUI/GUI/GUI/BrowserSource.h +++ b/GUI/GUI/GUI/BrowserSource.h @@ -6,15 +6,6 @@ #include #endif -#include "oatpp/core/macro/codegen.hpp" -#include "oatpp/core/macro/component.hpp" -#include "oatpp/core/Types.hpp" -#include "oatpp/network/tcp/server/ConnectionProvider.hpp" -#include "oatpp/parser/json/mapping/ObjectMapper.hpp" -#include "oatpp/web/server/api/ApiController.hpp" -#include "oatpp/web/server/HttpConnectionHandler.hpp" -#include "oatpp/web/protocol/http/incoming/Request.hpp" - #include "Transcript.h" #include @@ -22,55 +13,6 @@ #include #include -#include OATPP_CODEGEN_BEGIN(DTO) - -class AppDto : public oatpp::DTO -{ - DTO_INIT(AppDto, DTO) - - DTO_FIELD(Int32, statusCode); - DTO_FIELD(String, transcript); -}; - -#include OATPP_CODEGEN_END(DTO) - -#include OATPP_CODEGEN_BEGIN(ApiController) - -class AppController : public oatpp::web::server::api::ApiController -{ -public: - AppController(std::shared_ptr objectMapper, Transcript* transcript) - : oatpp::web::server::api::ApiController(objectMapper), transcript_(transcript) - {} - - ENDPOINT("GET", "/api/transcript", transcription) { - auto dto = AppDto::createShared(); - dto->statusCode = 200; - - std::ostringstream oss; - std::vector segments = transcript_->Get(); - for (const auto& seg : segments) { - oss << seg; - } - dto->transcript = oss.str(); - - return createDtoResponse(Status::CODE_200, dto); - } - - ENDPOINT("GET", "/", root) { - auto html_path = std::filesystem::path("Resources/BrowserSource/index.html"); - std::ifstream html_ifs(html_path); - std::vector resp(4096 * 16, 0); - html_ifs.read(resp.data(), resp.size()); - return createResponse(Status::CODE_200, resp.data()); - } - -private: - Transcript* const transcript_; -}; - -#include OATPP_CODEGEN_END(ApiController) - class BrowserSource { public: diff --git a/GUI/GUI/GUI/Config.cpp b/GUI/GUI/GUI/Config.cpp index 3e09a2e..1a5f6de 100644 --- a/GUI/GUI/GUI/Config.cpp +++ b/GUI/GUI/GUI/Config.cpp @@ -89,7 +89,7 @@ AppConfig::AppConfig() menu_path(), clear_osc(false), - whisper_model("ggml-base.en.bin"), + whisper_model("ggml-medium.bin"), whisper_mic(0), browser_src_port(9517), diff --git a/GUI/GUI/GUI/Frame.cpp b/GUI/GUI/GUI/Frame.cpp index e67240a..01b3140 100644 --- a/GUI/GUI/GUI/Frame.cpp +++ b/GUI/GUI/GUI/Frame.cpp @@ -256,11 +256,10 @@ namespace { "ggml-medium.bin", "ggml-medium.en.bin", "ggml-large.bin", - "ggml-large.en.bin", }; const size_t kNumWhisperModelChoices = sizeof(kWhisperModelChoices) / sizeof(kWhisperModelChoices[0]); - constexpr int kWhisperModelDefault = 3; // base.en + constexpr int kWhisperModelDefault = 6; // medium.bin const wxString kCharsPerSync[] = { "5", @@ -1346,7 +1345,11 @@ void Frame::PopulateDynamicInputFields() for (int i = 0; i < std::min(mics.size(), kNumWhisperMicChoices); i++) { contents[i] = mics[i]; } + int mic_idx = whisper_mic->GetSelection(); whisper_mic->Set(contents); + if (mic_idx < contents.size()) { + whisper_mic->SetSelection(mic_idx); + } } } } @@ -1359,15 +1362,14 @@ void Frame::OnExit(wxCloseEvent& event) void Frame::OnNavbarTranscribe(wxCommandEvent& event) { - // Initialize input fields using AppConfig. - ApplyConfigToInputFields(); - PopulateDynamicInputFields(); - transcribe_panel_->Hide(); unity_panel_->Hide(); debug_panel_->Hide(); whisper_panel_->Hide(); - Resize(); + + // Initialize input fields using AppConfig. + ApplyConfigToInputFields(); + PopulateDynamicInputFields(); transcribe_panel_->Show(); Resize(); @@ -1375,15 +1377,14 @@ void Frame::OnNavbarTranscribe(wxCommandEvent& event) void Frame::OnNavbarUnity(wxCommandEvent& event) { - // Initialize input fields using AppConfig. - ApplyConfigToInputFields(); - PopulateDynamicInputFields(); - transcribe_panel_->Hide(); unity_panel_->Hide(); debug_panel_->Hide(); whisper_panel_->Hide(); - Resize(); + + // Initialize input fields using AppConfig. + ApplyConfigToInputFields(); + PopulateDynamicInputFields(); unity_panel_->Show(); Resize(); @@ -1391,15 +1392,14 @@ void Frame::OnNavbarUnity(wxCommandEvent& event) void Frame::OnNavbarDebug(wxCommandEvent& event) { - // Initialize input fields using AppConfig. - ApplyConfigToInputFields(); - PopulateDynamicInputFields(); - transcribe_panel_->Hide(); unity_panel_->Hide(); debug_panel_->Hide(); whisper_panel_->Hide(); - Resize(); + + // Initialize input fields using AppConfig. + ApplyConfigToInputFields(); + PopulateDynamicInputFields(); debug_panel_->Show(); Resize(); @@ -1407,18 +1407,16 @@ void Frame::OnNavbarDebug(wxCommandEvent& event) void Frame::OnNavbarWhisper(wxCommandEvent& event) { - // Initialize input fields using AppConfig. - ApplyConfigToInputFields(); - PopulateDynamicInputFields(); - transcribe_panel_->Hide(); unity_panel_->Hide(); debug_panel_->Hide(); whisper_panel_->Hide(); - Resize(); - whisper_panel_->Show(); + // Initialize input fields using AppConfig. + ApplyConfigToInputFields(); + PopulateDynamicInputFields(); + whisper_panel_->Show(); Resize(); } @@ -2082,10 +2080,10 @@ void Frame::Resize() auto frame_sz = GetBestSize(); auto panel_sz = main_panel_->GetBestSize(); - auto ideal_sz = panel_sz; - ideal_sz.x += frame_sz.x; - ideal_sz.y += frame_sz.y; + //auto ideal_sz = panel_sz; + //ideal_sz.x += frame_sz.x; + //ideal_sz.y += frame_sz.y; - this->SetSize(ideal_sz); + this->SetSize(panel_sz); } diff --git a/GUI/GUI/GUI/GUI.vcxproj b/GUI/GUI/GUI/GUI.vcxproj index 568b06f..1f14088 100644 --- a/GUI/GUI/GUI/GUI.vcxproj +++ b/GUI/GUI/GUI/GUI.vcxproj @@ -127,7 +127,7 @@ Windows true - kernel32.lib;user32.lib;gdi32.lib;comdlg32.lib;winspool.lib;shell32.lib;shlwapi.lib;ole32.lib;oleaut32.lib;uuid.lib;advapi32.lib;version.lib;comctl32.lib;rpcrt4.lib;ws2_32.lib;wininet.lib;winmm.lib;Whisper.lib;oatpp.lib;%(AdditionalDependencies) + kernel32.lib;user32.lib;gdi32.lib;comdlg32.lib;winspool.lib;shell32.lib;shlwapi.lib;ole32.lib;oleaut32.lib;uuid.lib;advapi32.lib;version.lib;comctl32.lib;rpcrt4.lib;ws2_32.lib;wininet.lib;winmm.lib;Whisper.lib;%(AdditionalDependencies) @@ -146,7 +146,7 @@ true true true - kernel32.lib;user32.lib;gdi32.lib;comdlg32.lib;winspool.lib;shell32.lib;shlwapi.lib;ole32.lib;oleaut32.lib;uuid.lib;advapi32.lib;version.lib;comctl32.lib;rpcrt4.lib;ws2_32.lib;wininet.lib;winmm.lib;Whisper.lib;oatpp.lib;%(AdditionalDependencies) + kernel32.lib;user32.lib;gdi32.lib;comdlg32.lib;winspool.lib;shell32.lib;shlwapi.lib;ole32.lib;oleaut32.lib;uuid.lib;advapi32.lib;version.lib;comctl32.lib;rpcrt4.lib;ws2_32.lib;wininet.lib;winmm.lib;Whisper.lib;%(AdditionalDependencies) @@ -154,10 +154,12 @@ + + @@ -165,6 +167,7 @@ + @@ -172,6 +175,8 @@ + + diff --git a/GUI/GUI/GUI/GUI.vcxproj.filters b/GUI/GUI/GUI/GUI.vcxproj.filters index aa2e6d1..43f6791 100644 --- a/GUI/GUI/GUI/GUI.vcxproj.filters +++ b/GUI/GUI/GUI/GUI.vcxproj.filters @@ -13,6 +13,9 @@ {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + {a0953f26-cbc5-43b7-86c6-2d4b1030b13f} + @@ -42,6 +45,12 @@ Source Files + + WebServer + + + WebServer + @@ -80,6 +89,15 @@ Header Files + + WebServer + + + WebServer + + + WebServer + diff --git a/GUI/GUI/GUI/HTTPMapper.cpp b/GUI/GUI/GUI/HTTPMapper.cpp new file mode 100644 index 0000000..e69de29 diff --git a/GUI/GUI/GUI/HTTPMapper.h b/GUI/GUI/GUI/HTTPMapper.h new file mode 100644 index 0000000..50e9667 --- /dev/null +++ b/GUI/GUI/GUI/HTTPMapper.h @@ -0,0 +1 @@ +#pragma once diff --git a/GUI/GUI/GUI/WebCommon.h b/GUI/GUI/GUI/WebCommon.h new file mode 100644 index 0000000..e19223a --- /dev/null +++ b/GUI/GUI/GUI/WebCommon.h @@ -0,0 +1,8 @@ +#pragma once + +namespace WebServer { + enum ContentType { + HTTP, + JSON, + }; +}; diff --git a/GUI/GUI/GUI/WebServer.cpp b/GUI/GUI/GUI/WebServer.cpp new file mode 100644 index 0000000..6cce5d6 --- /dev/null +++ b/GUI/GUI/GUI/WebServer.cpp @@ -0,0 +1,94 @@ +#include + +#ifndef WX_PRECOMP +#include +#endif + +#include "ScopeGuard.h" +#include "WebServer.h" + +#include +#include +#include + +using ::Logging::Log; + +namespace WebServer { + WebServer::WebServer(wxTextCtrl* out, uint16_t port) + : out_(out), port_(port) + {} + + bool WebServer::RegisterPathHandler(const std::string& method, + const std::string& path, handler_t&& handler) { + dispatch_key_t key = GetDispatchKey(method, path); + if (dispatch_map_.contains(key)) { + Log(out_, "Failed to register path handler at {} {}: " + "Handler already exists!\n", method, path); + return false; + } + + dispatch_map_[key] = std::move(handler); + return true; + } + + bool WebServer::Run(volatile bool* run) { + WSADATA wsaData; + int result = WSAStartup(/*version=*/MAKEWORD(2, 2), &wsaData); + if (result) { + Log(out_, "Failed to start winsock: {}\n", result); + return false; + } + ScopeGuard wsa_cleanup([]() { WSACleanup(); }); + + SOCKET sock = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (sock == INVALID_SOCKET) { + Log(out_, "Failed to create socket: {}\n", WSAGetLastError()); + return false; + } + ScopeGuard sock_cleanup([sock]() { closesocket(sock); }); + + sockaddr_in saddr; + saddr.sin_family = AF_INET; + saddr.sin_addr.s_addr = INADDR_ANY; // TODO(yum) loopback? + saddr.sin_port = htons(port_); + if (bind(sock, (sockaddr*)&saddr, sizeof(saddr)) == SOCKET_ERROR) { + Log(out_, "Failed to bind to port {}: {}\n", port_, WSAGetLastError()); + return false; + } + + u_long enable_nonblock = 1; + if (ioctlsocket(sock, FIONBIO, &enable_nonblock) == SOCKET_ERROR) { + Log(out_, "Failed to enable non-blocking socket: {}\n", WSAGetLastError()); + return false; + } + + if (listen(sock, SOMAXCONN) == SOCKET_ERROR) { + Log(out_, "Failed to listen on port {}: {}\n", port_, WSAGetLastError()); + return false; + } + + Log(out_, "Server running on port {}\n", port_); + + sockaddr_in peer_addr; + while (*run) { + int peer_addr_sz = sizeof(peer_addr); + SOCKET csock = accept(sock, (sockaddr*)&peer_addr, &peer_addr_sz); + if (csock == INVALID_SOCKET) { + int err = WSAGetLastError(); + if (err == WSAEWOULDBLOCK) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + continue; + } + Log(out_, "Accept failed: {}\n", WSAGetLastError()); + return false; + } + ScopeGuard csock_cleanup([csock]() { closesocket(csock); }); + char peer_ip_str[INET_ADDRSTRLEN]{}; + inet_ntop(AF_INET, &peer_addr.sin_addr, peer_ip_str, sizeof(peer_ip_str)); + Log(out_, "Connection get: peer: {}:{}\n", peer_ip_str, ntohs(peer_addr.sin_port)); + // TODO(yum) parse and send a response + } + + return true; + } +} diff --git a/GUI/GUI/GUI/WebServer.h b/GUI/GUI/GUI/WebServer.h new file mode 100644 index 0000000..f66e0f7 --- /dev/null +++ b/GUI/GUI/GUI/WebServer.h @@ -0,0 +1,50 @@ +#pragma once + +#include + +#ifndef WX_PRECOMP +#include +#endif + +#include + +#include +#include +#include + +#include "Logging.h" +#include "WebCommon.h" + +namespace WebServer { + class WebServer { + public: + WebServer(wxTextCtrl *out, std::uint16_t port); + + typedef std::function handler_t; + + bool RegisterPathHandler(const std::string& method, + const std::string& path, handler_t&& handler); + + bool Run(volatile bool* run); + + private: + // Dispatch requests by mapping from (method, path) to handler. + // Dispatch key is (method, path) in that order. + typedef std::tuple dispatch_key_t; + static inline dispatch_key_t GetDispatchKey(const std::string& method, const std::string& path) + { + return dispatch_key_t(method, path); + } + + typedef std::map dispatch_map_t; + dispatch_map_t dispatch_map_; + + wxTextCtrl* const out_; + const uint16_t port_; + }; +} + diff --git a/GUI/GUI/GUI/WhisperCPP.cpp b/GUI/GUI/GUI/WhisperCPP.cpp index e8ed4ef..6028e26 100644 --- a/GUI/GUI/GUI/WhisperCPP.cpp +++ b/GUI/GUI/GUI/WhisperCPP.cpp @@ -31,7 +31,7 @@ namespace { return ""; } - std::string result(len, 0); + std::string result(len + 1, 0); size_t len_out; wcstombs_s(&len_out, result.data(), len, wc_str, _TRUNCATE); @@ -296,7 +296,7 @@ void WhisperCPP::Start(const AppConfig& c) { ScopeGuard context_cleanup([context]() { context->Release(); }); Whisper::sFullParams wparams{}; - context->fullDefaultParams(eSamplingStrategy::BeamSearch, &wparams); + context->fullDefaultParams(eSamplingStrategy::Greedy, &wparams); wparams.language = Whisper::makeLanguageKey("en"); // TODO(yum) use config // This must be set to keep memory usage from growing without bound. wparams.n_max_text_ctx = 100; @@ -318,10 +318,10 @@ void WhisperCPP::Start(const AppConfig& c) { return S_OK; } + // Scanning a vector is faster than using a hashtable up to ~1k + // entries (source: I heard it from someone once). static const std::vector banned_words{ - " [BLANK_AUDIO]", - " [SOUND]", - " [ Silence ]", + " -", }; const sSegment* const segments = results->getSegments(); @@ -335,26 +335,7 @@ void WhisperCPP::Start(const AppConfig& c) { std::string_view tok_str(tok.text); if (tok_str.starts_with("[") || tok_str.starts_with(" [")) { - if (tok_str.ends_with("]")) { - continue; - } - } - std::vector::const_iterator word_iter = - std::find(banned_words.cbegin(), banned_words.cend(), - tok_str); - if (word_iter != banned_words.end()) { - continue; - } -#if 0 - if (tok_str.starts_with("[") || - tok_str.starts_with(" [") || - tok_str.starts_with(" (")) { - if (tok_str.ends_with("]") || - tok_str.ends_with(")")) { - continue; - } is_metadata = true; - continue; } if (is_metadata) { if (tok_str.ends_with("]") || @@ -363,10 +344,12 @@ void WhisperCPP::Start(const AppConfig& c) { } continue; } - if (tok_str.ends_with("BLANK_AUDIO")) { + std::vector::const_iterator word_iter = + std::find(banned_words.cbegin(), banned_words.cend(), + tok_str); + if (word_iter != banned_words.end()) { continue; } -#endif Log(app->out_, "{}", tok.text); app->transcript_.Append(tok.text); } diff --git a/GUI/Libraries/fetch.ps1 b/GUI/Libraries/fetch.ps1 index 1495cc9..22b088b 100644 --- a/GUI/Libraries/fetch.ps1 +++ b/GUI/Libraries/fetch.ps1 @@ -16,15 +16,6 @@ $WHISPER_1_7_0_URL = "https://github.com/Const-me/Whisper/releases/download/1.7. $WHISPER_URL = $WHISPER_1_7_0_URL $WHISPER_FILE = $(Split-Path -Path $WHISPER_URL -Leaf) -$OATPP_1_3_0_URL = "https://github.com/oatpp/oatpp/archive/refs/tags/1.3.0.zip" -$OATPP_URL = $OATPP_1_3_0_URL -$OATPP_FILE = $(Split-Path -Path $OATPP_URL -Leaf) -$OATPP_VER = $OATPP_FILE -replace '\.[a-z\.]*$' -$OATPP_DIR = "oatpp-$OATPP_VER" - -$NPROC = $(Get-CimInstance Win32_Processor).NumberOfCores -echo "nproc: $NPROC" - pushd $PSScriptRoot # WX @@ -53,6 +44,7 @@ if (-Not (Test-Path rapidyaml)) { python3 tools/amalgamate.py ryml.h cp ryml.h ../../GUI/GUI/ryml.h + popd } if ((Test-Path whisper) -And ($overwrite)) { @@ -74,35 +66,5 @@ if (-Not (Test-Path whisper)) { popd > $null } -if ((Test-Path oatpp) -And ($overwrite)) { - rm -Recurse oatpp -} - -if (-Not (Test-Path oatpp)) { - mkdir oatpp - pushd oatpp > $null - Invoke-WebRequest $OATPP_URL -OutFile $OATPP_FILE - Expand-Archive $OATPP_FILE -DestinationPath . - if (Test-Path ../../GUI/GUI/oatpp/) { - rm -Recurse ../../GUI/GUI/oatpp/ - } - mkdir ../../GUI/GUI/oatpp/ - pushd $OATPP_DIR > $null - mkdir build - pushd build > $null - cmake.exe .. ` - -DCMAKE_BUILD_TYPE=$release ` - -DBUILD_SHARED_LIBS=OFF ` - -DOATPP_MSVC_LINK_STATIC_RUNTIME=ON ` - -DOATPP_BUILD_TESTS=OFF - cmake.exe --build . -j $NPROC --config $release - cp src/$release/oatpp.lib ../../../../GUI/GUI/oatpp/ - cp -Recurse ../src/oatpp/* ../../../../GUI/GUI/oatpp/ - popd > $null - popd > $null -} - -popd > $null # rapidyaml - popd > $null # $PSScriptRoot -- cgit v1.2.3