From 8d225cfd66dfb60998b4eab43d8aa3b287375695 Mon Sep 17 00:00:00 2001 From: yum Date: Tue, 20 Dec 2022 00:26:50 -0800 Subject: GUI: Begin work generating animator The GUI can now generate guid.map and animations. --- GUI/GUI/GUI/Frame.cpp | 83 ++++++++++++++++++++++++++++++++++++------ GUI/GUI/GUI/Frame.h | 2 + GUI/GUI/GUI/PythonWrapper.cpp | 65 +++++++++++++++++++++++++++++++++ GUI/GUI/GUI/PythonWrapper.h | 11 ++++++ Scripts/libtastt.py | 28 +++++++------- Scripts/libunity.py | 4 ++ Scripts/transcribe.py | 3 +- TaSTT.fbx | Bin 18172 -> 0 bytes TaSTT_Menu.asset | 56 ---------------------------- UnityAssets/TaSTT.fbx | Bin 0 -> 18172 bytes UnityAssets/TaSTT_Menu.asset | 56 ++++++++++++++++++++++++++++ 11 files changed, 226 insertions(+), 82 deletions(-) delete mode 100644 TaSTT.fbx delete mode 100644 TaSTT_Menu.asset create mode 100644 UnityAssets/TaSTT.fbx create mode 100644 UnityAssets/TaSTT_Menu.asset diff --git a/GUI/GUI/GUI/Frame.cpp b/GUI/GUI/GUI/Frame.cpp index d535f85..1645258 100644 --- a/GUI/GUI/GUI/Frame.cpp +++ b/GUI/GUI/GUI/Frame.cpp @@ -31,6 +31,7 @@ namespace { ID_UNITY_PANEL, ID_UNITY_CONFIG_PANEL, ID_UNITY_OUT, + ID_UNITY_ASSETS_FILE_PICKER, ID_UNITY_ANIMATOR_FILE_PICKER, ID_UNITY_PARAMETERS_FILE_PICKER, ID_UNITY_MENU_FILE_PICKER, @@ -281,6 +282,14 @@ Frame::Frame() { auto* unity_config_panel_pairs = new wxPanel(unity_config_panel, ID_UNITY_CONFIG_PANEL_PAIRS); { + auto* unity_assets_file_picker = new wxDirPickerCtrl( + unity_config_panel_pairs, + ID_UNITY_ASSETS_FILE_PICKER, + /*path=*/wxEmptyString, + /*message=*/"Unity Assets folder" + ); + unity_assets_file_picker_ = unity_assets_file_picker; + auto* unity_animator_file_picker = new wxFilePickerCtrl( unity_config_panel_pairs, ID_UNITY_ANIMATOR_FILE_PICKER, @@ -312,7 +321,7 @@ Frame::Frame() /*pos=*/wxDefaultPosition, /*size=*/wxDefaultSize ); - unity_parameters_file_picker_ = unity_parameters_file_picker; + unity_menu_file_picker_ = unity_menu_file_picker; auto* unity_animator_generated_dir = new wxTextCtrl(unity_config_panel_pairs, ID_UNITY_ANIMATOR_GENERATED_DIR, @@ -339,6 +348,9 @@ Frame::Frame() auto* sizer = new wxFlexGridSizer(/*cols=*/2); unity_config_panel_pairs->SetSizer(sizer); + sizer->Add(new wxStaticText(unity_config_panel_pairs, wxID_ANY, /*label=*/"Unity Assets folder:")); + sizer->Add(unity_assets_file_picker); + sizer->Add(new wxStaticText(unity_config_panel_pairs, wxID_ANY, /*label=*/"FX controller:")); sizer->Add(unity_animator_file_picker); @@ -477,7 +489,64 @@ void Frame::OnDumpMics(wxCommandEvent& event) transcribe_out_->AppendText(PythonWrapper::DumpMics()); } -void Frame::OnGenerateFX(wxCommandEvent& event) { +#define DEBUG + +void Frame::OnGenerateFX(wxCommandEvent& event) +{ + std::filesystem::path unity_assets_path = unity_assets_file_picker_->GetPath().ToStdString(); +#ifndef DEBUG + if (!std::filesystem::exists(unity_assets_path)) { + std::ostringstream oss; + oss << "Cannot generate FX layer: assets directory does not exist at " << unity_assets_path << std::endl; + wxLogError(oss.str().c_str()); + return; + } +#endif + std::filesystem::path unity_animator_path = unity_animator_file_picker_->GetPath().ToStdString(); +#ifndef DEBUG + if (!std::filesystem::exists(unity_animator_path)) { + std::ostringstream oss; + oss << "Cannot generate FX layer: animator does not exist at " << unity_animator_path << std::endl; + wxLogError(oss.str().c_str()); + return; + } +#endif + std::filesystem::path unity_parameters_path = unity_parameters_file_picker_->GetPath().ToStdString(); +#ifndef DEBUG + if (!std::filesystem::exists(unity_parameters_path)) { + std::ostringstream oss; + oss << "Cannot generate FX layer: parameters do not exist at " << unity_parameters_path << std::endl; + wxLogError(oss.str().c_str()); + return; + } +#endif + std::filesystem::path unity_menu_path = unity_menu_file_picker_->GetPath().ToStdString(); +#ifndef DEBUG + if (!std::filesystem::exists(unity_menu_path)) { + std::ostringstream oss; + oss << "Cannot generate FX layer: menu does not exist at " << unity_menu_path << std::endl; + wxLogError(oss.str().c_str()); + return; + } +#endif + std::string unity_animator_generated_dir = unity_animator_generated_dir_->GetLineText(0).ToStdString(); + std::string unity_animator_generated_name = unity_animator_generated_name_->GetLineText(0).ToStdString(); + std::string unity_parameters_generated_name = unity_parameters_generated_name_->GetLineText(0).ToStdString(); + std::string unity_menu_generated_name = unity_menu_generated_name_->GetLineText(0).ToStdString(); + + std::string out; + if (!PythonWrapper::GenerateAnimator( + unity_assets_path.string(), + unity_animator_path.string(), + unity_parameters_path.string(), + unity_menu_path.string(), + unity_animator_generated_dir, + unity_animator_generated_name, + unity_parameters_generated_name, + unity_menu_generated_name, + unity_out_)) { + wxLogError("Failed to generate animator:\n%s\n", out.c_str()); + } } void Frame::OnAppStart(wxCommandEvent& event) { @@ -582,22 +651,12 @@ void Frame::DrainApp(wxProcess* proc, std::ostringstream& oss) { return; } - bool first = true; while (proc->IsInputAvailable()) { - if (first) { - first = false; - oss << " " << "stdout:" << std::endl; - } wxTextInputStream iss(*(proc->GetInputStream())); oss << " " << iss.ReadLine() << std::endl; } - first = true; while (proc->IsErrorAvailable()) { - if (first) { - first = false; - oss << " " << "stderr:" << std::endl; - } wxTextInputStream iss(*(proc->GetErrorStream())); oss << " " << iss.ReadLine() << std::endl; } diff --git a/GUI/GUI/GUI/Frame.h b/GUI/GUI/GUI/Frame.h index 35922a1..1d847ca 100644 --- a/GUI/GUI/GUI/Frame.h +++ b/GUI/GUI/GUI/Frame.h @@ -29,8 +29,10 @@ private: wxTextCtrl* unity_parameters_generated_name_; wxTextCtrl* unity_menu_generated_name_; + wxDirPickerCtrl* unity_assets_file_picker_; wxFilePickerCtrl* unity_animator_file_picker_; wxFilePickerCtrl* unity_parameters_file_picker_; + wxFilePickerCtrl* unity_menu_file_picker_; wxChoice* py_app_mic_; wxChoice* py_app_lang_; diff --git a/GUI/GUI/GUI/PythonWrapper.cpp b/GUI/GUI/GUI/PythonWrapper.cpp index 0878560..62a002a 100644 --- a/GUI/GUI/GUI/PythonWrapper.cpp +++ b/GUI/GUI/GUI/PythonWrapper.cpp @@ -2,6 +2,7 @@ #include +#include #include class PythonProcess : public wxProcess { @@ -130,3 +131,67 @@ wxProcess* PythonWrapper::StartApp( std::move(exit_callback)); } +bool PythonWrapper::GenerateAnimator( + const std::string& unity_assets_path, + const std::string& unity_animator_path, + const std::string& unity_parameters_path, + const std::string& unity_menu_path, + const std::string& unity_animator_generated_dir, + const std::string& unity_animator_generated_name, + const std::string& unity_parameters_generated_name, + const std::string& unity_menu_generated_name, + wxTextCtrl* out) { + // Python script locations + std::string libunity_path = "Resources/Scripts/libunity.py"; + std::string libtastt_path = "Resources/Scripts/libtastt.py"; + + // Generated directory locations + std::filesystem::path tastt_generated_dir_path = + std::filesystem::path(unity_assets_path) / unity_animator_generated_dir; + std::filesystem::path guid_map_path = + tastt_generated_dir_path / "guid.map"; + std::filesystem::path tastt_animations_path = + tastt_generated_dir_path / "Animations"; + std::filesystem::path tastt_animator_path = + tastt_generated_dir_path / unity_animator_generated_name; + std::filesystem::path tastt_params_path = + tastt_generated_dir_path / unity_parameters_generated_name; + std::filesystem::path tastt_menu_path = + tastt_generated_dir_path / unity_menu_generated_name; + + { + out->AppendText("Generating guid.map... "); + std::string py_stdout, py_stderr; + if (InvokeWithArgs({ libunity_path, "guid_map", + "--project_root", unity_assets_path, + "--save_to", guid_map_path.string() }, + &py_stdout, &py_stderr)) { + out->AppendText("success!\n"); + out->AppendText(py_stdout.c_str()); + out->AppendText(py_stderr.c_str()); + } + else { + wxLogError("Failed to generate guid.map: %s", py_stderr.c_str()); + out->AppendText("failed!\n"); + } + } + { + out->AppendText("Generating animations... "); + std::string py_stdout, py_stderr; + if (InvokeWithArgs({ libtastt_path, "gen_anims", + "--gen_anim_dir", tastt_animations_path.string(), + "--guid_map", guid_map_path.string() }, + &py_stdout, &py_stderr)) { + out->AppendText("success!\n"); + out->AppendText(py_stdout.c_str()); + out->AppendText(py_stderr.c_str()); + } + else { + wxLogError("Failed to generate animations: %s", py_stderr.c_str()); + out->AppendText("failed!\n"); + } + } + + return true; +} + diff --git a/GUI/GUI/GUI/PythonWrapper.h b/GUI/GUI/GUI/PythonWrapper.h index 6c90087..de5a2e4 100644 --- a/GUI/GUI/GUI/PythonWrapper.h +++ b/GUI/GUI/GUI/PythonWrapper.h @@ -39,5 +39,16 @@ namespace PythonWrapper wxProcess* StartApp( std::function&& exit_callback, const std::string& mic, const std::string& lang, const std::string& model); + + bool GenerateAnimator( + const std::string& unity_assets_path, + const std::string& unity_animator_path, + const std::string& unity_parameters_path, + const std::string& unity_menu_path, + const std::string& unity_animator_generated_dir, + const std::string& unity_animator_generated_name, + const std::string& unity_parameters_generated_name, + const std::string& unity_menu_generated_name, + wxTextCtrl* out); }; diff --git a/Scripts/libtastt.py b/Scripts/libtastt.py index bee535f..f448117 100644 --- a/Scripts/libtastt.py +++ b/Scripts/libtastt.py @@ -179,7 +179,8 @@ def generateClearAnimation(anim_dir, guid_map): anim_clip.mapping['m_EditorCurves'].sequence.append(curve) # Serialize animation to file anim_name = generate_utils.getClearAnimationName() - anim_path = anim_dir + anim_name + ".anim" + anim_path = os.path.join(anim_dir, anim_name + ".anim") + print("Generating clear animation at {}".format(anim_path)) with open(anim_path, "w") as f: f.write(libunity.unityYamlToString([anim_node])) # Generate metadata @@ -221,7 +222,8 @@ def generateToggleAnimations(anim_dir, shader_param, guid_map): anim_suffix = "_Off" if shader_value == 1: anim_suffix = "_On" - anim_path = anim_dir + shader_param + anim_suffix + ".anim" + anim_path = os.path.join(anim_dir, shader_param + anim_suffix + + ".anim") with open(anim_path, "w") as f: f.write(libunity.unityYamlToString([anim_node])) # Generate metadata @@ -261,7 +263,7 @@ def generateFloatAnimation(anim_name: str, anim_dir: str, anim_clip.mapping['m_EditorCurves'].sequence.append(curve) # Serialize animation to file - anim_path = anim_dir + anim_name + ".anim" + anim_path = os.path.join(anim_dir, anim_name + ".anim") with open(anim_path, "w") as f: f.write(libunity.unityYamlToString([anim_node])) # Generate metadata @@ -275,10 +277,10 @@ def generateFloatAnimation(anim_name: str, anim_dir: str, return meta.guid def generateAnimations(anim_dir, guid_map): - generateClearAnimation(args.gen_anim_dir, guid_map) + generateClearAnimation(anim_dir, guid_map) - generateToggleAnimations(args.gen_anim_dir, generate_utils.getIndicator0Param(), guid_map) - generateToggleAnimations(args.gen_anim_dir, generate_utils.getIndicator1Param(), guid_map) + generateToggleAnimations(anim_dir, generate_utils.getIndicator0Param(), guid_map) + generateToggleAnimations(anim_dir, generate_utils.getIndicator1Param(), guid_map) print("Generating letter animations", file=sys.stderr) @@ -318,7 +320,7 @@ def generateAnimations(anim_dir, guid_map): clip.mapping['m_FloatCurves'].sequence.append(curve) clip.mapping['m_EditorCurves'].sequence.append(curve) # Serialize animation to file - anim_path = anim_dir + anim_name + ".anim" + anim_path = os.path.join(anim_dir, anim_name + ".anim") with open(anim_path, "w") as f: f.write(libunity.unityYamlToString([node])) # Generate metadata @@ -378,15 +380,15 @@ def generateFXLayer(which_layer: int, anim: libunity.UnityAnimator, layer: dy = 200 # Create blend tree for this region. - anim_lo_path = gen_anim_dir + \ + anim_lo_path = os.path.join(gen_anim_dir, generate_utils.getAnimationNameByLayerAndIndex( which_layer, i, 0, byte) + \ - ".anim" + ".anim") guid_lo = guid_map[anim_lo_path] - anim_hi_path = gen_anim_dir + \ + anim_hi_path = os.path.join(gen_anim_dir, generate_utils.getAnimationNameByLayerAndIndex( which_layer, i, generate_utils.CHARS_PER_CELL - 1, byte) + \ - ".anim" + ".anim") guid_hi = guid_map[anim_hi_path] select_states[i] = anim.addAnimatorBlendTree(layer, @@ -430,13 +432,13 @@ def generateToggle(layer_name: str, on_state = anim.addAnimatorState(layer, layer_name + "_On", dy=100) if off_anim_basename: - off_anim_path = gen_anim_dir + off_anim_basename + off_anim_path = os.path.join(gen_anim_dir, off_anim_basename) off_anim_meta = libunity.Metadata() off_anim_meta.load(off_anim_path) anim.setAnimatorStateAnimation(off_state, off_anim_meta.guid) if on_anim_basename: - on_anim_path = gen_anim_dir + on_anim_basename + on_anim_path = os.path.join(gen_anim_dir, on_anim_basename) on_anim_meta = libunity.Metadata() on_anim_meta.load(on_anim_path) anim.setAnimatorStateAnimation(on_state, on_anim_meta.guid) diff --git a/Scripts/libunity.py b/Scripts/libunity.py index f9e9e28..7223568 100644 --- a/Scripts/libunity.py +++ b/Scripts/libunity.py @@ -1259,6 +1259,10 @@ if __name__ == "__main__": print("Looking up GUIDs under {}".format(args.project_root), file=sys.stderr) guid_map = getGuidMap(args.project_root) + + save_to_dir = os.path.dirname(args.save_to) + os.makedirs(save_to_dir, exist_ok=True) + if args.guid_map_append: tmp_map = {} with open(args.save_to, "rb") as f: diff --git a/Scripts/transcribe.py b/Scripts/transcribe.py index 0f7ae37..831ae66 100644 --- a/Scripts/transcribe.py +++ b/Scripts/transcribe.py @@ -94,6 +94,8 @@ def onAudioFramesAvailable( frame_count, time_info, status_flags): + if audio_state.audio_paused: + return (frames, pyaudio.paContinue) # Reduce sample rate from mic rate to Whisper rate by dropping frames. decimated = b'' @@ -207,7 +209,6 @@ def transcribe(audio_state, model, frames): #for temp in (0.00, 0.05, 0.10, 0.15, 0.20): #for temp in (0.00, 0.05): for temp in (0.00,): - print("temp: {}".format(temp)) options = whisper.DecodingOptions(language = audio_state.language, beam_size = 5, temperature = temp, without_timestamps = True) result = whisper.decode(model, mel, options) diff --git a/TaSTT.fbx b/TaSTT.fbx deleted file mode 100644 index d6737d0..0000000 Binary files a/TaSTT.fbx and /dev/null differ diff --git a/TaSTT_Menu.asset b/TaSTT_Menu.asset deleted file mode 100644 index 77628bd..0000000 --- a/TaSTT_Menu.asset +++ /dev/null @@ -1,56 +0,0 @@ -%YAML 1.1 -%TAG !u! tag:unity3d.com,2011: ---- !u!114 &11400000 -MonoBehaviour: - m_ObjectHideFlags: 0 - m_CorrespondingSourceObject: {fileID: 0} - m_PrefabInstance: {fileID: 0} - m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 0} - m_Enabled: 1 - m_EditorHideFlags: 0 - m_Script: {fileID: -340790334, guid: 67cc4cb7839cd3741b63733d5adf0442, type: 3} - m_Name: TaSTT_Menu - m_EditorClassIdentifier: - controls: - - name: Show - icon: {fileID: 0} - type: 102 - parameter: - name: TaSTT_Toggle - value: 1 - style: 0 - subMenu: {fileID: 0} - subParameters: [] - labels: [] - - name: Lock World - icon: {fileID: 0} - type: 102 - parameter: - name: TaSTT_Lock_World - value: 1 - style: 0 - subMenu: {fileID: 0} - subParameters: [] - labels: [] - - name: Enable Beep - icon: {fileID: 0} - type: 102 - parameter: - name: TaSTT_Speech_Noise_Enable - value: 1 - style: 0 - subMenu: {fileID: 0} - subParameters: [] - labels: [] - - name: Scale - icon: {fileID: 0} - type: 203 - parameter: - name: - value: 1 - style: 0 - subMenu: {fileID: 0} - subParameters: - - name: TaSTT_Scale - labels: [] diff --git a/UnityAssets/TaSTT.fbx b/UnityAssets/TaSTT.fbx new file mode 100644 index 0000000..d6737d0 Binary files /dev/null and b/UnityAssets/TaSTT.fbx differ diff --git a/UnityAssets/TaSTT_Menu.asset b/UnityAssets/TaSTT_Menu.asset new file mode 100644 index 0000000..77628bd --- /dev/null +++ b/UnityAssets/TaSTT_Menu.asset @@ -0,0 +1,56 @@ +%YAML 1.1 +%TAG !u! tag:unity3d.com,2011: +--- !u!114 &11400000 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 0} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: -340790334, guid: 67cc4cb7839cd3741b63733d5adf0442, type: 3} + m_Name: TaSTT_Menu + m_EditorClassIdentifier: + controls: + - name: Show + icon: {fileID: 0} + type: 102 + parameter: + name: TaSTT_Toggle + value: 1 + style: 0 + subMenu: {fileID: 0} + subParameters: [] + labels: [] + - name: Lock World + icon: {fileID: 0} + type: 102 + parameter: + name: TaSTT_Lock_World + value: 1 + style: 0 + subMenu: {fileID: 0} + subParameters: [] + labels: [] + - name: Enable Beep + icon: {fileID: 0} + type: 102 + parameter: + name: TaSTT_Speech_Noise_Enable + value: 1 + style: 0 + subMenu: {fileID: 0} + subParameters: [] + labels: [] + - name: Scale + icon: {fileID: 0} + type: 203 + parameter: + name: + value: 1 + style: 0 + subMenu: {fileID: 0} + subParameters: + - name: TaSTT_Scale + labels: [] -- cgit v1.2.3