Continue work on in-game audio, revert steamvr.py

We now play arpeggiated *chords* of vowels instead of one, allowing for
a denser audio feedback mechanism.

4 files changed, 229 insertions, 127 deletions

diff --git a/Scripts/libtastt.py b/Scripts/libtastt.py
index 12f9056..c8d3958 100644
--- a/Scripts/libtastt.py
+++ b/Scripts/libtastt.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import argparse
+import array
 import generate_utils
 import libunity
 import os
@@ -220,7 +221,7 @@ AnimationClip:
     m_Level: 0
     m_CycleOffset: 0
     m_HasAdditiveReferencePose: 0
-    m_LoopTime: 1
+    m_LoopTime: 0
     m_LoopBlend: 0
     m_LoopBlendOrientation: 0
     m_LoopBlendPositionY: 0
@@ -472,9 +473,14 @@ def generateClearAnimation(anim_dir, guid_map):
     guid_map[anim_path] = meta.guid
     guid_map[meta.guid] = anim_path
 
+# sound_chord: whether to play a, e, i, o, u
 # value: 0 or 1
-def generateSoundAnimation(nth_sound: int, value: int, anim_name: str, anim_dir: str, guid_map: typing.Dict[str, str]):
-    print(f"Generating sound {nth_sound} animation", file=sys.stderr)
+def generateSoundAnimation(sound_chord: typing.Tuple[int,int,int,int,int],
+        value: int,
+        anim_name: str,
+        anim_dir: str, guid_map: typing.Dict[str, str],
+        anim_delay_frames = 2):
+    print(f"Generating sound animation {sound_chord} / {anim_name}", file=sys.stderr)
 
     parser = libunity.UnityParser()
     parser.parse(SOUND_ANIMATION_TEMPLATE)
@@ -485,17 +491,36 @@ def generateSoundAnimation(nth_sound: int, value: int, anim_name: str, anim_dir:
     anim_clip.mapping['m_FloatCurves'].sequence = []
     anim_clip.mapping['m_EditorCurves'].sequence = []
 
-    curve = curve_template.copy()
-    for keyframe in curve.mapping['curve'].mapping['m_Curve'].sequence:
-        keyframe.mapping['value'] = str(value)
-        curve.mapping['path'] = f"World Constraint/Container/TaSTT/Audio {nth_sound}"
-    # Add curve to animation
-    anim_clip.mapping['m_FloatCurves'].sequence.append(curve)
-    anim_clip.mapping['m_EditorCurves'].sequence.append(curve)
+    # Animate all notes.
+    for note_i in range(len(sound_chord)):
+        curve = curve_template.copy()
+
+        keyframe_template = curve.mapping['curve'].mapping['m_Curve'].sequence[0]
+        curve.mapping['curve'].mapping['m_Curve'].sequence = []
+
+        # First keyframe: zero all but first note
+        if note_i != 0:
+            keyframe = keyframe_template.copy()
+            keyframe.mapping['time'] = 0
+            keyframe.mapping['value'] = 0
+            curve.mapping['path'] = f"World Constraint/Container/TaSTT/Audio {note_i + 1}"
+            curve.mapping['curve'].mapping['m_Curve'].sequence.append(keyframe)
+
+        # Subsequent keyframes: animate as normal
+        keyframe = keyframe_template.copy()
+        keyframe.mapping['time']= str(note_i * anim_delay_frames * 1.0 / 60.0)
+        keyframe.mapping['value'] = str(sound_chord[note_i])
+        curve.mapping['path'] = f"World Constraint/Container/TaSTT/Audio {note_i + 1}"
+        curve.mapping['curve'].mapping['m_Curve'].sequence.append(keyframe)
+
+        # Add curve to animation
+        anim_clip.mapping['m_FloatCurves'].sequence.append(curve)
+        anim_clip.mapping['m_EditorCurves'].sequence.append(curve)
+
+    anim_clip.mapping['m_AnimationClipSettings'].mapping['m_StopTime'] = str((len(sound_chord)-1) * anim_delay_frames * 1.0 / 60.0)
 
     # Serialize animation to file
     anim_path = os.path.join(anim_dir, anim_name + ".anim")
-    print("Generating sound animation at {}".format(anim_path), file=sys.stderr)
     with open(anim_path, "w", encoding="utf-8") as f:
         f.write(libunity.unityYamlToString([anim_node]))
     # Generate metadata
@@ -598,11 +623,14 @@ def generateScaleAnimation(anim_name: str, anim_dir: str,
 def generateAnimations(anim_dir, guid_map):
     generateClearAnimation(anim_dir, guid_map)
 
-    for i in range(5):
-        anim_name = generate_utils.getSoundParam(i+1) + "_Off"
-        generateSoundAnimation(i+1, 0, anim_name, anim_dir, guid_map)
-        anim_name = generate_utils.getSoundParam(i+1) + "_On"
-        generateSoundAnimation(i+1, 1, anim_name, anim_dir, guid_map)
+    for chord_bits in range(2**5):
+        chord = [0, 0, 0, 0, 0]
+        for i in range(5):
+            if (chord_bits >> i) % 2 == 1:
+                chord[i] = 1
+        print(f"Generating chord {chord}", file=sys.stderr)
+        anim_name = f"Sound_a{chord[0]}_e{chord[1]}_i{chord[2]}_o{chord[3]}_u{chord[4]}"
+        generateSoundAnimation(chord, 0, anim_name, anim_dir, guid_map)
 
     print("Generating letter animations", file=sys.stderr)
 
@@ -809,6 +837,117 @@ def generateScaleLayer(anim: libunity.UnityAnimator,
 
     pass
 
+def generateSoundLayer(anim: libunity.UnityAnimator,
+        gen_anim_dir: str,
+        guid_map: typing.Dict[str, str],
+        anim_len_s = 12.0/60.0):
+
+    layer = anim.addLayer("TaSTT_Sound")
+
+    # Create `a` state.
+    a_state = anim.addAnimatorState(layer, "a", is_default_state=True)
+
+    for a_bool in range(2):
+        dy = 100
+        dx = a_bool * 800
+        # Create `e` state.
+        ax_e_state = anim.addAnimatorState(layer,
+                f"a{a_bool}_e",
+                dy=dy, dx=dx)
+        # Create transition based on whether `a` is set.
+        trans = anim.addTransition(ax_e_state)
+        param = generate_utils.getSoundParam(1)
+        anim.addTransitionBooleanCondition(a_state, trans, param, a_bool)
+
+        for e_bool in range(2):
+            dy = 200
+            dx = a_bool * 800 + e_bool * 400
+
+            # Create `i` state.
+            ax_ex_i_state = anim.addAnimatorState(layer,
+                    f"a{a_bool}_e{e_bool}_i",
+                    dy=dy, dx=dx)
+
+            # Create transition based on whether `e` is set.
+            trans = anim.addTransition(ax_ex_i_state)
+            param = generate_utils.getSoundParam(2)
+            anim.addTransitionBooleanCondition(ax_e_state, trans, param, e_bool)
+
+            for i_bool in range(2):
+                dy = 300
+                dx = a_bool * 800 + e_bool * 400 + i_bool * 200
+
+                # Create `o` state.
+                ax_ex_ix_o_state = anim.addAnimatorState(layer,
+                        f"a{a_bool}_e{e_bool}_i{i_bool}_o",
+                        dy=dy, dx=dx)
+                # Create transition based on whether `i` is set.
+                trans = anim.addTransition(ax_ex_ix_o_state)
+                param = generate_utils.getSoundParam(3)
+                anim.addTransitionBooleanCondition(ax_ex_i_state, trans, param, i_bool)
+
+                for o_bool in range(2):
+                    dy = 400
+                    dx = a_bool * 800 + e_bool * 400 + i_bool * 200 + o_bool * 100
+
+                    # Create `u` state.
+                    ax_ex_ix_ox_u_state = anim.addAnimatorState(layer,
+                            f"a{a_bool}_e{e_bool}_i{i_bool}_o{o_bool}_u",
+                            dy=dy, dx=dx)
+                    # Create transition based on whether `o` is set.
+                    trans = anim.addTransition(ax_ex_ix_ox_u_state)
+                    param = generate_utils.getSoundParam(4)
+                    anim.addTransitionBooleanCondition(ax_ex_ix_o_state,
+                            trans, param, o_bool)
+
+                    for u_bool in range(2):
+                        dy = 500
+                        dx = a_bool * 800 + e_bool * 400 + i_bool * 200 + o_bool * 100 + u_bool * 50
+                        if u_bool == 1:
+                            dy = 550
+
+                        # Create `u` state.
+                        ax_ex_ix_ox_ux_state = anim.addAnimatorState(layer,
+                                f"a{a_bool}_e{e_bool}_i{i_bool}_o{o_bool}_u{u_bool}",
+                                dy=dy, dx=dx)
+                        # Create transition based on whether `u` is set.
+                        trans = anim.addTransition(ax_ex_ix_ox_ux_state)
+                        param = generate_utils.getSoundParam(5)
+                        anim.addTransitionBooleanCondition(ax_ex_ix_ox_u_state,
+                                trans, param, u_bool)
+
+                        chord = [a_bool, e_bool, i_bool, o_bool, u_bool]
+                        anim_name = f"Sound_a{chord[0]}_e{chord[1]}_i{chord[2]}_o{chord[3]}_u{chord[4]}"
+                        anim_path = os.path.join(gen_anim_dir, anim_name + ".anim")
+                        anim_guid = guid_map[anim_path]
+                        anim.setAnimatorStateAnimation(ax_ex_ix_ox_ux_state, anim_guid)
+
+                        # Create return-home transitions.
+                        trans = anim.addTransition(a_state, dur_s = anim_len_s)
+                        trans.mapping['AnimatorStateTransition'].mapping['m_InterruptionSource'] = '0'
+                        param = generate_utils.getSoundParam(1)
+                        anim.addTransitionBooleanCondition(ax_ex_ix_ox_ux_state, trans, param, 1 - a_bool)
+
+                        trans = anim.addTransition(a_state, dur_s = anim_len_s)
+                        trans.mapping['AnimatorStateTransition'].mapping['m_InterruptionSource'] = '0'
+                        param = generate_utils.getSoundParam(2)
+                        anim.addTransitionBooleanCondition(ax_ex_ix_ox_ux_state, trans, param, 1 - e_bool)
+
+                        trans = anim.addTransition(a_state, dur_s = anim_len_s)
+                        trans.mapping['AnimatorStateTransition'].mapping['m_InterruptionSource'] = '0'
+                        param = generate_utils.getSoundParam(3)
+                        anim.addTransitionBooleanCondition(ax_ex_ix_ox_ux_state, trans, param, 1 - i_bool)
+
+                        trans = anim.addTransition(a_state, dur_s = anim_len_s)
+                        trans.mapping['AnimatorStateTransition'].mapping['m_InterruptionSource'] = '0'
+                        param = generate_utils.getSoundParam(4)
+                        anim.addTransitionBooleanCondition(ax_ex_ix_ox_ux_state, trans, param, 1 - o_bool)
+
+                        trans = anim.addTransition(a_state, dur_s = anim_len_s)
+                        trans.mapping['AnimatorStateTransition'].mapping['m_InterruptionSource'] = '0'
+                        param = generate_utils.getSoundParam(5)
+                        anim.addTransitionBooleanCondition(ax_ex_ix_ox_ux_state, trans, param, 1 - u_bool)
+
 def generateFX(guid_map, gen_anim_dir):
     anim = libunity.UnityAnimator()
 
@@ -852,16 +991,8 @@ def generateFX(guid_map, gen_anim_dir):
             "TaSTT_Emerge_100.anim",
             anim, guid_map, 0.5)
 
-    for i in range(5):
-        param_name = generate_utils.getSoundParam(i+1)
-        generateToggle(f"TaSTT_Audio{i+1}",
-                param_name,
-                gen_anim_dir,
-                param_name + "_Off.anim",
-                param_name + "_On.anim",
-                anim, guid_map)
-
     generateScaleLayer(anim, gen_anim_dir, guid_map)
+    generateSoundLayer(anim, gen_anim_dir, guid_map)
 
     return anim
 
diff --git a/Scripts/libunity.py b/Scripts/libunity.py
index cd8174d..f79cd6f 100644
--- a/Scripts/libunity.py
+++ b/Scripts/libunity.py
@@ -745,7 +745,7 @@ class UnityAnimator():
         node.class_id = "1101"
         node.anchor = str(new_id)
         state = node.mapping['AnimatorStateTransition']
-        state.mapping['m_DstState'].mapping['fileID'] = dst_state.anchor
+        state.mapping['m_DstState'].mapping['fileID'] = copy.copy(dst_state.anchor)
         state.mapping['m_TransitionDuration'] = dur_s
         self.nodes.append(node)
 
@@ -908,7 +908,7 @@ class UnityAnimator():
         # Register the transition with the `from_state`.
         if from_state:
             from_state_trans = from_state.mapping['AnimatorState'].mapping['m_Transitions'].addChildMapping()
-            from_state_trans.mapping['fileID'] = trans.anchor
+            from_state_trans.mapping['fileID'] = copy.copy(trans.anchor)
 
     def addTransitionIntegerEqualityCondition(self, from_state, trans, param, param_val):
         # Populate the transition's condition logic.
@@ -994,7 +994,8 @@ class UnityAnimator():
                 motion.mapping["guid"] = noop_anim_meta.guid
                 motion.mapping["type"] = "2"
             else:
-                print(f"Skipping state {anchor} / {name}")
+                #print(f"Skipping state {anchor} / {name}")
+                pass
 
 def unityYamlToString(nodes):
     lines = []
diff --git a/Scripts/osc_ctrl.py b/Scripts/osc_ctrl.py
index ad5667f..413e2ae 100644
--- a/Scripts/osc_ctrl.py
+++ b/Scripts/osc_ctrl.py
@@ -103,18 +103,12 @@ def pageMessage(osc_state: OscState, msg: str, estate: EmotesState) -> bool:
     sounds_to_make = set()
     letter_i = 1
     for letter in ["a", "e", "i", "o", "u"]:
-        if letter in msg_slice:
+        if letter in msg_slice.lower():
             sounds_to_make.add(letter_i)
         letter_i += 1
-    if len(sounds_to_make) == 0:
+    if len(sounds_to_make) > 0:
         for i in range(5):
-            playAudio(osc_state, i+1, False)
-    else:
-        sound_to_make = random.sample(sounds_to_make, 1)[0]
-        for i in range(5):
-            if i+1 == sound_to_make:
-                # TODO(yum) think about making this probabilistic
-                print(f"Playing sound {i+1}")
+            if i+1 in sounds_to_make:
                 playAudio(osc_state, i+1, True)
             else:
                 playAudio(osc_state, i+1, False)
diff --git a/Scripts/steamvr.py b/Scripts/steamvr.py
index 0f241ca..e0b59e3 100644
--- a/Scripts/steamvr.py
+++ b/Scripts/steamvr.py
@@ -1,109 +1,85 @@
-import openvr
+#!/usr/bin/env python3
+
+# python3 -m pip install openvr
+# License: BSD-3.0 (requires showing notice in binary distributions)
+import openvr as vr
 import sys
 import time
-import typing
 
-EVENT_RISING_EDGE = 0
-EVENT_FALLING_EDGE = 1
-EVENT_POSE = 2
+EVENT_NONE = 0
+EVENT_RISING_EDGE = 1
+EVENT_FALLING_EDGE = 2
 
 class InputEvent:
     def __init__(self,
-            opcode: int,
-            pos: typing.Tuple[float,float,float] = None):
+            opcode: int):
         self.opcode = opcode
-        self.pos = pos
 
-    def __str__(self):
-        if self.opcode == EVENT_RISING_EDGE:
-            return "EVENT_RISING_EDGE"
-        elif self.opcode == EVENT_FALLING_EDGE:
-            return "EVENT_FALLING_EDGE"
-        elif self.opcode == EVENT_POSE:
-            return f"EVENT_POSE: {self.pos}"
+# Checks if the given button on the given controller is pressed.
+def pollButtonPress(
+        hand: str = "right",
+        button: str = "b",
+        ) -> int:
+    hands = {}
+    hands["left"] = vr.TrackedControllerRole_LeftHand
+    hands["right"] = vr.TrackedControllerRole_RightHand
 
-def pollButtonPress(hand: str = "right", button: str = "b") -> InputEvent:
-    openvr.init(openvr.VRApplication_Overlay)
+    buttons = {}
+    buttons["a"] = vr.k_EButton_IndexController_A
+    buttons["b"] = vr.k_EButton_IndexController_B
+    buttons["thumbstick"] = vr.k_EButton_Axis0
 
-    system = openvr.VRSystem()
+    system = None
+    while not system:
+        try:
+            system = vr.init(vr.VRApplication_Background)
+        except Exception as e:
+            print(f"Failed to start steamVR input thread: {repr(e)}", file=sys.stderr)
+            time.sleep(5)
+    last_packet = 0
+    event_high = False
 
-    button_mapping = {
-        'a': k_EButton_Index_Controller_A,
-        'b': k_EButton_Index_Controller_B,
-        'thumbstick': k_EButton_SteamVR_Touchpad,
-    }
+    while True:
+        time.sleep(0.01)
 
-    print("SteamVR session created. Listening for controller input...")
+        lh_idx = system.getTrackedDeviceIndexForControllerRole(hand_id)
+        #print("left hand device idx: {}".format(lh_idx))
 
-    while True:
-        # Drain input events.
-        event = openvr.VREvent_t()
-        while system.pollNextEvent(event):
-            # Event processing, e.g. button presses, goes here
-            if event.eventType == openvr.VREvent_ButtonPress or \
-                    event.eventType == openvr.VREvent_ButtonUnpress:
-                print(f"event.data.controller.button: {event.data.controller.button}")
-                continue
-                print(f"event: {dir(event)}")
-                print(f"event.data: {dir(event.data)}")
-                print(f"event.data.controller: {dir(event.data.controller)}")
-                print(f"event.data.controller.button: {event.data.controller.button}")
-                print(f"event.data.keyboard: {dir(event.data.keyboard)}")
-                print(f"event.data.keyboard.cNewInput: {int.from_bytes(event.data.keyboard.cNewInput, byteorder='little')}")
-                print(f"event.data.keyboard.uUserValue: {event.data.keyboard.uUserValue}")
-                print(f"event.data.mouse: {dir(event.data.mouse)}")
-                print(f"event.data.mouse.button: {event.data.mouse.button}")
-                print(f"event.data.touchPadMove: {dir(event.data.touchPadMove)}")
-                print(f"event.data.touchPadMove.bFingerDown: {event.data.touchPadMove.bFingerDown}")
-                is_rising = event.eventType == openvr.VREvent_ButtonPress
-                # Check if the intended button is pressed
-                if button == 'thumbstick':
-                    _, controller_state = system.getControllerState(event.trackedDeviceIndex)
-                    mouse_x = controller_state.rAxis[0].x
-                    mouse_y = controller_state.rAxis[0].y
-                    print(f"mouse x/y: {mouse_x}/{mouse_y}")
-                    print(f"mouse rad: {mouse_x**2 + mouse_y**2}")
-                    dead_zone_radius = 0.05
-                    thumbstick_moved = mouse_x**2 + mouse_y**2 > dead_zone_radius**2
-                    if event.data.controller.button == button_mapping['thumbstick'] and not thumbstick_moved:
-                        if is_rising:
-                            yield InputEvent(EVENT_RISING_EDGE)
-                        else:
-                            yield InputEvent(EVENT_FALLING_EDGE)
-                elif event.data.controller.button == button_mapping[button]:
-                    if is_rising:
-                        yield InputEvent(EVENT_RISING_EDGE)
-                    else:
-                        yield InputEvent(EVENT_FALLING_EDGE)
-        # Check poses.
-        # TODO(yum) use this. Thinking about adding gestures: swipe to scale
-        # up/down, etc.
-        if False:
-            poses = (openvr.TrackedDevicePose_t * openvr.k_unMaxTrackedDeviceCount)()
-            system.getDeviceToAbsoluteTrackingPose(openvr.TrackingUniverseStanding, 0, poses)
-            pose = None
-            for i in range(openvr.k_unMaxTrackedDeviceCount):
-                if system.getControllerRoleForTrackedDeviceIndex(i) == openvr.TrackedControllerRole_RightHand:
-                    pose = poses[i]
-            if pose and pose.bPoseIsValid:
-                position = pose.mDeviceToAbsoluteTracking.m[0][3], \
-                        pose.mDeviceToAbsoluteTracking.m[1][3], \
-                        pose.mDeviceToAbsoluteTracking.m[2][3]
-                yield InputEvent(EVENT_POSE, pos=position)
+        got_state, state = system.getControllerState(lh_idx)
+        if not got_state:
+            continue
 
-        # Max out a 100 Hz.
-        time.sleep(0.01)
+        if state.unPacketNum == last_packet:
+            continue
 
-    openvr.shutdown()
+        # Clicking joysticks and moving joysticks fire the same events. To
+        # differentiate movement from clicking, we create a dead zone: if the event
+        # fires while the stick isn't moved far from center, we assume it's a
+        # click, not movement.
+        dead_zone_radius = 0.7
 
-if __name__ == "__main__":
-    if len(sys.argv) != 3:
-        print("Usage: script_name.py [left|right] [a|b|thumbstick]")
-        sys.exit(1)
+        button_mask = (1 << button_id)
+        ret = EVENT_NONE
+        if (state.ulButtonPressed & button_mask) != 0 and\
+                (state.rAxis[0].x**2 + state.rAxis[0].y**2 < dead_zone_radius**2):
+            #print("button pressed: %016x" % state.ulButtonPressed)
+            #for i in range(0, 5):
+            #    print("axis {} x: {} y: {}".format(i, state.rAxis[i].x, state.rAxis[i].y))
+            if not event_high:
+                yield InputEvent(EVENT_RISING_EDGE)
+            event_high = True
+        elif event_high:
+            event_high = False
+            yield InputEvent(EVENT_FALLING_EDGE)
 
-    hand = sys.argv[1]
-    button = sys.argv[2]
-    gen = pollButtonPress(hand, button)
+if __name__ == "__main__":
+    gen = pollButtonPress()
     while True:
-        print(next(gen))
+        time.sleep(0.1)
+
+        event = pollButtonPress(session_state, hand_id = hands["left"], button_id = buttons["joystick"])
+        if event == EVENT_RISING_EDGE:
+            print("rising edge")
+        elif event == EVENT_FALLING_EDGE:
+            print("falling edge")