diff options
| -rw-r--r-- | README.md | 3 | ||||
| -rw-r--r-- | Shaders/TaSTT.shader | 202 | ||||
| -rw-r--r-- | UnityAssets/Materials/TaSTT_Text.mat | 8 | ||||
| -rw-r--r-- | UnityAssets/TaSTT.fbx | bin | 18172 -> 16236 bytes | |||
| -rw-r--r-- | UnityAssets/black.png | bin | 0 -> 789 bytes | |||
| -rw-r--r-- | UnityAssets/black.png.meta | 92 |
6 files changed, 268 insertions, 37 deletions
@@ -167,6 +167,8 @@ Contributions welcome. Send a pull request to this repository. 5. Display text in overlay. Enables (1) lower latency view of TaSTT's transcription state; (2) checking transcriptions ahead of time; (3) checking transcriptions without having to see the board in game. + 6. TTS. Multiple people have requested this. See if there are open source + algorithms available; or, figure out how to integrate with 4. Optimization 1. ~~Utilize the avatar 3.0 SDK's ability to drive parameters to reduce the total # of parameters (and therefore OSC messages & sync events). Note @@ -191,6 +193,7 @@ Contributions welcome. Send a pull request to this repository. idea!) 5. Bugfixes 1. ~~The whisper STT says "Thank you." when there's no audio?~~ DONE + 2. JP and CN transcription does not work in the GUI due to encoding issues. 6. Shine 1. Smooth scrolling. 2. ~~Infinite scrolling.~~ DONE diff --git a/Shaders/TaSTT.shader b/Shaders/TaSTT.shader index d77d025..c3e6ae5 100644 --- a/Shaders/TaSTT.shader +++ b/Shaders/TaSTT.shader @@ -14,6 +14,7 @@ [MaterialToggle] Render_Margin("Render margin", float) = 1
[MaterialToggle] Render_Visual_Indicator("Render visual speech indicator", float) = 1
Margin_Scale("Margin scale", float) = 0.03
+ Margin_Rounding_Scale("Margin rounding scale", float) = 0.03
TaSTT_Backplate("TaSTT_Backplate", 2D) = "black" {}
@@ -420,7 +421,7 @@ #pragma fragment frag
#pragma multi_compile
- #include "UnityCG.cginc"
+ //#include "UnityCG.cginc"
struct appdata
{
@@ -449,6 +450,7 @@ float Render_Margin;
float Render_Visual_Indicator;
float Margin_Scale;
+ float Margin_Rounding_Scale;
float3 HUEtoRGB(in float H)
{
@@ -882,14 +884,6 @@ return clamp(lerp(lo, hi, uv), 0.0, 1.0);
}
- bool InMargin(float2 uv, float2 margin)
- {
- return uv.x < margin.x / 2 ||
- uv.x > 1 - margin.x / 2 ||
- uv.y < margin.y / 2 ||
- uv.y > 1 - margin.y / 2;
- }
-
// dist = sqrt(dx^2 + dy^2) = sqrt(<dx,dy> * <dx,dy>)
bool InRadius2(float2 uv, float2 pos, float radius2)
{
@@ -897,6 +891,71 @@ return dot(delta, delta) < radius2;
}
+ bool InMargin(float2 uv, float2 margin)
+ {
+ if (uv.x < margin.x ||
+ uv.x > 1 - margin.x ||
+ uv.y < margin.y ||
+ uv.y > 1 - margin.y) {
+ return true;
+ }
+
+ return false;
+ }
+
+ bool InSpeechIndicator(float2 uv, float2 margin)
+ {
+ if (!Render_Visual_Indicator) {
+ return false;
+ }
+
+ // Margin is uv_margin/2 wide/tall.
+ // We want a circle whose radius is ~80% of that.
+ float radius_factor = 0.95;
+ float radius = margin.x * radius_factor;
+ // We want this circle to be centered halfway through the margin
+ // vertically, and at 1.5x the margin width horizontally.
+ float2 indicator_center = float2(margin.x + radius, margin.y * 0.5);
+ // Finally, translate it to the top of the board instead of the
+ // bottom.
+ indicator_center.y = 1.0 - indicator_center.y;
+
+ if (InRadius2(uv, indicator_center, radius * radius)) {
+ return true;
+ }
+
+ return false;
+ }
+
+ bool InMarginRounding(float2 uv, float2 margin, float rounding, bool interior)
+ {
+ if (!interior) {
+ rounding += margin.x;
+ margin = float2(0, 0);
+ }
+
+ // This is the center of a circle whose perimeter touches the
+ // upper left corner of the margin.
+ float2 c0 = float2(rounding + margin.x, rounding + margin.y);
+ if (uv.x < c0.x && uv.y < c0.y && uv.x > margin.x && uv.y > margin.y && !InRadius2(uv, c0, rounding * rounding)) {
+ return true;
+ }
+ c0 = float2(rounding + margin.x, 1 - (rounding + margin.y));
+ if (uv.x < c0.x && uv.y > c0.y && uv.x > margin.x && uv.y < 1 - margin.y && !InRadius2(uv, c0, rounding * rounding)) {
+ return true;
+ }
+ c0 = float2(1 - (rounding + margin.x), 1 - (rounding + margin.y));
+ if (uv.x > c0.x && uv.y > c0.y && uv.x < 1 - margin.x && uv.y < 1 - margin.y && !InRadius2(uv, c0, rounding * rounding)) {
+ return true;
+ }
+ c0 = float2(1 - (rounding + margin.x), rounding + margin.y);
+ if (uv.x > c0.x && uv.y < c0.y && uv.x < 1 - margin.x && uv.y > margin.y && !InRadius2(uv, c0, rounding * rounding)) {
+ return true;
+ }
+
+ return false;
+ }
+
// Write the nth letter in the current cell and return the value of the
// pixel.
// `texture_rows` and `texture_cols` indicate how many rows and columns are
@@ -1383,6 +1442,85 @@ return float2(0, 0);
}
+ fixed sq_dist(fixed2 p0, fixed2 p1)
+ {
+ fixed2 delta = p1 - p0;
+ //return abs(delta.x) + abs(delta.y);
+ return max(abs(delta.x), abs(delta.y));
+ }
+
+ fixed4 effect_squares (v2f i)
+ {
+ const fixed time = _Time.y;
+
+ #define PI 3.1415926535
+ fixed theta = PI/4 + sin(time / 4) * 0.1;
+ fixed2x2 rot =
+ fixed2x2(cos(theta), -1 * sin(theta),
+ sin(theta), cos(theta));
+
+ #define NSQ_X 9.0
+ #define NSQ_Y 5.0
+
+ // Map uv from [0, 1] to [-.5, .5].
+ fixed2 p = i.uv - 0.5;
+ p *= fixed2(NSQ_X, NSQ_Y);
+ p = mul(rot, p);
+ p -= 0.5;
+
+ // See how far we are from the nearest grid point
+ fixed2 intra_pos = frac(p);
+ fixed2 intra_center = fixed2(0.5, 0.5);
+ fixed intra_dist = sq_dist(intra_pos, intra_center);
+
+ fixed st0 = (sin(time) + 1) / 2;
+ fixed st1 = (sin(time + PI/8) + 1) / 2;
+ fixed st2 = (sin(time + PI/2) + 1) / 2;
+ fixed st3 = (sin(time + PI/2 + PI/8) + 1) / 2;
+
+ fixed2 center = fixed2(0, 0);
+ center = mul(rot, center);
+ center -= 0.5;
+ fixed2 rot_lim = fixed2(NSQ_X, NSQ_Y);
+ rot_lim = mul(rot, rot_lim);
+ rot_lim -= 0.5;
+
+ float v = 0;
+ float x = 0;
+
+ if (intra_dist > 0.5 * (0.5 + sin(time * 1.5) * 0.1)) {
+ v = intra_dist;
+ } else {
+ v = 0;
+ }
+
+ fixed extra_dist = sq_dist(p, center);
+ fixed check = max(rot_lim.x, rot_lim.y) / 2;
+ if (extra_dist > check * st0) {
+ v = 1.0 - v;
+ }
+ if (extra_dist > check * st1) {
+ v = 1.0 - v;
+ }
+ if (extra_dist > check * st2) {
+ v = 1.0 - v;
+ }
+ if (extra_dist > check * st3) {
+ v = 1.0 - v;
+ } else {
+ x = 0.50;
+ }
+
+ fixed3 hsv;
+ hsv[0] = (v * 0.2 * (1 - x * .8) + 0.55) - x;
+ hsv[1] = 0.7;
+ hsv[2] = 0.8;
+
+ fixed3 col = HSVtoRGB(hsv);
+
+ return fixed4(col, 1.0);
+ }
+
fixed4 frag (v2f i) : SV_Target
{
float2 uv = i.uv;
@@ -1393,24 +1531,18 @@ uv.x = 1.0 - uv.x;
}
- float2 uv_margin = float2(Margin_Scale, Margin_Scale * 2);
- if (InMargin(uv, uv_margin)) {
- // Margin is uv_margin/2 wide/tall.
- // We want a circle whose radius is ~80% of that.
- if (Render_Visual_Indicator) {
- float radius_factor = 0.95;
- float radius = (uv_margin.x / 2) * radius_factor;
- // We want this circle to be centered halfway through the margin
- // vertically, and at 1.5x the margin width horizontally.
- float2 indicator_center = float2(
- uv_margin.x * 0.5 + radius,
- uv_margin.y * 0.5 * 0.5
- );
- // Finally, translate it to the top of the board instead of the
- // bottom.
- indicator_center.y = 1.0 - indicator_center.y;
-
- if (InRadius2(uv, indicator_center, radius * radius)) {
+ float2 uv_margin = float2(Margin_Scale, Margin_Scale * 2) / 2;
+ if (Render_Margin) {
+ if (Margin_Rounding_Scale > 0.0) {
+ if (InMarginRounding(uv, uv_margin, Margin_Rounding_Scale, /*interior=*/true)) {
+ return effect_squares(i);
+ }
+ if (InMarginRounding(uv, uv_margin, Margin_Rounding_Scale, /*interior=*/false)) {
+ return fixed4(0, 0, 0, 0);
+ }
+ }
+ if (InMargin(uv, uv_margin)) {
+ if (InSpeechIndicator(uv, uv_margin)) {
if (floor(TaSTT_Indicator_0) == 1.0) {
// Actively speaking
return float3tofixed4(TaSTT_Indicator_Color_2, 1.0);
@@ -1422,17 +1554,17 @@ return float3tofixed4(TaSTT_Indicator_Color_0, 1.0);
}
}
- }
- if (Render_Margin) {
- return fixed4(1,1,1,1);
+
+ if (Render_Margin) {
+ return effect_squares(i);
+ }
}
}
+ uv_margin *= 4;
float2 uv_with_margin = AddMarginToUV(uv, uv_margin);
- uv_margin *= 2;
- float2 uv_with_margin2 = AddMarginToUV(uv, uv_margin);
- int2 letter_bytes = (int2) floor(GetLetterParameter(uv_with_margin2));
+ int2 letter_bytes = (int2) floor(GetLetterParameter(uv_with_margin));
int letter = letter_bytes[0] | (letter_bytes[1] << 8);
float texture_cols;
@@ -1441,11 +1573,11 @@ if (letter < 0xE000) {
texture_cols = 128.0;
texture_rows = 64.0;
- letter_uv = GetLetter(uv_with_margin2, letter, texture_cols, texture_rows, 48, 4);
+ letter_uv = GetLetter(uv_with_margin, letter, texture_cols, texture_rows, 48, 4);
} else {
texture_cols = 8.0;
texture_rows = 8.0;
- letter_uv = GetLetter(uv_with_margin2, letter, texture_cols, texture_rows, 8, 4);
+ letter_uv = GetLetter(uv_with_margin, letter, texture_cols, texture_rows, 8, 4);
}
fixed4 background = TaSTT_Backplate.Sample(sampler_linear_repeat, uv);
diff --git a/UnityAssets/Materials/TaSTT_Text.mat b/UnityAssets/Materials/TaSTT_Text.mat index 8b1c64b..9eb680f 100644 --- a/UnityAssets/Materials/TaSTT_Text.mat +++ b/UnityAssets/Materials/TaSTT_Text.mat @@ -9,7 +9,7 @@ Material: m_PrefabAsset: {fileID: 0} m_Name: TaSTT_Text m_Shader: {fileID: 4800000, guid: feba055fa9e7f9543aaae032a30ec878, type: 3} - m_ShaderKeywords: + m_ShaderKeywords: RENDER_MARGIN_ON m_LightmapFlags: 4 m_EnableInstancingVariants: 0 m_DoubleSidedGI: 0 @@ -20,7 +20,7 @@ Material: serializedVersion: 3 m_TexEnvs: - TaSTT_Backplate: - m_Texture: {fileID: 0} + m_Texture: {fileID: 2800000, guid: 06c9223f22185d54b920542d45168c6b, type: 3} m_Scale: {x: 1, y: 1} m_Offset: {x: 0, y: 0} - _BumpMap: @@ -92,6 +92,10 @@ Material: m_Scale: {x: 1, y: 1} m_Offset: {x: 0, y: 0} m_Floats: + - Margin_Scale: 0.06 + - Margin_Rounding_Scale: 0.11 + - Render_Margin: 1 + - Render_Visual_Indicator: 0 - TaSTT_Indicator_0: 0 - TaSTT_Indicator_1: 0 - _BumpScale: 1 diff --git a/UnityAssets/TaSTT.fbx b/UnityAssets/TaSTT.fbx Binary files differindex d6737d0..68936d0 100644 --- a/UnityAssets/TaSTT.fbx +++ b/UnityAssets/TaSTT.fbx diff --git a/UnityAssets/black.png b/UnityAssets/black.png Binary files differnew file mode 100644 index 0000000..185328b --- /dev/null +++ b/UnityAssets/black.png diff --git a/UnityAssets/black.png.meta b/UnityAssets/black.png.meta new file mode 100644 index 0000000..8f12b6f --- /dev/null +++ b/UnityAssets/black.png.meta @@ -0,0 +1,92 @@ +fileFormatVersion: 2 +guid: 06c9223f22185d54b920542d45168c6b +TextureImporter: + internalIDToNameTable: [] + externalObjects: {} + serializedVersion: 11 + mipmaps: + mipMapMode: 0 + enableMipMap: 1 + sRGBTexture: 1 + linearTexture: 0 + fadeOut: 0 + borderMipMap: 0 + mipMapsPreserveCoverage: 0 + alphaTestReferenceValue: 0.5 + mipMapFadeDistanceStart: 1 + mipMapFadeDistanceEnd: 3 + bumpmap: + convertToNormalMap: 0 + externalNormalMap: 0 + heightScale: 0.25 + normalMapFilter: 0 + isReadable: 0 + streamingMipmaps: 0 + streamingMipmapsPriority: 0 + grayScaleToAlpha: 0 + generateCubemap: 6 + cubemapConvolution: 0 + seamlessCubemap: 0 + textureFormat: 1 + maxTextureSize: 2048 + textureSettings: + serializedVersion: 2 + filterMode: 1 + aniso: 1 + mipBias: 0 + wrapU: 0 + wrapV: 0 + wrapW: 0 + nPOTScale: 1 + lightmap: 0 + compressionQuality: 50 + spriteMode: 0 + spriteExtrude: 1 + spriteMeshType: 1 + alignment: 0 + spritePivot: {x: 0.5, y: 0.5} + spritePixelsToUnits: 100 + spriteBorder: {x: 0, y: 0, z: 0, w: 0} + spriteGenerateFallbackPhysicsShape: 1 + alphaUsage: 1 + alphaIsTransparency: 0 + spriteTessellationDetail: -1 + textureType: 0 + textureShape: 1 + singleChannelComponent: 0 + maxTextureSizeSet: 0 + compressionQualitySet: 0 + textureFormatSet: 0 + applyGammaDecoding: 0 + platformSettings: + - serializedVersion: 3 + buildTarget: DefaultTexturePlatform + maxTextureSize: 2048 + resizeAlgorithm: 0 + textureFormat: -1 + textureCompression: 1 + compressionQuality: 50 + crunchedCompression: 0 + allowsAlphaSplitting: 0 + overridden: 0 + androidETC2FallbackOverride: 0 + forceMaximumCompressionQuality_BC6H_BC7: 0 + spriteSheet: + serializedVersion: 2 + sprites: [] + outline: [] + physicsShape: [] + bones: [] + spriteID: + internalID: 0 + vertices: [] + indices: + edges: [] + weights: [] + secondaryTextures: [] + spritePackingTag: + pSDRemoveMatte: 0 + pSDShowRemoveMatteOption: 0 + userData: + assetBundleName: + assetBundleVariant: |
