Add controls for overlays on transparent materials

Add ability to constrain overlay to parts of the texture based on
opacity. Min/max opacity are configurable.

Add slider to multiply the overlay's alpha channel. This helps prevent
transparent overlays from disappearing in less detailed mip maps.

4 files changed, 143 insertions, 16 deletions

diff --git a/Editor/tooner.cs b/Editor/tooner.cs
index 4ad7fa7..f7bdc50 100644
--- a/Editor/tooner.cs
+++ b/Editor/tooner.cs
@@ -166,6 +166,23 @@ public class ToonerGUI : ShaderGUI {
         SetKeyword($"_PBR_OVERLAY{i}_MIX_MIN", mode == PbrAlbedoMixMode.Min);
         SetKeyword($"_PBR_OVERLAY{i}_MIX_MAX", mode == PbrAlbedoMixMode.Max);
 
+        bc = FindProperty($"_PBR_Overlay{i}_Constrain_By_Alpha");
+        enabled = bc.floatValue > 1E-6;
+        EditorGUI.BeginChangeCheck();
+        enabled = EditorGUILayout.Toggle("Constrain to transparent sections", enabled);
+        EditorGUI.EndChangeCheck();
+        bc.floatValue = enabled ? 1.0f : 0.0f;
+        if (enabled) {
+          EditorGUI.indentLevel += 1;
+          bc = FindProperty($"_PBR_Overlay{i}_Constrain_By_Alpha_Min");
+          editor.RangeProperty(bc, "Min");
+          bc = FindProperty($"_PBR_Overlay{i}_Constrain_By_Alpha_Max");
+          editor.RangeProperty(bc, "Max");
+          EditorGUI.indentLevel -= 1;
+        }
+        bc = FindProperty($"_PBR_Overlay{i}_Alpha_Multiplier");
+        editor.RangeProperty(bc, "Alpha multiplier");
+
         bc = FindProperty($"_PBR_Overlay{i}_Emission");
         bct = FindProperty($"_PBR_Overlay{i}_EmissionTex");
         editor.TexturePropertySingleLine(
@@ -223,6 +240,17 @@ public class ToonerGUI : ShaderGUI {
           EditorGUI.EndChangeCheck();
           bc.floatValue = enabled ? 1.0f : 0.0f;
         }
+      } else {
+        SetKeyword($"_PBR_OVERLAY{i}_BASECOLOR_MAP", false);
+        SetKeyword($"_PBR_OVERLAY{i}_MIX_ALPHA_BLEND", false);
+        SetKeyword($"_PBR_OVERLAY{i}_MIX_ADD", false);
+        SetKeyword($"_PBR_OVERLAY{i}_MIX_MIN", false);
+        SetKeyword($"_PBR_OVERLAY{i}_MIX_MAX", false);
+        SetKeyword($"_PBR_OVERLAY{i}_EMISSION_MAP", false);
+        SetKeyword($"_PBR_OVERLAY{i}_NORMAL_MAP", false);
+        SetKeyword($"_PBR_OVERLAY{i}_METALLIC_MAP", false);
+        SetKeyword($"_PBR_OVERLAY{i}_ROUGHNESS_MAP", false);
+        SetKeyword($"_PBR_OVERLAY{i}_MASK", false);
       }
       EditorGUI.indentLevel -= 1;
     }
@@ -1217,7 +1245,7 @@ public class ToonerGUI : ShaderGUI {
         case RenderingMode.TransClipping:
           queue = RenderQueue.AlphaTest;
           render_type = "Transparent";
-          src_blend = BlendMode.One;
+          src_blend = BlendMode.SrcAlpha;
           dst_blend = BlendMode.OneMinusSrcAlpha;
           zwrite = true;
           break;
diff --git a/globals.cginc b/globals.cginc
index e14b1a7..1f2d4a6 100644
--- a/globals.cginc
+++ b/globals.cginc
@@ -92,6 +92,10 @@ float4 _PBR_Overlay0_RoughnessTex_ST;
 float _PBR_Overlay0_Tex_NormalStr;
 texture2D _PBR_Overlay0_Mask;
 float _PBR_Overlay0_Mask_Invert;
+float _PBR_Overlay0_Constrain_By_Alpha;
+float _PBR_Overlay0_Constrain_By_Alpha_Min;
+float _PBR_Overlay0_Constrain_By_Alpha_Max;
+float _PBR_Overlay0_Alpha_Multiplier;
 #endif
 
 #if defined(_PBR_OVERLAY1)
@@ -112,6 +116,10 @@ float4 _PBR_Overlay1_RoughnessTex_ST;
 float _PBR_Overlay1_Tex_NormalStr;
 texture2D _PBR_Overlay1_Mask;
 float _PBR_Overlay1_Mask_Invert;
+float _PBR_Overlay1_Constrain_By_Alpha;
+float _PBR_Overlay1_Constrain_By_Alpha_Min;
+float _PBR_Overlay1_Constrain_By_Alpha_Max;
+float _PBR_Overlay1_Alpha_Multiplier;
 #endif
 
 #if defined(_PBR_OVERLAY2)
@@ -132,6 +140,10 @@ float4 _PBR_Overlay2_RoughnessTex_ST;
 float _PBR_Overlay2_Tex_NormalStr;
 texture2D _PBR_Overlay2_Mask;
 float _PBR_Overlay2_Mask_Invert;
+float _PBR_Overlay2_Constrain_By_Alpha;
+float _PBR_Overlay2_Constrain_By_Alpha_Min;
+float _PBR_Overlay2_Constrain_By_Alpha_Max;
+float _PBR_Overlay2_Alpha_Multiplier;
 #endif
 
 #if defined(_PBR_OVERLAY3)
@@ -152,6 +164,10 @@ float4 _PBR_Overlay3_RoughnessTex_ST;
 float _PBR_Overlay3_Tex_NormalStr;
 texture2D _PBR_Overlay3_Mask;
 float _PBR_Overlay3_Mask_Invert;
+float _PBR_Overlay3_Constrain_By_Alpha;
+float _PBR_Overlay3_Constrain_By_Alpha_Min;
+float _PBR_Overlay3_Constrain_By_Alpha_Max;
+float _PBR_Overlay3_Alpha_Multiplier;
 #endif
 
 #if defined(_DECAL0)
diff --git a/tooner.shader b/tooner.shader
index c58aedf..c7901fe 100644
--- a/tooner.shader
+++ b/tooner.shader
@@ -30,6 +30,10 @@ Shader "yum_food/tooner"
     _PBR_Overlay0_Mask("Mask", 2D) = "white" {}
     _PBR_Overlay0_Mask_Invert("Mask invert", Float) = 0.0
     _PBR_Overlay0_Mix("Mix mode", Float) = 0.0
+    _PBR_Overlay0_Constrain_By_Alpha("Constrain by alpha channel", Float) = 0.0
+    _PBR_Overlay0_Constrain_By_Alpha_Min("Constrain by alpha channel", Range(0, 1)) = 0
+    _PBR_Overlay0_Constrain_By_Alpha_Max("Constrain by alpha channel", Range(0, 1)) = 1
+    _PBR_Overlay0_Alpha_Multiplier("Constrain by alpha channel", Range(0, 5)) = 1
 
     _PBR_Overlay1_Enable("Enable PBR overlay", Float) = 0.0
     _PBR_Overlay1_BaseColor("Base color", Color) = (0.8, 0.8, 0.8, 1)
@@ -45,6 +49,10 @@ Shader "yum_food/tooner"
     _PBR_Overlay1_Mask("Mask", 2D) = "white" {}
     _PBR_Overlay1_Mask_Invert("Mask invert", Float) = 0.0
     _PBR_Overlay1_Mix("Mix mode", Float) = 0.0
+    _PBR_Overlay1_Constrain_By_Alpha("Constrain by alpha channel", Float) = 0.0
+    _PBR_Overlay1_Constrain_By_Alpha_Min("Constrain by alpha channel", Range(0, 1)) = 0
+    _PBR_Overlay1_Constrain_By_Alpha_Max("Constrain by alpha channel", Range(0, 1)) = 1
+    _PBR_Overlay1_Alpha_Multiplier("Constrain by alpha channel", Range(0, 5)) = 1
 
     _PBR_Overlay2_Enable("Enable PBR overlay", Float) = 0.0
     _PBR_Overlay2_BaseColor("Base color", Color) = (0.8, 0.8, 0.8, 1)
@@ -60,6 +68,10 @@ Shader "yum_food/tooner"
     _PBR_Overlay2_Mask("Mask", 2D) = "white" {}
     _PBR_Overlay2_Mask_Invert("Mask invert", Float) = 0.0
     _PBR_Overlay2_Mix("Mix mode", Float) = 0.0
+    _PBR_Overlay2_Constrain_By_Alpha("Constrain by alpha channel", Float) = 0.0
+    _PBR_Overlay2_Constrain_By_Alpha_Min("Constrain by alpha channel", Range(0, 1)) = 0
+    _PBR_Overlay2_Constrain_By_Alpha_Max("Constrain by alpha channel", Range(0, 1)) = 1
+    _PBR_Overlay2_Alpha_Multiplier("Constrain by alpha channel", Range(0, 5)) = 1
 
     _PBR_Overlay3_Enable("Enable PBR overlay", Float) = 0.0
     _PBR_Overlay3_BaseColor("Base color", Color) = (0.8, 0.8, 0.8, 1)
@@ -75,6 +87,10 @@ Shader "yum_food/tooner"
     _PBR_Overlay3_Mask("Mask", 2D) = "white" {}
     _PBR_Overlay3_Mask_Invert("Mask invert", Float) = 0.0
     _PBR_Overlay3_Mix("Mix mode", Float) = 0.0
+    _PBR_Overlay3_Constrain_By_Alpha("Constrain by alpha channel", Float) = 0.0
+    _PBR_Overlay3_Constrain_By_Alpha_Min("Constrain by alpha channel", Range(0, 1)) = 0
+    _PBR_Overlay3_Constrain_By_Alpha_Max("Constrain by alpha channel", Range(0, 1)) = 1
+    _PBR_Overlay3_Alpha_Multiplier("Constrain by alpha channel", Range(0, 5)) = 1
 
     _Decal0_Enable("Enable decal", Float) = 0.0
     _Decal0_BaseColor("Base color", 2D) = "white" {}
@@ -365,6 +381,7 @@ Shader "yum_food/tooner"
       #include "tooner_lighting.cginc"
       ENDCG
     }
+    /*
     Pass {
       Tags {
         "RenderType" = "Opaque"
@@ -435,6 +452,7 @@ Shader "yum_food/tooner"
       #include "mochie_shadow_caster.cginc"
 			ENDCG
 		}
+    */
   }
   CustomEditor "ToonerGUI"
 }
diff --git a/tooner_lighting.cginc b/tooner_lighting.cginc
index 4531931..8ea356c 100644
--- a/tooner_lighting.cginc
+++ b/tooner_lighting.cginc
@@ -614,7 +614,7 @@ void getOverlayAlbedo(inout PbrOverlay ov,
 }
 
 void applyDecalAlbedoImpl(
-    inout float3 albedo,
+    inout float4 albedo,
     inout float decal_emission,
     v2f i,
     texture2D tex,
@@ -645,10 +645,11 @@ void applyDecalAlbedoImpl(
   d0_c *= d0_in_range;
 
   albedo.rgb = lerp(albedo.rgb, d0_c.rgb, d0_c.a);
+  albedo.a = max(albedo.a, d0_c.a);
   decal_emission += d0_c.rgb * emission_strength;
 }
 
-void applyDecalAlbedo(inout float3 albedo,
+void applyDecalAlbedo(inout float4 albedo,
     inout float decal_emission,
     v2f i)
 {
@@ -682,10 +683,45 @@ void applyDecalAlbedo(inout float3 albedo,
 #endif  // _DECAL3
 }
 
-void mixOverlayAlbedo(inout float3 albedo, PbrOverlay ov) {
+void mixOverlayAlbedo(inout float4 albedo, PbrOverlay ov) {
+  // Calculate alpha masks before we start mutating alpha.
+#if defined(_PBR_OVERLAY0)
+  float a0 = saturate(ov.ov0_albedo.a * _PBR_Overlay0_Alpha_Multiplier);
+  if (_PBR_Overlay0_Constrain_By_Alpha) {
+    bool in_range = (albedo.a > _PBR_Overlay0_Constrain_By_Alpha_Min) *
+      (albedo.a < _PBR_Overlay0_Constrain_By_Alpha_Max);
+    a0 *= in_range;
+  }
+#endif
+#if defined(_PBR_OVERLAY1)
+  float a1 = saturate(ov.ov1_albedo.a * _PBR_Overlay1_Alpha_Multiplier);
+  if (_PBR_Overlay1_Constrain_By_Alpha) {
+    bool in_range = (albedo.a > _PBR_Overlay1_Constrain_By_Alpha_Min) *
+      (albedo.a < _PBR_Overlay1_Constrain_By_Alpha_Max);
+    a1 *= in_range;
+  }
+#endif
+#if defined(_PBR_OVERLAY2)
+  float a2 = saturate(ov.ov2_albedo.a * _PBR_Overlay2_Alpha_Multiplier);
+  if (_PBR_Overlay2_Constrain_By_Alpha) {
+    bool in_range = (albedo.a > _PBR_Overlay2_Constrain_By_Alpha_Min) *
+      (albedo.a < _PBR_Overlay2_Constrain_By_Alpha_Max);
+    a2 *= in_range;
+  }
+#endif
+#if defined(_PBR_OVERLAY3)
+  float a3 = saturate(ov.ov3_albedo.a * _PBR_Overlay3_Alpha_Multiplier);
+  if (_PBR_Overlay3_Constrain_By_Alpha) {
+    bool in_range = (albedo.a > _PBR_Overlay3_Constrain_By_Alpha_Min) *
+      (albedo.a < _PBR_Overlay3_Constrain_By_Alpha_Max);
+    a3 *= in_range;
+  }
+#endif
+
 #if defined(_PBR_OVERLAY0)
 #if defined(_PBR_OVERLAY0_MIX_ALPHA_BLEND)
-  albedo.rgb = lerp(albedo.rgb, ov.ov0_albedo.rgb, ov.ov0_albedo.a);
+  albedo.rgb = lerp(albedo.rgb, ov.ov0_albedo.rgb, a0);
+  albedo.a = max(albedo.a, a0);
 #elif defined(_PBR_OVERLAY0_MIX_ADD)
   albedo.rgb += ov.ov0_albedo;
 #elif defined(_PBR_OVERLAY0_MIX_MIN)
@@ -697,7 +733,8 @@ void mixOverlayAlbedo(inout float3 albedo, PbrOverlay ov) {
 
 #if defined(_PBR_OVERLAY1)
 #if defined(_PBR_OVERLAY1_MIX_ALPHA_BLEND)
-  albedo.rgb = lerp(albedo.rgb, ov.ov1_albedo.rgb, ov.ov1_albedo.a);
+  albedo.rgb = lerp(albedo.rgb, ov.ov1_albedo.rgb, a1);
+  albedo.a = max(albedo.a, a1);
 #elif defined(_PBR_OVERLAY1_MIX_ADD)
   albedo.rgb += ov.ov1_albedo;
 #elif defined(_PBR_OVERLAY1_MIX_MIN)
@@ -710,6 +747,7 @@ void mixOverlayAlbedo(inout float3 albedo, PbrOverlay ov) {
 #if defined(_PBR_OVERLAY2)
 #if defined(_PBR_OVERLAY2_MIX_ALPHA_BLEND)
   albedo.rgb = lerp(albedo.rgb, ov.ov2_albedo.rgb, ov.ov2_albedo.a);
+  albedo.a = max(albedo.a, a2);
 #elif defined(_PBR_OVERLAY2_MIX_ADD)
   albedo.rgb += ov.ov2_albedo;
 #elif defined(_PBR_OVERLAY2_MIX_MIN)
@@ -721,7 +759,8 @@ void mixOverlayAlbedo(inout float3 albedo, PbrOverlay ov) {
 
 #if defined(_PBR_OVERLAY3)
 #if defined(_PBR_OVERLAY3_MIX_ALPHA_BLEND)
-  albedo.rgb = lerp(albedo.rgb, ov.ov3_albedo.rgb, ov.ov3_albedo.a);
+  albedo.rgb = lerp(albedo.rgb, ov.ov3_albedo.rgb, a3);
+  albedo.a = max(albedo.a, a3);
 #elif defined(_PBR_OVERLAY3_MIX_ADD)
   albedo.rgb += ov.ov3_albedo;
 #elif defined(_PBR_OVERLAY3_MIX_MIN)
@@ -732,10 +771,16 @@ void mixOverlayAlbedo(inout float3 albedo, PbrOverlay ov) {
 #endif
 }
 
-void applyOverlayNormal(inout float3 raw_normal, PbrOverlay ov, v2f i, float iddx, float iddy)
+void applyOverlayNormal(inout float3 raw_normal, float4 albedo, PbrOverlay ov, v2f i, float iddx, float iddy)
 {
   float3 raw_normal_2;
 #if defined(_PBR_OVERLAY0) && defined(_PBR_OVERLAY0_NORMAL_MAP)
+  float a0 = 1;
+  if (_PBR_Overlay0_Constrain_By_Alpha) {
+    bool in_range = (albedo.a > _PBR_Overlay0_Constrain_By_Alpha_Min) *
+      (albedo.a < _PBR_Overlay0_Constrain_By_Alpha_Max);
+    a0 *= in_range;
+  }
   // Use UVs to smoothly blend between fully detailed normals when close up and
   // flat normals when far away. If we don't do this, then we see moire effects
   // on e.g. striped normal maps.
@@ -743,40 +788,58 @@ void applyOverlayNormal(inout float3 raw_normal, PbrOverlay ov, v2f i, float idd
         UV_SCOFF(i.uv, _PBR_Overlay0_NormalTex_ST),
         iddx * _PBR_Overlay0_NormalTex_ST.x,
         iddy * _PBR_Overlay0_NormalTex_ST.y),
-      _PBR_Overlay0_Tex_NormalStr * ov.ov0_mask);
+      _PBR_Overlay0_Tex_NormalStr * ov.ov0_mask * a0);
 
   raw_normal = BlendNormals(
       raw_normal,
       raw_normal_2);
 #endif  // _PBR_OVERLAY0 && _PBR_OVERLAY0_NORMAL_MAP
 #if defined(_PBR_OVERLAY1) && defined(_PBR_OVERLAY1_NORMAL_MAP)
+  float a1 = 1;
+  if (_PBR_Overlay1_Constrain_By_Alpha) {
+    bool in_range = (albedo.a > _PBR_Overlay1_Constrain_By_Alpha_Min) *
+      (albedo.a < _PBR_Overlay1_Constrain_By_Alpha_Max);
+    a1 *= in_range;
+  }
   raw_normal_2 = UnpackScaleNormal(_PBR_Overlay1_NormalTex.SampleGrad(linear_repeat_s,
         UV_SCOFF(i.uv, _PBR_Overlay1_NormalTex_ST),
         iddx * _PBR_Overlay1_NormalTex_ST.x,
         iddy * _PBR_Overlay1_NormalTex_ST.y),
-      _PBR_Overlay1_Tex_NormalStr * ov.ov0_mask);
+      _PBR_Overlay1_Tex_NormalStr * ov.ov1_mask * a1);
 
   raw_normal = BlendNormals(
       raw_normal,
       raw_normal_2);
 #endif  // _PBR_OVERLAY1 && _PBR_OVERLAY1_NORMAL_MAP
 #if defined(_PBR_OVERLAY2) && defined(_PBR_OVERLAY2_NORMAL_MAP)
+  float a2 = 1;
+  if (_PBR_Overlay2_Constrain_By_Alpha) {
+    bool in_range = (albedo.a > _PBR_Overlay2_Constrain_By_Alpha_Min) *
+      (albedo.a < _PBR_Overlay2_Constrain_By_Alpha_Max);
+    a2 *= in_range;
+  }
   raw_normal_2 = UnpackScaleNormal(_PBR_Overlay2_NormalTex.SampleGrad(linear_repeat_s,
         UV_SCOFF(i.uv, _PBR_Overlay2_NormalTex_ST),
         iddx * _PBR_Overlay2_NormalTex_ST.x,
         iddy * _PBR_Overlay2_NormalTex_ST.y),
-      _PBR_Overlay2_Tex_NormalStr * ov.ov0_mask);
+      _PBR_Overlay2_Tex_NormalStr * ov.ov2_mask * a2);
 
   raw_normal = BlendNormals(
       raw_normal,
       raw_normal_2);
 #endif  // _PBR_OVERLAY2 && _PBR_OVERLAY2_NORMAL_MAP
 #if defined(_PBR_OVERLAY3) && defined(_PBR_OVERLAY3_NORMAL_MAP)
+  float a3 = 1;
+  if (_PBR_Overlay3_Constrain_By_Alpha) {
+    bool in_range = (albedo.a > _PBR_Overlay3_Constrain_By_Alpha_Min) *
+      (albedo.a < _PBR_Overlay3_Constrain_By_Alpha_Max);
+    a3 *= in_range;
+  }
   raw_normal_2 = UnpackScaleNormal(_PBR_Overlay3_NormalTex.SampleGrad(linear_repeat_s,
         UV_SCOFF(i.uv, _PBR_Overlay3_NormalTex_ST),
         iddx * _PBR_Overlay3_NormalTex_ST.x,
         iddy * _PBR_Overlay3_NormalTex_ST.y),
-      _PBR_Overlay3_Tex_NormalStr * ov.ov0_mask);
+      _PBR_Overlay3_Tex_NormalStr * ov.ov3_mask * a3);
 
   raw_normal = BlendNormals(
       raw_normal,
@@ -910,7 +973,9 @@ float4 effect(inout v2f i)
   // flat normals when far away. If we don't do this, then we see moire effects
   // on e.g. striped normal maps.
   float fw = clamp(fwidth(i.uv), .001, 1) * 1200;
-  float3 raw_normal = UnpackScaleNormal(_NormalTex.SampleGrad(linear_repeat_s, UV_SCOFF(i.uv, _NormalTex_ST), iddx, iddy), _Tex_NormalStr);
+  float3 raw_normal = UnpackScaleNormal(_NormalTex.SampleGrad(linear_repeat_s, UV_SCOFF(i.uv, _NormalTex_ST),
+        iddx * _NormalTex_ST.x,
+        iddy * _NormalTex_ST.y), _Tex_NormalStr);
 
   raw_normal = BlendNormals(
       (1/fw) * raw_normal,
@@ -919,7 +984,7 @@ float4 effect(inout v2f i)
   float3 raw_normal = UnpackNormal(float4(0.5, 0.5, 1, 1));
 #endif  // _NORMAL_MAP
 
-  applyOverlayNormal(raw_normal, ov, i, iddx, iddy);
+  applyOverlayNormal(raw_normal, albedo, ov, i, iddx, iddy);
 
   float3 binormal = CreateBinormal(i.normal, i.tangent.xyz, i.tangent.w);
   // normalize is not necessary; result is already normalized
@@ -958,10 +1023,10 @@ float4 effect(inout v2f i)
   }
 #endif
 
-  mixOverlayAlbedo(albedo.rgb, ov);
+  mixOverlayAlbedo(albedo, ov);
 #if defined(_DECAL0) || defined(_DECAL1) || defined(_DECAL2) || defined(_DECAL3)
   float decal_emission = 0;
-  applyDecalAlbedo(albedo.rgb, decal_emission, i);
+  applyDecalAlbedo(albedo, decal_emission, i);
 #endif
 
 #if defined(_MATCAP0) || defined(_MATCAP1) || defined(_RIM_LIGHTING0) || defined(_RIM_LIGHTING1)