Fix IEEE 754 NaN comparisons in constant folding (#7721)

* Fix IEEE 754 NaN comparisons in constant folding

Added proper NaN handling in SCCP optimization pass to follow IEEE 754 standard:
- NaN \!= any value returns true
- All other NaN comparisons return false
- Added double precision NaN detection support
- Fixed type detection to check operands instead of result type

* Avoid differentiating NaN and non-NaN cases

* format code (#76)

---------

Co-authored-by: slangbot <ellieh+slangbot@nvidia.com>

2 files changed, 229 insertions, 0 deletions

diff --git a/tests/compute/ieee754-mixed-type-nan-comparisons.slang b/tests/compute/ieee754-mixed-type-nan-comparisons.slang
new file mode 100644
index 000000000..31ce6b05b
--- /dev/null
+++ b/tests/compute/ieee754-mixed-type-nan-comparisons.slang
@@ -0,0 +1,79 @@
+//TEST:COMPARE_COMPUTE(filecheck-buffer=CHECK): -shaderobj -output-using-type
+
+// Test IEEE 754 NaN comparison behavior with mixed int/float types
+// Tests the type promotion logic across integer and floating-point categories
+// Also tests both operand orders since implementation bugs could affect operand handling
+
+static const float fNAN = 0.0f / 0.0f;
+static const float fONE = 1.0f;
+static const int iONE = 1;
+static const int iZERO = 0;
+static const uint uONE = 1u;
+
+//TEST_INPUT:ubuffer(data=[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0], stride=4):out,name=outputBuffer
+RWStructuredBuffer<uint> outputBuffer;
+
+[numthreads(1, 1, 1)]
+void computeMain(uint3 dispatchThreadID : SV_DispatchThreadID)
+{
+    uint testIndex = 0;
+
+    // Test int compared with float NaN - all should follow IEEE 754
+    // CHECK: 0
+    // CHECK: 1
+    // CHECK: 0
+    // CHECK: 0
+    // CHECK: 0
+    // CHECK: 0
+    outputBuffer[testIndex++] = (iONE == fNAN) ? 1u : 0u;   // int 1 == float NaN -> false
+    outputBuffer[testIndex++] = (iONE != fNAN) ? 1u : 0u;   // int 1 != float NaN -> true
+    outputBuffer[testIndex++] = (iONE < fNAN) ? 1u : 0u;    // int 1 < float NaN -> false
+    outputBuffer[testIndex++] = (iONE > fNAN) ? 1u : 0u;    // int 1 > float NaN -> false
+    outputBuffer[testIndex++] = (iONE <= fNAN) ? 1u : 0u;   // int 1 <= float NaN -> false
+    outputBuffer[testIndex++] = (iONE >= fNAN) ? 1u : 0u;   // int 1 >= float NaN -> false
+
+    // Test float NaN compared with int - same results but different operand order
+    // CHECK: 0
+    // CHECK: 1
+    // CHECK: 0
+    // CHECK: 0
+    // CHECK: 0
+    // CHECK: 0
+    outputBuffer[testIndex++] = (fNAN == iONE) ? 1u : 0u;   // float NaN == int 1 -> false
+    outputBuffer[testIndex++] = (fNAN != iONE) ? 1u : 0u;   // float NaN != int 1 -> true
+    outputBuffer[testIndex++] = (fNAN < iONE) ? 1u : 0u;    // float NaN < int 1 -> false
+    outputBuffer[testIndex++] = (fNAN > iONE) ? 1u : 0u;    // float NaN > int 1 -> false
+    outputBuffer[testIndex++] = (fNAN <= iONE) ? 1u : 0u;   // float NaN <= int 1 -> false
+    outputBuffer[testIndex++] = (fNAN >= iONE) ? 1u : 0u;   // float NaN >= int 1 -> false
+
+    // Test with different int values to ensure consistent behavior
+    // CHECK: 0
+    // CHECK: 1
+    // CHECK: 0
+    // CHECK: 1
+    outputBuffer[testIndex++] = (iZERO == fNAN) ? 1u : 0u;  // int 0 == float NaN -> false
+    outputBuffer[testIndex++] = (iZERO != fNAN) ? 1u : 0u;  // int 0 != float NaN -> true
+    outputBuffer[testIndex++] = (fNAN == iZERO) ? 1u : 0u;  // float NaN == int 0 -> false
+    outputBuffer[testIndex++] = (fNAN != iZERO) ? 1u : 0u;  // float NaN != int 0 -> true
+
+    // Test unsigned int with float NaN
+    // CHECK: 0
+    // CHECK: 1
+    // CHECK: 0
+    // CHECK: 1
+    outputBuffer[testIndex++] = (uONE == fNAN) ? 1u : 0u;   // uint 1 == float NaN -> false
+    outputBuffer[testIndex++] = (uONE != fNAN) ? 1u : 0u;   // uint 1 != float NaN -> true
+    outputBuffer[testIndex++] = (fNAN == uONE) ? 1u : 0u;   // float NaN == uint 1 -> false
+    outputBuffer[testIndex++] = (fNAN != uONE) ? 1u : 0u;   // float NaN != uint 1 -> true
+
+    // Test normal int vs float comparisons (no NaN) to ensure type promotion works
+    // CHECK: 1
+    // CHECK: 1
+    // CHECK: 1
+    // CHECK: 1
+    outputBuffer[testIndex++] = (iONE == fONE) ? 1u : 0u;   // int 1 == float 1.0 -> true
+    outputBuffer[testIndex++] = (fONE == iONE) ? 1u : 0u;   // float 1.0 == int 1 -> true
+    outputBuffer[testIndex++] = (iZERO < fONE) ? 1u : 0u;   // int 0 < float 1.0 -> true
+    outputBuffer[testIndex++] = (fONE > iZERO) ? 1u : 0u;   // float 1.0 > int 0 -> true
+}
+
diff --git a/tests/compute/ieee754-nan-comparisons.slang b/tests/compute/ieee754-nan-comparisons.slang
new file mode 100644
index 000000000..70a252cb1
--- /dev/null
+++ b/tests/compute/ieee754-nan-comparisons.slang
@@ -0,0 +1,150 @@
+//TEST:COMPARE_COMPUTE(filecheck-buffer=CHECK): -shaderobj -output-using-type -compute
+
+// Test IEEE 754 NaN comparison behavior
+// According to IEEE 754 standard:
+// - Any comparison with NaN (except !=) should return false
+// - The != comparison with NaN should return true
+
+static const float fNAN = 0.0f / 0.0f;
+static const float fPOSITIVE_INFINITY = 1.0f / 0.0f;
+static const float fNEGATIVE_INFINITY = -1.0f / 0.0f;
+static const float fZERO = 0.0f;
+static const float fONE = 1.0f;
+static const float fNEG_ONE = -1.0f;
+
+//TEST_INPUT:ubuffer(data=[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0], stride=4):out,name=outputBuffer
+RWStructuredBuffer<uint> outputBuffer;
+
+[numthreads(1, 1, 1)]
+void computeMain(uint3 dispatchThreadID : SV_DispatchThreadID)
+{
+    uint testIndex = 0;
+
+    // Test 1: NaN == NaN should be false
+    // CHECK: 0
+    outputBuffer[testIndex++] = (fNAN == fNAN) ? 1u : 0u;
+
+    // Test 2: NaN != NaN should be true
+    // CHECK: 1
+    outputBuffer[testIndex++] = (fNAN != fNAN) ? 1u : 0u;
+
+    // Test 3: NaN > NaN should be false
+    // CHECK: 0
+    outputBuffer[testIndex++] = (fNAN > fNAN) ? 1u : 0u;
+
+    // Test 4: NaN < NaN should be false
+    // CHECK: 0
+    outputBuffer[testIndex++] = (fNAN < fNAN) ? 1u : 0u;
+
+    // Test 5: NaN >= NaN should be false
+    // CHECK: 0
+    outputBuffer[testIndex++] = (fNAN >= fNAN) ? 1u : 0u;
+
+    // Test 6: NaN <= NaN should be false
+    // CHECK: 0
+    outputBuffer[testIndex++] = (fNAN <= fNAN) ? 1u : 0u;
+
+    // Test 7: NaN == 0.0 should be false
+    // CHECK: 0
+    outputBuffer[testIndex++] = (fNAN == fZERO) ? 1u : 0u;
+
+    // Test 8: NaN != 0.0 should be true
+    // CHECK: 1
+    outputBuffer[testIndex++] = (fNAN != fZERO) ? 1u : 0u;
+
+    // Test 9: NaN > 0.0 should be false
+    // CHECK: 0
+    outputBuffer[testIndex++] = (fNAN > fZERO) ? 1u : 0u;
+
+    // Test 10: NaN < 0.0 should be false
+    // CHECK: 0
+    outputBuffer[testIndex++] = (fNAN < fZERO) ? 1u : 0u;
+
+    // Test 11: NaN >= 0.0 should be false
+    // CHECK: 0
+    outputBuffer[testIndex++] = (fNAN >= fZERO) ? 1u : 0u;
+
+    // Test 12: NaN <= 0.0 should be false
+    // CHECK: 0
+    outputBuffer[testIndex++] = (fNAN <= fZERO) ? 1u : 0u;
+
+    // Test 13: 0.0 == NaN should be false
+    // CHECK: 0
+    outputBuffer[testIndex++] = (fZERO == fNAN) ? 1u : 0u;
+
+    // Test 14: 0.0 != NaN should be true
+    // CHECK: 1
+    outputBuffer[testIndex++] = (fZERO != fNAN) ? 1u : 0u;
+
+    // Test 15: 0.0 > NaN should be false
+    // CHECK: 0
+    outputBuffer[testIndex++] = (fZERO > fNAN) ? 1u : 0u;
+
+    // Test 16: 0.0 < NaN should be false
+    // CHECK: 0
+    outputBuffer[testIndex++] = (fZERO < fNAN) ? 1u : 0u;
+
+    // Test 17: 0.0 >= NaN should be false
+    // CHECK: 0
+    outputBuffer[testIndex++] = (fZERO >= fNAN) ? 1u : 0u;
+
+    // Test 18: 0.0 <= NaN should be false
+    // CHECK: 0
+    outputBuffer[testIndex++] = (fZERO <= fNAN) ? 1u : 0u;
+
+    // Test 19: NaN == +infinity should be false
+    // CHECK: 0
+    outputBuffer[testIndex++] = (fNAN == fPOSITIVE_INFINITY) ? 1u : 0u;
+
+    // Test 20: NaN != +infinity should be true
+    // CHECK: 1
+    outputBuffer[testIndex++] = (fNAN != fPOSITIVE_INFINITY) ? 1u : 0u;
+
+    // Test 21: NaN > +infinity should be false
+    // CHECK: 0
+    outputBuffer[testIndex++] = (fNAN > fPOSITIVE_INFINITY) ? 1u : 0u;
+
+    // Test 22: NaN < +infinity should be false
+    // CHECK: 0
+    outputBuffer[testIndex++] = (fNAN < fPOSITIVE_INFINITY) ? 1u : 0u;
+
+    // Test 23: NaN >= +infinity should be false
+    // CHECK: 0
+    outputBuffer[testIndex++] = (fNAN >= fPOSITIVE_INFINITY) ? 1u : 0u;
+
+    // Test 24: NaN <= +infinity should be false
+    // CHECK: 0
+    outputBuffer[testIndex++] = (fNAN <= fPOSITIVE_INFINITY) ? 1u : 0u;
+
+    // Test 25: NaN == -infinity should be false
+    // CHECK: 0
+    outputBuffer[testIndex++] = (fNAN == fNEGATIVE_INFINITY) ? 1u : 0u;
+
+    // Test 26: NaN != -infinity should be true
+    // CHECK: 1
+    outputBuffer[testIndex++] = (fNAN != fNEGATIVE_INFINITY) ? 1u : 0u;
+
+    // Test 27: NaN > -infinity should be false
+    // CHECK: 0
+    outputBuffer[testIndex++] = (fNAN > fNEGATIVE_INFINITY) ? 1u : 0u;
+
+    // Test 28: NaN < -infinity should be false
+    // CHECK: 0
+    outputBuffer[testIndex++] = (fNAN < fNEGATIVE_INFINITY) ? 1u : 0u;
+
+    // Test 29: NaN >= -infinity should be false
+    // CHECK: 0
+    outputBuffer[testIndex++] = (fNAN >= fNEGATIVE_INFINITY) ? 1u : 0u;
+
+    // Test 30: NaN <= -infinity should be false
+    // CHECK: 0
+    outputBuffer[testIndex++] = (fNAN <= fNEGATIVE_INFINITY) ? 1u : 0u;
+
+    // Test 31: NaN == 1.0 should be false
+    // CHECK: 0
+    outputBuffer[testIndex++] = (fNAN == fONE) ? 1u : 0u;
+
+    // Test 32: NaN != 1.0 should be true
+    // CHECK: 1
+    outputBuffer[testIndex++] = (fNAN != fONE) ? 1u : 0u;
+}