Fix the sign-extending issue in right shift (#3820)

Fix issue  (#3637).

In constant folding of a right shift operation,slang always uses
signed interger as the operand no matter the input source code is
signed or unsigned, this could causes sign-extending issue if the
input source is unsigned integer with highest bit set to 1.

Fix the issue by checking the original type of the input and use
the unsigned type if the input is unsigned.

3 files changed, 39 insertions, 0 deletions

diff --git a/source/slang/slang-ir-sccp.cpp b/source/slang/slang-ir-sccp.cpp
index fc3766e63..80cee8ecb 100644
--- a/source/slang/slang-ir-sccp.cpp
+++ b/source/slang/slang-ir-sccp.cpp
@@ -676,11 +676,23 @@ struct SCCPContext
     }
     LatticeVal evalLsh(IRType* type, LatticeVal v0, LatticeVal v1)
     {
+        IntInfo info = getIntTypeInfo(type);
+        if (info.isSigned == false)
+        {
+            return evalBinaryIntImpl(
+               type, v0, v1, [](IRUnsignedIntegerValue c0, IRUnsignedIntegerValue c1) { return c0 << c1; });
+        }
         return evalBinaryIntImpl(
             type, v0, v1, [](IRIntegerValue c0, IRIntegerValue c1) { return c0 << c1; });
     }
     LatticeVal evalRsh(IRType* type, LatticeVal v0, LatticeVal v1)
     {
+        IntInfo info = getIntTypeInfo(type);
+        if (info.isSigned == false)
+        {
+            return evalBinaryIntImpl(
+               type, v0, v1, [](IRUnsignedIntegerValue c0, IRUnsignedIntegerValue c1) { return c0 >> c1; });
+        }
         return evalBinaryIntImpl(
             type, v0, v1, [](IRIntegerValue c0, IRIntegerValue c1) { return c0 >> c1; });
     }
diff --git a/source/slang/slang-ir.h b/source/slang/slang-ir.h
index eb77127c8..36e145c01 100644
--- a/source/slang/slang-ir.h
+++ b/source/slang/slang-ir.h
@@ -1059,6 +1059,7 @@ void findAllInstsBreadthFirst(IRInst* inst, List<IRInst*>& outInsts);
 // Constant Instructions
 
 typedef int64_t IRIntegerValue;
+typedef uint64_t IRUnsignedIntegerValue;
 typedef double IRFloatingPointValue;
 
 struct IRConstant : IRInst
diff --git a/tests/bugs/gh-3637.slang b/tests/bugs/gh-3637.slang
new file mode 100644
index 000000000..81aa58c73
--- /dev/null
+++ b/tests/bugs/gh-3637.slang
@@ -0,0 +1,26 @@
+//TEST(compute):COMPARE_COMPUTE_EX(filecheck-buffer=CHECK):-cuda -compute -output-using-type
+//TEST(compute, vulkan):COMPARE_COMPUTE_EX(filecheck-buffer=CHECK):-vk -compute -output-using-type
+//TEST(compute):COMPARE_COMPUTE_EX(filecheck-buffer=CHECK):-cpu -compute -output-using-type
+
+// CHECK: 9223372036854775808
+// CHECK: 1
+// CHECK: 18446744073709551615
+
+// This tests exhibits a bug in constant folding in which the right shift
+// operator unconditionally performs sign-extension.
+
+//TEST_INPUT:ubuffer(data=[0 0 0 0 0 0 0 0], stride=8):out,name=outputBuffer
+RWStructuredBuffer<uint64_t> outputBuffer;
+
+[numthreads(1, 1, 1)]
+void computeMain(uint3 dispatchThreadID : SV_DispatchThreadID)
+{
+    const uint64_t topBitSet = 0x8000000000000000ull;
+    const uint64_t bottomBitSet = topBitSet >> 63;
+    const uint64_t allBitsSet = int64_t(topBitSet) >> 63; // expect sign-extending which will set all bits.
+    const uint64_t noBitsSet = topBitSet >> 64; // This exhibits undefined behaviour, as the compiler shifts right by at least the number of bits in the first operand
+    outputBuffer[0] = topBitSet;
+    outputBuffer[1] = bottomBitSet;
+    outputBuffer[2] = allBitsSet;
+    outputBuffer[3] = noBitsSet;
+}