Print warning when operator<< shifting too much (#4255)

* Print warning when operator<< shifting too much

Closes #3944

For the given type of the left side operand to `operator<<` is not big
enough for the right side operand, print a warning that the result will
be always zero.

7 files changed, 126 insertions, 1 deletions

diff --git a/build/visual-studio/slang/slang.vcxproj b/build/visual-studio/slang/slang.vcxproj
index f8965a967..826ac3e38 100644
--- a/build/visual-studio/slang/slang.vcxproj
+++ b/build/visual-studio/slang/slang.vcxproj
@@ -451,6 +451,7 @@ IF EXIST ..\..\..\external\slang-glslang\bin\windows-aarch64\release\slang-glsla
     <ClInclude Include="..\..\..\source\slang\slang-ir-metal-legalize.h" />

     <ClInclude Include="..\..\..\source\slang\slang-ir-missing-return.h" />

     <ClInclude Include="..\..\..\source\slang\slang-ir-obfuscate-loc.h" />

+    <ClInclude Include="..\..\..\source\slang\slang-ir-operator-shift-overflow.h" />

     <ClInclude Include="..\..\..\source\slang\slang-ir-optix-entry-point-uniforms.h" />

     <ClInclude Include="..\..\..\source\slang\slang-ir-peephole.h" />

     <ClInclude Include="..\..\..\source\slang\slang-ir-propagate-func-properties.h" />

@@ -691,6 +692,7 @@ IF EXIST ..\..\..\external\slang-glslang\bin\windows-aarch64\release\slang-glsla
     <ClCompile Include="..\..\..\source\slang\slang-ir-metal-legalize.cpp" />

     <ClCompile Include="..\..\..\source\slang\slang-ir-missing-return.cpp" />

     <ClCompile Include="..\..\..\source\slang\slang-ir-obfuscate-loc.cpp" />

+    <ClCompile Include="..\..\..\source\slang\slang-ir-operator-shift-overflow.cpp" />

     <ClCompile Include="..\..\..\source\slang\slang-ir-optix-entry-point-uniforms.cpp" />

     <ClCompile Include="..\..\..\source\slang\slang-ir-peephole.cpp" />

     <ClCompile Include="..\..\..\source\slang\slang-ir-propagate-func-properties.cpp" />

diff --git a/build/visual-studio/slang/slang.vcxproj.filters b/build/visual-studio/slang/slang.vcxproj.filters
index c580aa7bc..59fdc4048 100644
--- a/build/visual-studio/slang/slang.vcxproj.filters
+++ b/build/visual-studio/slang/slang.vcxproj.filters
@@ -441,6 +441,9 @@
     <ClInclude Include="..\..\..\source\slang\slang-ir-obfuscate-loc.h">

       <Filter>Header Files</Filter>

     </ClInclude>

+    <ClInclude Include="..\..\..\source\slang\slang-ir-operator-shift-overflow.h">

+      <Filter>Header Files</Filter>

+    </ClInclude>

     <ClInclude Include="..\..\..\source\slang\slang-ir-optix-entry-point-uniforms.h">

       <Filter>Header Files</Filter>

     </ClInclude>

@@ -1157,6 +1160,9 @@
     <ClCompile Include="..\..\..\source\slang\slang-ir-obfuscate-loc.cpp">

       <Filter>Source Files</Filter>

     </ClCompile>

+    <ClCompile Include="..\..\..\source\slang\slang-ir-operator-shift-overflow.cpp">

+      <Filter>Source Files</Filter>

+    </ClCompile>

     <ClCompile Include="..\..\..\source\slang\slang-ir-optix-entry-point-uniforms.cpp">

       <Filter>Source Files</Filter>

     </ClCompile>

diff --git a/source/slang/slang-diagnostic-defs.h b/source/slang/slang-diagnostic-defs.h
index 7385d5e33..4cf6a899d 100644
--- a/source/slang/slang-diagnostic-defs.h
+++ b/source/slang/slang-diagnostic-defs.h
@@ -740,6 +740,8 @@ DIAGNOSTIC(41021, Error, differentiableFuncMustHaveOutput, "a differentiable fun
 DIAGNOSTIC(41022, Error, differentiableFuncMustHaveInput, "a differentiable function must have at least one differentiable input.")
 DIAGNOSTIC(41023, Error, getStringHashMustBeOnStringLiteral, "getStringHash can only be called when argument is statically resolvable to a string literal")
 
+DIAGNOSTIC(41030, Warning, operatorShiftLeftOverflow, "left shift amount exceeds the number of bits and the result will be always zero, (`$0` << `$1`).")
+
 DIAGNOSTIC(41901, Error, unsupportedUseOfLValueForAutoDiff, "unsupported use of L-value for auto differentiation.")
 DIAGNOSTIC(41902, Error, cannotDifferentiateDynamicallyIndexedData, "cannot auto-differentiate mixed read/write access to dynamically indexed data in '$0'.")
 
diff --git a/source/slang/slang-ir-operator-shift-overflow.cpp b/source/slang/slang-ir-operator-shift-overflow.cpp
new file mode 100644
index 000000000..33ee53262
--- /dev/null
+++ b/source/slang/slang-ir-operator-shift-overflow.cpp
@@ -0,0 +1,70 @@
+// slang-ir-operator-shift-overflow.cpp
+#include "slang-ir-operator-shift-overflow.h"
+
+#include "../../slang.h"
+#include "slang-ir.h"
+#include "slang-ir-insts.h"
+#include "slang-ir-layout.h"
+
+namespace Slang {
+
+    class DiagnosticSink;
+    struct IRModule;
+
+    void checkForOperatorShiftOverflowRecursive(
+        IRInst* inst,
+        CompilerOptionSet& optionSet,
+        DiagnosticSink* sink)
+    {
+        if (auto code = as<IRGlobalValueWithCode>(inst))
+        {
+            for (auto block : code->getBlocks())
+            {
+                for (auto opInst : block->getChildren())
+                {
+                    switch (opInst->getOp())
+                    {
+                    case kIROp_Lsh:
+                    {
+                        SLANG_ASSERT(opInst->getOperandCount() == 2);
+
+                        IRInst* rhs = opInst->getOperand(1);
+                        auto rhsLit = as<IRIntLit>(rhs);
+                        if (!rhsLit)
+                            continue;
+
+                        IRInst* lhs = opInst->getOperand(0);
+                        IRType* lhsType = lhs->getDataType();
+
+                        IRSizeAndAlignment sizeAlignment;
+                        if (SLANG_FAILED(getNaturalSizeAndAlignment(optionSet, lhsType, &sizeAlignment)))
+                            continue;
+
+                        IRIntegerValue shiftAmount = rhsLit->getValue();
+                        if (sizeAlignment.size * 8 <= shiftAmount)
+                        {
+                            sink->diagnose(opInst, Diagnostics::operatorShiftLeftOverflow, lhsType, shiftAmount);
+                        }
+                        break;
+                    }
+                    }
+                }
+            }
+        }
+
+        for (auto childInst : inst->getChildren())
+        {
+            checkForOperatorShiftOverflowRecursive(childInst, optionSet, sink);
+        }
+    }
+
+    void checkForOperatorShiftOverflow(
+        IRModule* module,
+        CompilerOptionSet& optionSet,
+        DiagnosticSink* sink)
+    {
+        // Look for `operator<<` instructions
+        checkForOperatorShiftOverflowRecursive(module->getModuleInst(), optionSet, sink);
+    }
+
+}
diff --git a/source/slang/slang-ir-operator-shift-overflow.h b/source/slang/slang-ir-operator-shift-overflow.h
new file mode 100644
index 000000000..59280d82c
--- /dev/null
+++ b/source/slang/slang-ir-operator-shift-overflow.h
@@ -0,0 +1,15 @@
+// slang-ir-operator-shift-overflow.h
+#pragma once
+
+#include "slang-compiler-options.h"
+
+namespace Slang
+{
+    class DiagnosticSink;
+    struct IRModule;
+
+    void checkForOperatorShiftOverflow(
+        IRModule* module,
+        CompilerOptionSet& optionSet,
+        DiagnosticSink* sink);
+}
diff --git a/source/slang/slang-lower-to-ir.cpp b/source/slang/slang-lower-to-ir.cpp
index 5c4196839..bfc4c444d 100644
--- a/source/slang/slang-lower-to-ir.cpp
+++ b/source/slang/slang-lower-to-ir.cpp
@@ -23,6 +23,7 @@
 #include "slang-ir-check-differentiability.h"
 #include "slang-ir-check-recursive-type.h"
 #include "slang-ir-missing-return.h"
+#include "slang-ir-operator-shift-overflow.h"
 #include "slang-ir-sccp.h"
 #include "slang-ir-ssa.h"
 #include "slang-ir-strip.h"
@@ -10917,13 +10918,13 @@ RefPtr<IRModule> generateIRForTranslationUnit(
             break;
     }
 
-    // Check for using uninitialized out parameters.
     if (compileRequest->getLinkage()->m_optionSet.shouldRunNonEssentialValidation())
     {
         // Propagate `constexpr`-ness through the dataflow graph (and the
         // call graph) based on constraints imposed by different instructions.
         propagateConstExpr(module, compileRequest->getSink());
 
+        // Check for using uninitialized out parameters.
         checkForUsingUninitializedOutParams(module, compileRequest->getSink());
         
         // TODO: give error messages if any `undefined` or
@@ -10936,6 +10937,8 @@ RefPtr<IRModule> generateIRForTranslationUnit(
 
         // Check for invalid differentiable function body.
         checkAutoDiffUsages(module, compileRequest->getSink());
+
+        checkForOperatorShiftOverflow(module, linkage->m_optionSet, compileRequest->getSink());
     }
 
     // The "mandatory" optimization passes may make use of the
diff --git a/tests/diagnostics/warning_operator_left_shift_overflow.slang b/tests/diagnostics/warning_operator_left_shift_overflow.slang
new file mode 100644
index 000000000..5c156a9b9
--- /dev/null
+++ b/tests/diagnostics/warning_operator_left_shift_overflow.slang
@@ -0,0 +1,27 @@
+//TEST:SIMPLE(filecheck=CHECK): -stage compute -entry computeMain -target hlsl
+
+RWStructuredBuffer<int> inputBuffer;
+RWStructuredBuffer<int> outputBuffer;
+
+[numthreads(4, 1, 1)]
+void computeMain(int3 dispatchThreadID: SV_DispatchThreadID)
+{
+    uint8_t u8 = uint8_t(inputBuffer[0]);
+
+    uint result = (u8 << 0)
+        + (u8 << 1)
+        + (u8 << 2)
+        + (u8 << 3)
+        + (u8 << 4)
+        + (u8 << 5)
+        + (u8 << 6)
+        + (u8 << 7)
+        // CHECK-NOT: warning 41030:
+        // CHECK: ([[#@LINE+1]]): warning 41030: {{.*}}uint8{{.*}}8
+        + (u8 << 8)
+        // CHECK: ([[#@LINE+1]]): warning 41030: {{.*}}uint8{{.*}}9
+        + (u8 << 9)
+        ;
+
+    outputBuffer[0] = int(result);
+}