Conform non-suffixed integer literals (#5717)

* Make non-suffixed integer literal type resolution conform to C

* Update integer literal tests

* Clean up integer literal implementation a bit

* Update docs on integer literals

* Clean up docs update

* Clean up docs update

* Add comment on INT64_MIN edge case

* Fixed failing test, fixed formatting and cleaned up code

---------

Co-authored-by: Yong He <yonghe@outlook.com>

5 files changed, 91 insertions, 37 deletions

diff --git a/tests/diagnostics/int-literal.slang b/tests/diagnostics/int-literal.slang
index 3724d0e14..920fb58eb 100644
--- a/tests/diagnostics/int-literal.slang
+++ b/tests/diagnostics/int-literal.slang
@@ -2,8 +2,8 @@
 
 int doSomething(int a)
 {
-    // Warning can't fit
-    int c0 = 0x800000000;
+    // No warning, literal will be interpreted as 64 bit.
+    uint64_t c0 = 0x800000000;
     
     // No warning as top bits are just ignored
     int c1 = -1ll;
@@ -13,12 +13,10 @@ int doSomething(int a)
     // Should sign extend 
     int c3 = 0x80000000;
     
-    // Should give a warning (ideally including the preceeding -)
-    // Currently we don't have the -, because the lexer lexes - independently
-    int c4 = -0xfffffffff;
+    // No warning, hex literal will be interpreted as an unsigned 64 integer then signed with negative operator.
+    int64_t c4 = -0xfffffffff;
     
-    // 
-    a += c0 + c1 + c2;
+    a += (int)c0 + c1 + c2;
     
     int64_t b = 0;
 
@@ -26,6 +24,19 @@ int doSomething(int a)
     b += 0x800000000ll;
     
     uint64_t c5 = -2ull;
+
+    // Warning, integer literal is too large for signed 64 bit, must be interpreted as unsigned.
+    uint64_t d0 = 18446744073709551615;
+
+    // Warning, integer literal is too small for signed 64 bit, must be interpreted as unsigned.
+    uint64_t d1 = -9223372036854775809;
+
+    // This is INT64_MIN and valid negative signed integer, but warning will be emitted as negative(-) is scanned
+    // separately in the lexer, and the positive literal portion will emit a warning.
+    // The final value will still be correctly set as INT64_MIN.
+    //
+    // To not have this warning the lexer must scan the negative operator and number together.
+    uint64_t d2 = -9223372036854775808;
     
     return a + int(b);
 }
diff --git a/tests/diagnostics/int-literal.slang.expected b/tests/diagnostics/int-literal.slang.expected
index 01e7785f6..39ab66d1d 100644
--- a/tests/diagnostics/int-literal.slang.expected
+++ b/tests/diagnostics/int-literal.slang.expected
@@ -1,11 +1,14 @@
 result code = 0
 standard error = {
-tests/diagnostics/int-literal.slang(6): warning 39999: integer literal '0x800000000' too large for type 'int' truncated to '0'
-    int c0 = 0x800000000;
-             ^~~~~~~~~~~
-tests/diagnostics/int-literal.slang(18): warning 39999: integer literal '0xfffffffff' too large for type 'int' truncated to '-1'
-    int c4 = -0xfffffffff;
-              ^~~~~~~~~~~
+tests/diagnostics/int-literal.slang(29): warning 39999: integer literal is too large to be represented in a signed integer type, interpreting as unsigned
+    uint64_t d0 = 18446744073709551615;
+                  ^~~~~~~~~~~~~~~~~~~~
+tests/diagnostics/int-literal.slang(32): warning 39999: integer literal is too large to be represented in a signed integer type, interpreting as unsigned
+    uint64_t d1 = -9223372036854775809;
+                   ^~~~~~~~~~~~~~~~~~~
+tests/diagnostics/int-literal.slang(39): warning 39999: integer literal is too large to be represented in a signed integer type, interpreting as unsigned
+    uint64_t d2 = -9223372036854775808;
+                   ^~~~~~~~~~~~~~~~~~~
 }
 standard output = {
 }
diff --git a/tests/hlsl-intrinsic/literal-int64.slang b/tests/hlsl-intrinsic/literal-int64.slang
index 977c85a9f..ab2a38e57 100644
--- a/tests/hlsl-intrinsic/literal-int64.slang
+++ b/tests/hlsl-intrinsic/literal-int64.slang
@@ -1,21 +1,27 @@
-//TEST(compute):COMPARE_COMPUTE_EX:-cpu -compute -shaderobj
+//TEST(compute):COMPARE_COMPUTE_EX(filecheck=CHK):-cpu -compute -shaderobj
 // No support for int64_t on D3D11 (no sm 6.0)
-//DISABLE_TEST(compute):COMPARE_COMPUTE_EX:-slang -compute -shaderobj
+//DISABLE_TEST(compute):COMPARE_COMPUTE_EX(filecheck=CHK):-slang -compute -shaderobj
 // No support with Dx12 with dxbc. Needs SM6.0 + dxil
-//DISABLE_TEST(compute):COMPARE_COMPUTE_EX:-slang -compute -dx12 -shaderobj
-//TEST(compute):COMPARE_COMPUTE_EX:-slang -compute -profile cs_6_0 -dx12 -use-dxil -shaderobj -render-feature hardware-device
-//TEST(compute, vulkan):COMPARE_COMPUTE_EX:-vk -compute -shaderobj -render-feature int64
-//TEST(compute):COMPARE_COMPUTE_EX:-cuda -compute -shaderobj
+//DISABLE_TEST(compute):COMPARE_COMPUTE_EX(filecheck=CHK):-slang -compute -dx12 -shaderobj
+//TEST(compute):COMPARE_COMPUTE_EX(filecheck=CHK):-slang -compute -profile cs_6_0 -dx12 -use-dxil -shaderobj -render-feature hardware-device
+//TEST(compute, vulkan):COMPARE_COMPUTE_EX(filecheck=CHK):-vk -compute -shaderobj -render-feature int64
+//TEST(compute):COMPARE_COMPUTE_EX(filecheck=CHK):-cuda -compute -shaderobj
 
-// Note that the behavior we expect here, is a int without suffix is assumed to 
-// be an int literal.
-// That when making a conversion to a uint64_t, we expect first it to be widened
-// to an int64_t, before becoming a uint64_t.
+// The expected behavior is that the literal type is the first from the following list
+// in which the value can fit:
+// - For decimal bases:
+//     - `int`
+//     - `int64_t`
+// - For non-decimal bases:
+//     - `int`
+//     - `uint`
+//     - `int64_t`
+//     - `uint64_t`
 
-//TEST_INPUT:ubuffer(data=[0 0 0 0 0 0 0 0 0 0 0 0 0 0], stride=4):out,name outputBuffer
+//TEST_INPUT:ubuffer(data=[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0], stride=4):out,name outputBuffer
 RWStructuredBuffer<uint64_t> outputBuffer;
 
-[numthreads(7, 1, 1)]
+[numthreads(10, 1, 1)]
 void computeMain(uint3 dispatchThreadID : SV_DispatchThreadID)
 {
     int idx = int(int64_t(dispatchThreadID.x));
@@ -24,38 +30,66 @@ void computeMain(uint3 dispatchThreadID : SV_DispatchThreadID)
     
     if (idx == 0)
     {
-        // Should be 0xffffffff
+        // Should be 0xffffffffffffffff
         v = -1;
     }
     else if (idx == 1)
     {
-        // Should be 0xffffffff
+        // Should be 0x00000000ffffffff
         v = -1u;
     }
     else if (idx == 2)
     {
-        // Should be 0x80000000
+        // Should be 0x0000000080000000
         v = 0x80000000u;
     }
     else if (idx == 3)
     {
-        
-        // Should be 0xffffffff80000000
+        // Should be 0x0000000080000000
         int64_t v0 = 0x80000000;
         v = v0;
     }
     else if (idx == 4)
-    {   // Should be 0xffffffff80000000
+    {
+        // Should be 0x0000000080000000. Shows hexadecimal literals can represent unsigned 32-bit integers.
         v = 0x80000000;
     }
     else if (idx == 5)
-    {   // Should be 0xffffffff
+    {
+        // Should be 0xffffffffffffffff
         v = ~0; 
     }
-    else 
-    {   // Should be 0xffffffffffffffff
+    else if (idx == 6)
+    {
+        // Should be 0xffffffffffffffff
         v = ~0LL; 
     }
+    else if (idx == 7)
+    {
+        // Should be 0x7fffffffffffffff. Shows decimal literals can represent 64-bit integers.
+        v = 9223372036854775807;
+    }
+    else if (idx == 8)
+    {
+        // Should be 1. Shows decimal literals can represent INT64_MIN or 0x8000000000000000.
+        // Warning will be emitted as negative(-) is scanned separately in the lexer, and the positive
+        // literal portion will emit a warning.
+        // The final value must be correctly set as INT64_MIN and must be negative.
+        if (-9223372036854775808 < 0L)
+        {
+            v = 1;
+        }
+        else
+        {
+            v = 0;
+        }
+        //CHK: warning 39999: integer literal is too large to be represented in a signed integer type, interpreting as unsigned
+    }
+    else
+    {
+        // Shoud be 0xfaaaaaaabaaaaaaa. Shows hexadecimal literals can represent unsigned 64-bit integers.
+        v = 0xfaaaaaaabaaaaaaa;
+    }
     
     outputBuffer[idx] = v;
-}
\ No newline at end of file
+}
diff --git a/tests/hlsl-intrinsic/literal-int64.slang.expected.txt b/tests/hlsl-intrinsic/literal-int64.slang.expected.txt
index 96557632e..64d20ab93 100644
--- a/tests/hlsl-intrinsic/literal-int64.slang.expected.txt
+++ b/tests/hlsl-intrinsic/literal-int64.slang.expected.txt
@@ -5,10 +5,16 @@ FFFFFFFF
 80000000
 0
 80000000
-FFFFFFFF
+0
 80000000
+0
 FFFFFFFF
 FFFFFFFF
 FFFFFFFF
 FFFFFFFF
 FFFFFFFF
+7FFFFFFF
+1
+0
+BAAAAAAA
+FAAAAAAA
diff --git a/tests/slang-extension/atomic-int64-byte-address-buffer.slang b/tests/slang-extension/atomic-int64-byte-address-buffer.slang
index 61e38069d..feb221f20 100644
--- a/tests/slang-extension/atomic-int64-byte-address-buffer.slang
+++ b/tests/slang-extension/atomic-int64-byte-address-buffer.slang
@@ -32,6 +32,6 @@ void computeMain(int3 dispatchThreadID : SV_DispatchThreadID)
     // Bit logical
     outputBuffer.InterlockedOrU64((idx << 3), (uint64_t(2) << 32) | (tid << 4));
     outputBuffer.InterlockedXorU64((idx << 3), tid << 8);
-    outputBuffer.InterlockedAndU64((idx << 3), (uint64_t(tid | 2) << 32) |  0xffffffff);
+    outputBuffer.InterlockedAndU64((idx << 3), (uint64_t(tid | 2) << 32) |  0xffffffffffffffff);
 }