Literal handling improvements (#1202)

* WIP: 64 literal diagnostic and truncation.

* Improve how integer truncation is handled/supported.
Added literal-int64.slang test.
Set a suffix on all literals.
Fixed problem on C++ based targets where l suffix was not the same as int() cast. So on C++ derived emitters, int() is used instead of l suffix to have same behavior across targets.

* Add literal diagnostic testing.

* Allow lexer to lex - in front of literals.

* Fix lexing and converting int literal with -.

* Too large small values of floats become inf.
Handling writing inf types out on different targets.
Add function to deterimine if a float literals kind.

* Roll back the support of lexer lexing negative literals.

* Fixed tests broken because of diagnostics numbers.
Improved _isFinite

* Fix compilation on linux.

* Fix problem with abs on linux - use Math::Abs.

* Fix typo.

* * Improve warnings for float literals zeroed
* Improved 64 bit type documentation
* Handle half
* Improved comments
* Fixed tests broken
* Use capital letters for suffixes.

* Make default behavior on outputting a int literal that is an 'int32_t' is cast (not suffix) to avoid platform inconsistencies.
Improve documentation for 64 bit types.
Make tests cover material in docs.

* Fixed tests.

* Rename FloatKind::Normal -> Finite

* Fix half zero check.

10 files changed, 148 insertions, 46 deletions

diff --git a/tests/compute/half-texture.slang.1.expected b/tests/compute/half-texture.slang.1.expected
index dc742a1ff..1e1a282fa 100644
--- a/tests/compute/half-texture.slang.1.expected
+++ b/tests/compute/half-texture.slang.1.expected
@@ -24,7 +24,7 @@ void computeMain(vector<uint,3> dispatchThreadID_0 : SV_DISPATCHTHREADID)
 #line 20
     vector<int,2> pos_0 = (vector<int,2>) dispatchThreadID_0.xy;
     float _S1 = 1.00000000000000000000 / 3.00000000000000000000;
-    vector<int,2> pos2_0 = vector<int,2>(3 - pos_0.y, 3 - pos_0.x);
+    vector<int,2> pos2_0 = vector<int,2>(int(3) - pos_0.y, int(3) - pos_0.x);
 
 #line 29
     half h_0 = halfTexture_0[(vector<uint,2>) pos2_0];
@@ -37,7 +37,7 @@ void computeMain(vector<uint,3> dispatchThreadID_0 : SV_DISPATCHTHREADID)
     halfTexture2_0[(vector<uint,2>) pos_0] = h4_0.xy;
     halfTexture4_0[(vector<uint,2>) pos_0] = vector<half,4>(h2_0, h_0, h_0);
 
-    int index_0 = pos_0.x + pos_0.y * 4;
+    int index_0 = pos_0.x + pos_0.y * int(4);
     outputBuffer_0[(uint) index_0] = index_0;
 
 #line 18
diff --git a/tests/diagnostics/float-literal.slang b/tests/diagnostics/float-literal.slang
new file mode 100644
index 000000000..46ce048b9
--- /dev/null
+++ b/tests/diagnostics/float-literal.slang
@@ -0,0 +1,23 @@
+//DIAGNOSTIC_TEST:SIMPLE:
+
+
+float doSomething(float a)
+{
+    // Too large with become +inf
+    a += 5e+40;
+    // Will be narrowed to 0
+    a += 9e-50;
+    
+    double b = 0.0f;
+    
+    // These shouldn't produce warning as they can fit 
+    b += 5e+40l;
+    b += 9e-50l;
+    
+    // Cos these don't have l suffix they should also produce warnings
+    // and produce -inf and 0
+    b += -5e+40;
+    b += -9e-50;
+    
+    return a + float(b);
+}
diff --git a/tests/diagnostics/float-literal.slang.expected b/tests/diagnostics/float-literal.slang.expected
new file mode 100644
index 000000000..347317666
--- /dev/null
+++ b/tests/diagnostics/float-literal.slang.expected
@@ -0,0 +1,9 @@
+result code = 0
+standard error = {
+tests/diagnostics/float-literal.slang(7): warning 39999: float literal '5e+40' unrepresentable, converted to 'inf'
+tests/diagnostics/float-literal.slang(9): warning 39999: '9e-50' is smaller than the smallest representable value for type float, converted to '0'
+tests/diagnostics/float-literal.slang(19): warning 39999: float literal '5e+40' unrepresentable, converted to 'inf'
+tests/diagnostics/float-literal.slang(20): warning 39999: '9e-50' is smaller than the smallest representable value for type float, converted to '0'
+}
+standard output = {
+}
diff --git a/tests/diagnostics/int-literal.slang b/tests/diagnostics/int-literal.slang
new file mode 100644
index 000000000..3724d0e14
--- /dev/null
+++ b/tests/diagnostics/int-literal.slang
@@ -0,0 +1,32 @@
+//DIAGNOSTIC_TEST:SIMPLE:
+
+int doSomething(int a)
+{
+    // Warning can't fit
+    int c0 = 0x800000000;
+    
+    // No warning as top bits are just ignored
+    int c1 = -1ll;
+    
+    int c2 = int(-1u);
+    
+    // Should sign extend 
+    int c3 = 0x80000000;
+    
+    // Should give a warning (ideally including the preceeding -)
+    // Currently we don't have the -, because the lexer lexes - independently
+    int c4 = -0xfffffffff;
+    
+    // 
+    a += c0 + c1 + c2;
+    
+    int64_t b = 0;
+
+    // Ok
+    b += 0x800000000ll;
+    
+    uint64_t c5 = -2ull;
+    
+    return a + int(b);
+}
+
diff --git a/tests/diagnostics/int-literal.slang.expected b/tests/diagnostics/int-literal.slang.expected
new file mode 100644
index 000000000..ffc5ff6d2
--- /dev/null
+++ b/tests/diagnostics/int-literal.slang.expected
@@ -0,0 +1,7 @@
+result code = 0
+standard error = {
+tests/diagnostics/int-literal.slang(6): warning 39999: integer literal '0x800000000' too large for type 'int' truncated to '0'
+tests/diagnostics/int-literal.slang(18): warning 39999: integer literal '0xfffffffff' too large for type 'int' truncated to '-1'
+}
+standard output = {
+}
diff --git a/tests/hlsl-intrinsic/literal-int64.slang b/tests/hlsl-intrinsic/literal-int64.slang
new file mode 100644
index 000000000..cb3cef5f2
--- /dev/null
+++ b/tests/hlsl-intrinsic/literal-int64.slang
@@ -0,0 +1,61 @@
+//TEST(compute):COMPARE_COMPUTE_EX:-cpu -compute 
+// No support for int64_t on D3D11 (no sm 6.0)
+//DISABLE_TEST(compute):COMPARE_COMPUTE_EX:-slang -compute
+// No support with Dx12 with dxbc. Needs SM6.0 + dxil
+//DISABLE_TEST(compute):COMPARE_COMPUTE_EX:-slang -compute -dx12 
+//TEST(compute):COMPARE_COMPUTE_EX:-slang -compute -profile cs_6_0 -dx12 -use-dxil
+//TEST(compute, vulkan):COMPARE_COMPUTE_EX:-vk -compute
+//TEST(compute):COMPARE_COMPUTE_EX:-cuda -compute
+
+// Note that the behavior we expect here, is a int without suffix is assumed to 
+// be an int literal.
+// That when making a conversion to a uint64_t, we expect first it to be widened
+// to an int64_t, before becoming a uint64_t.
+
+//TEST_INPUT:ubuffer(data=[0 0 0 0 0 0 0 0 0 0 0 0 0 0], stride=4):out,name outputBuffer
+RWStructuredBuffer<uint64_t> outputBuffer;
+
+[numthreads(7, 1, 1)]
+void computeMain(uint3 dispatchThreadID : SV_DispatchThreadID)
+{
+    int idx = int64_t(dispatchThreadID.x);
+    
+    uint64_t v = 0;
+    
+    if (idx == 0)
+    {
+        // Should be 0xffffffff
+        v = -1;
+    }
+    else if (idx == 1)
+    {
+        // Should be 0xffffffff
+        v = -1u;
+    }
+    else if (idx == 2)
+    {
+        // Should be 0x80000000
+        v = 0x80000000u;
+    }
+    else if (idx == 3)
+    {
+        
+        // Should be 0xffffffff80000000
+        int64_t v0 = 0x80000000;
+        v = v0;
+    }
+    else if (idx == 4)
+    {   // Should be 0xffffffff80000000
+        v = 0x80000000;
+    }
+    else if (idx == 5)
+    {   // Should be 0xffffffff
+        v = ~0; 
+    }
+    else 
+    {   // Should be 0xffffffffffffffff
+        v = ~0LL; 
+    }
+    
+    outputBuffer[idx] = v;
+}
\ No newline at end of file
diff --git a/tests/hlsl-intrinsic/literal-int64.slang.expected.txt b/tests/hlsl-intrinsic/literal-int64.slang.expected.txt
new file mode 100644
index 000000000..96557632e
--- /dev/null
+++ b/tests/hlsl-intrinsic/literal-int64.slang.expected.txt
@@ -0,0 +1,14 @@
+FFFFFFFF
+FFFFFFFF
+FFFFFFFF
+0
+80000000
+0
+80000000
+FFFFFFFF
+80000000
+FFFFFFFF
+FFFFFFFF
+FFFFFFFF
+FFFFFFFF
+FFFFFFFF
diff --git a/tests/hlsl-intrinsic/scalar-float.slang b/tests/hlsl-intrinsic/scalar-float.slang
index a1982bbb8..6c090cff1 100644
--- a/tests/hlsl-intrinsic/scalar-float.slang
+++ b/tests/hlsl-intrinsic/scalar-float.slang
@@ -14,7 +14,6 @@ void computeMain(uint3 dispatchThreadID : SV_DispatchThreadID)
     
     float f = idx * (1.0f / (4.0f));
 
-    int it = 0;
     float ft = 0.0f;
     
     // fmod
diff --git a/tests/hlsl-intrinsic/scalar-int64-literal-problem.slang b/tests/hlsl-intrinsic/scalar-int64-literal-problem.slang
deleted file mode 100644
index c315e84cd..000000000
--- a/tests/hlsl-intrinsic/scalar-int64-literal-problem.slang
+++ /dev/null
@@ -1,35 +0,0 @@
-// Disable the test in general, exists to check the output when the fix is in.
-
-//TEST(compute):COMPARE_COMPUTE_EX:-cpu -compute 
-// No support for int64_t on dx11 (no sm 6.0)
-//DISABLE_TEST(compute):COMPARE_COMPUTE_EX:-slang -compute
-// No support with Dx12 with dxbc. Needs SM6.0 + dxil
-//DISABLE_TEST(compute):COMPARE_COMPUTE_EX:-slang -compute -dx12 
-//DISABLE_TEST(compute):COMPARE_COMPUTE_EX:-slang -compute -profile cs_6_0 -dx12 -use-dxil
-// GLSL notices the narrowing of a 64 bit literal into 32 bits and for it this is an error.
-//DISABLE_TEST(compute, vulkan):COMPARE_COMPUTE_EX:-vk -compute
-//DISABLE_TEST(compute):COMPARE_COMPUTE_EX:-cuda -compute
-
-
-//TEST_INPUT:ubuffer(data=[0 0 0 0 0 0 0 0], stride=4):out,name outputBuffer
-RWStructuredBuffer<int> outputBuffer;
-
-[numthreads(4, 1, 1)]
-void computeMain(uint3 dispatchThreadID : SV_DispatchThreadID)
-{
-    uint idx = dispatchThreadID.x;
-    
-    // The multiply here will see dispatchThreadID.x as being uint, 
-    // and then the multiply will coerce the lhs 0x7fff... into being uint, 
-    // meaning 0x7fff.... will be interpreted as -1. int32_t(0x7ffff...) -> 0xffffffff 
-    int64_t m = 0x7fffffffffffffff * idx; 
-    
-    m += 0x7fffffffffffffff * int(idx);
-    
-    int64_t v = 0x7fffffffffffffff;
-    
-    int64_t r = m + v;
-    
-    outputBuffer[idx * 2] = int(uint64_t(r) >> 32); 
-    outputBuffer[idx * 2 + 1] = int(r); 
-}
\ No newline at end of file
diff --git a/tests/hlsl-intrinsic/scalar-int64-literal-problem.slang.expected.txt b/tests/hlsl-intrinsic/scalar-int64-literal-problem.slang.expected.txt
deleted file mode 100644
index 6c2c9dabe..000000000
--- a/tests/hlsl-intrinsic/scalar-int64-literal-problem.slang.expected.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-7FFFFFFF
-FFFFFFFF
-0
-FFFFFFFD
-80000000
-FFFFFFFB
-0
-FFFFFFF9