Add IREnumType to distinguish enums from ints and each other (#6973)

* Add IREnumType to distinguish enums from ints and each other

* Add issue example as test

* format code

* Add expected test output

* Fix peephole optimization hanging

No idea why this PR triggered this, but there seems to have been a clear bug
here anyway, so may just as well fix it now.

* Move enum lowering later

* Add linkage decoration to enum type

* Use filecheck-buffer instead of expected.txt

* Fix comment

* Make enum casts actually use IR enum casts

They were all BuiltinCasts by accident

* Lower enum type before VM

* Deal with rate-qualified types in enum cast

* Allow any value marshalling for enum types

* Handle new enum instructions in a couple more switches

* Fix formatting

---------

Co-authored-by: slangbot <186143334+slangbot@users.noreply.github.com>

1 files changed, 39 insertions, 3 deletions

diff --git a/source/slang/slang-ir.cpp b/source/slang/slang-ir.cpp
index c105a698a..c1cec36fe 100644
--- a/source/slang/slang-ir.cpp
+++ b/source/slang/slang-ir.cpp
@@ -4051,6 +4051,7 @@ enum class TypeCastStyle
     Float,
     Bool,
     Ptr,
+    Enum,
     Void
 };
 static TypeCastStyle _getTypeStyleId(IRType* type)
@@ -4083,6 +4084,8 @@ static TypeCastStyle _getTypeStyleId(IRType* type)
     case kIROp_RefType:
     case kIROp_ConstRefType:
         return TypeCastStyle::Ptr;
+    case kIROp_EnumType:
+        return TypeCastStyle::Enum;
     case kIROp_VoidType:
         return TypeCastStyle::Void;
     default:
@@ -4131,27 +4134,42 @@ IRInst* IRBuilder::emitCast(IRType* type, IRInst* value, bool fallbackToBuiltinC
         }
     };
 
-    static const OpSeq opMap[4][5] = {
-        /*      To:      Int, Float, Bool, Ptr, Void*/
+    static const OpSeq opMap[5][6] = {
+        /*      To:      Int, Float, Bool, Ptr, Enum, Void */
         /* From Int   */ {
             kIROp_IntCast,
             kIROp_CastIntToFloat,
             kIROp_IntCast,
             kIROp_CastIntToPtr,
+            kIROp_CastIntToEnum,
             kIROp_CastToVoid},
         /* From Float */
         {kIROp_CastFloatToInt,
          kIROp_FloatCast,
          {kIROp_Neq},
          {kIROp_CastFloatToInt, kIROp_CastIntToPtr},
+         {kIROp_CastFloatToInt, kIROp_CastIntToEnum},
          kIROp_CastToVoid},
         /* From Bool  */
-        {kIROp_IntCast, kIROp_CastIntToFloat, kIROp_Nop, kIROp_CastIntToPtr, kIROp_CastToVoid},
+        {kIROp_IntCast,
+         kIROp_CastIntToFloat,
+         kIROp_Nop,
+         kIROp_CastIntToPtr,
+         kIROp_CastIntToEnum,
+         kIROp_CastToVoid},
         /* From Ptr   */
         {kIROp_CastPtrToInt,
          {kIROp_CastPtrToInt, kIROp_CastIntToFloat},
          kIROp_CastPtrToBool,
          kIROp_BitCast,
+         {kIROp_CastPtrToInt, kIROp_CastIntToEnum},
+         kIROp_CastToVoid},
+        /* From Enum   */
+        {kIROp_CastEnumToInt,
+         {kIROp_CastEnumToInt, kIROp_CastIntToFloat},
+         {kIROp_CastEnumToInt, kIROp_IntCast},
+         {kIROp_CastEnumToInt, kIROp_CastIntToPtr},
+         kIROp_EnumCast,
          kIROp_CastToVoid},
     };
 
@@ -4252,6 +4270,11 @@ IRInst* IRBuilder::emitVectorReshape(IRType* type, IRInst* value)
                 }
                 return emitMakeVector(targetVectorType, args);
             }
+            else
+            {
+                // Sizes match, no need to reshape.
+                return value;
+            }
         }
         auto reshape = emitIntrinsicInst(
             getVectorType(sourceVectorType->getElementType(), targetVectorType->getElementCount()),
@@ -4807,6 +4830,13 @@ IRClassType* IRBuilder::createClassType()
     return classType;
 }
 
+IREnumType* IRBuilder::createEnumType(IRType* tagType)
+{
+    IREnumType* enumType = createInst<IREnumType>(this, kIROp_EnumType, getTypeKind(), tagType);
+    addGlobalValue(this, enumType);
+    return enumType;
+}
+
 IRGLSLShaderStorageBufferType* IRBuilder::createGLSLShaderStorableBufferType()
 {
     IRGLSLShaderStorageBufferType* ssboType = createInst<IRGLSLShaderStorageBufferType>(
@@ -8505,6 +8535,9 @@ bool IRInst::mightHaveSideEffects(SideEffectAnalysisOptions options)
     case kIROp_CastPtrToInt:
     case kIROp_CastIntToPtr:
     case kIROp_PtrCast:
+    case kIROp_CastEnumToInt:
+    case kIROp_CastIntToEnum:
+    case kIROp_EnumCast:
     case kIROp_CastUInt2ToDescriptorHandle:
     case kIROp_CastDescriptorHandleToUInt2:
     case kIROp_CastDescriptorHandleToResource:
@@ -8956,6 +8989,9 @@ bool isMovableInst(IRInst* inst)
     case kIROp_CastPtrToBool:
     case kIROp_CastPtrToInt:
     case kIROp_PtrCast:
+    case kIROp_CastEnumToInt:
+    case kIROp_CastIntToEnum:
+    case kIROp_EnumCast:
     case kIROp_CastDynamicResource:
     case kIROp_BitAnd:
     case kIROp_BitNot: