Be lenient on same-size unsigend->signed conversion. (#2913)

* Be lenient on same-size unsigend->signed conversion.

* Fix tests.

* Use 250.

* wip

* Fix.

* Fix tests.

* Fix.

---------

Co-authored-by: Yong He <yhe@nvidia.com>

4 files changed, 148 insertions, 20 deletions

diff --git a/source/slang/slang-ast-support-types.h b/source/slang/slang-ast-support-types.h
index 3ab9d1a01..7e65dd02e 100644
--- a/source/slang/slang-ast-support-types.h
+++ b/source/slang/slang-ast-support-types.h
@@ -80,6 +80,10 @@ namespace Slang
         // from one on `Foo`
         kConversionCost_GetRef = 5,
         kConversionCost_ImplicitDereference = 10,
+        kConversionCost_InRangeIntLitConversion = 23,
+        kConversionCost_InRangeIntLitSignedToUnsignedConversion = 32,
+        kConversionCost_InRangeIntLitUnsignedToSignedConversion = 81,
+
 
         // Conversions based on explicit sub-typing relationships are the cheapest
         //
@@ -97,6 +101,9 @@ namespace Slang
         // Conversions that are lossless, but change "kind"
         kConversionCost_UnsignedToSignedPromotion = 200,
 
+        // Same-size size unsigned->signed conversions are potentially lossy, but they are commonly allowed silently.
+        kConversionCost_SameSizeUnsignedToSignedConversion = 250,
+
         // Conversion from signed->unsigned integer of same or greater size
         kConversionCost_SignedToUnsignedConversion = 300,
 
diff --git a/source/slang/slang-check-conversion.cpp b/source/slang/slang-check-conversion.cpp
index 639ae7939..9ee5bca1f 100644
--- a/source/slang/slang-check-conversion.cpp
+++ b/source/slang/slang-check-conversion.cpp
@@ -628,6 +628,79 @@ namespace Slang
         }
     }
 
+    static bool isSigned(Type* t)
+    {
+        auto basicType = as<BasicExpressionType>(t);
+        if (!basicType) return false;
+        switch (basicType->baseType)
+        {
+        case BaseType::Int8:
+        case BaseType::Int16:
+        case BaseType::Int:
+        case BaseType::Int64:
+        case BaseType::IntPtr:
+            return true;
+        default:
+            return false;
+        }
+    }
+
+    static int getTypeBitSize(Type* t)
+    {
+        auto basicType = as<BasicExpressionType>(t);
+        if (!basicType) return 0;
+
+        switch (basicType->baseType)
+        {
+        case BaseType::Int8:
+        case BaseType::UInt8:
+            return 8;
+        case BaseType::Int16:
+        case BaseType::UInt16:
+            return 16;
+        case BaseType::Int:
+        case BaseType::UInt:
+            return 32;
+        case BaseType::Int64:
+        case BaseType::UInt64:
+            return 64;
+        case BaseType::IntPtr:
+        case BaseType::UIntPtr:
+#if SLANG_PTR_IS_32
+            return 32;
+#else
+            return 64;
+#endif
+        default:
+            return 0;
+        }
+    }
+
+    ConversionCost SemanticsVisitor::getImplicitConversionCostWithKnownArg(Decl* decl, Type* toType, Expr* arg)
+    {
+        ConversionCost candidateCost = getImplicitConversionCost(decl);
+
+        // Fix up the cost if the operand is a const lit.
+        if (isScalarIntegerType(toType))
+        {
+            auto knownVal = as<IntegerLiteralExpr>(arg);
+            if (!knownVal)
+                return candidateCost;
+            if (getIntValueBitSize(knownVal->value) <= getTypeBitSize(toType))
+            {
+                bool toTypeIsSigned = isSigned(toType);
+                bool fromTypeIsSigned = isSigned(knownVal->type);
+                if (toTypeIsSigned == fromTypeIsSigned)
+                    candidateCost = kConversionCost_InRangeIntLitConversion;
+                else if (toTypeIsSigned)
+                    candidateCost = kConversionCost_InRangeIntLitUnsignedToSignedConversion;
+                else
+                    candidateCost = kConversionCost_InRangeIntLitSignedToUnsignedConversion;
+            }
+        }
+        return candidateCost;
+    }
+
     bool SemanticsVisitor::_coerce(
         CoercionSite site,
         Type*    toType,
@@ -989,8 +1062,8 @@ namespace Slang
             ConversionCost bestCost = kConversionCost_Explicit;
             for(auto candidate : overloadContext.bestCandidates)
             {
-                ConversionCost candidateCost = getImplicitConversionCost(
-                    candidate.item.declRef.getDecl());
+                ConversionCost candidateCost = getImplicitConversionCostWithKnownArg(
+                    candidate.item.declRef.getDecl(), toType, fromExpr);
 
                 if(candidateCost < bestCost)
                     bestCost = candidateCost;
@@ -1027,8 +1100,8 @@ namespace Slang
             // Next, we need to look at the implicit conversion
             // cost associated with the initializer we are invoking.
             //
-            ConversionCost cost = getImplicitConversionCost(
-                    overloadContext.bestCandidate->item.declRef.getDecl());
+            ConversionCost cost = getImplicitConversionCostWithKnownArg(
+                overloadContext.bestCandidate->item.declRef.getDecl(), toType, fromExpr);
 
             // If the cost is too high to be usable as an
             // implicit conversion, then we will report the
@@ -1149,7 +1222,7 @@ namespace Slang
 
         BasicTypeKeyPair cacheKey;
         cacheKey.type1 = makeBasicTypeKey(toType);
-        cacheKey.type2 = makeBasicTypeKey(fromType);
+        cacheKey.type2 = makeBasicTypeKey(fromType, fromExpr);
     
         if( cacheKey.isValid())
         {
diff --git a/source/slang/slang-check-impl.h b/source/slang/slang-check-impl.h
index efc3989e2..ac4d51549 100644
--- a/source/slang/slang-check-impl.h
+++ b/source/slang/slang-check-impl.h
@@ -26,24 +26,60 @@ namespace Slang
         /// Note: this currently does not include PtrTypeBase.
     Type* getPointedToTypeIfCanImplicitDeref(Type* type);
 
+    inline int getIntValueBitSize(IntegerLiteralValue val)
+    {
+        uint64_t v = val > 0 ? (uint64_t)val : (uint64_t)-val;
+        int result = 1;
+        while (v >>= 1)
+        {
+            result++;
+        }
+        return result;
+    }
+
     // A flat representation of basic types (scalars, vectors and matrices)
     // that can be used as lookup key in caches
-    enum class BasicTypeKey : uint16_t
+    struct BasicTypeKey
     {
-        Invalid = 0xffff,                 ///< Value that can never be a valid type
+        uint32_t baseType : 8;
+        uint32_t dim1 : 4;
+        uint32_t dim2 : 4;
+        uint32_t knownConstantBitCount : 8;
+        uint32_t knownNegative : 1;
+        uint32_t reserved : 7;
+        uint32_t getRaw() const
+        {
+            uint32_t val;
+            memcpy(&val, this, sizeof(uint32_t));
+            return val;
+        }
+        bool operator==(BasicTypeKey other) const
+        {
+            return getRaw() == other.getRaw();
+        }
+        static BasicTypeKey invalid() { return BasicTypeKey{ 0xff, 0, 0 }; }
     };
 
     SLANG_FORCE_INLINE BasicTypeKey makeBasicTypeKey(BaseType baseType, IntegerLiteralValue dim1 = 0, IntegerLiteralValue dim2 = 0)
     {
         SLANG_ASSERT(dim1 >= 0 && dim2 >= 0);
-        return BasicTypeKey((uint8_t(baseType) << 8) | (uint8_t(dim1) << 4) | uint8_t(dim2));
+        return BasicTypeKey{ uint8_t(baseType), uint8_t(dim1), uint8_t(dim2) };
     }
 
-    inline BasicTypeKey makeBasicTypeKey(Type* typeIn)
+    inline BasicTypeKey makeBasicTypeKey(Type* typeIn, Expr* exprIn = nullptr)
     {
         if (auto basicType = as<BasicExpressionType>(typeIn))
         {
-            return makeBasicTypeKey(basicType->baseType);
+            auto rs = makeBasicTypeKey(basicType->baseType);
+            if (auto constInt = as<IntegerLiteralExpr>(exprIn))
+            {
+                if (constInt->value < 0)
+                {
+                    rs.knownNegative = 1;
+                }
+                rs.knownConstantBitCount = getIntValueBitSize(constInt->value);
+            }
+            return rs;
         }
         else if (auto vectorType = as<VectorExpressionType>(typeIn))
         {
@@ -68,7 +104,7 @@ namespace Slang
                 }
             }
         }
-        return BasicTypeKey::Invalid;
+        return BasicTypeKey::invalid();
     }
 
     struct BasicTypeKeyPair
@@ -77,11 +113,11 @@ namespace Slang
         bool operator==(const BasicTypeKeyPair& rhs) const { return type1 == rhs.type1 && type2 == rhs.type2; }
         bool operator!=(const BasicTypeKeyPair& rhs) const { return !(*this == rhs); }
 
-        bool isValid() const { return type1 != BasicTypeKey::Invalid && type2 != BasicTypeKey::Invalid; }
+        bool isValid() const { return type1.getRaw() != BasicTypeKey::invalid().getRaw() && type2.getRaw() != BasicTypeKey::invalid().getRaw(); }
 
         HashCode getHashCode()
         {
-            return HashCode(int(type1) << 16 | int(type2));
+            return combineHash(type1.getRaw(), type2.getRaw());
         }
     };
 
@@ -95,25 +131,25 @@ namespace Slang
         }
         HashCode getHashCode()
         {
-            return HashCode(((int)(UInt64)(void*)(operatorName) << 16) ^ (int(args[0]) << 8) ^ (int(args[1])));
+            return combineHash((int)(UInt64)(void*)(operatorName), args[0].getRaw(), args[1].getRaw());
         }
         bool fromOperatorExpr(OperatorExpr* opExpr)
         {
             // First, lets see if the argument types are ones
             // that we can encode in our space of keys.
-            args[0] = BasicTypeKey::Invalid;
-            args[1] = BasicTypeKey::Invalid;
+            args[0] = BasicTypeKey::invalid();
+            args[1] = BasicTypeKey::invalid();
             if (opExpr->arguments.getCount() > 2)
                 return false;
 
             for (Index i = 0; i < opExpr->arguments.getCount(); i++)
             {
-                auto key = makeBasicTypeKey(opExpr->arguments[i]->type.Ptr());
-                if (key == BasicTypeKey::Invalid)
+                auto key = makeBasicTypeKey(opExpr->arguments[i]->type.Ptr(), opExpr->arguments[i]);
+                if (key.getRaw() == BasicTypeKey::invalid().getRaw())
                 {
                     return false;
                 }
-                args[i]=  key;
+                args[i] = key;
             }
 
             // Next, lets see if we can find an intrinsic opcode
@@ -136,7 +172,7 @@ namespace Slang
                     // Look at a candidate definition to be called and
                     // see if it gives us a key to work with.
                     //
-                    Decl* funcDecl = overloadedBase->lookupResult2.item.declRef.decl;
+                    Decl* funcDecl = item.declRef.decl;
                     if (auto genDecl = as<GenericDecl>(funcDecl))
                         funcDecl = genDecl->inner;
 
@@ -783,6 +819,9 @@ namespace Slang
         ConversionCost getImplicitConversionCost(
             Decl* decl);
 
+        ConversionCost getImplicitConversionCostWithKnownArg(Decl* decl, Type* toType, Expr* arg);
+
+
         BuiltinConversionKind getImplicitConversionBuiltinKind(
             Decl* decl);
 
diff --git a/source/slang/slang-stdlib.cpp b/source/slang/slang-stdlib.cpp
index 1f8d0a97a..a31c77985 100644
--- a/source/slang/slang-stdlib.cpp
+++ b/source/slang/slang-stdlib.cpp
@@ -156,6 +156,15 @@ namespace Slang
         {
             return kConversionCost_UnsignedToSignedPromotion;
         }
+        // Same-size unsigned to signed integer conversion.
+        else if (toInfo.conversionKind == kBaseTypeConversionKind_Signed
+            && fromInfo.conversionKind == kBaseTypeConversionKind_Unsigned
+            && toInfo.conversionRank == fromInfo.conversionRank
+            && toInfo.conversionRank != kBaseTypeConversionRank_IntPtr
+            && fromInfo.conversionRank != kBaseTypeConversionRank_IntPtr)
+        {
+            return kConversionCost_SameSizeUnsignedToSignedConversion;
+        }
 
         // Conversion from signed to unsigned is always lossy,
         // but it is preferred over conversions from unsigned