Relax the inst definition order rule (#8588)

Close #8572.

The root cause of the issue is that in `_replaceInstUsesWith` call,
if the use of the inst is a generic parameter, and the inst is the data
type of that generic parameter, we could end up of moving the data type
before the generic parameter. This will break the layout of generic
parameters, where all the generic parameters should be laid consecutively
at the beginning of the first block of the generic.

Therefore, we don't make that relocation for such case.

4 files changed, 38 insertions, 2 deletions

diff --git a/source/slang/slang-ir-util.cpp b/source/slang/slang-ir-util.cpp
index 551a72fc7..12e37f42a 100644
--- a/source/slang/slang-ir-util.cpp
+++ b/source/slang/slang-ir-util.cpp
@@ -2744,4 +2744,23 @@ bool isPointerToImmutableLocation(IRInst* loc)
     }
     return false;
 }
+
+bool isGenericParameter(IRInst* inst)
+{
+    // The generic parameter must be in the first block
+    bool isParam = inst->getOp() == kIROp_Param;
+    bool isGeneric = false;
+    if (auto irBlock = as<IRBlock>(inst->parent))
+    {
+        isGeneric = as<IRGeneric>(irBlock->getParent()) != nullptr;
+    }
+    return isParam && isGeneric;
+}
+
+bool canRelaxInstOrderRule(IRInst* inst, IRInst* useOfInst)
+{
+    bool isSameBlock = (inst->getParent() == useOfInst->getParent());
+    return isSameBlock && isGenericParameter(useOfInst) && (useOfInst->getDataType() == inst);
+}
+
 } // namespace Slang
diff --git a/source/slang/slang-ir-util.h b/source/slang/slang-ir-util.h
index b8937d569..c12be51d6 100644
--- a/source/slang/slang-ir-util.h
+++ b/source/slang/slang-ir-util.h
@@ -446,6 +446,14 @@ inline bool isUseBaseAddrOperand(IRUse* use, IRInst* user)
     return user->getOperandUse(0) == use;
 }
 
+// Check if the inst is a generic parameter.
+bool isGenericParameter(IRInst* inst);
+
+// Usually we want to enforce that an instruction is defined before any of its uses, but
+// if the use is a generic parameter, we can relax this rule when the instruction is the data type
+// of the generic parameter.
+bool canRelaxInstOrderRule(IRInst* instToCheck, IRInst* otherInst);
+
 } // namespace Slang
 
 #endif
diff --git a/source/slang/slang-ir-validate.cpp b/source/slang/slang-ir-validate.cpp
index 55f3ad227..e7bb7b548 100644
--- a/source/slang/slang-ir-validate.cpp
+++ b/source/slang/slang-ir-validate.cpp
@@ -201,9 +201,13 @@ void validateIRInstOperand(IRValidateContext* context, IRInst* inst, IRUse* oper
                 // in order.
                 if (context)
                 {
+                    // There is exception that the use of an inst is defined before the inst,
+                    // e.g. generic parameter can be defined before its data type in some cases.
+                    // In those cases we allow relaxing the rule.
                     validate(
                         context,
-                        context->seenInsts.contains(operandValue),
+                        context->seenInsts.contains(operandValue) ||
+                            canRelaxInstOrderRule(operandValue, inst),
                         inst,
                         "def must come before use in same block");
                 }
diff --git a/source/slang/slang-ir.cpp b/source/slang/slang-ir.cpp
index 92543c952..e54238a9c 100644
--- a/source/slang/slang-ir.cpp
+++ b/source/slang/slang-ir.cpp
@@ -8164,7 +8164,12 @@ static void _maybeHoistOperand(IRUse* use)
                 continue;
 
             // If the operand is defined after user, move it to before user.
-            if (_isInstDefinedAfter(operand, user))
+            // There is exception that the use of an inst can be defined before the inst,
+            // e.g. generic parameter can be defined before its data type in some cases.
+            // And moving the datatype of a generic parameter before the parameter will result
+            // in incorrect IR layout, because we require that generic parameters are laid
+            // consecutively at first block of a generic.
+            if (_isInstDefinedAfter(operand, user) && !canRelaxInstOrderRule(operand, user))
             {
                 operand->insertBefore(user);
                 for (UInt i = 0; i < operand->getOperandCount(); i++)