Emit errors when seeing a matrix with row/col count of 1 (#6044)

Such matrices aren't well supported except by D3D targets.
Therefore, generate an error rather than outputting invalid code for non-D3D targets.

This closes #5987.

Co-authored-by: Yong He <yonghe@outlook.com>

2 files changed, 28 insertions, 0 deletions

diff --git a/source/slang/slang-diagnostic-defs.h b/source/slang/slang-diagnostic-defs.h
index d86cd8be2..eacc345a7 100644
--- a/source/slang/slang-diagnostic-defs.h
+++ b/source/slang/slang-diagnostic-defs.h
@@ -1658,6 +1658,12 @@ DIAGNOSTIC(
     overloadedParameterToHigherOrderFunction,
     "passing overloaded functions to higher order functions is not supported")
 
+DIAGNOSTIC(
+    39999,
+    Error,
+    matrixColumnOrRowCountIsOne,
+    "matrices with 1 column or row are not supported by the current code generation target")
+
 // 38xxx
 
 DIAGNOSTIC(
diff --git a/source/slang/slang-emit.cpp b/source/slang/slang-emit.cpp
index ee2582267..b17bc03f3 100644
--- a/source/slang/slang-emit.cpp
+++ b/source/slang/slang-emit.cpp
@@ -555,6 +555,24 @@ static void unexportNonEmbeddableIR(CodeGenTarget target, IRModule* irModule)
     }
 }
 
+static void validateMatrixDimensions(DiagnosticSink* sink, IRModule* module)
+{
+    for (auto globalInst : module->getGlobalInsts())
+    {
+        if (auto matrixType = as<IRMatrixType>(globalInst))
+        {
+            auto colCount = as<IRIntLit>(matrixType->getColumnCount());
+            auto rowCount = as<IRIntLit>(matrixType->getRowCount());
+
+            if ((rowCount && (rowCount->getValue() == 1)) ||
+                (colCount && (colCount->getValue() == 1)))
+            {
+                sink->diagnose(matrixType->sourceLoc, Diagnostics::matrixColumnOrRowCountIsOne);
+            }
+        }
+    }
+}
+
 Result linkAndOptimizeIR(
     CodeGenContext* codeGenContext,
     LinkingAndOptimizationOptions const& options,
@@ -1504,6 +1522,10 @@ Result linkAndOptimizeIR(
 #endif
     validateIRModuleIfEnabled(codeGenContext, irModule);
 
+    // Make sure there are no matrices with 1 row/column, except for D3D targets where it's allowed.
+    if (!isD3DTarget(targetRequest))
+        validateMatrixDimensions(sink, irModule);
+
     // The resource-based specialization pass above
     // may create specialized versions of functions, but
     // it does not try to completely eliminate the original