Fix `sessionDesc.defaultMatrixLayoutMode` being ineffective. (#3753)

* Fix `sessionDesc.defaultMatrixLayoutMode` being ineffective.

* Fix matrix layout in buffer pointer.

* Attempt to fix.

* Fix buffer element type lowering for buffer pointers.

* Add comment.

* Fix test.

* Fix member lookup in `Ref<T>`.

* Fix validation error.

* Enhance test.

1 files changed, 84 insertions, 0 deletions

diff --git a/tools/slang-unit-test/unit-test-default-matrix-layout.cpp b/tools/slang-unit-test/unit-test-default-matrix-layout.cpp
new file mode 100644
index 000000000..468b7d986
--- /dev/null
+++ b/tools/slang-unit-test/unit-test-default-matrix-layout.cpp
@@ -0,0 +1,84 @@
+// unit-test-default-matrix-layout.cpp
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "tools/unit-test/slang-unit-test.h"
+
+#include "../../slang.h"
+#include "../../slang-com-helper.h"
+#include "../../slang-com-ptr.h"
+
+#include "../../source/core/slang-list.h"
+
+namespace {
+
+using namespace Slang;
+
+struct DefaultMatrixLayoutTestContext
+{
+    DefaultMatrixLayoutTestContext(UnitTestContext* context):
+        m_unitTestContext(context)
+    {
+        slang::IGlobalSession* slangSession = m_unitTestContext->slangGlobalSession;
+    }
+
+    SlangResult runTests()
+    {
+        slang::IGlobalSession* slangSession = m_unitTestContext->slangGlobalSession;
+        ComPtr<slang::ISession> session;
+        slang::SessionDesc sessionDesc{};
+        sessionDesc.targetCount = 1;
+        slang::TargetDesc targetDesc{};
+        targetDesc.format = SLANG_GLSL;
+        targetDesc.profile = slangSession->findProfile("glsl_460");
+        sessionDesc.targets = &targetDesc;
+        sessionDesc.defaultMatrixLayoutMode = SLANG_MATRIX_LAYOUT_COLUMN_MAJOR;
+        SLANG_RETURN_ON_FAIL(slangSession->createSession(sessionDesc, session.writeRef()));
+
+        auto module = session->loadModuleFromSourceString("mymodule", "mymodule.slang",
+            R"(
+            RWStructuredBuffer<float> output;
+            [numthreads(1,1,1)] [shader("compute")]
+            void main(uniform float3x4 m)
+            {
+                output[0] = m[0][0];
+            })");
+        if (!module)
+            return SLANG_FAIL;
+
+        ComPtr<slang::IEntryPoint> entryPoint;
+        SLANG_RETURN_ON_FAIL(module->findEntryPointByName("main", entryPoint.writeRef()));
+
+        if (!entryPoint)
+            return SLANG_FAIL;
+
+        slang::IComponentType* components[] = { module, entryPoint.get() };
+        ComPtr<slang::IComponentType> composedProgram;
+        SLANG_RETURN_ON_FAIL(session->createCompositeComponentType(components, 2, composedProgram.writeRef()));
+
+        ComPtr<slang::IComponentType> linkedProgram;
+        SLANG_RETURN_ON_FAIL(composedProgram->link(linkedProgram.writeRef()));
+
+        ComPtr<slang::IBlob> outCode;
+        SLANG_RETURN_ON_FAIL(linkedProgram->getEntryPointCode(0, 0, outCode.writeRef()));
+
+        const char* code = (const char*)outCode->getBufferPointer();
+        if (strstr(code, "row_major") != nullptr)
+            return SLANG_OK;
+        return SLANG_FAIL;
+    }
+
+    UnitTestContext* m_unitTestContext;
+};
+
+} // anonymous
+
+SLANG_UNIT_TEST(defaultMatrixLayout)
+{
+    DefaultMatrixLayoutTestContext context(unitTestContext);
+
+    const auto result = context.runTests();
+    
+    SLANG_CHECK(SLANG_SUCCEEDED(result));
+}