Fix `sessionDesc.defaultMatrixLayoutMode` being ineffective. (#3753)

* Fix `sessionDesc.defaultMatrixLayoutMode` being ineffective.

* Fix matrix layout in buffer pointer.

* Attempt to fix.

* Fix buffer element type lowering for buffer pointers.

* Add comment.

* Fix test.

* Fix member lookup in `Ref<T>`.

* Fix validation error.

* Enhance test.

4 files changed, 55 insertions, 1 deletions

diff --git a/tests/front-end/typedef-matrix.slang b/tests/front-end/typedef-matrix.slang
new file mode 100644
index 000000000..81188f806
--- /dev/null
+++ b/tests/front-end/typedef-matrix.slang
@@ -0,0 +1,13 @@
+//TEST:SIMPLE(filecheck=CHECK): -target spirv -emit-spirv-directly -entry main -stage compute -matrix-layout-row-major
+
+// CHECK: ColMajor
+
+typedef column_major float3x4 Mat;
+
+RWStructuredBuffer<float> output;
+
+[numthreads(1,1,1)]
+void main(uniform Mat m)
+{
+    output[0] = m[0][0];
+}
\ No newline at end of file
diff --git a/tests/spirv/buffer-pointer-matrix-layout.slang b/tests/spirv/buffer-pointer-matrix-layout.slang
new file mode 100644
index 000000000..2ccde7b71
--- /dev/null
+++ b/tests/spirv/buffer-pointer-matrix-layout.slang
@@ -0,0 +1,34 @@
+//TEST:SIMPLE(filecheck=CHECK): -target spirv -emit-spirv-directly -stage compute -entry main -matrix-layout-column-major
+
+// CHECK: OpLoad %_MatrixStorage_float3x4_ColMajornatural {{.*}} Aligned 4
+// CHECK: OpLoad %_MatrixStorage_float3x4_ColMajornatural {{.*}} Aligned 4
+
+struct Push
+{
+    float3x4* ptr;
+};
+
+[[vk::push_constant]] Push push;
+[shader("compute")]
+[numthreads(1, 1, 1)]
+void main(uint3 dtid : SV_DispatchThreadID)
+{    
+    // This matrix is in memry column major. Slang respects this here and load it properly!
+    float3x4 correctly_read_matrix = *push.ptr;
+    printf("(%f,%f,%f,%f)\n(%f,%f,%f,%f)\n",
+        correctly_read_matrix[0][0], correctly_read_matrix[0][1], correctly_read_matrix[0][2], correctly_read_matrix[0][3],
+        correctly_read_matrix[1][0], correctly_read_matrix[1][1], correctly_read_matrix[1][2], correctly_read_matrix[1][3]
+    );
+    printf("(%f,%f,%f,%f)\n\n",
+        correctly_read_matrix[2][0], correctly_read_matrix[2][1], correctly_read_matrix[2][2], correctly_read_matrix[2][3]
+    );
+    // With this syntax however, Slang ignores the column major setting and loads it as it it was row major!
+    float3x4 broken_matrix = push.ptr[0];
+    printf("(%f,%f,%f,%f)\n(%f,%f,%f,%f)\n",
+        broken_matrix[0][0], broken_matrix[0][1], broken_matrix[0][2], broken_matrix[0][3],
+        broken_matrix[1][0], broken_matrix[1][1], broken_matrix[1][2], broken_matrix[1][3]
+    );
+    printf("(%f,%f,%f,%f)\n\n",
+        broken_matrix[2][0], broken_matrix[2][1], broken_matrix[2][2], broken_matrix[2][3]
+    );
+}
\ No newline at end of file
diff --git a/tests/spirv/pointer-bug.slang b/tests/spirv/pointer-bug.slang
index 1668cec13..404da286f 100644
--- a/tests/spirv/pointer-bug.slang
+++ b/tests/spirv/pointer-bug.slang
@@ -7,7 +7,7 @@ struct Params {
   Foo *foo;
 };
 
-// CHECK: %_ptr_PhysicalStorageBuffer_Foo = OpTypePointer PhysicalStorageBuffer %Foo
+// CHECK: OpTypePointer PushConstant %_ptr_PhysicalStorageBuffer_Foo_natural
 
 [[vk::push_constant]] Params params;
 
diff --git a/tests/spirv/pointer.slang b/tests/spirv/pointer.slang
index cb2d56f66..cd4845e4d 100644
--- a/tests/spirv/pointer.slang
+++ b/tests/spirv/pointer.slang
@@ -21,6 +21,11 @@ int* funcThatReturnsPointer(PP* p)
     return &p.data;
 }
 
+void funcWithInOutParam(inout PP p)
+{
+    p.data = 0;
+}
+
 // CHECK: OpEntryPoint
 
 StructuredBuffer<Data> buffer;
@@ -44,5 +49,7 @@ void main(int id : SV_DispatchThreadID)
     if (pData1 > pData)
     {
         funcThatTakesPointer(buffer[0].pNext);
+        output[1] = (*buffer[0].pNext).data;
     }
+    funcWithInOutParam(*buffer[0].pNext);
 }