Fix the issue of name mangling (#4691)

* Fix the issue of name mangle

During our name mangling, we should add the direction of the parameter
in the name, otherwise it could have the name collision which will
result in invalid code generation:

e.g.

// in slang-module.slang
export func(float a) { ...}

// in test.slang
extern func(inout float a);

when we compile test.slang, slang will pass a pointer type to the
'func', however, in the slang-module.slang, `func` expects a value
instead of pointer. This will lead the wrong spirv code.

So we should add the parameter direction into the mangle name such
that above two symbols will have the different mangled names, and
we will catch this during IR-link stage.

* Change to use  to get param direction

* Address few comments

3 files changed, 111 insertions, 0 deletions

diff --git a/source/slang/slang-mangle.cpp b/source/slang/slang-mangle.cpp
index ad1a9caca..779a6f090 100644
--- a/source/slang/slang-mangle.cpp
+++ b/source/slang/slang-mangle.cpp
@@ -547,6 +547,33 @@ namespace Slang
 
             for(auto paramDeclRef : parameters)
             {
+                // parameter modifier makes big difference in the spirv code generation, for example
+                // "out"/"inout" parameter will be passed by pointer. Therefore, we need to
+                // distinguish them in the mangled name to avoid name collision.
+                ParameterDirection paramDirection = getParameterDirection(paramDeclRef.getDecl());
+                switch (paramDirection)
+                {
+                case kParameterDirection_Ref:
+                    emitRaw(context, "r_");
+                    break;
+                case kParameterDirection_ConstRef:
+                    emitRaw(context, "c_");
+                    break;
+                case kParameterDirection_Out:
+                    emitRaw(context, "o_");
+                    break;
+                case kParameterDirection_InOut:
+                    emitRaw(context, "io_");
+                    break;
+                case kParameterDirection_In:
+                    emitRaw(context, "i_");
+                    break;
+                default:
+                    StringBuilder errMsg;
+                    errMsg << "Unknown parameter direction: " << paramDirection;
+                    SLANG_ABORT_COMPILATION(errMsg.toString().begin());
+                    break;
+                }
                 emitType(context, getType(context->astBuilder, paramDeclRef));
             }
 
diff --git a/tests/diagnostics/illegal-func-decl-module.slang b/tests/diagnostics/illegal-func-decl-module.slang
new file mode 100644
index 000000000..97b66d129
--- /dev/null
+++ b/tests/diagnostics/illegal-func-decl-module.slang
@@ -0,0 +1,26 @@
+//TEST_IGNORE_FILE:
+
+export float libraryFunction(float a)
+{
+    a = a + a;
+    return a;
+}
+
+export float libraryFunction1(inout float a)
+{
+    a = a * a;
+    return a;
+}
+
+export float libraryFunction2(inout float a, in float b, out float c)
+{
+    a = a * a;
+    a = a + b;
+    return a;
+}
+
+export float libraryFunction3(in float a)
+{
+    a = a + a;
+    return a;
+}
diff --git a/tests/diagnostics/illegal-func-decl.slang b/tests/diagnostics/illegal-func-decl.slang
new file mode 100644
index 000000000..0ec73dc27
--- /dev/null
+++ b/tests/diagnostics/illegal-func-decl.slang
@@ -0,0 +1,58 @@
+// illegal-func-decl.slang
+
+// This test checks that the in/out/inout modifiers in function declarations must
+// be consistent with the function's definition, and slang can diagnose the inconsistency.
+
+//TEST:COMPILE: tests/diagnostics/illegal-func-decl-module.slang -o tests/diagnostics/illegal-func-decl-module.slang-module
+
+//DIAGNOSTIC_TEST:SIMPLE(filecheck=CHECK1): -r tests/diagnostics/illegal-func-decl-module.slang-module -DTEST1 -target spirv -o illegal-func-decl.spv
+//DIAGNOSTIC_TEST:SIMPLE(filecheck=CHECK2): -r tests/diagnostics/illegal-func-decl-module.slang-module -DTEST2 -target spirv -o illegal-func-decl.spv
+//DIAGNOSTIC_TEST:SIMPLE(filecheck=CHECK3): -r tests/diagnostics/illegal-func-decl-module.slang-module -DTEST3 -target spirv -o illegal-func-decl.spv
+//DIAGNOSTIC_TEST:SIMPLE(filecheck=CHECK4): -r tests/diagnostics/illegal-func-decl-module.slang-module -DTEST4 -target spirv -o illegal-func-decl.spv
+
+#ifdef TEST1
+// CHECK1: ([[# @LINE+1]]): error 45001: unresolved external symbol 'libraryFunction'.
+extern float libraryFunction(inout float a);    // invalid: 'a' is 'in'
+#endif
+
+#ifdef TEST2
+// CHECK2-NOT: ([[# @LINE+1]]): error 45001: unresolved external symbol 'libraryFunction1'.
+extern float libraryFunction1(inout float b);   // valid
+#endif
+
+#ifdef TEST3
+// CHECK3: ([[# @LINE+1]]): error 45001: unresolved external symbol 'libraryFunction2'.
+extern float libraryFunction2(inout float a, in float b, float c);   // valid: 'c' is 'inout'
+#endif
+
+#ifdef TEST4
+// CHECK4-NOT: ([[# @LINE+1]]): error 45001: unresolved external symbol 'libraryFunction3'.
+export float libraryFunction3(float a);         // valid: 'in' is the default is not specified
+#endif
+
+[shader("compute")]
+[numthreads(1, 1, 1)]
+void main(out float4 col : SV_Target0, bool isFrontHit)
+{
+    float a = 5;
+    float b = 7;
+    float c = 7;
+
+#ifdef TEST1
+    col.x = libraryFunction(a);
+#endif
+
+#ifdef TEST2
+    col.y = libraryFunction1(b);
+#endif
+
+#ifdef TEST3
+    col.z = libraryFunction2(a, b, c);
+#endif
+
+#ifdef TEST4
+    col.w = libraryFunction3(a);
+#endif
+}
+
+