Add a better error message for common global generic failure (#634)

A common mistake that seems to come up when using global generic type parameters:

```hlsl
interface IHero { ... }
type_param H : IHero;
ParameterBlock<H> gHero;
```

is to accidentally try to specialize the type parameter `H` using `H` itself as the argument (instead of some concrete type like `Batman`). The current front-end checks naively let this pass, because `H` satisfies all the requirements (it sure does declare that it implements `IHero`, which is the only requirement we have). This currently leads to downstream failure when we generate code with generic type parameters still left in the IR.

This change implements a simple fix which is to:

- Check when we are trying to specialize a global generic parameter using another global generic parameter, since this is currently always a mistake

- Add a special-case diagnostic for the 99% case of this failure, which is specializing a type parameter to itself

This fix is primarily motivated by the way generics support will initially be implemented in Falcor.

1 files changed, 36 insertions, 0 deletions

diff --git a/source/slang/check.cpp b/source/slang/check.cpp
index 42bcfe4b5..7f91ac0c3 100644
--- a/source/slang/check.cpp
+++ b/source/slang/check.cpp
@@ -8635,6 +8635,42 @@ namespace Slang
                 SLANG_ASSERT(argIndex < globalGenericArgs.Count());
                 auto globalGenericArg = globalGenericArgs[argIndex];
 
+                // As a quick sanity check, see if the argument that is being supplied for a parameter
+                // is just the parameter itself, because this should always be an error:
+                //
+                if( auto argDeclRefType = globalGenericArg->As<DeclRefType>() )
+                {
+                    auto argDeclRef = argDeclRefType->declRef;
+                    if(auto argGenericParamDeclRef = argDeclRef.As<GlobalGenericParamDecl>())
+                    {
+                        if(argGenericParamDeclRef.getDecl() == globalGenericParam)
+                        {
+                            // We are trying to specialize a generic parameter using itself.
+                            sink->diagnose(globalGenericParam,
+                                Diagnostics::cannotSpecializeGlobalGenericToItself,
+                                globalGenericParam->getName());
+                            sink->diagnose(entryPointFuncDecl,
+                                Diagnostics::noteWhenCompilingEntryPoint,
+                                entryPointFuncDecl->getName());
+                            continue;
+                        }
+                        else
+                        {
+                            // We are trying to specialize a generic parameter using a *different*
+                            // global generic type parameter.
+                            sink->diagnose(globalGenericParam,
+                                Diagnostics::cannotSpecializeGlobalGenericToAnotherGenericParam,
+                                globalGenericParam->getName(),
+                                argGenericParamDeclRef.GetName());
+                            sink->diagnose(entryPointFuncDecl,
+                                Diagnostics::noteWhenCompilingEntryPoint,
+                                entryPointFuncDecl->getName());
+                            continue;
+                        }
+                    }
+                }
+
+
                 // Create a substitution for this parameter/argument.
                 RefPtr<GlobalGenericParamSubstitution> subst = new GlobalGenericParamSubstitution();
                 subst->paramDecl = globalGenericParam;