Strip IR after front-end steps are done (#1092)

* Strip IR after front-end steps are done

The main feature of this change is to unconditonally strip out the `IRHighLevelDeclDecoration`s in an IR module once the "mandatory" IR passes in the front end have run. This ensures that later IR passes (e.g., code emission) *cannot* rely on AST-level information to get their job done.

Since I was already writing a pass to remove some instructions at the end of the front-end passes, I went ahead and also made the `-obfuscate` flag apply to the front-end IR generation by causing it to strip `IRNameHintDecoration`s while it is doing the other stripping. With this, the main identifying information left in IR modules (other than semantics and entry-point names) is mangled name strings for imported/exported symbols.

A few other things got changes along the way:

* Removed the `.expected` file for one of the tests, where that file seemingly shouldn't have been checked in at all.

* Updated the signature of the DCE pass both so that it doesn't require a back-end compile request (it wasn't using it anyway), and so that it takes some options to decide whether to keep symbols marked `[export(...)]` alive (the front-end wants to keep these, while back-end passes currently need to be able to eliminate them).

* Moved the `obfuscateCode` flag from the back-end compile request to the base class shared between front- and back-end requests, and updated the options and repro logic to set both as needed. An obvious improvement in the future would be to have the front- and back-end requests share these settings by referencing a single common object in the end-to-end case, rather than each having their own copy.

* Removed logic that was keeping layout instructions alive in DCE, even if they weren't used. This seems to have been a vestige of an intermediate step between AST and IR layout.

* fixup: add the new files

13 files changed, 162 insertions, 105 deletions

diff --git a/source/slang/slang-compiler.h b/source/slang/slang-compiler.h
index 1e1f166de..a5a1b7d93 100644
--- a/source/slang/slang-compiler.h
+++ b/source/slang/slang-compiler.h
@@ -1331,6 +1331,10 @@ namespace Slang
         bool shouldDumpIR = false;
         bool shouldValidateIR = false;
 
+        // Remove name hints to help obfuscate code
+        //
+        bool obfuscateCode = false;
+
     protected:
         CompileRequestBase(
             Linkage*        linkage,
@@ -1671,9 +1675,6 @@ namespace Slang
         //
         bool useUnknownImageFormatAsDefault = false;
 
-        // Remove name hints to help obfuscate code
-        //
-        bool obfuscateCode = false;
     private:
         RefPtr<ComponentType> m_program;
     };
diff --git a/source/slang/slang-emit.cpp b/source/slang/slang-emit.cpp
index f60febe0b..280822094 100644
--- a/source/slang/slang-emit.cpp
+++ b/source/slang/slang-emit.cpp
@@ -339,7 +339,7 @@ String emitEntryPoint(
         // TODO: Are there other cleanup optimizations we should
         // apply at this point?
         //
-        eliminateDeadCode(compileRequest, irModule);
+        eliminateDeadCode(irModule);
 #if 0
         dumpIRIfEnabled(compileRequest, irModule, "AFTER DCE");
 #endif
@@ -376,7 +376,7 @@ String emitEntryPoint(
             legalizeExistentialTypeLayout(
                 irModule,
                 sink);
-            eliminateDeadCode(compileRequest, irModule);
+            eliminateDeadCode(irModule);
         
 #if 0
             dumpIRIfEnabled(compileRequest, irModule, "EXISTENTIALS LEGALIZED");
@@ -397,7 +397,7 @@ String emitEntryPoint(
             legalizeResourceTypes(
                 irModule,
                 sink);
-            eliminateDeadCode(compileRequest, irModule);
+            eliminateDeadCode(irModule);
         }
 
         //  Debugging output of legalization
@@ -482,7 +482,7 @@ String emitEntryPoint(
         // dead-code-elimination (DCE) pass that only retains
         // whatever code is "live."
         //
-        eliminateDeadCode(compileRequest, irModule);
+        eliminateDeadCode(irModule);
 #if 0
         dumpIRIfEnabled(compileRequest, irModule, "AFTER DCE");
 #endif
diff --git a/source/slang/slang-ir-dce.cpp b/source/slang/slang-ir-dce.cpp
index 4d58947d4..4359e7e92 100644
--- a/source/slang/slang-ir-dce.cpp
+++ b/source/slang/slang-ir-dce.cpp
@@ -16,8 +16,8 @@ struct DeadCodeEliminationContext
     // the parameters that were passed to the top-level
     // `eliminateDeadCode` function.
     //
-    BackEndCompileRequest*  compileRequest;
-    IRModule*               module;
+    IRModule*                       module;
+    IRDeadCodeEliminationOptions    options;
 
     // Our overall process is going to be to determine
     // which instructions in the module are "live"
@@ -222,11 +222,6 @@ struct DeadCodeEliminationContext
         //
         if(inst->mightHaveSideEffects())
             return true;
-
-        // If it's a layout instruction we don't want to remove it
-        if (as<IRLayout>(inst))
-            return true;
-
         //
         // The `mightHaveSideEffects` query is conservative, and will
         // return `true` as its default mode, so once we are past that
@@ -245,10 +240,21 @@ struct DeadCodeEliminationContext
         if(inst->findDecorationImpl(kIROp_KeepAliveDecoration))
             return true;
         //
-        // TODO: Eventually it would make sense to consider everything
-        // with an `[export(...)]` decoration as live, but our current
-        // approach to linking for back-end compilation leaves many
-        // linkage decorations in place that we seemingly don't need/want.
+        // We also consider anything with an `[export(...)]` as live,
+        // when the appropriate option has been set.
+        //
+        // Note: our current approach to linking for back-end compilation
+        // leaves many linakge decorations in place that we seemingly
+        // don't need/want, so this option currently can't be enabled
+        // unconditionally.
+        //
+        if( options.keepExportsAlive )
+        {
+            if( inst->findDecoration<IRExportDecoration>() )
+            {
+                return true;
+            }
+        }
 
         // A basic block is an interesting case. Knowing that a function
         // is live means that its entry block is live, but the liveness
@@ -317,12 +323,12 @@ struct DeadCodeEliminationContext
 // and then defer to it for the real work.
 //
 void eliminateDeadCode(
-    BackEndCompileRequest* compileRequest,
-    IRModule*       module)
+    IRModule*                           module,
+    IRDeadCodeEliminationOptions const& options)
 {
     DeadCodeEliminationContext context;
-    context.compileRequest = compileRequest;
     context.module = module;
+    context.options = options;
 
     context.processModule();
 }
diff --git a/source/slang/slang-ir-dce.h b/source/slang/slang-ir-dce.h
index b568d9883..6d4ae9d08 100644
--- a/source/slang/slang-ir-dce.h
+++ b/source/slang/slang-ir-dce.h
@@ -3,9 +3,13 @@
 
 namespace Slang
 {
-    class BackEndCompileRequest;
     struct IRModule;
 
+    struct IRDeadCodeEliminationOptions
+    {
+        bool keepExportsAlive = false;
+    };
+
         /// Eliminate "dead" code from the given IR module.
         ///
         /// This pass is primarily designed for flow-insensitive
@@ -14,6 +18,6 @@ namespace Slang
         /// etc.
         ///
     void eliminateDeadCode(
-        BackEndCompileRequest*  compileRequest,
-        IRModule*               module);
+        IRModule*                           module,
+        IRDeadCodeEliminationOptions const& options = IRDeadCodeEliminationOptions());
 }
diff --git a/source/slang/slang-ir-strip.cpp b/source/slang/slang-ir-strip.cpp
new file mode 100644
index 000000000..1cf8267d1
--- /dev/null
+++ b/source/slang/slang-ir-strip.cpp
@@ -0,0 +1,54 @@
+// slang-ir-strip.cpp
+#include "slang-ir-strip.h"
+
+#include "slang-ir.h"
+#include "slang-ir-insts.h"
+
+namespace Slang {
+
+    /// Should `inst` be stripped, given the current `options`?
+static bool _shouldStripInst(
+    IRInst*                 inst,
+    IRStripOptions const&   options)
+{
+    switch( inst->op )
+    {
+    default:
+        return false;
+
+    case kIROp_HighLevelDeclDecoration:
+        return true;
+
+    case kIROp_NameHintDecoration:
+        return options.shouldStripNameHints;
+    }
+}
+
+    /// Recursively strip `inst` and its children according to `options`.
+static void _stripFrontEndOnlyInstructionsRec(
+    IRInst*                 inst,
+    IRStripOptions const&   options)
+{
+    if( _shouldStripInst(inst, options) )
+    {
+        inst->removeAndDeallocate();
+        return;
+    }
+
+    IRInst* nextChild = nullptr;
+    for( IRInst* child = inst->getFirstDecorationOrChild(); child; child = nextChild )
+    {
+        nextChild = child->getNextInst();
+
+        _stripFrontEndOnlyInstructionsRec(child, options);
+    }
+}
+
+void stripFrontEndOnlyInstructions(
+    IRModule*               module,
+    IRStripOptions const&   options)
+{
+    _stripFrontEndOnlyInstructionsRec(module->getModuleInst(), options);
+}
+
+}
diff --git a/source/slang/slang-ir-strip.h b/source/slang/slang-ir-strip.h
new file mode 100644
index 000000000..50f10ba7d
--- /dev/null
+++ b/source/slang/slang-ir-strip.h
@@ -0,0 +1,18 @@
+// slang-ir-strip.h
+#pragma once
+
+namespace Slang
+{
+    struct IRModule;
+
+    struct IRStripOptions
+    {
+        bool shouldStripNameHints = false;
+    };
+
+        /// Strip out instructions that should only be used by the front-end.
+    void stripFrontEndOnlyInstructions(
+        IRModule*               module,
+        IRStripOptions const&   options);
+}
+#pragma once
diff --git a/source/slang/slang-ir.cpp b/source/slang/slang-ir.cpp
index 89e49816f..31224fde2 100644
--- a/source/slang/slang-ir.cpp
+++ b/source/slang/slang-ir.cpp
@@ -4040,26 +4040,6 @@ namespace Slang
     {
         for(auto dd : inst->getDecorations())
         {
-            // Certain decorations aren't helpful to appear
-            // in output dumps, so we will only include them
-            // in the "detailed" dumping mode.
-            //
-            // For all other modes, we will check the opcode
-            // and skip selected decorations.
-            //
-            if(context->mode != IRDumpMode::Detailed)
-            {
-                switch(dd->op)
-                {
-                default:
-                    break;
-
-                case kIROp_HighLevelDeclDecoration:
-                case kIROp_LayoutDecoration:
-                    continue;
-                }
-            }
-
             dump(context, "[");
             dumpInstBody(context, dd);
             dump(context, "]\n");
diff --git a/source/slang/slang-lower-to-ir.cpp b/source/slang/slang-lower-to-ir.cpp
index 113dcbfad..eabe08de3 100644
--- a/source/slang/slang-lower-to-ir.cpp
+++ b/source/slang/slang-lower-to-ir.cpp
@@ -6,10 +6,12 @@
 #include "slang-check.h"
 #include "slang-ir.h"
 #include "slang-ir-constexpr.h"
+#include "slang-ir-dce.h"
 #include "slang-ir-insts.h"
 #include "slang-ir-missing-return.h"
 #include "slang-ir-sccp.h"
 #include "slang-ir-ssa.h"
+#include "slang-ir-strip.h"
 #include "slang-ir-validate.h"
 #include "slang-mangle.h"
 #include "slang-type-layout.h"
@@ -6553,6 +6555,46 @@ IRModule* generateIRForTranslationUnit(
 
     checkForMissingReturns(module, compileRequest->getSink());
 
+    // The "mandatory" optimization passes may make use of the
+    // `IRHighLevelDeclDecoration` type to relate IR instructions
+    // back to AST-level code in order to improve the quality
+    // of diagnostics that are emitted.
+    //
+    // While it is important for these passes to have access
+    // to AST-level information, allowing that information to
+    // flow into later steps (e.g., code generation) could lead
+    // to unclean layering of the parts of the compiler.
+    // In principle, back-end steps should not need to know where
+    // IR code came from.
+    //
+    // In order to avoid problems, we run a pass here to strip
+    // out any decorations that should not be relied upon by
+    // later passes.
+    //
+    {
+        // Because we are already stripping out undesired decorations,
+        // this is also a convenient place to remove any `IRNameHintDecoration`s
+        // in the case where we are obfuscating code. We handle this
+        // by setting up the options for the stripping pass appropriately.
+        //
+        IRStripOptions stripOptions;
+        stripOptions.shouldStripNameHints = compileRequest->obfuscateCode;
+
+        stripFrontEndOnlyInstructions(module, stripOptions);
+
+        // Stripping out decorations could leave some dead code behind
+        // in the module, and in some cases that extra code is also
+        // undesirable (e.g., the string literals referenced by name-hint
+        // decorations are just as undesirable as the decorations themselves).
+        // To clean up after ourselves we also run a dead-code elimination
+        // pass here, but make sure to set our options so that we don't
+        // eliminate anything that has been marked for export.
+        //
+        IRDeadCodeEliminationOptions options;
+        options.keepExportsAlive = true;
+        eliminateDeadCode(module, options);
+    }
+
     // TODO: consider doing some more aggressive optimizations
     // (in particular specialization of generics) here, so
     // that we can avoid doing them downstream.
@@ -6564,7 +6606,7 @@ IRModule* generateIRForTranslationUnit(
 
     validateIRModuleIfEnabled(compileRequest, module);
 
-    // If we are being sked to dump IR during compilation,
+    // If we are being asked to dump IR during compilation,
     // then we can dump the initial IR for the module here.
     if(compileRequest->shouldDumpIR)
     {
diff --git a/source/slang/slang-options.cpp b/source/slang/slang-options.cpp
index 6b09179db..3e2bec07d 100644
--- a/source/slang/slang-options.cpp
+++ b/source/slang/slang-options.cpp
@@ -823,6 +823,7 @@ struct OptionsParser
                 }
                 else if (argStr == "-obfuscate")
                 {
+                    requestImpl->getFrontEndReq()->obfuscateCode = true;
                     requestImpl->getBackEndReq()->obfuscateCode = true;
                 }
                 else if (argStr == "-file-system")
diff --git a/source/slang/slang-state-serialize.cpp b/source/slang/slang-state-serialize.cpp
index 13dd8b132..f7e472643 100644
--- a/source/slang/slang-state-serialize.cpp
+++ b/source/slang/slang-state-serialize.cpp
@@ -763,6 +763,8 @@ static void _loadDefines(const Relative32Array<StateSerializeUtil::StringPair>&
         request->getBackEndReq()->useUnknownImageFormatAsDefault = requestState->useUnknownImageFormatAsDefault;
         request->getBackEndReq()->obfuscateCode = requestState->obfuscateCode;
 
+        request->getFrontEndReq()->obfuscateCode = requestState->obfuscateCode;
+
         linkage->setMatrixLayoutMode(requestState->defaultMatrixLayoutMode);
     }
 
diff --git a/source/slang/slang.vcxproj b/source/slang/slang.vcxproj
index 36f28705f..cedbde34f 100644
--- a/source/slang/slang.vcxproj
+++ b/source/slang/slang.vcxproj
@@ -208,6 +208,7 @@
     <ClInclude Include="slang-ir-specialize-resources.h" />
     <ClInclude Include="slang-ir-specialize.h" />
     <ClInclude Include="slang-ir-ssa.h" />
+    <ClInclude Include="slang-ir-strip.h" />
     <ClInclude Include="slang-ir-union.h" />
     <ClInclude Include="slang-ir-validate.h" />
     <ClInclude Include="slang-ir.h" />
@@ -273,6 +274,7 @@
     <ClCompile Include="slang-ir-specialize-resources.cpp" />
     <ClCompile Include="slang-ir-specialize.cpp" />
     <ClCompile Include="slang-ir-ssa.cpp" />
+    <ClCompile Include="slang-ir-strip.cpp" />
     <ClCompile Include="slang-ir-union.cpp" />
     <ClCompile Include="slang-ir-validate.cpp" />
     <ClCompile Include="slang-ir.cpp" />
diff --git a/source/slang/slang.vcxproj.filters b/source/slang/slang.vcxproj.filters
index 474f4c2bb..7b4be61e8 100644
--- a/source/slang/slang.vcxproj.filters
+++ b/source/slang/slang.vcxproj.filters
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <ItemGroup>
     <Filter Include="Header Files">
@@ -222,6 +222,9 @@
     <ClInclude Include="slang-visitor.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="slang-ir-strip.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="slang-check.cpp">
@@ -386,11 +389,9 @@
     <ClCompile Include="slang.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
-  </ItemGroup>
-  <ItemGroup>
-    <None Include="slang.natvis">
+    <ClCompile Include="slang-ir-strip.cpp">
       <Filter>Source Files</Filter>
-    </None>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <CustomBuild Include="core.meta.slang">
@@ -404,5 +405,6 @@
     <Natvis Include="..\core\core.natvis">
       <Filter>Source Files</Filter>
     </Natvis>
+    <Natvis Include="slang.natvis" />
   </ItemGroup>
 </Project>
\ No newline at end of file
diff --git a/tests/bugs/texture2d-ms.hlsl.expected b/tests/bugs/texture2d-ms.hlsl.expected
deleted file mode 100644
index 2202bbbec..000000000
--- a/tests/bugs/texture2d-ms.hlsl.expected
+++ /dev/null
@@ -1,55 +0,0 @@
-result code = 0
-standard error = {
-}
-standard output = {
-// Module Version 10000
-// Generated by (magic number): 80007
-// Id's are bound by 30
-
-                              Capability Shader
-               1:             ExtInstImport  "GLSL.std.450"
-                              MemoryModel Logical GLSL450
-                              EntryPoint GLCompute 4  "main" 17
-                              ExecutionMode 4 LocalSize 4 4 1
-                              Source GLSL 450
-                              SourceExtension  "GL_EXT_samplerless_texture_functions"
-                              SourceExtension  "GL_GOOGLE_cpp_style_line_directive"
-                              Name 4  "main"
-                              Name 9  "_S1"
-                              Name 12  "tex_0"
-                              Name 17  "gl_WorkGroupID"
-                              Decorate 12(tex_0) DescriptorSet 0
-                              Decorate 12(tex_0) Binding 0
-                              Decorate 17(gl_WorkGroupID) BuiltIn WorkgroupId
-                              Decorate 29 BuiltIn WorkgroupSize
-               2:             TypeVoid
-               3:             TypeFunction 2
-               6:             TypeFloat 32
-               7:             TypeVector 6(float) 4
-               8:             TypePointer Function 7(fvec4)
-              10:             TypeImage 6(float) 2D multi-sampled sampled format:Unknown
-              11:             TypePointer UniformConstant 10
-       12(tex_0):     11(ptr) Variable UniformConstant
-              14:             TypeInt 32 0
-              15:             TypeVector 14(int) 3
-              16:             TypePointer Input 15(ivec3)
-17(gl_WorkGroupID):     16(ptr) Variable Input
-              18:             TypeVector 14(int) 2
-              21:             TypeInt 32 1
-              22:             TypeVector 21(int) 2
-              24:     21(int) Constant 0
-              27:     14(int) Constant 4
-              28:     14(int) Constant 1
-              29:   15(ivec3) ConstantComposite 27 27 28
-         4(main):           2 Function None 3
-               5:             Label
-          9(_S1):      8(ptr) Variable Function
-              13:          10 Load 12(tex_0)
-              19:   15(ivec3) Load 17(gl_WorkGroupID)
-              20:   18(ivec2) VectorShuffle 19 19 0 1
-              23:   22(ivec2) Bitcast 20
-              25:    7(fvec4) ImageFetch 13 23 Sample 24
-                              Store 9(_S1) 25
-                              Return
-                              FunctionEnd
-}