From 6fc2c37d9a4c408331db1b33deb3b46e74d2a7d2 Mon Sep 17 00:00:00 2001 From: jsmall-nvidia Date: Mon, 12 Aug 2019 15:41:41 -0400 Subject: Callable CPU code support (#1014) * First pass support for compiling to a loaded shared library. * Improve documentation for cpu target. * Removed the SLANG_COMPILE_FLAG_LOAD_SHARED_LIBRARY flag. Use the SLANG_HOST_CALLABLE code target Document mechanism. * Fix typo in cpp-resource.slang In test code if the target is 'callable' we don't need to compile (indeed there is no source file). * Small refactor using CommandLineCPPCompiler as base class to implement VisualStudioCPPCompiler and GCCCPPCompiler. * Improvements around CPPCompiler. Mechanism to know products produced. Cleaning up products after execution. * Fix multiple definition of 'SourceType' --- source/slang/slang-compiler.cpp | 143 ++++++++++++++++++++----- source/slang/slang-compiler.h | 11 +- source/slang/slang-ir-entry-point-uniforms.cpp | 1 + source/slang/slang-parameter-binding.cpp | 1 + source/slang/slang-type-layout.cpp | 2 +- source/slang/slang.cpp | 61 ++++++++--- 6 files changed, 176 insertions(+), 43 deletions(-) (limited to 'source/slang') diff --git a/source/slang/slang-compiler.cpp b/source/slang/slang-compiler.cpp index 1c0bed065..8ead63784 100644 --- a/source/slang/slang-compiler.cpp +++ b/source/slang/slang-compiler.cpp @@ -87,7 +87,8 @@ namespace Slang x("c", CSource) \ x("cpp", CPPSource) \ x("exe,executable", Executable) \ - x("sharedlib,sharedlibrary,dll", SharedLibrary) + x("sharedlib,sharedlibrary,dll", SharedLibrary) \ + x("callable,host-callable", HostCallable) #define SLANG_CODE_GEN_INFO(names, e) \ { CodeGenTarget::e, UnownedStringSlice::fromLiteral(names) }, @@ -149,28 +150,64 @@ namespace Slang { outputBinary.addRange(result.outputBinary.getBuffer(), result.outputBinary.getCount()); } + else if (appendTo == ResultFormat::SharedLibrary) + { + SLANG_ASSERT(!"Trying to append to a shared library"); + } + } + + SlangResult CompileResult::getSharedLibrary(ComPtr& outSharedLibrary) + { + if (format == ResultFormat::SharedLibrary && sharedLibrary) + { + outSharedLibrary = sharedLibrary; + return SLANG_OK; + } + return SLANG_FAIL; } - ComPtr CompileResult::getBlob() + SlangResult CompileResult::getBlob(ComPtr& outBlob) { if(!blob) { switch(format) { - case ResultFormat::None: - default: - break; + case ResultFormat::None: + default: + break; - case ResultFormat::Text: - blob = StringUtil::createStringBlob(outputString); - break; + case ResultFormat::Text: + blob = StringUtil::createStringBlob(outputString); + break; - case ResultFormat::Binary: - blob = createRawBlob(outputBinary.getBuffer(), outputBinary.getCount()); - break; + case ResultFormat::Binary: + blob = createRawBlob(outputBinary.getBuffer(), outputBinary.getCount()); + break; + case ResultFormat::SharedLibrary: + { + // TODO(JS): Could do something more sophisticated to check this cast is safe (like have an interface to get the path), but this + // is simple and works with current impl + TemporarySharedLibrary* tempLibrary = static_cast(sharedLibrary.get()); + + SLANG_ASSERT(sharedLibrary); + List contents; + try + { + // We can read it from the shared library... + contents = File::readAllBytes(tempLibrary->getPath()); + } + catch (const Slang::IOException&) + { + return SLANG_E_CANNOT_OPEN; + } + blob = createRawBlob(contents.getBuffer(), contents.getCount()); + break; + } } } - return blob; + + outBlob = blob; + return SLANG_OK; } // @@ -478,6 +515,7 @@ namespace Slang // Don't need an external compiler to output C and C++ code return PassThroughMode::None; } + case CodeGenTarget::HostCallable: case CodeGenTarget::SharedLibrary: case CodeGenTarget::Executable: { @@ -1103,13 +1141,15 @@ SlangResult dissassembleDXILUsingDXC( Int entryPointIndex, TargetRequest* targetReq, EndToEndCompileRequest* endToEndReq, - List& binOut) + ComPtr& outSharedLib, + List& outBin) { auto sink = slangRequest->getSink(); const String originalSourcePath = calcSourcePathForEntryPoint(endToEndReq, entryPointIndex); - binOut.clear(); + outBin.clear(); + outSharedLib.setNull(); CPPCompilerSet* compilerSet = slangRequest->getSession()->requireCPPCompilerSet(); @@ -1311,6 +1351,7 @@ SlangResult dissassembleDXILUsingDXC( // Set what kind of target we should build switch (targetReq->target) { + case CodeGenTarget::HostCallable: case CodeGenTarget::SharedLibrary: { options.targetType = CPPCompiler::TargetType::SharedLibrary; @@ -1332,8 +1373,13 @@ SlangResult dissassembleDXILUsingDXC( moduleFilePath = builder.ProduceString(); } - // Add so it's deleted at the end - temporaryFileSet.add(moduleFilePath); + // Find all the files that will be produced + TemporaryFileSet productFileSet; + { + List paths; + SLANG_RETURN_ON_FAIL(compiler->calcCompileProducts(options, CPPCompiler::ProductFlag::All, paths)); + productFileSet.add(paths); + } // Need to configure for the compilation @@ -1457,16 +1503,48 @@ SlangResult dissassembleDXILUsingDXC( return SLANG_FAIL; } - // Read the binary - try + // If callable we need to load the shared library + if (targetReq->target == CodeGenTarget::HostCallable) { - // Read the contents of the binary - binOut = File::readAllBytes(moduleFilePath); + // Try loading the shared library + SharedLibrary::Handle handle; + if (SLANG_FAILED(SharedLibrary::loadWithPlatformPath(moduleFilePath.getBuffer(), handle))) + { + sink->diagnose(SourceLoc(), Diagnostics::unableToReadFile, moduleFilePath); + return SLANG_FAIL; + } + RefPtr sharedLib(new TemporarySharedLibrary(handle, moduleFilePath)); + sharedLib->m_temporaryFileSet = productFileSet; + productFileSet.clear(); } - catch (const Slang::IOException&) + else { - sink->diagnose(SourceLoc(), Diagnostics::unableToReadFile, moduleFilePath); - return SLANG_FAIL; + // Read the binary + try + { + // TODO(JS): We have a problem here.. productFileSet will clear up all temporaries + // and although we return the binary here (through outBin), we don't return debug info + // which is separate (say with a pdb). To work around this we reevaluate productFileSet, + // so we don't include debug info. The executable will presumably be reconstructed from + // outBin + // The problem is that these files have no specific lifetime (unlike with HostCallable). + + CPPCompiler::ProductFlags flags = CPPCompiler::ProductFlag::All; + flags &= ~CPPCompiler::ProductFlag::Debug; + + List paths; + SLANG_RETURN_ON_FAIL(compiler->calcCompileProducts(options, flags, paths)); + productFileSet.clear(); + productFileSet.add(paths); + + // Read the contents of the binary + outBin = File::readAllBytes(moduleFilePath); + } + catch (const Slang::IOException&) + { + sink->diagnose(SourceLoc(), Diagnostics::unableToReadFile, moduleFilePath); + return SLANG_FAIL; + } } return SLANG_OK; @@ -1550,9 +1628,12 @@ SlangResult dissassembleDXILUsingDXC( switch (target) { + case CodeGenTarget::HostCallable: case CodeGenTarget::SharedLibrary: case CodeGenTarget::Executable: { + ComPtr sharedLibrary; + List code; if (SLANG_SUCCEEDED(emitCPUBinaryForEntryPoint( compileRequest, @@ -1560,10 +1641,18 @@ SlangResult dissassembleDXILUsingDXC( entryPointIndex, targetReq, endToEndReq, + sharedLibrary, code))) { - maybeDumpIntermediate(compileRequest, code.getBuffer(), code.getCount(), target); - result = CompileResult(code); + if (target == CodeGenTarget::HostCallable) + { + result = CompileResult(sharedLibrary); + } + else + { + maybeDumpIntermediate(compileRequest, code.getBuffer(), code.getCount(), target); + result = CompileResult(code); + } } } break; @@ -1860,6 +1949,9 @@ SlangResult dissassembleDXILUsingDXC( writeOutputToConsole(writer, result.outputString); break; + case ResultFormat::SharedLibrary: + break; + case ResultFormat::Binary: { auto& data = result.outputBinary; @@ -2232,6 +2324,7 @@ SlangResult dissassembleDXILUsingDXC( // for now dumpIntermediateBinary(compileRequest, data, size, ".exe"); break; + case CodeGenTarget::HostCallable: case CodeGenTarget::SharedLibrary: dumpIntermediateBinary(compileRequest, data, size, ".shared-lib"); break; diff --git a/source/slang/slang-compiler.h b/source/slang/slang-compiler.h index 8f9c20044..bbc6505b0 100644 --- a/source/slang/slang-compiler.h +++ b/source/slang/slang-compiler.h @@ -61,6 +61,7 @@ namespace Slang CPPSource = SLANG_CPP_SOURCE, Executable = SLANG_EXECUTABLE, SharedLibrary = SLANG_SHARED_LIBRARY, + HostCallable = SLANG_HOST_CALLABLE, }; CodeGenTarget calcCodeGenTargetFromName(const UnownedStringSlice& name); @@ -84,7 +85,8 @@ namespace Slang { None, Text, - Binary + Binary, + SharedLibrary, }; // When storing the layout for a matrix-type @@ -122,22 +124,25 @@ namespace Slang class TranslationUnitRequest; // Result of compiling an entry point. - // Should only ever be string OR binary. + // Should only ever be string, binary or shared library class CompileResult { public: CompileResult() = default; CompileResult(String const& str) : format(ResultFormat::Text), outputString(str) {} CompileResult(List const& buffer) : format(ResultFormat::Binary), outputBinary(buffer) {} + CompileResult(ISlangSharedLibrary* inSharedLibrary) : format(ResultFormat::SharedLibrary), sharedLibrary(inSharedLibrary) {} void append(CompileResult const& result); - ComPtr getBlob(); + SlangResult getBlob(ComPtr& outBlob); + SlangResult getSharedLibrary(ComPtr& outSharedLibrary); ResultFormat format = ResultFormat::None; String outputString; List outputBinary; + ComPtr sharedLibrary; ComPtr blob; }; diff --git a/source/slang/slang-ir-entry-point-uniforms.cpp b/source/slang/slang-ir-entry-point-uniforms.cpp index da036d798..822174620 100644 --- a/source/slang/slang-ir-entry-point-uniforms.cpp +++ b/source/slang/slang-ir-entry-point-uniforms.cpp @@ -441,6 +441,7 @@ void moveEntryPointUniformParamsToGlobalScope( case CodeGenTarget::CSource: case CodeGenTarget::Executable: case CodeGenTarget::SharedLibrary: + case CodeGenTarget::HostCallable: { context.targetNeedsConstantBuffer = false; break; diff --git a/source/slang/slang-parameter-binding.cpp b/source/slang/slang-parameter-binding.cpp index f9657e776..6e5838da5 100644 --- a/source/slang/slang-parameter-binding.cpp +++ b/source/slang/slang-parameter-binding.cpp @@ -2640,6 +2640,7 @@ static bool _isCPUTarget(CodeGenTarget target) case CodeGenTarget::CSource: case CodeGenTarget::Executable: case CodeGenTarget::SharedLibrary: + case CodeGenTarget::HostCallable: { return true; } diff --git a/source/slang/slang-type-layout.cpp b/source/slang/slang-type-layout.cpp index 3b14b74c2..9551cfe4e 100644 --- a/source/slang/slang-type-layout.cpp +++ b/source/slang/slang-type-layout.cpp @@ -982,7 +982,7 @@ LayoutRulesFamilyImpl* getDefaultLayoutRulesFamilyForTarget(TargetRequest* targe case CodeGenTarget::SPIRVAssembly: return &kGLSLLayoutRulesFamilyImpl; - + case CodeGenTarget::HostCallable: case CodeGenTarget::Executable: case CodeGenTarget::SharedLibrary: case CodeGenTarget::CPPSource: diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp index cae9855e0..bb7e705e6 100644 --- a/source/slang/slang.cpp +++ b/source/slang/slang.cpp @@ -1716,7 +1716,9 @@ SLANG_NO_THROW SlangResult SLANG_MCALL ComponentType::getEntryPointCode( if(entryPointResult.format == ResultFormat::None ) return SLANG_FAIL; - *outCode = entryPointResult.getBlob().detach(); + ComPtr blob; + SLANG_RETURN_ON_FAIL(entryPointResult.getBlob(blob)); + *outCode = blob.detach(); return SLANG_OK; } @@ -3140,29 +3142,28 @@ SLANG_API void const* spGetEntryPointCode( return data; } -SLANG_API SlangResult spGetEntryPointCodeBlob( - SlangCompileRequest* request, - int entryPointIndex, - int targetIndex, - ISlangBlob** outBlob) +static SlangResult _getEntryPointResult( + SlangCompileRequest* request, + int entryPointIndex, + int targetIndex, + Slang::CompileResult** outCompileResult) { using namespace Slang; - if(!request) return SLANG_ERROR_INVALID_PARAMETER; - if(!outBlob) return SLANG_ERROR_INVALID_PARAMETER; - + if (!request) return SLANG_ERROR_INVALID_PARAMETER; + auto req = Slang::asInternal(request); auto linkage = req->getLinkage(); auto program = req->getSpecializedGlobalAndEntryPointsComponentType(); Index targetCount = linkage->targets.getCount(); - if((targetIndex < 0) || (targetIndex >= targetCount)) + if ((targetIndex < 0) || (targetIndex >= targetCount)) { return SLANG_ERROR_INVALID_PARAMETER; } auto targetReq = linkage->targets[targetIndex]; Index entryPointCount = req->entryPoints.getCount(); - if((entryPointIndex < 0) || (entryPointIndex >= entryPointCount)) + if ((entryPointIndex < 0) || (entryPointIndex >= entryPointCount)) { return SLANG_ERROR_INVALID_PARAMETER; } @@ -3170,15 +3171,47 @@ SLANG_API SlangResult spGetEntryPointCodeBlob( auto targetProgram = program->getTargetProgram(targetReq); - if(!targetProgram) + if (!targetProgram) return SLANG_FAIL; - Slang::CompileResult& result = targetProgram->getExistingEntryPointResult(entryPointIndex); + *outCompileResult = &targetProgram->getExistingEntryPointResult(entryPointIndex); + return SLANG_OK; +} + +SLANG_API SlangResult spGetEntryPointCodeBlob( + SlangCompileRequest* request, + int entryPointIndex, + int targetIndex, + ISlangBlob** outBlob) +{ + using namespace Slang; + if(!outBlob) return SLANG_ERROR_INVALID_PARAMETER; + Slang::CompileResult* compileResult = nullptr; + SLANG_RETURN_ON_FAIL(_getEntryPointResult(request, entryPointIndex, targetIndex, &compileResult)); - auto blob = result.getBlob(); + ComPtr blob; + SLANG_RETURN_ON_FAIL(compileResult->getBlob(blob)); *outBlob = blob.detach(); return SLANG_OK; } +SLANG_API SlangResult spGetEntryPointHostCallable( + SlangCompileRequest* request, + int entryPointIndex, + int targetIndex, + ISlangSharedLibrary** outSharedLibrary) +{ + using namespace Slang; + if (!outSharedLibrary) return SLANG_ERROR_INVALID_PARAMETER; + + Slang::CompileResult* compileResult = nullptr; + SLANG_RETURN_ON_FAIL(_getEntryPointResult(request, entryPointIndex, targetIndex, &compileResult)); + + ComPtr sharedLibrary; + SLANG_RETURN_ON_FAIL(compileResult->getSharedLibrary(sharedLibrary)); + *outSharedLibrary = sharedLibrary.detach(); + return SLANG_OK; +} + SLANG_API char const* spGetEntryPointSource( SlangCompileRequest* request, int entryPointIndex) -- cgit v1.2.3