diff options
| author | jsmall-nvidia <jsmall@nvidia.com> | 2021-04-22 09:32:25 -0400 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2021-04-22 09:32:25 -0400 |
| commit | da0d295d6c8b6fb03245dea0583437c198890349 (patch) | |
| tree | ed17baba750b15f6ace1427f04cf19690269161e | |
| parent | 34fba7b5e726136c6eee8a318ab9a75381399c00 (diff) | |
C++ extractor improvements (#1803)
* #include an absolute path didn't work - because paths were taken to always be relative.
* Split of NodeTree.
Split out FileUtil.
Split out MacroWriter.
* Rename slang-cpp-extractor-main.cpp -> cpp-extractor-main.cpp
* First pass at extractor unit-tests
* Initial parsing of enum.
* Ability to disable/enable parsing of scope types.
* Initial support for typedef.
* Added operator== != to ArrayVIew.
Added test for splitting to unit tests.
* Improve comment in StringUtil.
* Fix comment.
* Fix typo.
27 files changed, 2124 insertions, 1068 deletions
diff --git a/build/visual-studio/slang-cpp-extractor/slang-cpp-extractor.vcxproj b/build/visual-studio/slang-cpp-extractor/slang-cpp-extractor.vcxproj index 6e5d1cdb2..707327ba7 100644 --- a/build/visual-studio/slang-cpp-extractor/slang-cpp-extractor.vcxproj +++ b/build/visual-studio/slang-cpp-extractor/slang-cpp-extractor.vcxproj @@ -164,18 +164,26 @@ <ItemGroup> <ClInclude Include="..\..\..\tools\slang-cpp-extractor\diagnostic-defs.h" /> <ClInclude Include="..\..\..\tools\slang-cpp-extractor\diagnostics.h" /> + <ClInclude Include="..\..\..\tools\slang-cpp-extractor\file-util.h" /> <ClInclude Include="..\..\..\tools\slang-cpp-extractor\identifier-lookup.h" /> + <ClInclude Include="..\..\..\tools\slang-cpp-extractor\macro-writer.h" /> + <ClInclude Include="..\..\..\tools\slang-cpp-extractor\node-tree.h" /> <ClInclude Include="..\..\..\tools\slang-cpp-extractor\node.h" /> <ClInclude Include="..\..\..\tools\slang-cpp-extractor\options.h" /> <ClInclude Include="..\..\..\tools\slang-cpp-extractor\parser.h" /> + <ClInclude Include="..\..\..\tools\slang-cpp-extractor\unit-test.h" /> </ItemGroup> <ItemGroup> + <ClCompile Include="..\..\..\tools\slang-cpp-extractor\cpp-extractor-main.cpp" /> <ClCompile Include="..\..\..\tools\slang-cpp-extractor\diagnostics.cpp" /> + <ClCompile Include="..\..\..\tools\slang-cpp-extractor\file-util.cpp" /> <ClCompile Include="..\..\..\tools\slang-cpp-extractor\identifier-lookup.cpp" /> + <ClCompile Include="..\..\..\tools\slang-cpp-extractor\macro-writer.cpp" /> + <ClCompile Include="..\..\..\tools\slang-cpp-extractor\node-tree.cpp" /> <ClCompile Include="..\..\..\tools\slang-cpp-extractor\node.cpp" /> <ClCompile Include="..\..\..\tools\slang-cpp-extractor\options.cpp" /> <ClCompile Include="..\..\..\tools\slang-cpp-extractor\parser.cpp" /> - <ClCompile Include="..\..\..\tools\slang-cpp-extractor\slang-cpp-extractor-main.cpp" /> + <ClCompile Include="..\..\..\tools\slang-cpp-extractor\unit-test.cpp" /> </ItemGroup> <ItemGroup> <ProjectReference Include="..\core\core.vcxproj"> diff --git a/build/visual-studio/slang-cpp-extractor/slang-cpp-extractor.vcxproj.filters b/build/visual-studio/slang-cpp-extractor/slang-cpp-extractor.vcxproj.filters index c11f860de..ed10cd2c3 100644 --- a/build/visual-studio/slang-cpp-extractor/slang-cpp-extractor.vcxproj.filters +++ b/build/visual-studio/slang-cpp-extractor/slang-cpp-extractor.vcxproj.filters @@ -15,9 +15,18 @@ <ClInclude Include="..\..\..\tools\slang-cpp-extractor\diagnostics.h"> <Filter>Header Files</Filter> </ClInclude> + <ClInclude Include="..\..\..\tools\slang-cpp-extractor\file-util.h"> + <Filter>Header Files</Filter> + </ClInclude> <ClInclude Include="..\..\..\tools\slang-cpp-extractor\identifier-lookup.h"> <Filter>Header Files</Filter> </ClInclude> + <ClInclude Include="..\..\..\tools\slang-cpp-extractor\macro-writer.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="..\..\..\tools\slang-cpp-extractor\node-tree.h"> + <Filter>Header Files</Filter> + </ClInclude> <ClInclude Include="..\..\..\tools\slang-cpp-extractor\node.h"> <Filter>Header Files</Filter> </ClInclude> @@ -27,14 +36,29 @@ <ClInclude Include="..\..\..\tools\slang-cpp-extractor\parser.h"> <Filter>Header Files</Filter> </ClInclude> + <ClInclude Include="..\..\..\tools\slang-cpp-extractor\unit-test.h"> + <Filter>Header Files</Filter> + </ClInclude> </ItemGroup> <ItemGroup> + <ClCompile Include="..\..\..\tools\slang-cpp-extractor\cpp-extractor-main.cpp"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\..\..\tools\slang-cpp-extractor\diagnostics.cpp"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\..\..\tools\slang-cpp-extractor\file-util.cpp"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\..\..\tools\slang-cpp-extractor\identifier-lookup.cpp"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\..\..\tools\slang-cpp-extractor\macro-writer.cpp"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\..\..\tools\slang-cpp-extractor\node-tree.cpp"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\..\..\tools\slang-cpp-extractor\node.cpp"> <Filter>Source Files</Filter> </ClCompile> @@ -44,7 +68,7 @@ <ClCompile Include="..\..\..\tools\slang-cpp-extractor\parser.cpp"> <Filter>Source Files</Filter> </ClCompile> - <ClCompile Include="..\..\..\tools\slang-cpp-extractor\slang-cpp-extractor-main.cpp"> + <ClCompile Include="..\..\..\tools\slang-cpp-extractor\unit-test.cpp"> <Filter>Source Files</Filter> </ClCompile> </ItemGroup> diff --git a/source/compiler-core/slang-diagnostic-sink.cpp b/source/compiler-core/slang-diagnostic-sink.cpp index 2adf31f69..727c322a5 100644 --- a/source/compiler-core/slang-diagnostic-sink.cpp +++ b/source/compiler-core/slang-diagnostic-sink.cpp @@ -385,6 +385,23 @@ static void formatDiagnostic( } } +void DiagnosticSink::init(SourceManager* sourceManager, SourceLocationLexer sourceLocationLexer) +{ + m_errorCount = 0; + m_internalErrorLocsNoted = 0; + + m_flags = 0; + + m_sourceManager = sourceManager; + m_sourceLocationLexer = sourceLocationLexer; + + // If we have a source location lexer, we'll by default enable source location output + if (sourceLocationLexer) + { + setFlag(Flag::SourceLocationLine); + } +} + void DiagnosticSink::diagnoseImpl(SourceLoc const& pos, DiagnosticInfo const& info, int argCount, DiagnosticArg const* const* args) { StringBuilder sb; diff --git a/source/compiler-core/slang-diagnostic-sink.h b/source/compiler-core/slang-diagnostic-sink.h index 84001e6f5..347ff761c 100644 --- a/source/compiler-core/slang-diagnostic-sink.h +++ b/source/compiler-core/slang-diagnostic-sink.h @@ -217,16 +217,16 @@ public: /// character caret at location SourceLocationLexer getSourceLocationLexer() const { return m_sourceLocationLexer; } + /// Initialize state. + void init(SourceManager* sourceManager, SourceLocationLexer sourceLocationLexer); + /// Ctor - DiagnosticSink(SourceManager* sourceManager, SourceLocationLexer sourceLocationLexer) - : m_sourceManager(sourceManager), - m_sourceLocationLexer(sourceLocationLexer) + DiagnosticSink(SourceManager* sourceManager, SourceLocationLexer sourceLocationLexer) { init(sourceManager, sourceLocationLexer); } + /// Default Ctor + DiagnosticSink(): + m_sourceManager(nullptr), + m_sourceLocationLexer (nullptr) { - // If we have a source location lexer, we'll by default enable source location output - if (sourceLocationLexer) - { - setFlag(Flag::SourceLocationLine); - } } // Public members diff --git a/source/core/slang-array-view.h b/source/core/slang-array-view.h index 56c936073..c67a53337 100644 --- a/source/core/slang-array-view.h +++ b/source/core/slang-array-view.h @@ -12,6 +12,8 @@ namespace Slang class ConstArrayView { public: + typedef ConstArrayView ThisType; + const T* begin() const { return m_buffer; } const T* end() const { return m_buffer + m_count; } @@ -70,6 +72,30 @@ namespace Slang return -1; } + bool operator==(const ThisType& rhs) const + { + if (&rhs == this) + { + return true; + } + const Index count = getCount(); + if (count != rhs.getCount()) + { + return false; + } + const T* thisEle = getBuffer(); + const T* rhsEle = rhs.getBuffer(); + for (Index i = 0; i < count; ++i) + { + if (thisEle[i] != rhsEle[i]) + { + return false; + } + } + return true; + } + SLANG_FORCE_INLINE bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } + ConstArrayView() : m_buffer(nullptr), m_count(0) @@ -111,6 +137,8 @@ namespace Slang class ArrayView: public ConstArrayView<T> { public: + typedef ArrayView ThisType; + typedef ConstArrayView<T> Super; using Super::m_buffer; @@ -147,6 +175,7 @@ namespace Slang { return ArrayView<T>(buffer, count); } + } #endif diff --git a/source/core/slang-string-util.cpp b/source/core/slang-string-util.cpp index b4c767c87..8108bdc98 100644 --- a/source/core/slang-string-util.cpp +++ b/source/core/slang-string-util.cpp @@ -56,6 +56,52 @@ namespace Slang { } } +/* static */void StringUtil::split(const UnownedStringSlice& in, const UnownedStringSlice& splitSlice, List<UnownedStringSlice>& outSlices) +{ + const Index splitLen = splitSlice.getLength(); + + if (splitLen == 1) + { + return split(in, splitSlice[0], outSlices); + } + + outSlices.clear(); + + SLANG_ASSERT(splitLen > 0); + if (splitLen <= 0) + { + return; + } + + const char* start = in.begin(); + const char* end = in.end(); + + const char splitChar = splitSlice[0]; + + while (start < end) + { + // Move cur so it's either at the end or at next splitSlice + const char* cur = start; + while (cur < end) + { + if (*cur == splitChar && + (cur + splitLen <= end && UnownedStringSlice(cur, splitLen) == splitSlice)) + { + // We hit a split + break; + } + + cur++; + } + + // Add to output + outSlices.add(UnownedStringSlice(start, cur)); + + // Skip the split, if at end we are okay anyway + start = cur + splitLen; + } +} + /* static */Index StringUtil::split(const UnownedStringSlice& in, char splitChar, Index maxSlices, UnownedStringSlice* outSlices) { Index index = 0; diff --git a/source/core/slang-string-util.h b/source/core/slang-string-util.h index 2850c3b4f..8031f5c8c 100644 --- a/source/core/slang-string-util.h +++ b/source/core/slang-string-util.h @@ -24,6 +24,9 @@ struct StringUtil /// Split in, by specified splitChar into slices out /// Slices contents will directly address into in, so contents will only stay valid as long as in does. static void split(const UnownedStringSlice& in, char splitChar, List<UnownedStringSlice>& slicesOut); + /// Split in by the specified splitSlice + /// Slices contents will directly address into in, so contents will only stay valid as long as in does. + static void split(const UnownedStringSlice& in, const UnownedStringSlice& splitSlice, List<UnownedStringSlice>& slicesOut); /// Splits in into outSlices, up to maxSlices. May not consume all of in (for example if it runs out of space). static Index split(const UnownedStringSlice& in, char splitChar, Index maxSlices, UnownedStringSlice* outSlices); diff --git a/tools/slang-cpp-extractor/cpp-extractor-main.cpp b/tools/slang-cpp-extractor/cpp-extractor-main.cpp new file mode 100644 index 000000000..d1ec47e69 --- /dev/null +++ b/tools/slang-cpp-extractor/cpp-extractor-main.cpp @@ -0,0 +1,250 @@ +// main.cpp + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include "../../source/core/slang-secure-crt.h" + +#include "../../slang-com-helper.h" + +#include "../../source/core/slang-list.h" +#include "../../source/core/slang-string.h" +#include "../../source/core/slang-string-util.h" +#include "../../source/core/slang-io.h" +#include "../../source/core/slang-string-slice-pool.h" +#include "../../source/core/slang-writer.h" +#include "../../source/core/slang-file-system.h" + +#include "../../source/compiler-core/slang-source-loc.h" +#include "../../source/compiler-core/slang-lexer.h" +#include "../../source/compiler-core/slang-diagnostic-sink.h" +#include "../../source/compiler-core/slang-name.h" +#include "../../source/compiler-core/slang-name-convention-util.h" + +#include "node.h" +#include "diagnostics.h" +#include "options.h" +#include "parser.h" +#include "macro-writer.h" +#include "file-util.h" +#include "unit-test.h" + +/* +Some command lines: + +-d source/slang slang-ast-support-types.h slang-ast-base.h slang-ast-decl.h slang-ast-expr.h slang-ast-modifier.h slang-ast-stmt.h slang-ast-type.h slang-ast-val.h -strip-prefix slang- -o slang-generated -output-fields -mark-suffix _CLASS +*/ + +namespace CppExtract +{ + +using namespace Slang; + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! App !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +class App +{ +public: + + SlangResult execute(const Options& options); + + /// Execute + SlangResult executeWithArgs(int argc, const char*const* argv); + + const Options& getOptions() const { return m_options; } + + App(DiagnosticSink* sink, SourceManager* sourceManager, RootNamePool* rootNamePool): + m_sink(sink), + m_sourceManager(sourceManager), + m_slicePool(StringSlicePool::Style::Default) + { + m_namePool.setRootNamePool(rootNamePool); + } + +protected: + + NamePool m_namePool; + + Options m_options; + DiagnosticSink* m_sink; + SourceManager* m_sourceManager; + + StringSlicePool m_slicePool; +}; + + +SlangResult App::execute(const Options& options) +{ + m_options = options; + + if (options.m_runUnitTests) + { + SLANG_RETURN_ON_FAIL(UnitTestUtil::run()); + } + + IdentifierLookup identifierLookup; + identifierLookup.initDefault(options.m_markPrefix.getUnownedSlice()); + + NodeTree tree(&m_slicePool, &m_namePool, &identifierLookup); + + // Read in each of the input files + for (Index i = 0; i < m_options.m_inputPaths.getCount(); ++i) + { + String inputPath; + + if (m_options.m_inputDirectory.getLength()) + { + inputPath = Path::combine(m_options.m_inputDirectory, m_options.m_inputPaths[i]); + } + else + { + inputPath = m_options.m_inputPaths[i]; + } + + // Read the input file + String contents; + SLANG_RETURN_ON_FAIL(FileUtil::readAllText(inputPath, m_sink, contents)); + + PathInfo pathInfo = PathInfo::makeFromString(inputPath); + + SourceFile* sourceFile = m_sourceManager->createSourceFileWithString(pathInfo, contents); + + SourceOrigin* sourceOrigin = tree.addSourceOrigin(sourceFile, options); + + Parser parser(&tree, m_sink); + SLANG_RETURN_ON_FAIL(parser.parse(sourceOrigin, &m_options)); + } + + SLANG_RETURN_ON_FAIL(tree.calcDerivedTypes(m_sink)); + + // Okay let's check out the typeSets + { + for (TypeSet* typeSet : tree.getTypeSets()) + { + // The macro name is in upper snake, so split it + List<UnownedStringSlice> slices; + NameConventionUtil::split(typeSet->m_macroName, slices); + + if (typeSet->m_fileMark.getLength() == 0) + { + StringBuilder buf; + // Let's guess a 'fileMark' (it becomes part of the filename) based on the macro name. Use lower kabab. + NameConventionUtil::join(slices.getBuffer(), slices.getCount(), CharCase::Lower, NameConvention::Kabab, buf); + typeSet->m_fileMark = buf.ProduceString(); + } + + if (typeSet->m_typeName.getLength() == 0) + { + // Let's guess a typename if not set -> go with upper camel + StringBuilder buf; + NameConventionUtil::join(slices.getBuffer(), slices.getCount(), CharCase::Upper, NameConvention::Camel, buf); + typeSet->m_typeName = buf.ProduceString(); + } + } + } + + // Dump out the tree + if (options.m_dump) + { + { + StringBuilder buf; + tree.getRootNode()->dump(0, buf); + m_sink->writer->write(buf.getBuffer(), buf.getLength()); + } + + for (TypeSet* typeSet : tree.getTypeSets()) + { + const List<ClassLikeNode*>& baseTypes = typeSet->m_baseTypes; + + for (ClassLikeNode* baseType : baseTypes) + { + StringBuilder buf; + baseType->dumpDerived(0, buf); + m_sink->writer->write(buf.getBuffer(), buf.getLength()); + } + } + } + + if (options.m_defs) + { + MacroWriter macroWriter(m_sink, &m_options); + SLANG_RETURN_ON_FAIL(macroWriter.writeDefs(&tree)); + } + + if (options.m_outputPath.getLength()) + { + MacroWriter macroWriter(m_sink, &m_options); + SLANG_RETURN_ON_FAIL(macroWriter.writeOutput(&tree)); + } + + return SLANG_OK; +} + +/// Execute +SlangResult App::executeWithArgs(int argc, const char*const* argv) +{ + Options options; + OptionsParser optionsParser; + SLANG_RETURN_ON_FAIL(optionsParser.parse(argc, argv, m_sink, options)); + SLANG_RETURN_ON_FAIL(execute(options)); + return SLANG_OK; +} + +} // namespace CppExtract + +int main(int argc, const char*const* argv) +{ + using namespace CppExtract; + using namespace Slang; + + { + ComPtr<ISlangWriter> writer(new FileWriter(stderr, WriterFlag::AutoFlush)); + + RootNamePool rootNamePool; + + SourceManager sourceManager; + sourceManager.initialize(nullptr, nullptr); + + DiagnosticSink sink(&sourceManager, Lexer::sourceLocationLexer); + sink.writer = writer; + + // Set to true to see command line that initiated C++ extractor. Helpful when finding issues from solution building failing, and then so + // being able to repeat the issue + bool dumpCommandLine = false; + + if (dumpCommandLine) + { + StringBuilder builder; + + for (Index i = 1; i < argc; ++i) + { + builder << argv[i] << " "; + } + + sink.diagnose(SourceLoc(), CPPDiagnostics::commandLine, builder); + } + + App app(&sink, &sourceManager, &rootNamePool); + + try + { + if (SLANG_FAILED(app.executeWithArgs(argc - 1, argv + 1))) + { + sink.diagnose(SourceLoc(), CPPDiagnostics::extractorFailed); + return 1; + } + if (sink.getErrorCount()) + { + sink.diagnose(SourceLoc(), CPPDiagnostics::extractorFailed); + return 1; + } + } + catch (...) + { + sink.diagnose(SourceLoc(), CPPDiagnostics::internalError); + return 1; + } + } + return 0; +} + diff --git a/tools/slang-cpp-extractor/diagnostic-defs.h b/tools/slang-cpp-extractor/diagnostic-defs.h index 284e02d19..34b444206 100644 --- a/tools/slang-cpp-extractor/diagnostic-defs.h +++ b/tools/slang-cpp-extractor/diagnostic-defs.h @@ -41,7 +41,10 @@ DIAGNOSTIC(100009, Error, unexpectedUnbalancedToken, "Unexpected unbalanced toke DIAGNOSTIC(100010, Error, unexpectedEndOfFile, "Unexpected end of file") DIAGNOSTIC(100011, Error, expectingTypeKeyword, "Expecting type keyword - struct or class, found $0") -DIAGNOSTIC(100011, Error, typeInDifferentTypeSet, "Type $0 in different type set $1 from super class $2") +DIAGNOSTIC(100012, Error, typeInDifferentTypeSet, "Type $0 in different type set $1 from super class $2") +DIAGNOSTIC(100013, Error, expectingIdentifier, "Expecting an identifier, found $0") +DIAGNOSTIC(100014, Error, cannotDeclareTypeInScope, "Cannot declare types in this scope") +DIAGNOSTIC(100015, Error, identifierAlreadyDefined, "Identifier already defined '$0'") // Command line errors 100100 diff --git a/tools/slang-cpp-extractor/file-util.cpp b/tools/slang-cpp-extractor/file-util.cpp new file mode 100644 index 000000000..1d37df650 --- /dev/null +++ b/tools/slang-cpp-extractor/file-util.cpp @@ -0,0 +1,72 @@ +#include "file-util.h" + +#include "../../source/core/slang-io.h" + +namespace CppExtract { +using namespace Slang; + +/* static */SlangResult FileUtil::readAllText(const Slang::String& fileName, DiagnosticSink* sink, String& outRead) +{ + try + { + StreamReader reader(new FileStream(fileName, FileMode::Open, FileAccess::Read, FileShare::ReadWrite)); + outRead = reader.ReadToEnd(); + } + catch (const IOException&) + { + if (sink) + { + sink->diagnose(SourceLoc(), CPPDiagnostics::cannotOpenFile, fileName); + } + return SLANG_FAIL; + } + catch (...) + { + if (sink) + { + sink->diagnose(SourceLoc(), CPPDiagnostics::cannotOpenFile, fileName); + } + return SLANG_FAIL; + } + + return SLANG_OK; +} + +/* static */SlangResult FileUtil::writeAllText(const Slang::String& fileName, DiagnosticSink* sink, const UnownedStringSlice& text) +{ + try + { + if (File::exists(fileName)) + { + String existingText; + + if (readAllText(fileName, nullptr, existingText) == SLANG_OK) + { + if (existingText == text) + return SLANG_OK; + } + } + StreamWriter writer(new FileStream(fileName, FileMode::Create)); + writer.Write(text); + } + catch (const IOException&) + { + if (sink) + { + sink->diagnose(SourceLoc(), CPPDiagnostics::cannotOpenFile, fileName); + } + return SLANG_FAIL; + } + + return SLANG_OK; +} + +/* static */ void FileUtil::indent(Index indentCount, StringBuilder& out) +{ + for (Index i = 0; i < indentCount; ++i) + { + out << CPP_EXTRACT_INDENT_STRING; + } +} + +} // namespace CppExtract diff --git a/tools/slang-cpp-extractor/file-util.h b/tools/slang-cpp-extractor/file-util.h new file mode 100644 index 000000000..01aafeedf --- /dev/null +++ b/tools/slang-cpp-extractor/file-util.h @@ -0,0 +1,25 @@ +#ifndef CPP_EXTRACT_FILE_UTIL_H +#define CPP_EXTRACT_FILE_UTIL_H + +#include "diagnostics.h" + +namespace CppExtract { +using namespace Slang; + +// A macro to define a single indent as a string +#define CPP_EXTRACT_INDENT_STRING " " + +struct FileUtil +{ + /// Read text into outRead. Any failures written to sink (can be passed as nullptr, for no output) + static SlangResult readAllText(const Slang::String& fileName, DiagnosticSink* sink, String& outRead); + /// Write text to filename. Any failures written to sink. (can be passed as nullptr, for no output) + static SlangResult writeAllText(const Slang::String& fileName, DiagnosticSink* sink, const UnownedStringSlice& text); + + /// Appends CPP_EXTRACT_INDENT_STRING indentCount number of times to out + static void indent(Index indentCount, StringBuilder& out); +}; + +} // CppExtract + +#endif diff --git a/tools/slang-cpp-extractor/identifier-lookup.cpp b/tools/slang-cpp-extractor/identifier-lookup.cpp index 07f155248..c7be75d82 100644 --- a/tools/slang-cpp-extractor/identifier-lookup.cpp +++ b/tools/slang-cpp-extractor/identifier-lookup.cpp @@ -11,15 +11,19 @@ using namespace Slang; 0, /// Type set IdentifierFlag::Keyword, /// TypeModifier IdentifierFlag::Keyword, /// Keyword + IdentifierFlag::Keyword | IdentifierFlag::StartScope | IdentifierFlag::ClassLike, /// Class IdentifierFlag::Keyword | IdentifierFlag::StartScope | IdentifierFlag::ClassLike, /// Struct IdentifierFlag::Keyword | IdentifierFlag::StartScope, /// Namespace + IdentifierFlag::Keyword | IdentifierFlag::StartScope, /// Enum + + IdentifierFlag::Keyword, /// Typedef + IdentifierFlag::Keyword, /// Access IdentifierFlag::Reflection, /// Reflected IdentifierFlag::Reflection, /// Unreflected }; - void IdentifierLookup::set(const UnownedStringSlice& name, IdentifierStyle style) { StringSlicePool::Handle handle; @@ -44,5 +48,65 @@ void IdentifierLookup::set(const char*const* names, size_t namesCount, Identifie } } +void IdentifierLookup::set(const Pair* pairs, Index pairsCount) +{ + for (Index i = 0; i < pairsCount; ++i) + { + const auto& pair = pairs[i]; + set(UnownedStringSlice(pair.name), pair.style); + } +} + +void IdentifierLookup::initDefault(const UnownedStringSlice& markPrefix) +{ + reset(); + + // Some keywords + { + const char* names[] = { "virtual", "continue", "if", "case", "break", "catch", "default", "delete", "do", "else", "for", "new", "goto", "return", "switch", "throw", "using", "while", "operator" }; + set(names, SLANG_COUNT_OF(names), IdentifierStyle::Keyword); + } + + // Type modifier keywords + { + const char* names[] = { "const", "volatile" }; + set(names, SLANG_COUNT_OF(names), IdentifierStyle::TypeModifier); + } + + // Special markers + { + const char* names[] = { "PRE_DECLARE", "TYPE_SET", "REFLECTED", "UNREFLECTED" }; + const IdentifierStyle styles[] = { IdentifierStyle::PreDeclare, IdentifierStyle::TypeSet, IdentifierStyle::Reflected, IdentifierStyle::Unreflected }; + SLANG_COMPILE_TIME_ASSERT(SLANG_COUNT_OF(names) == SLANG_COUNT_OF(styles)); + + StringBuilder buf; + for (Index i = 0; i < SLANG_COUNT_OF(names); ++i) + { + buf.Clear(); + buf << markPrefix << names[i]; + set(buf.getUnownedSlice(), styles[i]); + } + } + + // Keywords which introduce types/scopes + { + const Pair pairs[] = + { + { "struct", IdentifierStyle::Struct }, + { "class", IdentifierStyle::Class }, + { "namespace", IdentifierStyle::Namespace }, + { "enum", IdentifierStyle::Enum }, + { "typedef", IdentifierStyle::TypeDef }, + }; + + set(pairs, SLANG_COUNT_OF(pairs)); + } + + // Keywords that control access + { + const char* names[] = { "private", "protected", "public" }; + set(names, SLANG_COUNT_OF(names), IdentifierStyle::Access); + } +} } // namespace CppExtract diff --git a/tools/slang-cpp-extractor/identifier-lookup.h b/tools/slang-cpp-extractor/identifier-lookup.h index b845f804c..3cee909ef 100644 --- a/tools/slang-cpp-extractor/identifier-lookup.h +++ b/tools/slang-cpp-extractor/identifier-lookup.h @@ -17,9 +17,14 @@ enum class IdentifierStyle TypeModifier, ///< const, volatile etc Keyword, ///< A keyword C/C++ keyword that is not another type + Class, ///< class Struct, ///< struct Namespace, ///< namespace + Enum, ///< enum + + TypeDef, ///< typedef + Access, ///< public, protected, private Reflected, @@ -45,6 +50,12 @@ class IdentifierLookup { public: + struct Pair + { + const char* name; + IdentifierStyle style; + }; + IdentifierStyle get(const UnownedStringSlice& slice) const { Index index = m_pool.findIndex(slice); @@ -60,12 +71,16 @@ public: void set(const char*const* names, size_t namesCount, IdentifierStyle style); + void set(const Pair* pairs, Index pairsCount); + void reset() { m_styles.clear(); m_pool.clear(); } + void initDefault(const UnownedStringSlice& markPrefix); + IdentifierLookup() : m_pool(StringSlicePool::Style::Empty) { diff --git a/tools/slang-cpp-extractor/macro-writer.cpp b/tools/slang-cpp-extractor/macro-writer.cpp new file mode 100644 index 000000000..2259f1800 --- /dev/null +++ b/tools/slang-cpp-extractor/macro-writer.cpp @@ -0,0 +1,436 @@ +#include "macro-writer.h" + +#include "../../slang-com-helper.h" + +#include "../../source/core/slang-list.h" +#include "../../source/core/slang-string.h" +//#include "../../source/core/slang-string-util.h" +#include "../../source/core/slang-io.h" + +#include "../../source/core/slang-writer.h" + +#include "../../source/compiler-core/slang-diagnostic-sink.h" +//#include "../../source/compiler-core/slang-name.h" + +#include "diagnostics.h" +#include "options.h" +#include "node-tree.h" +#include "file-util.h" + +namespace CppExtract +{ +using namespace Slang; + +SLANG_FORCE_INLINE static void _indent(Index indentCount, StringBuilder& out) { return FileUtil::indent(indentCount, out); } + +SlangResult MacroWriter::calcDef(NodeTree* tree, SourceOrigin* origin, StringBuilder& out) +{ + Node* currentScope = nullptr; + + for (Node* node : origin->m_nodes) + { + if (node->isReflected()) + { + if (auto classLikeNode = as<ClassLikeNode>(node)) + { + if (classLikeNode->m_marker.getContent().indexOf(UnownedStringSlice::fromLiteral("ABSTRACT")) >= 0) + { + out << "ABSTRACT_"; + } + + out << "SYNTAX_CLASS(" << node->m_name.getContent() << ", " << classLikeNode->m_super.getContent() << ")\n"; + out << "END_SYNTAX_CLASS()\n\n"; + } + } + } + return SLANG_OK; +} + +SlangResult MacroWriter::calcChildrenHeader(NodeTree* tree, TypeSet* typeSet, StringBuilder& out) +{ + const List<ClassLikeNode*>& baseTypes = typeSet->m_baseTypes; + const String& reflectTypeName = typeSet->m_typeName; + + out << "#pragma once\n\n"; + out << "// Do not edit this file is generated from slang-cpp-extractor tool\n\n"; + + List<ClassLikeNode*> classNodes; + for (Index i = 0; i < baseTypes.getCount(); ++i) + { + ClassLikeNode* baseType = baseTypes[i]; + baseType->calcDerivedDepthFirst(classNodes); + } + + //Node::filter(Node::isClassLike, nodes); + + List<ClassLikeNode*> derivedTypes; + + out << "\n\n /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!! CHILDREN !!!!!!!!!!!!!!!!!!!!!!!!!!!! */ \n\n"; + + // Now the children + for (ClassLikeNode* classNode : classNodes) + { + classNode->getReflectedDerivedTypes(derivedTypes); + + // Define the derived types + out << "#define " << m_options->m_markPrefix << "CHILDREN_" << reflectTypeName << "_" << classNode->m_name.getContent() << "(x, param)"; + + if (derivedTypes.getCount()) + { + out << " \\\n"; + for (Index j = 0; j < derivedTypes.getCount(); ++j) + { + Node* derivedType = derivedTypes[j]; + _indent(1, out); + out << m_options->m_markPrefix << "ALL_" << reflectTypeName << "_" << derivedType->m_name.getContent() << "(x, param)"; + if (j < derivedTypes.getCount() - 1) + { + out << "\\\n"; + } + } + } + out << "\n\n"; + } + + out << "\n\n /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!! ALL !!!!!!!!!!!!!!!!!!!!!!!!!!!! */\n\n"; + + for (ClassLikeNode* classNode : classNodes) + { + // Define the derived types + out << "#define " << m_options->m_markPrefix << "ALL_" << reflectTypeName << "_" << classNode->m_name.getContent() << "(x, param) \\\n"; + _indent(1, out); + out << m_options->m_markPrefix << reflectTypeName << "_" << classNode->m_name.getContent() << "(x, param)"; + + // If has derived types output them + if (classNode->hasReflectedDerivedType()) + { + out << " \\\n"; + _indent(1, out); + out << m_options->m_markPrefix << "CHILDREN_" << reflectTypeName << "_" << classNode->m_name.getContent() << "(x, param)"; + } + out << "\n\n"; + } + + if (m_options->m_outputFields) + { + out << "\n\n /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!! FIELDS !!!!!!!!!!!!!!!!!!!!!!!!!!!! */\n\n"; + + for (ClassLikeNode* classNode : classNodes) + { + // Define the derived types + out << "#define " << m_options->m_markPrefix << "FIELDS_" << reflectTypeName << "_" << classNode->m_name.getContent() << "(_x_, _param_)"; + + // Find all of the fields + List<FieldNode*> fields; + for (Node* child : classNode->m_children) + { + if (auto field = as<FieldNode>(child)) + { + fields.add(field); + } + } + + if (fields.getCount() > 0) + { + out << "\\\n"; + + const Index fieldsCount = fields.getCount(); + bool previousField = false; + for (Index j = 0; j < fieldsCount; ++j) + { + const FieldNode* field = fields[j]; + + if (field->isReflected()) + { + if (previousField) + { + out << "\\\n"; + } + + _indent(1, out); + + // NOTE! We put the type field in brackets, such that there is no issue with templates containing a comma. + // If stringified + out << "_x_(" << field->m_name.getContent() << ", (" << field->m_fieldType << "), _param_)"; + previousField = true; + } + } + } + + out << "\n\n"; + } + } + + return SLANG_OK; +} + +SlangResult MacroWriter::calcOriginHeader(NodeTree* tree, StringBuilder& out) +{ + // Do macros by origin + + out << "// Origin macros\n\n"; + + for (SourceOrigin* origin : tree->getSourceOrigins()) + { + out << "#define " << m_options->m_markPrefix << "ORIGIN_" << origin->m_macroOrigin << "(x, param) \\\n"; + + for (Node* node : origin->m_nodes) + { + if (!(node->isReflected() && node->isClassLike())) + { + continue; + } + + _indent(1, out); + out << "x(" << node->m_name.getContent() << ", param) \\\n"; + } + out << "/* */\n\n"; + } + + return SLANG_OK; +} + +SlangResult MacroWriter::calcTypeHeader(NodeTree* tree, TypeSet* typeSet, StringBuilder& out) +{ + const List<ClassLikeNode*>& baseTypes = typeSet->m_baseTypes; + const String& reflectTypeName = typeSet->m_typeName; + + out << "#pragma once\n\n"; + out << "// Do not edit this file is generated from slang-cpp-extractor tool\n\n"; + + if (baseTypes.getCount() == 0) + { + return SLANG_OK; + } + + // Set up the scope + List<Node*> baseScopePath; + baseTypes[0]->calcScopePath(baseScopePath); + + // Remove the global scope + baseScopePath.removeAt(0); + // Remove the type itself + baseScopePath.removeLast(); + + for (Node* scopeNode : baseScopePath) + { + SLANG_ASSERT(scopeNode->m_type == Node::Type::Namespace); + out << "namespace " << scopeNode->m_name.getContent() << " {\n"; + } + + // Add all the base types, with in order traversals + List<ClassLikeNode*> nodes; + for (Index i = 0; i < baseTypes.getCount(); ++i) + { + ClassLikeNode* baseType = baseTypes[i]; + baseType->calcDerivedDepthFirst(nodes); + } + + Node::filter(Node::isClassLikeAndReflected, nodes); + + // Write out the types + { + out << "\n"; + out << "enum class " << reflectTypeName << "Type\n"; + out << "{\n"; + + Index typeIndex = 0; + for (ClassLikeNode* node : nodes) + { + // Okay first we are going to output the enum values + const Index depth = node->calcDerivedDepth() - 1; + _indent(depth, out); + out << node->m_name.getContent() << " = " << typeIndex << ",\n"; + typeIndex++; + } + + _indent(1, out); + out << "CountOf\n"; + + out << "};\n\n"; + } + + // TODO(JS): + // Strictly speaking if we wanted the types to be in different scopes, we would have to + // change the namespaces here + + // Predeclare the classes + { + out << "// Predeclare\n\n"; + for (ClassLikeNode* node : nodes) + { + // If it's not reflected we don't output, in the enum list + if (node->isReflected()) + { + const char* type = (node->m_type == Node::Type::ClassType) ? "class" : "struct"; + out << type << " " << node->m_name.getContent() << ";\n"; + } + } + } + + // Do the macros for each of the types + + { + out << "// Type macros\n\n"; + + out << "// Order is (NAME, SUPER, ORIGIN, LAST, MARKER, TYPE, param) \n"; + out << "// NAME - is the class name\n"; + out << "// SUPER - is the super class name (or NO_SUPER)\n"; + out << "// ORIGIN - where the definition was found\n"; + out << "// LAST - is the class name for the last in the range (or NO_LAST)\n"; + out << "// MARKER - is the text inbetween in the prefix/postix (like ABSTRACT). If no inbetween text is is 'NONE'\n"; + out << "// TYPE - Can be BASE, INNER or LEAF for the overall base class, an INNER class, or a LEAF class\n"; + out << "// param is a user defined parameter that can be parsed to the invoked x macro\n\n"; + + // Output all of the definitions for each type + for (ClassLikeNode* node : nodes) + { + out << "#define " << m_options->m_markPrefix << reflectTypeName << "_" << node->m_name.getContent() << "(x, param) "; + + // Output the X macro part + _indent(1, out); + out << "x(" << node->m_name.getContent() << ", "; + + if (node->m_superNode) + { + out << node->m_superNode->m_name.getContent() << ", "; + } + else + { + out << "NO_SUPER, "; + } + + // Output the (file origin) + out << node->m_origin->m_macroOrigin; + out << ", "; + + // The last type + Node* lastDerived = node->findLastDerived(); + if (lastDerived) + { + out << lastDerived->m_name.getContent() << ", "; + } + else + { + out << "NO_LAST, "; + } + + // Output any specifics of the markup + UnownedStringSlice marker = node->m_marker.getContent(); + // Need to extract the name + if (marker.getLength() > m_options->m_markPrefix.getLength() + m_options->m_markSuffix.getLength()) + { + marker = UnownedStringSlice(marker.begin() + m_options->m_markPrefix.getLength(), marker.end() - m_options->m_markSuffix.getLength()); + } + else + { + marker = UnownedStringSlice::fromLiteral("NONE"); + } + out << marker << ", "; + + if (node->m_superNode == nullptr) + { + out << "BASE, "; + } + else if (node->hasReflectedDerivedType()) + { + out << "INNER, "; + } + else + { + out << "LEAF, "; + } + out << "param)\n"; + } + } + + // Now pop the scope in revers + for (Index j = baseScopePath.getCount() - 1; j >= 0; j--) + { + Node* scopeNode = baseScopePath[j]; + out << "} // namespace " << scopeNode->m_name.getContent() << "\n"; + } + + return SLANG_OK; +} + +SlangResult MacroWriter::writeDefs(NodeTree* tree) +{ + const auto& origins = tree->getSourceOrigins(); + + for (SourceOrigin* origin : origins) + { + const String path = origin->m_sourceFile->getPathInfo().foundPath; + + // We need to work out the name of the def file + + String ext = Path::getPathExt(path); + String pathWithoutExt = Path::getPathWithoutExt(path); + + // The output path + + StringBuilder outPath; + outPath << pathWithoutExt << "-defs." << ext; + + StringBuilder content; + SLANG_RETURN_ON_FAIL(calcDef(tree, origin, content)); + + // Write the defs file + SLANG_RETURN_ON_FAIL(FileUtil::writeAllText(outPath, m_sink, content.getUnownedSlice())); + } + + return SLANG_OK; +} + +SlangResult MacroWriter::writeOutput(NodeTree* tree) +{ + String path; + if (m_options->m_inputDirectory.getLength()) + { + path = Path::combine(m_options->m_inputDirectory, m_options->m_outputPath); + } + else + { + path = m_options->m_outputPath; + } + + // Get the ext + String ext = Path::getPathExt(path); + if (ext.getLength() == 0) + { + // Default to .h if not specified + ext = "h"; + } + + // Strip the extension if set + path = Path::getPathWithoutExt(path); + + for (TypeSet* typeSet : tree->getTypeSets()) + { + { + /// Calculate the header + StringBuilder header; + SLANG_RETURN_ON_FAIL(calcTypeHeader(tree, typeSet, header)); + + // Write it out + + StringBuilder headerPath; + headerPath << path << "-" << typeSet->m_fileMark << "." << ext; + SLANG_RETURN_ON_FAIL(FileUtil::writeAllText(headerPath, m_sink, header.getUnownedSlice())); + } + + { + StringBuilder childrenHeader; + SLANG_RETURN_ON_FAIL(calcChildrenHeader(tree, typeSet, childrenHeader)); + + StringBuilder headerPath; + headerPath << path << "-" << typeSet->m_fileMark << "-macro." + ext; + SLANG_RETURN_ON_FAIL(FileUtil::writeAllText(headerPath, m_sink, childrenHeader.getUnownedSlice())); + } + } + + return SLANG_OK; +} + +} // namespace CppExtract + diff --git a/tools/slang-cpp-extractor/macro-writer.h b/tools/slang-cpp-extractor/macro-writer.h new file mode 100644 index 000000000..b1754741d --- /dev/null +++ b/tools/slang-cpp-extractor/macro-writer.h @@ -0,0 +1,47 @@ +#ifndef CPP_EXTRACT_MACRO_WRITER_H +#define CPP_EXTRACT_MACRO_WRITER_H + +#include "diagnostics.h" + +#include "options.h" +#include "node-tree.h" + +#include "../../source/compiler-core/slang-diagnostic-sink.h" + +namespace CppExtract { +using namespace Slang; + +/* A class that writes out macros that define type hierarchies, as well as fields of types */ +class MacroWriter +{ +public: + + /// Write output + SlangResult writeOutput(NodeTree* tree); + + /// Write def files + SlangResult writeDefs(NodeTree* tree); + + /// Calculate the header + SlangResult calcTypeHeader(NodeTree* tree, TypeSet* typeSet, StringBuilder& out); + SlangResult calcChildrenHeader(NodeTree* tree, TypeSet* typeSet, StringBuilder& out); + SlangResult calcOriginHeader(NodeTree* tree, StringBuilder& out); + + SlangResult calcDef(NodeTree* tree, SourceOrigin* origin, StringBuilder& out); + + /// Ctor. + MacroWriter(DiagnosticSink* sink, const Options* options): + m_sink(sink), + m_options(options) + { + } + +protected: + + const Options* m_options = nullptr; + DiagnosticSink* m_sink; +}; + +} // CppExtract + +#endif diff --git a/tools/slang-cpp-extractor/node-tree.cpp b/tools/slang-cpp-extractor/node-tree.cpp new file mode 100644 index 000000000..3002a7abd --- /dev/null +++ b/tools/slang-cpp-extractor/node-tree.cpp @@ -0,0 +1,159 @@ +#include "node-tree.h" + +#include "options.h" +#include "identifier-lookup.h" + +#include "../../source/compiler-core/slang-name-convention-util.h" + +#include "../../source/core/slang-io.h" + +namespace CppExtract { +using namespace Slang; + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!! NodeTree !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +NodeTree::NodeTree(StringSlicePool* typePool, NamePool* namePool, IdentifierLookup* identifierLookup): + m_typePool(typePool), + m_namePool(namePool), + m_identifierLookup(identifierLookup), + m_typeSetPool(StringSlicePool::Style::Empty) +{ + m_rootNode = new ScopeNode(Node::Type::Namespace); + m_rootNode->m_reflectionType = ReflectionType::Reflected; +} + +TypeSet* NodeTree::getTypeSet(const UnownedStringSlice& slice) +{ + Index index = m_typeSetPool.findIndex(slice); + if (index < 0) + { + return nullptr; + } + return m_typeSets[index]; +} + +TypeSet* NodeTree::getOrAddTypeSet(const UnownedStringSlice& slice) +{ + const Index index = Index(m_typeSetPool.add(slice)); + if (index >= m_typeSets.getCount()) + { + SLANG_ASSERT(m_typeSets.getCount() == index); + TypeSet* typeSet = new TypeSet; + + m_typeSets.add(typeSet); + typeSet->m_macroName = m_typeSetPool.getSlice(StringSlicePool::Handle(index)); + return typeSet; + } + else + { + return m_typeSets[index]; + } +} + +SourceOrigin* NodeTree::addSourceOrigin(SourceFile* sourceFile, const Options& options) +{ + // Calculate from the path, a 'macro origin' name. + const String macroOrigin = calcMacroOrigin(sourceFile->getPathInfo().foundPath, options); + + SourceOrigin* origin = new SourceOrigin(sourceFile, macroOrigin); + m_sourceOrigins.add(origin); + return origin; +} + +/* static */String NodeTree::calcMacroOrigin(const String& filePath, const Options& options) +{ + // Get the filename without extension + String fileName = Path::getFileNameWithoutExt(filePath); + + // We can work on just the slice + UnownedStringSlice slice = fileName.getUnownedSlice(); + + // Filename prefix + if (options.m_stripFilePrefix.getLength() && slice.startsWith(options.m_stripFilePrefix.getUnownedSlice())) + { + const Index len = options.m_stripFilePrefix.getLength(); + slice = UnownedStringSlice(slice.begin() + len, slice.end()); + } + + // Trim - + slice = slice.trim('-'); + + StringBuilder out; + NameConventionUtil::convert(slice, CharCase::Upper, NameConvention::Snake, out); + return out; +} + +SlangResult NodeTree::_calcDerivedTypesRec(ScopeNode* inScopeNode, DiagnosticSink* sink) +{ + if (inScopeNode->isClassLike()) + { + ClassLikeNode* classLikeNode = static_cast<ClassLikeNode*>(inScopeNode); + + if (classLikeNode->m_super.hasContent()) + { + ScopeNode* parentScope = classLikeNode->m_parentScope; + if (parentScope == nullptr) + { + sink->diagnoseRaw(Severity::Error, UnownedStringSlice::fromLiteral("Can't lookup in scope if there is none!")); + return SLANG_FAIL; + } + + Node* superNode = Node::lookup(parentScope, classLikeNode->m_super.getContent()); + + if (!superNode) + { + if (classLikeNode->isReflected()) + { + sink->diagnose(classLikeNode->m_name, CPPDiagnostics::superTypeNotFound, classLikeNode->getAbsoluteName()); + return SLANG_FAIL; + } + } + else + { + ClassLikeNode* superType = as<ClassLikeNode>(superNode); + + if (!superType) + { + sink->diagnose(classLikeNode->m_name, CPPDiagnostics::superTypeNotAType, classLikeNode->getAbsoluteName()); + return SLANG_FAIL; + } + + if (superType->m_typeSet != classLikeNode->m_typeSet) + { + sink->diagnose(classLikeNode->m_name, CPPDiagnostics::typeInDifferentTypeSet, classLikeNode->m_name.getContent(), classLikeNode->m_typeSet->m_macroName, superType->m_typeSet->m_macroName); + return SLANG_FAIL; + } + + // The base class must be defined in same scope (as we didn't allow different scopes for base classes) + superType->addDerived(classLikeNode); + } + } + else + { + // Add to it's own typeset + if (classLikeNode->isReflected()) + { + classLikeNode->m_typeSet->m_baseTypes.add(classLikeNode); + } + } + } + + for (Node* child : inScopeNode->m_children) + { + ScopeNode* childScope = as<ScopeNode>(child); + if (childScope) + { + SLANG_RETURN_ON_FAIL(_calcDerivedTypesRec(childScope, sink)); + } + } + + return SLANG_OK; +} + +SlangResult NodeTree::calcDerivedTypes(DiagnosticSink* sink) +{ + return _calcDerivedTypesRec(m_rootNode, sink); +} + + +} // namespace CppExtract diff --git a/tools/slang-cpp-extractor/node-tree.h b/tools/slang-cpp-extractor/node-tree.h new file mode 100644 index 000000000..b54321b09 --- /dev/null +++ b/tools/slang-cpp-extractor/node-tree.h @@ -0,0 +1,102 @@ +#ifndef CPP_EXTRACT_NODE_TREE_H +#define CPP_EXTRACT_NODE_TREE_H + +#include "diagnostics.h" +#include "node.h" +#include "identifier-lookup.h" + +#include "../../source/compiler-core/slang-lexer.h" + +namespace CppExtract { +using namespace Slang; + +class TypeSet : public RefObject +{ +public: + /// This is the looked up name. + UnownedStringSlice m_macroName; ///< The name extracted from the macro SLANG_ABSTRACT_AST_CLASS -> AST + + String m_typeName; ///< The enum type name associated with this type for AST it is ASTNode + String m_fileMark; ///< This 'mark' becomes of the output filename + + List<ClassLikeNode*> m_baseTypes; ///< The base types for this type set +}; + +class SourceOrigin : public RefObject +{ +public: + + void addNode(Node* node) + { + if (auto classLike = as<ClassLikeNode>(node)) + { + SLANG_ASSERT(classLike->m_origin == nullptr); + classLike->m_origin = this; + } + + m_nodes.add(node); + } + + SourceOrigin(SourceFile* sourceFile, const String& macroOrigin) : + m_sourceFile(sourceFile), + m_macroOrigin(macroOrigin) + {} + + String m_macroOrigin; ///< The macro text is inserted into the macro to identify the origin. It is based on the filename + SourceFile* m_sourceFile; ///< The source file - also holds the path information + + /// All of the nodes defined in this file in the order they were defined + /// Note that the same namespace may be listed multiple times. + List<RefPtr<Node> > m_nodes; +}; + +struct Options; +class IdentifierLookup; + +/* NodeTree holds nodes that have been parsed into a tree rooted on the 'rootNode'. +Also contains other state associated with or useful to a node tree */ +class NodeTree +{ +public: + friend class Parser; + /// Get all of the parsed source origins + const List<RefPtr<SourceOrigin> >& getSourceOrigins() const { return m_sourceOrigins; } + + TypeSet* getTypeSet(const UnownedStringSlice& slice); + TypeSet* getOrAddTypeSet(const UnownedStringSlice& slice); + + SourceOrigin* addSourceOrigin(SourceFile* sourceFile, const Options& options); + + /// Get all of the type sets + const List<RefPtr<TypeSet>>& getTypeSets() const { return m_typeSets; } + + /// Get the root node + Node* getRootNode() const { return m_rootNode; } + + /// When parsing we don't lookup all up super types/add derived types. This is because + /// we allow files to be processed in any order, so we have to do the type lookup as a separate operation + SlangResult calcDerivedTypes(DiagnosticSink* sink); + + NodeTree(StringSlicePool* typePool, NamePool* namePool, IdentifierLookup* identifierLookup); + + static String calcMacroOrigin(const String& filePath, const Options& options); + +protected: + SlangResult _calcDerivedTypesRec(ScopeNode* node, DiagnosticSink* sink); + + StringSlicePool m_typeSetPool; ///< Pool for type set names + List<RefPtr<TypeSet> > m_typeSets; ///< The type sets + + IdentifierLookup* m_identifierLookup; + StringSlicePool* m_typePool; ///< Pool for just types + + NamePool* m_namePool; + + RefPtr<ScopeNode> m_rootNode; ///< The root scope + + List<RefPtr<SourceOrigin>> m_sourceOrigins; +}; + +} // CppExtract + +#endif diff --git a/tools/slang-cpp-extractor/node.cpp b/tools/slang-cpp-extractor/node.cpp index 3e259a81e..3b2403816 100644 --- a/tools/slang-cpp-extractor/node.cpp +++ b/tools/slang-cpp-extractor/node.cpp @@ -1,14 +1,29 @@ #include "node.h" +#include "file-util.h" + +#include "../../source/core/slang-string-util.h" + namespace CppExtract { // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Node Impl !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -static void _indent(Index indentCount, StringBuilder& out) +SLANG_FORCE_INLINE static void _indent(Index indentCount, StringBuilder& out) { FileUtil::indent(indentCount, out); } + +ScopeNode* Node::getRootScope() { - for (Index i = 0; i < indentCount; ++i) + if (m_parentScope) { - out << CPP_EXTRACT_INDENT_STRING; + ScopeNode* scope = m_parentScope; + while (scope->m_parentScope) + { + scope = scope->m_parentScope; + } + return scope; + } + else + { + return as<ScopeNode>(this); } } @@ -77,29 +92,158 @@ void Node::calcAbsoluteName(StringBuilder& outName) const } } -/* static */Node* Node::findNode(ScopeNode* scope, const UnownedStringSlice& name) +/* static */Node* Node::lookupNameInScope(ScopeNode* scope, const UnownedStringSlice& name) { - // TODO(JS): We may want to lookup based on the path. - // If the name is qualified, we give up for not - if (String(name).indexOf("::") >= 0) + // TODO(JS): Doesn't handle 'using namespace'. + + // Must be unqualified name + SLANG_ASSERT(name.indexOf(UnownedStringSlice::fromLiteral("::")) < 0); + + Node* childNode = scope->findChild(name); + if (childNode) { - return nullptr; + return childNode; } - // Okay try in all scopes up to the root - while (scope) + // If we have an anonymous namespace in this scope, try looking up in there.. + if (scope->m_anonymousNamespace) { - if (Node* node = scope->findChild(name)) + Node* childNode = scope->m_anonymousNamespace->findChild(name); + if (childNode) + { + return childNode; + } + } + + // I could have an enum (that's not an enum class) + for (Node* node : scope->m_children) + { + EnumNode* enumNode = as<EnumNode>(node); + if (enumNode && enumNode->m_type == Node::Type::Enum) + { + Node** nodePtr = enumNode->m_childMap.TryGetValue(name); + if (nodePtr) + { + return *nodePtr; + } + } + } + + return nullptr; +} + +/* static */Node* Node::lookupFromScope(ScopeNode* scope, const UnownedStringSlice* parts, Index partsCount) +{ + SLANG_ASSERT(partsCount > 0); + if (partsCount == 1) + { + return lookupNameInScope(scope, parts[0]); + } + + for (Index i = 0; i < partsCount; ++i) + { + const UnownedStringSlice& part = parts[i]; + + Node* node = lookupNameInScope(scope, part); + if (node == nullptr) + { + return node; + } + // If at end, then we are done + if (i == partsCount - 1) { return node; } - scope = scope->m_parentScope; + // If there are more elements, then node must be some kind of scope, + // if we are going to find it + scope = as<ScopeNode>(node); + if (scope == nullptr) + { + break; + } } return nullptr; } +/* static */void Node::splitPath(const UnownedStringSlice& inPath, List<UnownedStringSlice>& outParts) +{ + if (inPath.indexOf(UnownedStringSlice::fromLiteral("::")) >= 0) + { + StringUtil::split(inPath, UnownedStringSlice::fromLiteral("::"), outParts); + // Remove any whitespace + for (auto& part : outParts) + { + part = part.trim(); + } + } + else + { + outParts.clear(); + outParts.add(inPath.trim()); + } +} + +/* static */Node* Node::lookupFromScope(ScopeNode* scope, const UnownedStringSlice& inPath) +{ + if (inPath.indexOf(UnownedStringSlice::fromLiteral("::")) >= 0) + { + List<UnownedStringSlice> parts; + splitPath(inPath, parts); + + return lookupFromScope(scope, parts.getBuffer(), parts.getCount()); + } + else + { + return lookupNameInScope(scope, inPath); + } +} + +/* static */Node* Node::lookup(ScopeNode* scope, const UnownedStringSlice& inPath) +{ + if (inPath.indexOf(UnownedStringSlice::fromLiteral("::")) >= 0) + { + List<UnownedStringSlice> parts; + splitPath(inPath, parts); + + if (parts[0].getLength() == 0) + { + // It's a lookup from global scope + ScopeNode* rootScope = scope->getRootScope(); + return lookupFromScope(rootScope, parts.getBuffer() + 1, parts.getCount() + 1); + } + + // Okay lets try a lookup from each scope up to the global scope + while (scope) + { + Node* node = lookupFromScope(scope, parts.getBuffer(), parts.getCount()); + if (node) + { + return node; + } + + scope = scope->m_parentScope; + } + } + else + { + while (scope) + { + // Lookup in this scope + Node* node = lookupNameInScope(scope, inPath); + if (node) + { + return node; + } + + // Try parent scope + scope = scope->m_parentScope; + } + } + + return nullptr; +} // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ScopeNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -133,7 +277,14 @@ void ScopeNode::addChild(Node* child) Node* ScopeNode::findChild(const UnownedStringSlice& name) const { Node** nodePtr = m_childMap.TryGetValue(name); - return (nodePtr) ? *nodePtr : nullptr; + if (nodePtr) + { + return *nodePtr; + } + + + + return nullptr; } void ScopeNode::calcScopeDepthFirst(List<Node*>& outNodes) @@ -178,6 +329,85 @@ void ScopeNode::dump(int indentCount, StringBuilder& out) out << "}\n"; } +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! EnumCaseNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +void EnumCaseNode::dump(int indent, StringBuilder& out) +{ + if (isReflected()) + { + _indent(indent, out); + out << m_name.getContent(); + + if (m_value.type != TokenType::Invalid) + { + out << " = "; + out << m_value.getContent(); + } + + out << ",\n"; + } +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! EnumNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +void TypeDefNode::dump(int indent, StringBuilder& out) +{ + if (isReflected()) + { + _indent(indent, out); + + out << "typedef "; + + for (auto& tok : m_targetTypeTokens) + { + out << tok.getContent() << " "; + } + + out << m_name.getContent() << ";\n"; + } +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! EnumNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +void EnumNode::dump(int indent, StringBuilder& out) +{ + if (!isReflected()) + { + return; + } + + _indent(indent, out); + + out << "enum "; + + if (m_type == Type::EnumClass) + { + out << "class "; + } + + if (m_name.type != TokenType::Invalid) + { + out << m_name.getContent(); + } + + if (m_backingToken.type != TokenType::Invalid) + { + out << " : " << m_backingToken.getContent(); + } + + out << "\n"; + _indent(indent, out); + out << "{\n"; + + for (Node* child : m_children) + { + child->dump(indent + 1, out); + } + + _indent(indent, out); + out << "}\n"; +} + /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! FieldNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ void FieldNode::dump(int indent, StringBuilder& out) diff --git a/tools/slang-cpp-extractor/node.h b/tools/slang-cpp-extractor/node.h index c98a9204d..c741024e4 100644 --- a/tools/slang-cpp-extractor/node.h +++ b/tools/slang-cpp-extractor/node.h @@ -28,10 +28,18 @@ public: StructType, ClassType, + Enum, + EnumClass, + Namespace, AnonymousNamespace, Field, + EnumCase, + + TypeDef, + + CountOf, }; enum class TypeRange @@ -41,10 +49,29 @@ public: ClassLikeStart = int(Type::StructType), ClassLikeEnd = int(Type::ClassType), + + EnumStart = int(Type::Enum), + EnumEnd = int(Type::EnumClass), }; static bool isScopeType(Type type) { return int(type) >= int(TypeRange::ScopeStart) && int(type) <= int(TypeRange::ScopeEnd); } static bool isClassLikeType(Type type) { return int(type) >= int(TypeRange::ClassLikeStart) && int(type) <= int(TypeRange::ClassLikeEnd); } + static bool isEnumLikeType(Type type) { return int(type) >= int(TypeRange::EnumStart) && int(type) <= int(TypeRange::EnumEnd); } + static bool canAcceptTypes(Type type) + { + switch (type) + { + case Type::StructType: + case Type::ClassType: + case Type::Namespace: + case Type::AnonymousNamespace: + { + return true; + } + default: break; + } + return false; + } static bool isType(Type type) { return true; } @@ -67,6 +94,8 @@ public: /// True if reflected bool isReflected() const { return m_reflectionType == ReflectionType::Reflected; } + ScopeNode* getRootScope(); + typedef bool(*Filter)(Node* node); static bool isClassLikeAndReflected(Node* node) { return node->isClassLike() && node->isReflected(); } @@ -79,8 +108,19 @@ public: static void calcScopePath(Node* node, List<Node*>& outPath); - /// Find the name starting in specified scope - static Node* findNode(ScopeNode* scope, const UnownedStringSlice& name); + /// Lookup a name in just the specified scope + /// Handles anonymous namespaces, or name lookups that are in the parents space + static Node* lookupNameInScope(ScopeNode* scope, const UnownedStringSlice& name); + + /// Lookup from a path + static Node* lookupFromScope(ScopeNode* scope, const UnownedStringSlice* path, Index pathCount); + /// Looks up *just* from the specified scope. + static Node* lookupFromScope(ScopeNode* scope, const UnownedStringSlice& slice); + + /// Look up name (which can contain ::) + static Node* lookup(ScopeNode* scope, const UnownedStringSlice& name); + + static void splitPath(const UnownedStringSlice& slice, List<UnownedStringSlice>& outSplitPath); Node(Type type) : m_type(type), @@ -108,6 +148,8 @@ struct ScopeNode : public Node /// True if can accept fields (class like types can) bool acceptsFields() const { return isClassLike(); } + /// True if the scope can accept types + bool acceptsTypes() const { return canAcceptTypes(m_type); } /// Gets the reflection for any contained types ReflectionType getContainedReflectionType() const { return m_reflectionType == ReflectionType::NotReflected ? ReflectionType::NotReflected : m_reflectionOverride; } @@ -210,12 +252,56 @@ struct ClassLikeNode : public ScopeNode ClassLikeNode* m_superNode; ///< If this is a class/struct, the type it is derived from (or nullptr if base) }; +struct EnumCaseNode : public Node +{ + typedef Node Super; + + static bool isType(Type type) { return type == Type::EnumCase; } + + virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; + + EnumCaseNode(): + Super(Type::EnumCase) + { + } + + Token m_value; ///< If not defined will be invalid +}; + +struct EnumNode : public ScopeNode +{ + typedef ScopeNode Super; + static bool isType(Type type) { return isEnumLikeType(type); } + + virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; + + EnumNode(Type type): + Super(type) + { + SLANG_ASSERT(isEnumLikeType(type)); + } + + Token m_backingToken; +}; + +struct TypeDefNode : public Node +{ + typedef Node Super; + static bool isType(Type type) { return type == Type::TypeDef; } + + virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; + + TypeDefNode(): + Super(Type::TypeDef) + { + } + + List<Token> m_targetTypeTokens; +}; + template <typename T> T* as(Node* node) { return (node && T::isType(node->m_type)) ? static_cast<T*>(node) : nullptr; } -// A macro to define a single indent as a string -#define CPP_EXTRACT_INDENT_STRING " " - } // CppExtract #endif diff --git a/tools/slang-cpp-extractor/options.cpp b/tools/slang-cpp-extractor/options.cpp index c7b6a9df5..9ec671652 100644 --- a/tools/slang-cpp-extractor/options.cpp +++ b/tools/slang-cpp-extractor/options.cpp @@ -81,8 +81,7 @@ SlangResult OptionsParser::parse(int argc, const char*const* argv, DiagnosticSin } else if (arg == "-dump") { - outOptions.m_dump = true; - m_index++; + SLANG_RETURN_ON_FAIL(_parseArgFlag("-dump", outOptions.m_dump)); continue; } else if (arg == "-mark-prefix") @@ -110,13 +109,18 @@ SlangResult OptionsParser::parse(int argc, const char*const* argv, DiagnosticSin SLANG_RETURN_ON_FAIL(_parseArgWithValue("-strip-prefix", outOptions.m_stripFilePrefix)); continue; } + else if (arg == "-unit-test") + { + SLANG_RETURN_ON_FAIL(_parseArgFlag("-unit-test", outOptions.m_runUnitTests)); + continue; + } m_sink->diagnose(SourceLoc(), CPPDiagnostics::unknownOption, arg); return SLANG_FAIL; } else { - // If it starts with - then it an unknown option + // If it doesn't start with - then it's assumed to be an input path outOptions.m_inputPaths.add(arg); m_index++; } diff --git a/tools/slang-cpp-extractor/options.h b/tools/slang-cpp-extractor/options.h index ff88a3974..14ef9ecfc 100644 --- a/tools/slang-cpp-extractor/options.h +++ b/tools/slang-cpp-extractor/options.h @@ -24,6 +24,7 @@ struct Options bool m_defs = false; ///< If set will output a '-defs.h' file for each of the input files, that corresponds to previous defs files (although doesn't have fields/RAW) bool m_dump = false; ///< If true will dump to stderr the types/fields and hierarchy it extracted + bool m_runUnitTests = false; ///< If true will run internal unit tests bool m_outputFields = false; ///< When dumping macros also dump field definitions diff --git a/tools/slang-cpp-extractor/parser.cpp b/tools/slang-cpp-extractor/parser.cpp index 0d2cc31d2..9a2227b21 100644 --- a/tools/slang-cpp-extractor/parser.cpp +++ b/tools/slang-cpp-extractor/parser.cpp @@ -10,12 +10,46 @@ namespace CppExtract { using namespace Slang; -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! CPPExtractor !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +// If fails then we need more bits to identify types +SLANG_COMPILE_TIME_ASSERT(int(Node::Type::CountOf) <= 8 * sizeof(uint32_t)); + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Parser !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Parser::Parser(NodeTree* nodeTree, DiagnosticSink* sink) : m_sink(sink), - m_nodeTree(nodeTree) + m_nodeTree(nodeTree), + m_nodeTypeEnabled(0) { + // Enable types by default + const Node::Type defaultEnabled[] = + { + Node::Type::ClassType, + Node::Type::StructType, + Node::Type::Namespace, + Node::Type::AnonymousNamespace, + Node::Type::Field, + }; + setTypesEnabled(defaultEnabled, SLANG_COUNT_OF(defaultEnabled)); +} + +void Parser::setTypeEnabled(Node::Type type, bool isEnabled ) +{ + if (isEnabled) + { + m_nodeTypeEnabled |= (NodeTypeBitType(1) << int(type)); + } + else + { + m_nodeTypeEnabled &= ~(NodeTypeBitType(1) << int(type)); + } +} + +void Parser::setTypesEnabled(const Node::Type* types, Index typesCount, bool isEnabled) +{ + for (Index i = 0; i < typesCount; ++i) + { + setTypeEnabled(types[i], isEnabled); + } } bool Parser::_isMarker(const UnownedStringSlice& name) @@ -163,6 +197,31 @@ SlangResult Parser::popScope() return SLANG_OK; } +SlangResult Parser::_maybeConsumeScope() +{ + // Look for either ; or { to open scope + while (true) + { + const TokenType type = m_reader.peekTokenType(); + if (type == TokenType::Semicolon) + { + m_reader.advanceToken(); + return SLANG_OK; + } + else if (type == TokenType::LBrace) + { + m_reader.advanceToken(); + return consumeToClosingBrace(); + } + else if (type == TokenType::EndOfFile) + { + return SLANG_OK; + } + + m_reader.advanceToken(); + } +} + SlangResult Parser::consumeToClosingBrace(const Token* inOpenBraceToken) { Token openToken; @@ -204,6 +263,165 @@ SlangResult Parser::consumeToClosingBrace(const Token* inOpenBraceToken) } } + +SlangResult Parser::_parseEnum() +{ + // We are looking for + // enum ([class name] | [name]) [: base] ( { | ; ) + + Token enumToken; + + // consume enum + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &enumToken)); + + if (!m_currentScope->acceptsTypes()) + { + m_sink->diagnose(enumToken.loc, CPPDiagnostics::cannotDeclareTypeInScope); + return SLANG_FAIL; + } + + Node::Type type = Node::Type::Enum; + + Token nameToken; + if (advanceIfToken(TokenType::Identifier, &nameToken)) + { + const IdentifierStyle style = m_nodeTree->m_identifierLookup->get(nameToken.getContent()); + + if (style == IdentifierStyle::Class) + { + type = Node::Type::EnumClass; + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &nameToken)); + } + else if (style == IdentifierStyle::None) + { + // It holds the name then + } + else + { + m_sink->diagnose(nameToken.loc, CPPDiagnostics::expectingIdentifier, nameToken.getContent()); + return SLANG_FAIL; + } + } + + RefPtr<EnumNode> node = new EnumNode(type); + node->m_name = nameToken; + + if (advanceIfToken(TokenType::Colon)) + { + // We may have tokens up to { or ; + List<Token> backingTokens; + + while (true) + { + TokenType tokenType = m_reader.peekTokenType(); + if (tokenType == TokenType::Semicolon || + tokenType == TokenType::LBrace || + tokenType == TokenType::EndOfFile) + { + break; + } + + backingTokens.add(m_reader.advanceToken()); + } + + // TODO - Look up the backing type. It can only be an integral. We can assume it must be defined before lookup + // for our uses here. + // If we can't find the type, we could assume it's size is undefined + + if (backingTokens.getCount() == 1) + { + node->m_backingToken = backingTokens[0]; + } + } + + pushScope(node); + + if (advanceIfToken(TokenType::Semicolon)) + { + if (nameToken.type != TokenType::Invalid) + { + Node* node = m_currentScope->findChild(nameToken.getContent()); + if (node) + { + // Strictly speaking we should check the backing type etc, match, but for now ignore and assume it's ok + + if (node->m_type == type) + { + return SLANG_OK; + } + m_sink->diagnose(nameToken.loc, CPPDiagnostics::typeAlreadyDeclared, nameToken.getContent()); + return SLANG_FAIL; + } + return popScope(); + } + } + + SLANG_RETURN_ON_FAIL(expect(TokenType::LBrace)); + + while (true) + { + TokenType tokenType = m_reader.peekTokenType(); + if (tokenType == TokenType::RBrace) + { + break; + } + + RefPtr<EnumCaseNode> caseNode(new EnumCaseNode); + + // We could also check if the name is a valid identifier for name, for now just assume. + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &caseNode->m_name)); + + if (node->findChild(caseNode->m_name.getContent())) + { + m_sink->diagnose(caseNode->m_name.loc, CPPDiagnostics::identifierAlreadyDefined, caseNode->m_name.getContent()); + return SLANG_FAIL; + } + + // Add the value + node->addChild(caseNode); + + // TODO(JS): + // This could be better. We could lookup the value etc. For now just assume only one token is valid. + + if (advanceIfToken(TokenType::OpAssign)) + { + List<Token> valueTokens; + // Consume up to } or , + while (true) + { + TokenType assignType = m_reader.peekTokenType(); + + if (assignType == TokenType::Comma || + assignType == TokenType::RBrace || + assignType == TokenType::EndOfFile) + { + break; + } + valueTokens.add(m_reader.advanceToken()); + } + + if (valueTokens.getCount() == 1) + { + caseNode->m_value = valueTokens[0]; + } + } + + tokenType = m_reader.peekTokenType(); + if (tokenType == TokenType::Comma) + { + m_reader.advanceToken(); + continue; + } + + break; + } + + SLANG_RETURN_ON_FAIL(expect(TokenType::RBrace)); + SLANG_RETURN_ON_FAIL(expect(TokenType::Semicolon)); + + return popScope(); +} + SlangResult Parser::_maybeParseNode(Node::Type type) { // We are looking for @@ -234,6 +452,10 @@ SlangResult Parser::_maybeParseNode(Node::Type type) // Just ignore it then return SLANG_OK; } + else if (Node::isEnumLikeType(type)) + { + return _parseEnum(); + } // Must be class | struct @@ -407,8 +629,7 @@ SlangResult Parser::_maybeParseTemplateArg(Index& ioTemplateDepth) { case TokenType::Identifier: { - UnownedStringSlice name; - SLANG_RETURN_ON_FAIL(_maybeParseType(name, ioTemplateDepth)); + SLANG_RETURN_ON_FAIL(_maybeParseType(ioTemplateDepth)); return SLANG_OK; } case TokenType::IntegerLiteral: @@ -548,10 +769,8 @@ UnownedStringSlice Parser::_concatTokens(TokenReader::ParsingCursor start) return typePool->getSlice(typePool->add(buf)); } -SlangResult Parser::_maybeParseType(UnownedStringSlice& outType, Index& ioTemplateDepth) +SlangResult Parser::_maybeParseType(Index& ioTemplateDepth) { - auto startCursor = m_reader.getCursor(); - _consumeTypeModifiers(); advanceIfToken(TokenType::Scope); @@ -601,15 +820,43 @@ SlangResult Parser::_maybeParseType(UnownedStringSlice& outType, Index& ioTempla break; } - // We can build up the out type, from the tokens we found - outType = _concatTokens(startCursor); + return SLANG_OK; +} + +SlangResult Parser::_maybeParseType(List<Token>& outToks) +{ + auto startCursor = m_reader.getCursor(); + + Index templateDepth = 0; + SlangResult res = _maybeParseType(templateDepth); + if (SLANG_FAILED(res) && m_sink->getErrorCount()) + { + return res; + } + + if (templateDepth != 0) + { + m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::unexpectedTemplateClose); + return SLANG_FAIL; + } + + auto endCursor = m_reader.getCursor(); + m_reader.setCursor(startCursor); + + while (!m_reader.isAtCursor(endCursor)) + { + outToks.add(m_reader.advanceToken()); + } + return SLANG_OK; } SlangResult Parser::_maybeParseType(UnownedStringSlice& outType) { + auto startCursor = m_reader.getCursor(); + Index templateDepth = 0; - SlangResult res = _maybeParseType(outType, templateDepth); + SlangResult res = _maybeParseType(templateDepth); if (SLANG_FAILED(res) && m_sink->getErrorCount()) { return res; @@ -620,6 +867,9 @@ SlangResult Parser::_maybeParseType(UnownedStringSlice& outType) m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::unexpectedTemplateClose); return SLANG_FAIL; } + + // We can build up the out type, from the tokens we found + outType = _concatTokens(startCursor); return SLANG_OK; } @@ -710,6 +960,45 @@ SlangResult Parser::_parseBalanced(DiagnosticSink* sink) } } +SlangResult Parser::_parseTypeDef() +{ + if (!m_currentScope->acceptsTypes()) + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::cannotDeclareTypeInScope); + return SLANG_FAIL; + } + + // Consume the typedef + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier)); + + // Parse the type + List<Token> toks; + SLANG_RETURN_ON_FAIL(_maybeParseType(toks)); + + Token name; + + // Get the name + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &name)); + + if (Node::lookupNameInScope(m_currentScope, name.getContent())) + { + m_sink->diagnose(name.loc, CPPDiagnostics::identifierAlreadyDefined, name.getContent()); + return SLANG_FAIL; + } + + SLANG_RETURN_ON_FAIL(expect(TokenType::Semicolon)); + + RefPtr<TypeDefNode> node = new TypeDefNode; + node->m_name = name; + + // Set what aliases too + node->m_targetTypeTokens.swapWith(toks); + + m_currentScope->addChild(node); + + return SLANG_OK; +} + SlangResult Parser::_maybeParseField() { // Can only add a field if we are in a class @@ -793,9 +1082,11 @@ SlangResult Parser::_maybeParseField() { switch (style) { - case IdentifierStyle::Class: return Node::Type::ClassType; - case IdentifierStyle::Struct: return Node::Type::StructType; - case IdentifierStyle::Namespace: return Node::Type::Namespace; + case IdentifierStyle::Class: return Node::Type::ClassType; + case IdentifierStyle::Struct: return Node::Type::StructType; + case IdentifierStyle::Namespace: return Node::Type::Namespace; + case IdentifierStyle::Enum: return Node::Type::Enum; + case IdentifierStyle::TypeDef: return Node::Type::TypeDef; default: return Node::Type::Invalid; } } @@ -914,14 +1205,6 @@ SlangResult Parser::parse(SourceOrigin* sourceOrigin, const Options* options) SLANG_ASSERT(options); m_options = options; -#if 0 - // Calculate from the path, a 'macro origin' name. - const String macroOrigin = calcMacroOrigin(sourceFile->getPathInfo().foundPath, *options); - - RefPtr<SourceOrigin> origin = new SourceOrigin(sourceFile, macroOrigin); - m_sourceOrigins.add(origin); -#endif - // Set the current origin m_sourceOrigin = sourceOrigin; @@ -989,6 +1272,19 @@ SlangResult Parser::parse(SourceOrigin* sourceOrigin, const Options* options) SLANG_RETURN_ON_FAIL(expect(TokenType::Colon)); break; } + case IdentifierStyle::TypeDef: + { + if (isTypeEnabled(Node::Type::TypeDef)) + { + SLANG_RETURN_ON_FAIL(_parseTypeDef()); + } + else + { + m_reader.advanceToken(); + SLANG_RETURN_ON_FAIL(_consumeToSync()); + } + break; + } default: { IdentifierFlags flags = getFlags(style); @@ -996,7 +1292,16 @@ SlangResult Parser::parse(SourceOrigin* sourceOrigin, const Options* options) if (flags & IdentifierFlag::StartScope) { Node::Type type = _toNodeType(style); - SLANG_RETURN_ON_FAIL(_maybeParseNode(type)); + SLANG_ASSERT(type != Node::Type::Invalid); + + if (isTypeEnabled(type)) + { + SLANG_RETURN_ON_FAIL(_maybeParseNode(type)); + } + else + { + SLANG_RETURN_ON_FAIL(_maybeConsumeScope()); + } } else { @@ -1065,150 +1370,4 @@ SlangResult Parser::parse(SourceOrigin* sourceOrigin, const Options* options) } } -/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!! ParseSharedState !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ - -NodeTree::NodeTree(StringSlicePool* typePool, NamePool* namePool, IdentifierLookup* identifierLookup): - m_typePool(typePool), - m_namePool(namePool), - m_identifierLookup(identifierLookup), - m_typeSetPool(StringSlicePool::Style::Empty) -{ - m_rootNode = new ScopeNode(Node::Type::Namespace); - m_rootNode->m_reflectionType = ReflectionType::Reflected; -} - -TypeSet* NodeTree::getTypeSet(const UnownedStringSlice& slice) -{ - Index index = m_typeSetPool.findIndex(slice); - if (index < 0) - { - return nullptr; - } - return m_typeSets[index]; -} - -TypeSet* NodeTree::getOrAddTypeSet(const UnownedStringSlice& slice) -{ - const Index index = Index(m_typeSetPool.add(slice)); - if (index >= m_typeSets.getCount()) - { - SLANG_ASSERT(m_typeSets.getCount() == index); - TypeSet* typeSet = new TypeSet; - - m_typeSets.add(typeSet); - typeSet->m_macroName = m_typeSetPool.getSlice(StringSlicePool::Handle(index)); - return typeSet; - } - else - { - return m_typeSets[index]; - } -} - -SourceOrigin* NodeTree::addSourceOrigin(SourceFile* sourceFile, const Options& options) -{ - // Calculate from the path, a 'macro origin' name. - const String macroOrigin = calcMacroOrigin(sourceFile->getPathInfo().foundPath, options); - - SourceOrigin* origin = new SourceOrigin(sourceFile, macroOrigin); - m_sourceOrigins.add(origin); - return origin; -} - -/* static */String NodeTree::calcMacroOrigin(const String& filePath, const Options& options) -{ - // Get the filename without extension - String fileName = Path::getFileNameWithoutExt(filePath); - - // We can work on just the slice - UnownedStringSlice slice = fileName.getUnownedSlice(); - - // Filename prefix - if (options.m_stripFilePrefix.getLength() && slice.startsWith(options.m_stripFilePrefix.getUnownedSlice())) - { - const Index len = options.m_stripFilePrefix.getLength(); - slice = UnownedStringSlice(slice.begin() + len, slice.end()); - } - - // Trim - - slice = slice.trim('-'); - - StringBuilder out; - NameConventionUtil::convert(slice, CharCase::Upper, NameConvention::Snake, out); - return out; -} - -SlangResult NodeTree::_calcDerivedTypesRec(ScopeNode* inScopeNode, DiagnosticSink* sink) -{ - if (inScopeNode->isClassLike()) - { - ClassLikeNode* classLikeNode = static_cast<ClassLikeNode*>(inScopeNode); - - if (classLikeNode->m_super.hasContent()) - { - ScopeNode* parentScope = classLikeNode->m_parentScope; - if (parentScope == nullptr) - { - sink->diagnoseRaw(Severity::Error, UnownedStringSlice::fromLiteral("Can't lookup in scope if there is none!")); - return SLANG_FAIL; - } - - Node* superNode = Node::findNode(parentScope, classLikeNode->m_super.getContent()); - - if (!superNode) - { - if (classLikeNode->isReflected()) - { - sink->diagnose(classLikeNode->m_name, CPPDiagnostics::superTypeNotFound, classLikeNode->getAbsoluteName()); - return SLANG_FAIL; - } - } - else - { - ClassLikeNode* superType = as<ClassLikeNode>(superNode); - - if (!superType) - { - sink->diagnose(classLikeNode->m_name, CPPDiagnostics::superTypeNotAType, classLikeNode->getAbsoluteName()); - return SLANG_FAIL; - } - - if (superType->m_typeSet != classLikeNode->m_typeSet) - { - sink->diagnose(classLikeNode->m_name, CPPDiagnostics::typeInDifferentTypeSet, classLikeNode->m_name.getContent(), classLikeNode->m_typeSet->m_macroName, superType->m_typeSet->m_macroName); - return SLANG_FAIL; - } - - // The base class must be defined in same scope (as we didn't allow different scopes for base classes) - superType->addDerived(classLikeNode); - } - } - else - { - // Add to it's own typeset - if (classLikeNode->isReflected()) - { - classLikeNode->m_typeSet->m_baseTypes.add(classLikeNode); - } - } - } - - for (Node* child : inScopeNode->m_children) - { - ScopeNode* childScope = as<ScopeNode>(child); - if (childScope) - { - SLANG_RETURN_ON_FAIL(_calcDerivedTypesRec(childScope, sink)); - } - } - - return SLANG_OK; -} - -SlangResult NodeTree::calcDerivedTypes(DiagnosticSink* sink) -{ - return _calcDerivedTypesRec(m_rootNode, sink); -} - - } // namespace CppExtract diff --git a/tools/slang-cpp-extractor/parser.h b/tools/slang-cpp-extractor/parser.h index 58a13c19f..f0febb728 100644 --- a/tools/slang-cpp-extractor/parser.h +++ b/tools/slang-cpp-extractor/parser.h @@ -4,103 +4,19 @@ #include "diagnostics.h" #include "node.h" #include "identifier-lookup.h" +#include "node-tree.h" #include "../../source/compiler-core/slang-lexer.h" namespace CppExtract { using namespace Slang; -class TypeSet : public RefObject -{ -public: - /// This is the looked up name. - UnownedStringSlice m_macroName; ///< The name extracted from the macro SLANG_ABSTRACT_AST_CLASS -> AST - - String m_typeName; ///< The enum type name associated with this type for AST it is ASTNode - String m_fileMark; ///< This 'mark' becomes of the output filename - - List<ClassLikeNode*> m_baseTypes; ///< The base types for this type set -}; - -class SourceOrigin : public RefObject -{ -public: - - void addNode(Node* node) - { - if (auto classLike = as<ClassLikeNode>(node)) - { - SLANG_ASSERT(classLike->m_origin == nullptr); - classLike->m_origin = this; - } - - m_nodes.add(node); - } - - SourceOrigin(SourceFile* sourceFile, const String& macroOrigin) : - m_sourceFile(sourceFile), - m_macroOrigin(macroOrigin) - {} - - String m_macroOrigin; ///< The macro text is inserted into the macro to identify the origin. It is based on the filename - SourceFile* m_sourceFile; ///< The source file - also holds the path information - - /// All of the nodes defined in this file in the order they were defined - /// Note that the same namespace may be listed multiple times. - List<RefPtr<Node> > m_nodes; -}; - -struct Options; -class IdentifierLookup; - -/* NodeTree holds nodes that have been parsed into a tree rooted on the 'rootNode'. -Also contains other state associated with or useful to a node tree */ -class NodeTree -{ -public: - friend class Parser; - /// Get all of the parsed source origins - const List<RefPtr<SourceOrigin> >& getSourceOrigins() const { return m_sourceOrigins; } - - TypeSet* getTypeSet(const UnownedStringSlice& slice); - TypeSet* getOrAddTypeSet(const UnownedStringSlice& slice); - - SourceOrigin* addSourceOrigin(SourceFile* sourceFile, const Options& options); - - /// Get all of the type sets - const List<RefPtr<TypeSet>>& getTypeSets() const { return m_typeSets; } - - /// Get the root node - Node* getRootNode() const { return m_rootNode; } - - /// When parsing we don't lookup all up super types/add derived types. This is because - /// we allow files to be processed in any order, so we have to do the type lookup as a separate operation - SlangResult calcDerivedTypes(DiagnosticSink* sink); - - NodeTree(StringSlicePool* typePool, NamePool* namePool, IdentifierLookup* identifierLookup); - - static String calcMacroOrigin(const String& filePath, const Options& options); - -protected: - SlangResult _calcDerivedTypesRec(ScopeNode* node, DiagnosticSink* sink); - - StringSlicePool m_typeSetPool; ///< Pool for type set names - List<RefPtr<TypeSet> > m_typeSets; ///< The type sets - - IdentifierLookup* m_identifierLookup; - StringSlicePool* m_typePool; ///< Pool for just types - - NamePool* m_namePool; - - RefPtr<ScopeNode> m_rootNode; ///< The root scope - - List<RefPtr<SourceOrigin>> m_sourceOrigins; -}; - class Parser { public: + typedef uint32_t NodeTypeBitType; + SlangResult expect(TokenType type, Token* outToken = nullptr); bool advanceIfMarker(Token* outToken = nullptr); @@ -115,6 +31,10 @@ public: /// Parse the contents of the source file SlangResult parse(SourceOrigin* sourceOrigin, const Options* options); + void setTypeEnabled(Node::Type type, bool isEnabled = true); + bool isTypeEnabled(Node::Type type) { return (m_nodeTypeEnabled & (NodeTypeBitType(1) << int(type))) != 0; } + void setTypesEnabled(const Node::Type* types, Index typesCount, bool isEnabled = true); + Parser(NodeTree* nodeTree, DiagnosticSink* sink); protected: @@ -122,15 +42,21 @@ protected: bool _isMarker(const UnownedStringSlice& name); + SlangResult _maybeConsumeScope(); + SlangResult _parsePreDeclare(); SlangResult _parseTypeSet(); SlangResult _maybeParseNode(Node::Type type); SlangResult _maybeParseField(); + SlangResult _parseTypeDef(); + SlangResult _parseEnum(); + + SlangResult _maybeParseType(List<Token>& outToks); SlangResult _maybeParseType(UnownedStringSlice& outType); - SlangResult _maybeParseType(UnownedStringSlice& outType, Index& ioTemplateDepth); + SlangResult _maybeParseType(Index& ioTemplateDepth); SlangResult _maybeParseTemplateArgs(Index& ioTemplateDepth); SlangResult _maybeParseTemplateArg(Index& ioTemplateDepth); @@ -144,6 +70,8 @@ protected: SlangResult _consumeToSync(); + NodeTypeBitType m_nodeTypeEnabled; + TokenList m_tokenList; TokenReader m_reader; diff --git a/tools/slang-cpp-extractor/slang-cpp-extractor-main.cpp b/tools/slang-cpp-extractor/slang-cpp-extractor-main.cpp deleted file mode 100644 index 7ec53e29f..000000000 --- a/tools/slang-cpp-extractor/slang-cpp-extractor-main.cpp +++ /dev/null @@ -1,767 +0,0 @@ -// main.cpp - -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include "../../source/core/slang-secure-crt.h" - -#include "../../slang-com-helper.h" - -#include "../../source/core/slang-list.h" -#include "../../source/core/slang-string.h" -#include "../../source/core/slang-string-util.h" -#include "../../source/core/slang-io.h" -#include "../../source/core/slang-string-slice-pool.h" -#include "../../source/core/slang-writer.h" -#include "../../source/core/slang-file-system.h" - -#include "../../source/compiler-core/slang-source-loc.h" -#include "../../source/compiler-core/slang-lexer.h" -#include "../../source/compiler-core/slang-diagnostic-sink.h" -#include "../../source/compiler-core/slang-name.h" -#include "../../source/compiler-core/slang-name-convention-util.h" - -#include "node.h" -#include "diagnostics.h" -#include "options.h" -#include "parser.h" - -/* -Some command lines: - --d source/slang slang-ast-support-types.h slang-ast-base.h slang-ast-decl.h slang-ast-expr.h slang-ast-modifier.h slang-ast-stmt.h slang-ast-type.h slang-ast-val.h -strip-prefix slang- -o slang-generated -output-fields -mark-suffix _CLASS -*/ - -namespace CppExtract -{ - -using namespace Slang; - -static void _indent(Index indentCount, StringBuilder& out) -{ - for (Index i = 0; i < indentCount; ++i) - { - out << CPP_EXTRACT_INDENT_STRING; - } -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! CPPExtractorApp !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -class App -{ -public: - - SlangResult readAllText(const Slang::String& fileName, String& outRead); - SlangResult writeAllText(const Slang::String& fileName, const UnownedStringSlice& text); - - SlangResult execute(const Options& options); - - /// Execute - SlangResult executeWithArgs(int argc, const char*const* argv); - - /// Write output - SlangResult writeOutput(NodeTree* tree); - - /// Write def files - SlangResult writeDefs(NodeTree* tree); - - /// Calculate the header - SlangResult calcTypeHeader(NodeTree* tree, TypeSet* typeSet, StringBuilder& out); - SlangResult calcChildrenHeader(NodeTree* tree, TypeSet* typeSet, StringBuilder& out); - SlangResult calcOriginHeader(NodeTree* tree, StringBuilder& out); - - SlangResult calcDef(NodeTree* tree, SourceOrigin* origin, StringBuilder& out); - - const Options& getOptions() const { return m_options; } - - App(DiagnosticSink* sink, SourceManager* sourceManager, RootNamePool* rootNamePool): - m_sink(sink), - m_sourceManager(sourceManager), - m_slicePool(StringSlicePool::Style::Default) - { - m_namePool.setRootNamePool(rootNamePool); - } - -protected: - - /// Called to set up identifier lookup. Must be performed after options are initials - static void _initIdentifierLookup(const Options& options, IdentifierLookup& outLookup); - - NamePool m_namePool; - - Options m_options; - DiagnosticSink* m_sink; - SourceManager* m_sourceManager; - - StringSlicePool m_slicePool; -}; - -SlangResult App::readAllText(const Slang::String& fileName, String& outRead) -{ - try - { - StreamReader reader(new FileStream(fileName, FileMode::Open, FileAccess::Read, FileShare::ReadWrite)); - outRead = reader.ReadToEnd(); - } - catch (const IOException&) - { - m_sink->diagnose(SourceLoc(), CPPDiagnostics::cannotOpenFile, fileName); - return SLANG_FAIL; - } - catch (...) - { - m_sink->diagnose(SourceLoc(), CPPDiagnostics::cannotOpenFile, fileName); - return SLANG_FAIL; - } - - return SLANG_OK; -} - -SlangResult App::writeAllText(const Slang::String& fileName, const UnownedStringSlice& text) -{ - try - { - if (File::exists(fileName)) - { - String existingText; - if (readAllText(fileName, existingText) == SLANG_OK) - { - if (existingText == text) - return SLANG_OK; - } - } - StreamWriter writer(new FileStream(fileName, FileMode::Create)); - writer.Write(text); - } - catch (const IOException&) - { - m_sink->diagnose(SourceLoc(), CPPDiagnostics::cannotOpenFile, fileName); - return SLANG_FAIL; - } - - return SLANG_OK; -} - -SlangResult App::calcDef(NodeTree* tree, SourceOrigin* origin, StringBuilder& out) -{ - Node* currentScope = nullptr; - - for (Node* node : origin->m_nodes) - { - if (node->isReflected()) - { - if (auto classLikeNode = as<ClassLikeNode>(node)) - { - if (classLikeNode->m_marker.getContent().indexOf(UnownedStringSlice::fromLiteral("ABSTRACT")) >= 0) - { - out << "ABSTRACT_"; - } - - out << "SYNTAX_CLASS(" << node->m_name.getContent() << ", " << classLikeNode->m_super.getContent() << ")\n"; - out << "END_SYNTAX_CLASS()\n\n"; - } - } - } - return SLANG_OK; -} - -SlangResult App::calcChildrenHeader(NodeTree* tree, TypeSet* typeSet, StringBuilder& out) -{ - const List<ClassLikeNode*>& baseTypes = typeSet->m_baseTypes; - const String& reflectTypeName = typeSet->m_typeName; - - out << "#pragma once\n\n"; - out << "// Do not edit this file is generated from slang-cpp-extractor tool\n\n"; - - List<ClassLikeNode*> classNodes; - for (Index i = 0; i < baseTypes.getCount(); ++i) - { - ClassLikeNode* baseType = baseTypes[i]; - baseType->calcDerivedDepthFirst(classNodes); - } - - //Node::filter(Node::isClassLike, nodes); - - List<ClassLikeNode*> derivedTypes; - - out << "\n\n /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!! CHILDREN !!!!!!!!!!!!!!!!!!!!!!!!!!!! */ \n\n"; - - // Now the children - for (ClassLikeNode* classNode : classNodes) - { - classNode->getReflectedDerivedTypes(derivedTypes); - - // Define the derived types - out << "#define " << m_options.m_markPrefix << "CHILDREN_" << reflectTypeName << "_" << classNode->m_name.getContent() << "(x, param)"; - - if (derivedTypes.getCount()) - { - out << " \\\n"; - for (Index j = 0; j < derivedTypes.getCount(); ++j) - { - Node* derivedType = derivedTypes[j]; - _indent(1, out); - out << m_options.m_markPrefix << "ALL_" << reflectTypeName << "_" << derivedType->m_name.getContent() << "(x, param)"; - if (j < derivedTypes.getCount() - 1) - { - out << "\\\n"; - } - } - } - out << "\n\n"; - } - - out << "\n\n /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!! ALL !!!!!!!!!!!!!!!!!!!!!!!!!!!! */\n\n"; - - for (ClassLikeNode* classNode : classNodes) - { - // Define the derived types - out << "#define " << m_options.m_markPrefix << "ALL_" << reflectTypeName << "_" << classNode->m_name.getContent() << "(x, param) \\\n"; - _indent(1, out); - out << m_options.m_markPrefix << reflectTypeName << "_" << classNode->m_name.getContent() << "(x, param)"; - - // If has derived types output them - if (classNode->hasReflectedDerivedType()) - { - out << " \\\n"; - _indent(1, out); - out << m_options.m_markPrefix << "CHILDREN_" << reflectTypeName << "_" << classNode->m_name.getContent() << "(x, param)"; - } - out << "\n\n"; - } - - if (m_options.m_outputFields) - { - out << "\n\n /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!! FIELDS !!!!!!!!!!!!!!!!!!!!!!!!!!!! */\n\n"; - - for (ClassLikeNode* classNode : classNodes) - { - // Define the derived types - out << "#define " << m_options.m_markPrefix << "FIELDS_" << reflectTypeName << "_" << classNode->m_name.getContent() << "(_x_, _param_)"; - - // Find all of the fields - List<FieldNode*> fields; - for (Node* child : classNode->m_children) - { - if (auto field = as<FieldNode>(child)) - { - fields.add(field); - } - } - - if (fields.getCount() > 0) - { - out << "\\\n"; - - const Index fieldsCount = fields.getCount(); - bool previousField = false; - for (Index j = 0; j < fieldsCount; ++j) - { - const FieldNode* field = fields[j]; - - if (field->isReflected()) - { - if (previousField) - { - out << "\\\n"; - } - - _indent(1, out); - - // NOTE! We put the type field in brackets, such that there is no issue with templates containing a comma. - // If stringified - out << "_x_(" << field->m_name.getContent() << ", (" << field->m_fieldType << "), _param_)"; - previousField = true; - } - } - } - - out << "\n\n"; - } - } - - return SLANG_OK; -} - -SlangResult App::calcOriginHeader(NodeTree* tree, StringBuilder& out) -{ - // Do macros by origin - - out << "// Origin macros\n\n"; - - for (SourceOrigin* origin : tree->getSourceOrigins()) - { - out << "#define " << m_options.m_markPrefix << "ORIGIN_" << origin->m_macroOrigin << "(x, param) \\\n"; - - for (Node* node : origin->m_nodes) - { - if (!(node->isReflected() && node->isClassLike())) - { - continue; - } - - _indent(1, out); - out << "x(" << node->m_name.getContent() << ", param) \\\n"; - } - out << "/* */\n\n"; - } - - return SLANG_OK; -} - -SlangResult App::calcTypeHeader(NodeTree* tree, TypeSet* typeSet, StringBuilder& out) -{ - const List<ClassLikeNode*>& baseTypes = typeSet->m_baseTypes; - const String& reflectTypeName = typeSet->m_typeName; - - out << "#pragma once\n\n"; - out << "// Do not edit this file is generated from slang-cpp-extractor tool\n\n"; - - if (baseTypes.getCount() == 0) - { - return SLANG_OK; - } - - // Set up the scope - List<Node*> baseScopePath; - baseTypes[0]->calcScopePath(baseScopePath); - - // Remove the global scope - baseScopePath.removeAt(0); - // Remove the type itself - baseScopePath.removeLast(); - - for (Node* scopeNode : baseScopePath) - { - SLANG_ASSERT(scopeNode->m_type == Node::Type::Namespace); - out << "namespace " << scopeNode->m_name.getContent() << " {\n"; - } - - // Add all the base types, with in order traversals - List<ClassLikeNode*> nodes; - for (Index i = 0; i < baseTypes.getCount(); ++i) - { - ClassLikeNode* baseType = baseTypes[i]; - baseType->calcDerivedDepthFirst(nodes); - } - - Node::filter(Node::isClassLikeAndReflected, nodes); - - // Write out the types - { - out << "\n"; - out << "enum class " << reflectTypeName << "Type\n"; - out << "{\n"; - - Index typeIndex = 0; - for (ClassLikeNode* node : nodes) - { - // Okay first we are going to output the enum values - const Index depth = node->calcDerivedDepth() - 1; - _indent(depth, out); - out << node->m_name.getContent() << " = " << typeIndex << ",\n"; - typeIndex++; - } - - _indent(1, out); - out << "CountOf\n"; - - out << "};\n\n"; - } - - // TODO(JS): - // Strictly speaking if we wanted the types to be in different scopes, we would have to - // change the namespaces here - - // Predeclare the classes - { - out << "// Predeclare\n\n"; - for (ClassLikeNode* node : nodes) - { - // If it's not reflected we don't output, in the enum list - if (node->isReflected()) - { - const char* type = (node->m_type == Node::Type::ClassType) ? "class" : "struct"; - out << type << " " << node->m_name.getContent() << ";\n"; - } - } - } - - // Do the macros for each of the types - - { - out << "// Type macros\n\n"; - - out << "// Order is (NAME, SUPER, ORIGIN, LAST, MARKER, TYPE, param) \n"; - out << "// NAME - is the class name\n"; - out << "// SUPER - is the super class name (or NO_SUPER)\n"; - out << "// ORIGIN - where the definition was found\n"; - out << "// LAST - is the class name for the last in the range (or NO_LAST)\n"; - out << "// MARKER - is the text inbetween in the prefix/postix (like ABSTRACT). If no inbetween text is is 'NONE'\n"; - out << "// TYPE - Can be BASE, INNER or LEAF for the overall base class, an INNER class, or a LEAF class\n"; - out << "// param is a user defined parameter that can be parsed to the invoked x macro\n\n"; - - // Output all of the definitions for each type - for (ClassLikeNode* node : nodes) - { - out << "#define " << m_options.m_markPrefix << reflectTypeName << "_" << node->m_name.getContent() << "(x, param) "; - - // Output the X macro part - _indent(1, out); - out << "x(" << node->m_name.getContent() << ", "; - - if (node->m_superNode) - { - out << node->m_superNode->m_name.getContent() << ", "; - } - else - { - out << "NO_SUPER, "; - } - - // Output the (file origin) - out << node->m_origin->m_macroOrigin; - out << ", "; - - // The last type - Node* lastDerived = node->findLastDerived(); - if (lastDerived) - { - out << lastDerived->m_name.getContent() << ", "; - } - else - { - out << "NO_LAST, "; - } - - // Output any specifics of the markup - UnownedStringSlice marker = node->m_marker.getContent(); - // Need to extract the name - if (marker.getLength() > m_options.m_markPrefix.getLength() + m_options.m_markSuffix.getLength()) - { - marker = UnownedStringSlice(marker.begin() + m_options.m_markPrefix.getLength(), marker.end() - m_options.m_markSuffix.getLength()); - } - else - { - marker = UnownedStringSlice::fromLiteral("NONE"); - } - out << marker << ", "; - - if (node->m_superNode == nullptr) - { - out << "BASE, "; - } - else if (node->hasReflectedDerivedType()) - { - out << "INNER, "; - } - else - { - out << "LEAF, "; - } - out << "param)\n"; - } - } - - // Now pop the scope in revers - for (Index j = baseScopePath.getCount() - 1; j >= 0; j--) - { - Node* scopeNode = baseScopePath[j]; - out << "} // namespace " << scopeNode->m_name.getContent() << "\n"; - } - - return SLANG_OK; -} - -SlangResult App::writeDefs(NodeTree* tree) -{ - const auto& origins = tree->getSourceOrigins(); - - for (SourceOrigin* origin : origins) - { - const String path = origin->m_sourceFile->getPathInfo().foundPath; - - // We need to work out the name of the def file - - String ext = Path::getPathExt(path); - String pathWithoutExt = Path::getPathWithoutExt(path); - - // The output path - - StringBuilder outPath; - outPath << pathWithoutExt << "-defs." << ext; - - StringBuilder content; - SLANG_RETURN_ON_FAIL(calcDef(tree, origin, content)); - - // Write the defs file - SLANG_RETURN_ON_FAIL(writeAllText(outPath, content.getUnownedSlice())); - } - - return SLANG_OK; -} - -SlangResult App::writeOutput(NodeTree* tree) -{ - String path; - if (m_options.m_inputDirectory.getLength()) - { - path = Path::combine(m_options.m_inputDirectory, m_options.m_outputPath); - } - else - { - path = m_options.m_outputPath; - } - - // Get the ext - String ext = Path::getPathExt(path); - if (ext.getLength() == 0) - { - // Default to .h if not specified - ext = "h"; - } - - // Strip the extension if set - path = Path::getPathWithoutExt(path); - - for (TypeSet* typeSet : tree->getTypeSets()) - { - { - /// Calculate the header - StringBuilder header; - SLANG_RETURN_ON_FAIL(calcTypeHeader(tree, typeSet, header)); - - // Write it out - - StringBuilder headerPath; - headerPath << path << "-" << typeSet->m_fileMark << "." << ext; - SLANG_RETURN_ON_FAIL(writeAllText(headerPath, header.getUnownedSlice())); - } - - { - StringBuilder childrenHeader; - SLANG_RETURN_ON_FAIL(calcChildrenHeader(tree, typeSet, childrenHeader)); - - StringBuilder headerPath; - headerPath << path << "-" << typeSet->m_fileMark << "-macro." + ext; - SLANG_RETURN_ON_FAIL(writeAllText(headerPath, childrenHeader.getUnownedSlice())); - } - } - - return SLANG_OK; -} - -/* static */void App::_initIdentifierLookup(const Options& options, IdentifierLookup& outLookup) -{ - outLookup.reset(); - - // Some keywords - { - const char* names[] = { "virtual", "typedef", "continue", "if", "case", "break", "catch", "default", "delete", "do", "else", "for", "new", "goto", "return", "switch", "throw", "using", "while", "operator" }; - outLookup.set(names, SLANG_COUNT_OF(names), IdentifierStyle::Keyword); - } - - // Type modifier keywords - { - const char* names[] = { "const", "volatile" }; - outLookup.set(names, SLANG_COUNT_OF(names), IdentifierStyle::TypeModifier); - } - - // Special markers - { - const char* names[] = {"PRE_DECLARE", "TYPE_SET", "REFLECTED", "UNREFLECTED"}; - const IdentifierStyle styles[] = { IdentifierStyle::PreDeclare, IdentifierStyle::TypeSet, IdentifierStyle::Reflected, IdentifierStyle::Unreflected }; - SLANG_COMPILE_TIME_ASSERT(SLANG_COUNT_OF(names) == SLANG_COUNT_OF(styles)); - - StringBuilder buf; - for (Index i = 0; i < SLANG_COUNT_OF(names); ++i) - { - buf.Clear(); - buf << options.m_markPrefix << names[i]; - outLookup.set(buf.getUnownedSlice(), styles[i]); - } - } - - // Keywords which introduce types/scopes - { - outLookup.set("struct", IdentifierStyle::Struct); - outLookup.set("class", IdentifierStyle::Class); - outLookup.set("namespace", IdentifierStyle::Namespace); - } - - // Keywords that control access - { - const char* names[] = { "private", "protected", "public" }; - outLookup.set(names, SLANG_COUNT_OF(names), IdentifierStyle::Access); - } -} - -SlangResult App::execute(const Options& options) -{ - m_options = options; - - IdentifierLookup identifierLookup; - _initIdentifierLookup(options, identifierLookup); - - NodeTree tree(&m_slicePool, &m_namePool, &identifierLookup); - - - // Read in each of the input files - for (Index i = 0; i < m_options.m_inputPaths.getCount(); ++i) - { - String inputPath; - - if (m_options.m_inputDirectory.getLength()) - { - inputPath = Path::combine(m_options.m_inputDirectory, m_options.m_inputPaths[i]); - } - else - { - inputPath = m_options.m_inputPaths[i]; - } - - // Read the input file - String contents; - SLANG_RETURN_ON_FAIL(readAllText(inputPath, contents)); - - PathInfo pathInfo = PathInfo::makeFromString(inputPath); - - SourceFile* sourceFile = m_sourceManager->createSourceFileWithString(pathInfo, contents); - - SourceOrigin* sourceOrigin = tree.addSourceOrigin(sourceFile, options); - - Parser parser(&tree, m_sink); - SLANG_RETURN_ON_FAIL(parser.parse(sourceOrigin, &m_options)); - } - - SLANG_RETURN_ON_FAIL(tree.calcDerivedTypes(m_sink)); - - // Okay let's check out the typeSets - { - for (TypeSet* typeSet : tree.getTypeSets()) - { - // The macro name is in upper snake, so split it - List<UnownedStringSlice> slices; - NameConventionUtil::split(typeSet->m_macroName, slices); - - if (typeSet->m_fileMark.getLength() == 0) - { - StringBuilder buf; - // Let's guess a 'fileMark' (it becomes part of the filename) based on the macro name. Use lower kabab. - NameConventionUtil::join(slices.getBuffer(), slices.getCount(), CharCase::Lower, NameConvention::Kabab, buf); - typeSet->m_fileMark = buf.ProduceString(); - } - - if (typeSet->m_typeName.getLength() == 0) - { - // Let's guess a typename if not set -> go with upper camel - StringBuilder buf; - NameConventionUtil::join(slices.getBuffer(), slices.getCount(), CharCase::Upper, NameConvention::Camel, buf); - typeSet->m_typeName = buf.ProduceString(); - } - } - } - - // Dump out the tree - if (options.m_dump) - { - { - StringBuilder buf; - tree.getRootNode()->dump(0, buf); - m_sink->writer->write(buf.getBuffer(), buf.getLength()); - } - - for (TypeSet* typeSet : tree.getTypeSets()) - { - const List<ClassLikeNode*>& baseTypes = typeSet->m_baseTypes; - - for (ClassLikeNode* baseType : baseTypes) - { - StringBuilder buf; - baseType->dumpDerived(0, buf); - m_sink->writer->write(buf.getBuffer(), buf.getLength()); - } - } - } - - if (options.m_defs) - { - SLANG_RETURN_ON_FAIL(writeDefs(&tree)); - } - - if (options.m_outputPath.getLength()) - { - SLANG_RETURN_ON_FAIL(writeOutput(&tree)); - } - - return SLANG_OK; -} - -/// Execute -SlangResult App::executeWithArgs(int argc, const char*const* argv) -{ - Options options; - OptionsParser optionsParser; - SLANG_RETURN_ON_FAIL(optionsParser.parse(argc, argv, m_sink, options)); - SLANG_RETURN_ON_FAIL(execute(options)); - return SLANG_OK; -} - -} // namespace CppExtract - -int main(int argc, const char*const* argv) -{ - using namespace CppExtract; - using namespace Slang; - - { - ComPtr<ISlangWriter> writer(new FileWriter(stderr, WriterFlag::AutoFlush)); - - RootNamePool rootNamePool; - - SourceManager sourceManager; - sourceManager.initialize(nullptr, nullptr); - - DiagnosticSink sink(&sourceManager, Lexer::sourceLocationLexer); - sink.writer = writer; - - // Set to true to see command line that initiated C++ extractor. Helpful when finding issues from solution building failing, and then so - // being able to repeat the issue - bool dumpCommandLine = false; - - if (dumpCommandLine) - { - StringBuilder builder; - - for (Index i = 1; i < argc; ++i) - { - builder << argv[i] << " "; - } - - sink.diagnose(SourceLoc(), CPPDiagnostics::commandLine, builder); - } - - App app(&sink, &sourceManager, &rootNamePool); - - try - { - if (SLANG_FAILED(app.executeWithArgs(argc - 1, argv + 1))) - { - sink.diagnose(SourceLoc(), CPPDiagnostics::extractorFailed); - return 1; - } - if (sink.getErrorCount()) - { - sink.diagnose(SourceLoc(), CPPDiagnostics::extractorFailed); - return 1; - } - } - catch (...) - { - sink.diagnose(SourceLoc(), CPPDiagnostics::internalError); - return 1; - } - } - return 0; -} - diff --git a/tools/slang-cpp-extractor/unit-test.cpp b/tools/slang-cpp-extractor/unit-test.cpp new file mode 100644 index 000000000..7648e4e50 --- /dev/null +++ b/tools/slang-cpp-extractor/unit-test.cpp @@ -0,0 +1,80 @@ +#include "unit-test.h" + + +#include "../../source/compiler-core/slang-name-convention-util.h" + +#include "../../source/core/slang-io.h" + +#include "../../source/compiler-core/slang-source-loc.h" +#include "../../source/compiler-core/slang-lexer.h" + +#include "identifier-lookup.h" +#include "node-tree.h" +#include "parser.h" +#include "options.h" + +namespace CppExtract { +using namespace Slang; + + +struct TestState +{ + TestState(): + m_slicePool(StringSlicePool::Style::Default) + { + m_identifierLookup.initDefault(UnownedStringSlice::fromLiteral("SLANG_")); + + m_sourceManager.initialize(nullptr, nullptr); + + m_sink.init(&m_sourceManager, Lexer::sourceLocationLexer); + + m_namePool.setRootNamePool(&m_rootNamePool); + } + + RootNamePool m_rootNamePool; + Options m_options; + SourceManager m_sourceManager; + DiagnosticSink m_sink; + NamePool m_namePool; + StringSlicePool m_slicePool; + IdentifierLookup m_identifierLookup; +}; + +static const char someSource[] = +"enum SomeEnum\n" +"{\n" +" Value,\n" +" Another = 10,\n" +"};\n" +"typedef SomeEnum AliasEnum;\n"; + + +/* static */SlangResult UnitTestUtil::run() +{ + { + TestState state; + + NodeTree tree(&state.m_slicePool, &state.m_namePool, &state.m_identifierLookup); + + UnownedStringSlice contents = UnownedStringSlice::fromLiteral(someSource); + PathInfo pathInfo = PathInfo::makeFromString("source.h"); + + SourceManager* sourceManager = &state.m_sourceManager; + + SourceFile* sourceFile = sourceManager->createSourceFileWithString(pathInfo, contents); + SourceOrigin* sourceOrigin = tree.addSourceOrigin(sourceFile, state.m_options); + + Parser parser(&tree, &state.m_sink); + + { + const Node::Type enableTypes[] = { Node::Type::Enum, Node::Type::EnumClass, Node::Type::EnumCase, Node::Type::TypeDef }; + parser.setTypesEnabled(enableTypes, SLANG_COUNT_OF(enableTypes)); + } + + SLANG_RETURN_ON_FAIL(parser.parse(sourceOrigin, &state.m_options)); + } + + return SLANG_OK; +} + +} // namespace CppExtract diff --git a/tools/slang-cpp-extractor/unit-test.h b/tools/slang-cpp-extractor/unit-test.h new file mode 100644 index 000000000..9c8d9b08c --- /dev/null +++ b/tools/slang-cpp-extractor/unit-test.h @@ -0,0 +1,16 @@ +#ifndef CPP_EXTRACT_UNIT_TEST_H +#define CPP_EXTRACT_UNIT_TEST_H + +#include "diagnostics.h" + +namespace CppExtract { +using namespace Slang; + +struct UnitTestUtil +{ + static SlangResult run(); +}; + +} // CppExtract + +#endif diff --git a/tools/slang-test/unit-test-string.cpp b/tools/slang-test/unit-test-string.cpp index d585132c8..e27411b3e 100644 --- a/tools/slang-test/unit-test-string.cpp +++ b/tools/slang-test/unit-test-string.cpp @@ -84,24 +84,43 @@ static void stringUnitTest() { UnownedStringSlice values[] = { UnownedStringSlice("hello"), UnownedStringSlice("world"), UnownedStringSlice("!") }; + ArrayView<UnownedStringSlice> valuesView(values, SLANG_COUNT_OF(values)); + List<UnownedStringSlice> checkValues; StringBuilder builder; - builder.Clear(); - StringUtil::join(values, 0, ',', builder); - SLANG_CHECK(builder == ""); - builder.Clear(); - StringUtil::join(values, 1, ',', builder); - SLANG_CHECK(builder == "hello"); + { + builder.Clear(); + StringUtil::join(values, 0, ',', builder); + SLANG_CHECK(builder == ""); + } + + { + builder.Clear(); + StringUtil::join(values, 1, ',', builder); + SLANG_CHECK(builder == "hello"); + StringUtil::split(builder.getUnownedSlice(), ',', checkValues); + SLANG_CHECK(checkValues.getArrayView() == ArrayView<UnownedStringSlice>(values, 1)); + } - builder.Clear(); - StringUtil::join(values, 2, ',', builder); - SLANG_CHECK(builder == "hello,world"); + { + builder.Clear(); + StringUtil::join(values, 2, ',', builder); + SLANG_CHECK(builder == "hello,world"); - builder.Clear(); - StringUtil::join(values, 3, UnownedStringSlice("ab"), builder); - SLANG_CHECK(builder == "helloabworldab!"); + StringUtil::split(builder.getUnownedSlice(), ',', checkValues); + SLANG_CHECK(checkValues.getArrayView() == ArrayView<UnownedStringSlice>(values, 2)); + } + + { + builder.Clear(); + StringUtil::join(values, 3, UnownedStringSlice("ab"), builder); + SLANG_CHECK(builder == "helloabworldab!"); + + StringUtil::split(builder.getUnownedSlice(), UnownedStringSlice("ab"), checkValues); + SLANG_CHECK(checkValues.getArrayView() == ArrayView<UnownedStringSlice>(values, 3)); + } } } |