diff options
14 files changed, 2339 insertions, 2162 deletions
diff --git a/build/visual-studio/slang-cpp-extractor/slang-cpp-extractor.vcxproj b/build/visual-studio/slang-cpp-extractor/slang-cpp-extractor.vcxproj index eef3ca1ac..6e5d1cdb2 100644 --- a/build/visual-studio/slang-cpp-extractor/slang-cpp-extractor.vcxproj +++ b/build/visual-studio/slang-cpp-extractor/slang-cpp-extractor.vcxproj @@ -162,11 +162,19 @@ </Link> </ItemDefinitionGroup> <ItemGroup> - <ClInclude Include="..\..\..\tools\slang-cpp-extractor\slang-cpp-extractor-diagnostic-defs.h" /> - <ClInclude Include="..\..\..\tools\slang-cpp-extractor\slang-cpp-extractor-diagnostics.h" /> + <ClInclude Include="..\..\..\tools\slang-cpp-extractor\diagnostic-defs.h" /> + <ClInclude Include="..\..\..\tools\slang-cpp-extractor\diagnostics.h" /> + <ClInclude Include="..\..\..\tools\slang-cpp-extractor\identifier-lookup.h" /> + <ClInclude Include="..\..\..\tools\slang-cpp-extractor\node.h" /> + <ClInclude Include="..\..\..\tools\slang-cpp-extractor\options.h" /> + <ClInclude Include="..\..\..\tools\slang-cpp-extractor\parser.h" /> </ItemGroup> <ItemGroup> - <ClCompile Include="..\..\..\tools\slang-cpp-extractor\slang-cpp-extractor-diagnostics.cpp" /> + <ClCompile Include="..\..\..\tools\slang-cpp-extractor\diagnostics.cpp" /> + <ClCompile Include="..\..\..\tools\slang-cpp-extractor\identifier-lookup.cpp" /> + <ClCompile Include="..\..\..\tools\slang-cpp-extractor\node.cpp" /> + <ClCompile Include="..\..\..\tools\slang-cpp-extractor\options.cpp" /> + <ClCompile Include="..\..\..\tools\slang-cpp-extractor\parser.cpp" /> <ClCompile Include="..\..\..\tools\slang-cpp-extractor\slang-cpp-extractor-main.cpp" /> </ItemGroup> <ItemGroup> diff --git a/build/visual-studio/slang-cpp-extractor/slang-cpp-extractor.vcxproj.filters b/build/visual-studio/slang-cpp-extractor/slang-cpp-extractor.vcxproj.filters index 56ba63ca4..c11f860de 100644 --- a/build/visual-studio/slang-cpp-extractor/slang-cpp-extractor.vcxproj.filters +++ b/build/visual-studio/slang-cpp-extractor/slang-cpp-extractor.vcxproj.filters @@ -9,15 +9,39 @@ </Filter> </ItemGroup> <ItemGroup> - <ClInclude Include="..\..\..\tools\slang-cpp-extractor\slang-cpp-extractor-diagnostic-defs.h"> + <ClInclude Include="..\..\..\tools\slang-cpp-extractor\diagnostic-defs.h"> <Filter>Header Files</Filter> </ClInclude> - <ClInclude Include="..\..\..\tools\slang-cpp-extractor\slang-cpp-extractor-diagnostics.h"> + <ClInclude Include="..\..\..\tools\slang-cpp-extractor\diagnostics.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="..\..\..\tools\slang-cpp-extractor\identifier-lookup.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="..\..\..\tools\slang-cpp-extractor\node.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="..\..\..\tools\slang-cpp-extractor\options.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="..\..\..\tools\slang-cpp-extractor\parser.h"> <Filter>Header Files</Filter> </ClInclude> </ItemGroup> <ItemGroup> - <ClCompile Include="..\..\..\tools\slang-cpp-extractor\slang-cpp-extractor-diagnostics.cpp"> + <ClCompile Include="..\..\..\tools\slang-cpp-extractor\diagnostics.cpp"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\..\..\tools\slang-cpp-extractor\identifier-lookup.cpp"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\..\..\tools\slang-cpp-extractor\node.cpp"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\..\..\tools\slang-cpp-extractor\options.cpp"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\..\..\tools\slang-cpp-extractor\parser.cpp"> <Filter>Source Files</Filter> </ClCompile> <ClCompile Include="..\..\..\tools\slang-cpp-extractor\slang-cpp-extractor-main.cpp"> diff --git a/tools/slang-cpp-extractor/slang-cpp-extractor-diagnostic-defs.h b/tools/slang-cpp-extractor/diagnostic-defs.h index 284e02d19..284e02d19 100644 --- a/tools/slang-cpp-extractor/slang-cpp-extractor-diagnostic-defs.h +++ b/tools/slang-cpp-extractor/diagnostic-defs.h diff --git a/tools/slang-cpp-extractor/slang-cpp-extractor-diagnostics.cpp b/tools/slang-cpp-extractor/diagnostics.cpp index d07b125cb..3abbe1b38 100644 --- a/tools/slang-cpp-extractor/slang-cpp-extractor-diagnostics.cpp +++ b/tools/slang-cpp-extractor/diagnostics.cpp @@ -1,13 +1,13 @@ -#include "slang-cpp-extractor-diagnostics.h" +#include "diagnostics.h" -namespace SlangExperimental { +namespace CppExtract { namespace CPPDiagnostics { using namespace Slang; #define DIAGNOSTIC(id, severity, name, messageFormat) const DiagnosticInfo name = { id, Severity::severity, #name, messageFormat }; -#include "slang-cpp-extractor-diagnostic-defs.h" +#include "diagnostic-defs.h" } -} // namespace SlangExperimental +} // namespace CppExtract diff --git a/tools/slang-cpp-extractor/slang-cpp-extractor-diagnostics.h b/tools/slang-cpp-extractor/diagnostics.h index 419559fa8..3a98beee3 100644 --- a/tools/slang-cpp-extractor/slang-cpp-extractor-diagnostics.h +++ b/tools/slang-cpp-extractor/diagnostics.h @@ -1,17 +1,17 @@ -#ifndef SLANG_CPP_EXTRACTOR_DIAGNOSTICS_H -#define SLANG_CPP_EXTRACTOR_DIAGNOSTICS_H +#ifndef CPP_EXTRACT_DIAGNOSTICS_H +#define CPP_EXTRACT_DIAGNOSTICS_H #include "../../source/slang/slang-diagnostics.h" -namespace SlangExperimental { +namespace CppExtract { using namespace Slang; namespace CPPDiagnostics { #define DIAGNOSTIC(id, severity, name, messageFormat) extern const DiagnosticInfo name; -#include "slang-cpp-extractor-diagnostic-defs.h" +#include "diagnostic-defs.h" } // CPPDiagnostics -} // SlangExperimental +} // CppExtract #endif diff --git a/tools/slang-cpp-extractor/identifier-lookup.cpp b/tools/slang-cpp-extractor/identifier-lookup.cpp new file mode 100644 index 000000000..07f155248 --- /dev/null +++ b/tools/slang-cpp-extractor/identifier-lookup.cpp @@ -0,0 +1,48 @@ +#include "identifier-lookup.h" + +namespace CppExtract { +using namespace Slang; + +/* static */const IdentifierFlags IdentifierLookup::kIdentifierFlags[Index(IdentifierStyle::CountOf)] = +{ + 0, /// None + 0, /// Identifier + 0, /// Declare type + 0, /// Type set + IdentifierFlag::Keyword, /// TypeModifier + IdentifierFlag::Keyword, /// Keyword + IdentifierFlag::Keyword | IdentifierFlag::StartScope | IdentifierFlag::ClassLike, /// Class + IdentifierFlag::Keyword | IdentifierFlag::StartScope | IdentifierFlag::ClassLike, /// Struct + IdentifierFlag::Keyword | IdentifierFlag::StartScope, /// Namespace + IdentifierFlag::Keyword, /// Access + IdentifierFlag::Reflection, /// Reflected + IdentifierFlag::Reflection, /// Unreflected +}; + + +void IdentifierLookup::set(const UnownedStringSlice& name, IdentifierStyle style) +{ + StringSlicePool::Handle handle; + if (m_pool.findOrAdd(name, handle)) + { + // Add the extra flags + m_styles[Index(handle)] = style; + } + else + { + Index index = Index(handle); + SLANG_ASSERT(index == m_styles.getCount()); + m_styles.add(style); + } +} + +void IdentifierLookup::set(const char*const* names, size_t namesCount, IdentifierStyle style) +{ + for (size_t i = 0; i < namesCount; ++i) + { + set(UnownedStringSlice(names[i]), style); + } +} + + +} // namespace CppExtract diff --git a/tools/slang-cpp-extractor/identifier-lookup.h b/tools/slang-cpp-extractor/identifier-lookup.h new file mode 100644 index 000000000..b845f804c --- /dev/null +++ b/tools/slang-cpp-extractor/identifier-lookup.h @@ -0,0 +1,95 @@ +#ifndef CPP_EXTRACT_IDENTIFIER_LOOKUP_H +#define CPP_EXTRACT_IDENTIFIER_LOOKUP_H + +#include "diagnostics.h" + +namespace CppExtract { +using namespace Slang; + +enum class IdentifierStyle +{ + None, ///< It's not an identifier + + Identifier, ///< Just an identifier + + PreDeclare, ///< Declare a type (not visible in C++ code) + TypeSet, ///< TypeSet + + TypeModifier, ///< const, volatile etc + Keyword, ///< A keyword C/C++ keyword that is not another type + Class, ///< class + Struct, ///< struct + Namespace, ///< namespace + Access, ///< public, protected, private + + Reflected, + Unreflected, + + CountOf, +}; + +typedef uint32_t IdentifierFlags; +struct IdentifierFlag +{ + enum Enum : IdentifierFlags + { + StartScope = 0x1, ///< namespace, struct or class + ClassLike = 0x2, ///< Struct or class + Keyword = 0x4, + Reflection = 0x8, + }; +}; + + +class IdentifierLookup +{ +public: + + IdentifierStyle get(const UnownedStringSlice& slice) const + { + Index index = m_pool.findIndex(slice); + return (index >= 0) ? m_styles[index] : IdentifierStyle::None; + } + + void set(const char* name, IdentifierStyle style) + { + set(UnownedStringSlice(name), style); + } + + void set(const UnownedStringSlice& name, IdentifierStyle style); + + void set(const char*const* names, size_t namesCount, IdentifierStyle style); + + void reset() + { + m_styles.clear(); + m_pool.clear(); + } + + IdentifierLookup() : + m_pool(StringSlicePool::Style::Empty) + { + SLANG_ASSERT(m_pool.getSlicesCount() == 0); + } + + static const IdentifierFlags kIdentifierFlags[Index(IdentifierStyle::CountOf)]; + +protected: + List<IdentifierStyle> m_styles; + StringSlicePool m_pool; +}; + + +SLANG_FORCE_INLINE IdentifierFlags getFlags(IdentifierStyle style) +{ + return IdentifierLookup::kIdentifierFlags[Index(style)]; +} + +SLANG_FORCE_INLINE bool hasFlag(IdentifierStyle style, IdentifierFlag::Enum flag) +{ + return (getFlags(style) & flag) != 0; +} + +} // CppExtract + +#endif diff --git a/tools/slang-cpp-extractor/node.cpp b/tools/slang-cpp-extractor/node.cpp new file mode 100644 index 000000000..3e259a81e --- /dev/null +++ b/tools/slang-cpp-extractor/node.cpp @@ -0,0 +1,311 @@ +#include "node.h" + +namespace CppExtract { + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Node Impl !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +static void _indent(Index indentCount, StringBuilder& out) +{ + for (Index i = 0; i < indentCount; ++i) + { + out << CPP_EXTRACT_INDENT_STRING; + } +} + +void Node::calcScopeDepthFirst(List<Node*>& outNodes) +{ + outNodes.add(this); +} + +void Node::calcAbsoluteName(StringBuilder& outName) const +{ + List<Node*> path; + calcScopePath(const_cast<Node*>(this), path); + + // 1 so we skip the global scope + for (Index i = 1; i < path.getCount(); ++i) + { + Node* node = path[i]; + + if (i > 1) + { + outName << "::"; + } + + if (node->m_type == Type::AnonymousNamespace) + { + outName << "{Anonymous}"; + } + else + { + outName << node->m_name.getContent(); + } + } +} + +/* static */void Node::calcScopePath(Node* node, List<Node*>& outPath) +{ + outPath.clear(); + + while (node) + { + outPath.add(node); + node = node->m_parentScope; + } + + // reverse the order, so we go from root to the node + outPath.reverse(); +} + +/* static */void Node::filterImpl(Filter inFilter, List<Node*>& ioNodes) +{ + // Filter out all the unreflected nodes + Index count = ioNodes.getCount(); + for (Index j = 0; j < count; ) + { + Node* node = ioNodes[j]; + + if (!inFilter(node)) + { + ioNodes.removeAt(j); + count--; + } + else + { + j++; + } + } +} + +/* static */Node* Node::findNode(ScopeNode* scope, const UnownedStringSlice& name) +{ + // TODO(JS): We may want to lookup based on the path. + // If the name is qualified, we give up for not + if (String(name).indexOf("::") >= 0) + { + return nullptr; + } + + // Okay try in all scopes up to the root + while (scope) + { + if (Node* node = scope->findChild(name)) + { + return node; + } + + scope = scope->m_parentScope; + } + + return nullptr; +} + + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ScopeNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +ScopeNode* ScopeNode::getAnonymousNamespace() +{ + if (!m_anonymousNamespace) + { + m_anonymousNamespace = new ScopeNode(Type::AnonymousNamespace); + m_anonymousNamespace->m_parentScope = this; + m_children.add(m_anonymousNamespace); + } + + return m_anonymousNamespace; +} + +void ScopeNode::addChild(Node* child) +{ + SLANG_ASSERT(child->m_parentScope == nullptr); + // Can't add anonymous namespace this way - should be added via getAnonymousNamespace + SLANG_ASSERT(child->m_type != Type::AnonymousNamespace); + + child->m_parentScope = this; + m_children.add(child); + + if (child->m_name.hasContent()) + { + m_childMap.Add(child->m_name.getContent(), child); + } +} + +Node* ScopeNode::findChild(const UnownedStringSlice& name) const +{ + Node** nodePtr = m_childMap.TryGetValue(name); + return (nodePtr) ? *nodePtr : nullptr; +} + +void ScopeNode::calcScopeDepthFirst(List<Node*>& outNodes) +{ + outNodes.add(this); + for (Node* child : m_children) + { + child->calcScopeDepthFirst(outNodes); + } +} + +void ScopeNode::dump(int indentCount, StringBuilder& out) +{ + _indent(indentCount, out); + + switch (m_type) + { + case Type::AnonymousNamespace: + { + out << "namespace {\n"; + } + case Type::Namespace: + { + if (m_name.hasContent()) + { + out << "namespace " << m_name.getContent() << " {\n"; + } + else + { + out << "{\n"; + } + break; + } + } + + for (Node* child : m_children) + { + child->dump(indentCount + 1, out); + } + + _indent(indentCount, out); + out << "}\n"; +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! FieldNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +void FieldNode::dump(int indent, StringBuilder& out) +{ + if (isReflected()) + { + _indent(indent, out); + out << m_fieldType << " " << m_name.getContent() << "\n"; + } +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ClassLikeNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +/// Add a node that is derived from this +void ClassLikeNode::addDerived(ClassLikeNode* derived) +{ + SLANG_ASSERT(derived->m_superNode == nullptr); + derived->m_superNode = this; + m_derivedTypes.add(derived); +} + +void ClassLikeNode::calcDerivedDepthFirst(List<ClassLikeNode*>& outNodes) +{ + outNodes.add(this); + for (ClassLikeNode* derivedType : m_derivedTypes) + { + derivedType->calcDerivedDepthFirst(outNodes); + } +} + +void ClassLikeNode::dumpDerived(int indentCount, StringBuilder& out) +{ + if (isClassLike() && isReflected() && m_name.hasContent()) + { + _indent(indentCount, out); + out << m_name.getContent() << "\n"; + } + + for (ClassLikeNode* derivedType : m_derivedTypes) + { + derivedType->dumpDerived(indentCount + 1, out); + } +} + +Index ClassLikeNode::calcDerivedDepth() const +{ + const ClassLikeNode* node = this; + Index count = 0; + + while (node) + { + count++; + node = node->m_superNode; + } + + return count; +} + +ClassLikeNode* ClassLikeNode::findLastDerived() +{ + for (Index i = m_derivedTypes.getCount() - 1; i >= 0; --i) + { + ClassLikeNode* derivedType = m_derivedTypes[i]; + ClassLikeNode* found = derivedType->findLastDerived(); + if (found) + { + return found; + } + } + return this; +} + +bool ClassLikeNode::hasReflectedDerivedType() const +{ + for (ClassLikeNode* type : m_derivedTypes) + { + if (type->isReflected()) + { + return true; + } + } + return false; +} + +void ClassLikeNode::getReflectedDerivedTypes(List<ClassLikeNode*>& out) const +{ + out.clear(); + for (ClassLikeNode* type : m_derivedTypes) + { + if (type->isReflected()) + { + out.add(type); + } + } +} + +void ClassLikeNode::dump(int indentCount, StringBuilder& out) +{ + _indent(indentCount, out); + + const char* typeName = (m_type == Type::StructType) ? "struct" : "class"; + + out << typeName << " "; + + if (!isReflected()) + { + out << " ("; + } + out << m_name.getContent(); + if (!isReflected()) + { + out << ") "; + } + + if (m_super.hasContent()) + { + out << " : " << m_super.getContent(); + } + + out << " {\n"; + + for (Node* child : m_children) + { + child->dump(indentCount + 1, out); + } + + _indent(indentCount, out); + out << "}\n"; +} + +} // namespace CppExtract diff --git a/tools/slang-cpp-extractor/node.h b/tools/slang-cpp-extractor/node.h new file mode 100644 index 000000000..c98a9204d --- /dev/null +++ b/tools/slang-cpp-extractor/node.h @@ -0,0 +1,221 @@ +#ifndef CPP_EXTRACT_NODE_H +#define CPP_EXTRACT_NODE_H + +#include "diagnostics.h" + +namespace CppExtract { +using namespace Slang; + +enum class ReflectionType : uint8_t +{ + NotReflected, + Reflected, +}; + +// Pre-declare +class TypeSet; +class SourceOrigin; + +struct ScopeNode; + +class Node : public RefObject +{ +public: + enum class Type : uint8_t + { + Invalid, + + StructType, + ClassType, + + Namespace, + AnonymousNamespace, + + Field, + }; + + enum class TypeRange + { + ScopeStart = int(Type::StructType), + ScopeEnd = int(Type::AnonymousNamespace), + + ClassLikeStart = int(Type::StructType), + ClassLikeEnd = int(Type::ClassType), + }; + + static bool isScopeType(Type type) { return int(type) >= int(TypeRange::ScopeStart) && int(type) <= int(TypeRange::ScopeEnd); } + static bool isClassLikeType(Type type) { return int(type) >= int(TypeRange::ClassLikeStart) && int(type) <= int(TypeRange::ClassLikeEnd); } + + static bool isType(Type type) { return true; } + + bool isClassLike() const { return isClassLikeType(m_type); } + + virtual void dump(int indent, StringBuilder& out) = 0; + + /// Do depth first traversal of nodes in scopes + virtual void calcScopeDepthFirst(List<Node*>& outNodes); + + /// Calculate the absolute name for this namespace/type + void calcAbsoluteName(StringBuilder& outName) const; + + /// Get the absolute name + String getAbsoluteName() const { StringBuilder buf; calcAbsoluteName(buf); return buf.ProduceString(); } + + /// Calculate the scope path to this node, from the root + void calcScopePath(List<Node*>& outPath) { calcScopePath(this, outPath); } + + /// True if reflected + bool isReflected() const { return m_reflectionType == ReflectionType::Reflected; } + + typedef bool(*Filter)(Node* node); + + static bool isClassLikeAndReflected(Node* node) { return node->isClassLike() && node->isReflected(); } + static bool isClassLike(Node* node) { return isClassLikeType(node->m_type); } + + template <typename T> + static void filter(Filter filter, List<T*>& io) { const Node* _isNodeDerived = (T*)nullptr; SLANG_UNUSED(_isNodeDerived); filterImpl(filter, reinterpret_cast<List<Node*>&>(io)); } + + static void filterImpl(Filter filter, List<Node*>& io); + + static void calcScopePath(Node* node, List<Node*>& outPath); + + /// Find the name starting in specified scope + static Node* findNode(ScopeNode* scope, const UnownedStringSlice& name); + + Node(Type type) : + m_type(type), + m_parentScope(nullptr), + m_reflectionType(ReflectionType::NotReflected) + { + } + + Type m_type; ///< The type of node this is + ReflectionType m_reflectionType; /// Classes can be traversed, but not reflected. To be reflected they have to contain the marker + + Token m_name; ///< The name of this scope/type + + ScopeNode* m_parentScope; ///< The scope this type/scope is defined in +}; + +struct ScopeNode : public Node +{ + typedef Node Super; + + static bool isType(Type type) { return isScopeType(type); } + + virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; + virtual void calcScopeDepthFirst(List<Node*>& outNodes) SLANG_OVERRIDE; + + /// True if can accept fields (class like types can) + bool acceptsFields() const { return isClassLike(); } + + /// Gets the reflection for any contained types + ReflectionType getContainedReflectionType() const { return m_reflectionType == ReflectionType::NotReflected ? ReflectionType::NotReflected : m_reflectionOverride; } + + /// Add a child node to this nodes scope + void addChild(Node* child); + + /// Find a child node in this scope with the specified name. Return nullptr if not found + Node* findChild(const UnownedStringSlice& name) const; + + /// Gets the anonymous namespace associated with this scope + ScopeNode* getAnonymousNamespace(); + + ScopeNode(Type type) : + Super(type), + m_reflectionOverride(ReflectionType::Reflected), + m_anonymousNamespace(nullptr) + { + } + + /// For child types, fields, how reflection is handled. If this type is not reflected + ReflectionType m_reflectionOverride; + + /// All of the types and namespaces in this *scope* + List<RefPtr<Node>> m_children; + + /// Map from a name (in this scope) to the Node + Dictionary<UnownedStringSlice, Node*> m_childMap; + + /// There can only be one anonymousNamespace for a scope. If there is one it's held here + ScopeNode* m_anonymousNamespace; +}; + +struct FieldNode : public Node +{ + typedef Node Super; + + static bool isType(Type type) { return type == Type::Field; } + + virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; + + FieldNode() : + Super(Type::Field) + { + } + + UnownedStringSlice m_fieldType; + + // We may want to add initializer tokens +}; + +struct ClassLikeNode : public ScopeNode +{ + typedef ScopeNode Super; + + static bool isType(Type type) { return isClassLikeType(type); } + + /// Add a node that is derived from this + void addDerived(ClassLikeNode* derived); + + /// Dump all of the derived types + void dumpDerived(int indentCount, StringBuilder& out); + + /// Calculates the derived depth + Index calcDerivedDepth() const; + + /// Find the last (reflected) derived type + ClassLikeNode* findLastDerived(); + + /// Traverse the hierarchy of derived nodes, in depth first order + void calcDerivedDepthFirst(List<ClassLikeNode*>& outNodes); + + /// True if has a derived type that is reflected + bool hasReflectedDerivedType() const; + + /// Stores in out any reflected derived types + void getReflectedDerivedTypes(List<ClassLikeNode*>& out) const; + + // Node Impl + virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; + + ClassLikeNode(Type type) : + Super(type), + m_origin(nullptr), + m_typeSet(nullptr), + m_superNode(nullptr) + { + SLANG_ASSERT(type == Type::ClassType || type == Type::StructType); + } + + SourceOrigin* m_origin; ///< Defines where this was uniquely defined. + + Token m_marker; ///< The marker associated with this scope (typically the marker is SLANG_CLASS etc, that is used to identify reflectedType) + + List<RefPtr<ClassLikeNode>> m_derivedTypes; ///< All of the types derived from this type + + TypeSet* m_typeSet; ///< The typeset this type belongs to. + + Token m_super; ///< Super class name + ClassLikeNode* m_superNode; ///< If this is a class/struct, the type it is derived from (or nullptr if base) +}; + +template <typename T> +T* as(Node* node) { return (node && T::isType(node->m_type)) ? static_cast<T*>(node) : nullptr; } + +// A macro to define a single indent as a string +#define CPP_EXTRACT_INDENT_STRING " " + +} // CppExtract + +#endif diff --git a/tools/slang-cpp-extractor/options.cpp b/tools/slang-cpp-extractor/options.cpp new file mode 100644 index 000000000..c7b6a9df5 --- /dev/null +++ b/tools/slang-cpp-extractor/options.cpp @@ -0,0 +1,134 @@ +#include "options.h" + +#include "diagnostics.h" + +namespace CppExtract { + +SlangResult OptionsParser::_parseArgFlag(const char* option, bool& outFlag) +{ + SLANG_ASSERT(UnownedStringSlice(m_args[m_index]) == option); + SLANG_ASSERT(m_index < m_argCount); + + m_index++; + outFlag = true; + return SLANG_OK; +} + +SlangResult OptionsParser::_parseArgWithValue(const char* option, String& ioValue) +{ + SLANG_ASSERT(UnownedStringSlice(m_args[m_index]) == option); + if (m_index + 1 < m_argCount) + { + // Next parameter is the output path, there can only be one + if (ioValue.getLength()) + { + // There already is output + m_sink->diagnose(SourceLoc(), CPPDiagnostics::optionAlreadyDefined, option, ioValue); + return SLANG_FAIL; + } + } + else + { + m_sink->diagnose(SourceLoc(), CPPDiagnostics::requireValueAfterOption, option); + return SLANG_FAIL; + } + + ioValue = m_args[m_index + 1]; + m_index += 2; + return SLANG_OK; +} + +SlangResult OptionsParser::_parseArgReplaceValue(const char* option, String& ioValue) +{ + SLANG_ASSERT(UnownedStringSlice(m_args[m_index]) == option); + if (m_index + 1 >= m_argCount) + { + m_sink->diagnose(SourceLoc(), CPPDiagnostics::requireValueAfterOption, option); + return SLANG_FAIL; + } + + ioValue = m_args[m_index + 1]; + m_index += 2; + return SLANG_OK; +} + +SlangResult OptionsParser::parse(int argc, const char*const* argv, DiagnosticSink* sink, Options& outOptions) +{ + outOptions.reset(); + + m_index = 0; + m_argCount = argc; + m_args = argv; + m_sink = sink; + + outOptions.reset(); + + while (m_index < m_argCount) + { + const UnownedStringSlice arg = UnownedStringSlice(argv[m_index]); + + if (arg.getLength() > 0 && arg[0] == '-') + { + if (arg == "-d") + { + SLANG_RETURN_ON_FAIL(_parseArgWithValue("-d", outOptions.m_inputDirectory)); + continue; + } + else if (arg == "-o") + { + SLANG_RETURN_ON_FAIL(_parseArgWithValue("-o", outOptions.m_outputPath)); + continue; + } + else if (arg == "-dump") + { + outOptions.m_dump = true; + m_index++; + continue; + } + else if (arg == "-mark-prefix") + { + SLANG_RETURN_ON_FAIL(_parseArgReplaceValue("-mark-prefix", outOptions.m_markPrefix)); + continue; + } + else if (arg == "-mark-suffix") + { + SLANG_RETURN_ON_FAIL(_parseArgReplaceValue("-mark-suffix", outOptions.m_markSuffix)); + continue; + } + else if (arg == "-defs") + { + SLANG_RETURN_ON_FAIL(_parseArgFlag("-defs", outOptions.m_defs)); + continue; + } + else if (arg == "-output-fields") + { + SLANG_RETURN_ON_FAIL(_parseArgFlag("-output-fields", outOptions.m_outputFields)); + continue; + } + else if (arg == "-strip-prefix") + { + SLANG_RETURN_ON_FAIL(_parseArgWithValue("-strip-prefix", outOptions.m_stripFilePrefix)); + continue; + } + + m_sink->diagnose(SourceLoc(), CPPDiagnostics::unknownOption, arg); + return SLANG_FAIL; + } + else + { + // If it starts with - then it an unknown option + outOptions.m_inputPaths.add(arg); + m_index++; + } + } + + if (outOptions.m_inputPaths.getCount() < 0) + { + m_sink->diagnose(SourceLoc(), CPPDiagnostics::noInputPathsSpecified); + return SLANG_FAIL; + } + + return SLANG_OK; +} + +} // namespace CppExtract diff --git a/tools/slang-cpp-extractor/options.h b/tools/slang-cpp-extractor/options.h new file mode 100644 index 000000000..ff88a3974 --- /dev/null +++ b/tools/slang-cpp-extractor/options.h @@ -0,0 +1,60 @@ +#ifndef CPP_EXTRACT_OPTIONS_H +#define CPP_EXTRACT_OPTIONS_H + +#include "../../source/slang/slang-diagnostics.h" + +namespace CppExtract { +using namespace Slang; + + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Options !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +struct Options +{ + void reset() + { + *this = Options(); + } + + Options() + { + m_markPrefix = "SLANG_"; + m_markSuffix = "_CLASS"; + } + + bool m_defs = false; ///< If set will output a '-defs.h' file for each of the input files, that corresponds to previous defs files (although doesn't have fields/RAW) + bool m_dump = false; ///< If true will dump to stderr the types/fields and hierarchy it extracted + + bool m_outputFields = false; ///< When dumping macros also dump field definitions + + List<String> m_inputPaths; ///< The input paths to the files to be processed + + String m_outputPath; ///< The output path. Note that the extractor can generate multiple output files, and this will actually be the 'stem' of several files + + String m_inputDirectory; ///< The input directory that is by default used for reading m_inputPaths from. + String m_markPrefix; ///< The prefix of the 'marker' used to identify a reflected type + String m_markSuffix; ///< The postfix of the 'marker' used to identify a reflected type + String m_stripFilePrefix; ///< Used for the 'origin' information, this is stripped from the source filename, and the remainder of the filename (without extension) is 'macroized' +}; + +struct OptionsParser +{ + /// Parse the parameters. NOTE! Must have the program path removed + SlangResult parse(int argc, const char*const* argv, DiagnosticSink* sink, Options& outOptions); + + SlangResult _parseArgWithValue(const char* option, String& outValue); + SlangResult _parseArgReplaceValue(const char* option, String& outValue); + SlangResult _parseArgFlag(const char* option, bool& outFlag); + + String m_reflectType; + + Index m_index; + Int m_argCount; + const char*const* m_args; + DiagnosticSink* m_sink; +}; + + +} // CppExtract + +#endif diff --git a/tools/slang-cpp-extractor/parser.cpp b/tools/slang-cpp-extractor/parser.cpp new file mode 100644 index 000000000..0d2cc31d2 --- /dev/null +++ b/tools/slang-cpp-extractor/parser.cpp @@ -0,0 +1,1214 @@ +#include "parser.h" + +#include "options.h" +#include "identifier-lookup.h" + +#include "../../source/compiler-core/slang-name-convention-util.h" + +#include "../../source/core/slang-io.h" + +namespace CppExtract { +using namespace Slang; + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! CPPExtractor !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +Parser::Parser(NodeTree* nodeTree, DiagnosticSink* sink) : + m_sink(sink), + m_nodeTree(nodeTree) +{ +} + +bool Parser::_isMarker(const UnownedStringSlice& name) +{ + return name.startsWith(m_options->m_markPrefix.getUnownedSlice()) && name.endsWith(m_options->m_markSuffix.getUnownedSlice()); +} + +SlangResult Parser::expect(TokenType type, Token* outToken) +{ + if (m_reader.peekTokenType() != type) + { + m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::expectingToken, type); + return SLANG_FAIL; + } + + if (outToken) + { + *outToken = m_reader.advanceToken(); + } + else + { + m_reader.advanceToken(); + } + return SLANG_OK; +} + +bool Parser::advanceIfToken(TokenType type, Token* outToken) +{ + if (m_reader.peekTokenType() == type) + { + Token token = m_reader.advanceToken(); + if (outToken) + { + *outToken = token; + } + return true; + } + return false; +} + +bool Parser::advanceIfMarker(Token* outToken) +{ + const Token peekToken = m_reader.peekToken(); + if (peekToken.type == TokenType::Identifier && _isMarker(peekToken.getContent())) + { + m_reader.advanceToken(); + if (outToken) + { + *outToken = peekToken; + } + return true; + } + return false; +} + +bool Parser::advanceIfStyle(IdentifierStyle style, Token* outToken) +{ + if (m_reader.peekTokenType() == TokenType::Identifier) + { + IdentifierStyle readStyle = m_nodeTree->m_identifierLookup->get(m_reader.peekToken().getContent()); + if (readStyle == style) + { + Token token = m_reader.advanceToken(); + if (outToken) + { + *outToken = token; + } + return true; + } + } + return false; +} + + +SlangResult Parser::pushAnonymousNamespace() +{ + m_currentScope = m_currentScope->getAnonymousNamespace(); + + if (m_sourceOrigin) + { + m_sourceOrigin->addNode(m_currentScope); + } + + return SLANG_OK; +} + +SlangResult Parser::pushScope(ScopeNode* scopeNode) +{ + if (m_sourceOrigin) + { + m_sourceOrigin->addNode(scopeNode); + } + + if (scopeNode->m_name.hasContent()) + { + // For anonymous namespace, we should look if we already have one and just reopen that. Doing so will mean will + // find anonymous namespace clashes + + if (Node* foundNode = m_currentScope->findChild(scopeNode->m_name.getContent())) + { + if (scopeNode->isClassLike()) + { + m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::typeAlreadyDeclared, scopeNode->m_name.getContent()); + m_sink->diagnose(foundNode->m_name, CPPDiagnostics::seeDeclarationOf, scopeNode->m_name.getContent()); + return SLANG_FAIL; + } + + if (foundNode->m_type == Node::Type::Namespace) + { + if (foundNode->m_type != scopeNode->m_type) + { + // Different types can't work + m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::typeAlreadyDeclared, scopeNode->m_name.getContent()); + return SLANG_FAIL; + } + + ScopeNode* foundScopeNode = as<ScopeNode>(foundNode); + SLANG_ASSERT(foundScopeNode); + + // Make sure the node is empty, as we are *not* going to add it, we are just going to use + // the pre-existing namespace + SLANG_ASSERT(scopeNode->m_children.getCount() == 0); + + // We can just use the pre-existing namespace + m_currentScope = foundScopeNode; + return SLANG_OK; + } + } + } + + m_currentScope->addChild(scopeNode); + m_currentScope = scopeNode; + return SLANG_OK; +} + +SlangResult Parser::popScope() +{ + if (m_currentScope->m_parentScope == nullptr) + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::scopeNotClosed); + return SLANG_FAIL; + } + + m_currentScope = m_currentScope->m_parentScope; + return SLANG_OK; +} + +SlangResult Parser::consumeToClosingBrace(const Token* inOpenBraceToken) +{ + Token openToken; + if (inOpenBraceToken) + { + openToken = *inOpenBraceToken; + } + else + { + openToken = m_reader.advanceToken(); + } + + while (true) + { + switch (m_reader.peekTokenType()) + { + case TokenType::EndOfFile: + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::didntFindMatchingBrace); + m_sink->diagnose(openToken, CPPDiagnostics::seeOpen); + return SLANG_FAIL; + } + case TokenType::LBrace: + { + SLANG_RETURN_ON_FAIL(consumeToClosingBrace()); + break; + } + case TokenType::RBrace: + { + m_reader.advanceToken(); + return SLANG_OK; + } + default: + { + m_reader.advanceToken(); + break; + } + } + } +} + +SlangResult Parser::_maybeParseNode(Node::Type type) +{ + // We are looking for + // struct/class identifier [: [public|private|protected] Identifier ] { [public|private|proctected:]* marker ( identifier ); + + if (type == Node::Type::Namespace) + { + // consume namespace + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier)); + + Token name; + if (advanceIfToken(TokenType::LBrace)) + { + return pushAnonymousNamespace(); + } + else if (advanceIfToken(TokenType::Identifier, &name)) + { + if (advanceIfToken(TokenType::LBrace)) + { + // Okay looks like we are opening a namespace + RefPtr<ScopeNode> node(new ScopeNode(Node::Type::Namespace)); + node->m_name = name; + // Push the node + return pushScope(node); + } + } + + // Just ignore it then + return SLANG_OK; + } + + // Must be class | struct + + SLANG_ASSERT(type == Node::Type::ClassType || type == Node::Type::StructType); + + Token name; + + // consume class | struct + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier)); + // Next is the class name + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &name)); + + if (m_reader.peekTokenType() == TokenType::Semicolon) + { + // pre declaration; + return SLANG_OK; + } + + RefPtr<ClassLikeNode> node(new ClassLikeNode(type)); + node->m_name = name; + + // Defaults to not reflected + SLANG_ASSERT(!node->isReflected()); + + if (advanceIfToken(TokenType::Colon)) + { + // Could have public + advanceIfStyle(IdentifierStyle::Access); + + if (!advanceIfToken(TokenType::Identifier, &node->m_super)) + { + return SLANG_OK; + } + } + + if (m_reader.peekTokenType() != TokenType::LBrace) + { + // Consume up until we see a brace else it's an error + while (true) + { + const TokenType peekTokenType = m_reader.peekTokenType(); + if (peekTokenType == TokenType::EndOfFile) + { + // Expecting brace + m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::expectingToken, TokenType::LBrace); + return SLANG_FAIL; + } + else if (peekTokenType == TokenType::LBrace) + { + break; + } + m_reader.advanceToken(); + } + + return pushScope(node); + } + + Token braceToken = m_reader.advanceToken(); + + while (true) + { + // Okay now we are looking for the markers, or visibility qualifiers + if (advanceIfStyle(IdentifierStyle::Access)) + { + // Consume it and a colon + if (SLANG_FAILED(expect(TokenType::Colon))) + { + consumeToClosingBrace(&braceToken); + return SLANG_OK; + } + continue; + } + + switch (m_reader.peekTokenType()) + { + case TokenType::Identifier: break; + case TokenType::RBrace: + { + SLANG_RETURN_ON_FAIL(pushScope(node)); + SLANG_RETURN_ON_FAIL(popScope()); + m_reader.advanceToken(); + return SLANG_OK; + } + default: + { + SLANG_RETURN_ON_FAIL(pushScope(node)); + return SLANG_OK; + } + } + + // If it's one of the markers, then we continue to extract parameter + if (advanceIfMarker(&node->m_marker)) + { + break; + } + + // We still need to add the node, + SLANG_RETURN_ON_FAIL(pushScope(node)); + return SLANG_OK; + } + + // Let's extract the type set + { + UnownedStringSlice slice(node->m_marker.getContent()); + + SLANG_ASSERT(_isMarker(slice)); + + // Strip the prefix and suffix + slice = UnownedStringSlice(slice.begin() + m_options->m_markPrefix.getLength(), slice.end() - m_options->m_markSuffix.getLength()); + + // Strip ABSTRACT_ if it's there + UnownedStringSlice abstractSlice("ABSTRACT_"); + if (slice.startsWith(abstractSlice)) + { + slice = UnownedStringSlice(slice.begin() + abstractSlice.getLength(), slice.end()); + } + + // TODO: We could strip other stuff or have other heuristics there, but this is + // probably okay for now + + // Set the typeSet + node->m_typeSet = m_nodeTree->getOrAddTypeSet(slice); + } + + // Okay now looking for ( identifier) + Token typeNameToken; + + SLANG_RETURN_ON_FAIL(expect(TokenType::LParent)); + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &typeNameToken)); + SLANG_RETURN_ON_FAIL(expect(TokenType::RParent)); + + if (typeNameToken.getContent() != node->m_name.getContent()) + { + m_sink->diagnose(typeNameToken, CPPDiagnostics::typeNameDoesntMatch, node->m_name.getContent()); + return SLANG_FAIL; + } + + node->m_reflectionType = ReflectionType::Reflected; + return pushScope(node); +} + +SlangResult Parser::_consumeToSync() +{ + while (true) + { + TokenType type = m_reader.peekTokenType(); + + switch (type) + { + case TokenType::Semicolon: + { + m_reader.advanceToken(); + return SLANG_OK; + } + case TokenType::Pound: + case TokenType::EndOfFile: + case TokenType::LBrace: + case TokenType::RBrace: + { + return SLANG_OK; + } + } + + m_reader.advanceToken(); + } +} + +SlangResult Parser::_maybeParseTemplateArg(Index& ioTemplateDepth) +{ + switch (m_reader.peekTokenType()) + { + case TokenType::Identifier: + { + UnownedStringSlice name; + SLANG_RETURN_ON_FAIL(_maybeParseType(name, ioTemplateDepth)); + return SLANG_OK; + } + case TokenType::IntegerLiteral: + { + m_reader.advanceToken(); + return SLANG_OK; + } + default: break; + } + return SLANG_FAIL; +} + +SlangResult Parser::_maybeParseTemplateArgs(Index& ioTemplateDepth) +{ + if (!advanceIfToken(TokenType::OpLess)) + { + return SLANG_FAIL; + } + + ioTemplateDepth++; + + while (true) + { + if (ioTemplateDepth == 0) + { + return SLANG_OK; + } + + switch (m_reader.peekTokenType()) + { + case TokenType::OpGreater: + { + if (ioTemplateDepth <= 0) + { + m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::unexpectedTemplateClose); + return SLANG_FAIL; + } + ioTemplateDepth--; + m_reader.advanceToken(); + return SLANG_OK; + } + case TokenType::OpRsh: + { + if (ioTemplateDepth <= 1) + { + m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::unexpectedTemplateClose); + return SLANG_FAIL; + } + ioTemplateDepth -= 2; + m_reader.advanceToken(); + return SLANG_OK; + } + default: + { + while (true) + { + SLANG_RETURN_ON_FAIL(_maybeParseTemplateArg(ioTemplateDepth)); + + if (m_reader.peekTokenType() == TokenType::Comma) + { + m_reader.advanceToken(); + // If there is a comma parse another arg + continue; + } + break; + } + break; + } + } + } +} + +void Parser::_consumeTypeModifiers() +{ + while (advanceIfStyle(IdentifierStyle::TypeModifier)); +} + +// True if two of these token types of the same type placed immediately after one another +// produce a different token. Can be conservative, as if not strictly required +// it will just mean more spacing in the output +static bool _canRepeatTokenType(TokenType type) +{ + switch (type) + { + case TokenType::OpAdd: + case TokenType::OpSub: + case TokenType::OpAnd: + case TokenType::OpOr: + case TokenType::OpGreater: + case TokenType::OpLess: + case TokenType::Identifier: + case TokenType::OpAssign: + case TokenType::Colon: + { + return false; + } + default: break; + } + return true; +} + +// Returns true if there needs to be a space between the previous token type, and the current token +// type for correct output. It is assumed that the token stream is appropriate. +// The implementation might need more sophistication, but this at least avoids Blah const * -> Blahconst* +static bool _tokenConcatNeedsSpace(TokenType prev, TokenType cur) +{ + if ((cur == TokenType::OpAssign) || + (prev == cur && !_canRepeatTokenType(cur))) + { + return true; + } + return false; +} + +UnownedStringSlice Parser::_concatTokens(TokenReader::ParsingCursor start) +{ + auto endCursor = m_reader.getCursor(); + m_reader.setCursor(start); + + TokenType prevTokenType = TokenType::Unknown; + + StringBuilder buf; + while (!m_reader.isAtCursor(endCursor)) + { + const Token token = m_reader.advanceToken(); + // Check if we need a space between tokens + if (_tokenConcatNeedsSpace(prevTokenType, token.type)) + { + buf << " "; + } + buf << token.getContent(); + + prevTokenType = token.type; + } + + StringSlicePool* typePool = m_nodeTree->m_typePool; + return typePool->getSlice(typePool->add(buf)); +} + +SlangResult Parser::_maybeParseType(UnownedStringSlice& outType, Index& ioTemplateDepth) +{ + auto startCursor = m_reader.getCursor(); + + _consumeTypeModifiers(); + + advanceIfToken(TokenType::Scope); + while (true) + { + Token identifierToken; + if (!advanceIfToken(TokenType::Identifier, &identifierToken)) + { + return SLANG_FAIL; + } + + const IdentifierStyle style = m_nodeTree->m_identifierLookup->get(identifierToken.getContent()); + if (hasFlag(style, IdentifierFlag::Keyword)) + { + return SLANG_FAIL; + } + + if (advanceIfToken(TokenType::Scope)) + { + continue; + } + break; + } + + if (m_reader.peekTokenType() == TokenType::OpLess) + { + SLANG_RETURN_ON_FAIL(_maybeParseTemplateArgs(ioTemplateDepth)); + } + + // Strip all the consts etc modifiers + _consumeTypeModifiers(); + + // It's a reference and we are done + if (advanceIfToken(TokenType::OpBitAnd)) + { + return SLANG_OK; + } + + while (true) + { + if (advanceIfToken(TokenType::OpMul)) + { + // Strip all the consts + _consumeTypeModifiers(); + continue; + } + break; + } + + // We can build up the out type, from the tokens we found + outType = _concatTokens(startCursor); + return SLANG_OK; +} + +SlangResult Parser::_maybeParseType(UnownedStringSlice& outType) +{ + Index templateDepth = 0; + SlangResult res = _maybeParseType(outType, templateDepth); + if (SLANG_FAILED(res) && m_sink->getErrorCount()) + { + return res; + } + + if (templateDepth != 0) + { + m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::unexpectedTemplateClose); + return SLANG_FAIL; + } + return SLANG_OK; +} + +static bool _isBalancedOpen(TokenType tokenType) +{ + return tokenType == TokenType::LBrace || + tokenType == TokenType::LParent || + tokenType == TokenType::LBracket; +} + +static bool _isBalancedClose(TokenType tokenType) +{ + return tokenType == TokenType::RBrace || + tokenType == TokenType::RParent || + tokenType == TokenType::RBracket; +} + +static TokenType _getBalancedClose(TokenType tokenType) +{ + SLANG_ASSERT(_isBalancedOpen(tokenType)); + switch (tokenType) + { + case TokenType::LBrace: return TokenType::RBrace; + case TokenType::LParent: return TokenType::RParent; + case TokenType::LBracket: return TokenType::RBracket; + default: return TokenType::Unknown; + } +} + +SlangResult Parser::_parseBalanced(DiagnosticSink* sink) +{ + const TokenType openTokenType = m_reader.peekTokenType(); + if (!_isBalancedOpen(openTokenType)) + { + return SLANG_FAIL; + } + + // Save the start token + const Token startToken = m_reader.advanceToken(); + // Get the token type that would close the open + const TokenType closeTokenType = _getBalancedClose(openTokenType); + + while (true) + { + const TokenType tokenType = m_reader.peekTokenType(); + + // If we hit the closing token, we are done + if (tokenType == closeTokenType) + { + m_reader.advanceToken(); + return SLANG_OK; + } + + // If we hit a balanced open, recurse + if (_isBalancedOpen(tokenType)) + { + SLANG_RETURN_ON_FAIL(_parseBalanced(sink)); + continue; + } + + // If we hit a close token that doesn't match, then the balancing has gone wrong + if (_isBalancedClose(tokenType)) + { + // Only diagnose if required + if (sink) + { + sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::unexpectedUnbalancedToken); + sink->diagnose(startToken, CPPDiagnostics::seeOpen); + } + return SLANG_FAIL; + } + + // If we hit the end of the file and have not hit the closing token, then + // somethings gone wrong + if (tokenType == TokenType::EndOfFile) + { + if (sink) + { + sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::unexpectedEndOfFile); + sink->diagnose(startToken, CPPDiagnostics::seeOpen); + } + + return SLANG_FAIL; + } + + // Skip the token + m_reader.advanceToken(); + } +} + +SlangResult Parser::_maybeParseField() +{ + // Can only add a field if we are in a class + SLANG_ASSERT(m_currentScope->isClassLike()); + + UnownedStringSlice typeName; + if (SLANG_FAILED(_maybeParseType(typeName))) + { + if (m_sink->getErrorCount()) + { + return SLANG_FAIL; + } + + _consumeToSync(); + return SLANG_OK; + } + + if (m_reader.peekTokenType() != TokenType::Identifier) + { + _consumeToSync(); + return SLANG_OK; + } + + Token fieldName = m_reader.advanceToken(); + + if (m_reader.peekTokenType() == TokenType::LBracket) + { + auto startCursor = m_reader.getCursor(); + + // If it's not balanced we just assume it's not correct - and ignore + if (SLANG_FAILED(_parseBalanced(nullptr))) + { + _consumeToSync(); + return SLANG_OK; + } + + UnownedStringSlice arraySuffix = _concatTokens(startCursor); + + // The overall type is the typename concated with the arraySuffix + StringBuilder buf; + buf << typeName << arraySuffix; + + StringSlicePool* typePool = m_nodeTree->m_typePool; + + typeName = typePool->getSlice(typePool->add(buf)); + } + + switch (m_reader.peekTokenType()) + { + case TokenType::OpAssign: + { + // Special case to handle + // Type operator=(... + + m_reader.advanceToken(); + if (m_reader.peekTokenType() == TokenType::LParent) + { + // Not a field + break; + } + } + case TokenType::Semicolon: + { + FieldNode* fieldNode = new FieldNode; + + fieldNode->m_fieldType = typeName; + fieldNode->m_name = fieldName; + fieldNode->m_reflectionType = m_currentScope->getContainedReflectionType(); + + m_currentScope->addChild(fieldNode); + break; + } + default: break; + } + + _consumeToSync(); + return SLANG_OK; +} + +/* static */Node::Type Parser::_toNodeType(IdentifierStyle style) +{ + switch (style) + { + case IdentifierStyle::Class: return Node::Type::ClassType; + case IdentifierStyle::Struct: return Node::Type::StructType; + case IdentifierStyle::Namespace: return Node::Type::Namespace; + default: return Node::Type::Invalid; + } +} + +static UnownedStringSlice _trimUnderscorePrefix(const UnownedStringSlice& slice) +{ + if (slice.getLength() && slice[0] == '_') + { + return UnownedStringSlice(slice.begin() + 1, slice.end()); + } + else + { + return slice; + } +} + +SlangResult Parser::_parsePreDeclare() +{ + // Skip the declare type token + m_reader.advanceToken(); + + SLANG_RETURN_ON_FAIL(expect(TokenType::LParent)); + + // Get the typeSet + Token typeSetToken; + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &typeSetToken)); + TypeSet* typeSet = m_nodeTree->getOrAddTypeSet(typeSetToken.getContent()); + + SLANG_RETURN_ON_FAIL(expect(TokenType::Comma)); + + // Get the type of type + Node::Type nodeType; + { + Token typeToken; + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &typeToken)); + + const IdentifierStyle style = m_nodeTree->m_identifierLookup->get(typeToken.getContent()); + + if (style != IdentifierStyle::Struct && style != IdentifierStyle::Class) + { + m_sink->diagnose(typeToken, CPPDiagnostics::expectingTypeKeyword, typeToken.getContent()); + return SLANG_FAIL; + } + nodeType = _toNodeType(style); + } + + Token name; + Token super; + + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &name)); + + if (advanceIfToken(TokenType::Colon)) + { + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &super)); + } + + SLANG_RETURN_ON_FAIL(expect(TokenType::RParent)); + + switch (nodeType) + { + case Node::Type::ClassType: + case Node::Type::StructType: + { + RefPtr<ClassLikeNode> node(new ClassLikeNode(nodeType)); + + node->m_name = name; + node->m_super = super; + node->m_typeSet = typeSet; + + // Assume it is reflected + node->m_reflectionType = ReflectionType::Reflected; + + SLANG_RETURN_ON_FAIL(pushScope(node)); + // Pop out of the node + popScope(); + break; + } + default: + { + return SLANG_FAIL; + } + } + + + return SLANG_OK; +} + +SlangResult Parser::_parseTypeSet() +{ + // Skip the declare type token + m_reader.advanceToken(); + + SLANG_RETURN_ON_FAIL(expect(TokenType::LParent)); + + Token typeSetToken; + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &typeSetToken)); + + TypeSet* typeSet = m_nodeTree->getOrAddTypeSet(typeSetToken.getContent()); + + SLANG_RETURN_ON_FAIL(expect(TokenType::Comma)); + + // Get the type of type + Token typeToken; + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &typeToken)); + + SLANG_RETURN_ON_FAIL(expect(TokenType::RParent)); + + // Set the typename + typeSet->m_typeName = typeToken.getContent(); + + return SLANG_OK; +} + +SlangResult Parser::parse(SourceOrigin* sourceOrigin, const Options* options) +{ + SLANG_ASSERT(options); + m_options = options; + +#if 0 + // Calculate from the path, a 'macro origin' name. + const String macroOrigin = calcMacroOrigin(sourceFile->getPathInfo().foundPath, *options); + + RefPtr<SourceOrigin> origin = new SourceOrigin(sourceFile, macroOrigin); + m_sourceOrigins.add(origin); +#endif + + // Set the current origin + m_sourceOrigin = sourceOrigin; + + SourceFile* sourceFile = sourceOrigin->m_sourceFile; + + SourceManager* manager = sourceFile->getSourceManager(); + + SourceView* sourceView = manager->createSourceView(sourceFile, nullptr, SourceLoc::fromRaw(0)); + + Lexer lexer; + + m_currentScope = m_nodeTree->m_rootNode; + + lexer.initialize(sourceView, m_sink, m_nodeTree->m_namePool, manager->getMemoryArena()); + m_tokenList = lexer.lexAllTokens(); + // See if there were any errors + if (m_sink->getErrorCount()) + { + return SLANG_FAIL; + } + + m_reader = TokenReader(m_tokenList); + + while (true) + { + switch (m_reader.peekTokenType()) + { + case TokenType::Identifier: + { + const IdentifierStyle style = m_nodeTree->m_identifierLookup->get(m_reader.peekToken().getContent()); + + switch (style) + { + case IdentifierStyle::PreDeclare: + { + SLANG_RETURN_ON_FAIL(_parsePreDeclare()); + break; + } + case IdentifierStyle::TypeSet: + { + SLANG_RETURN_ON_FAIL(_parseTypeSet()); + break; + } + case IdentifierStyle::Reflected: + { + m_reader.advanceToken(); + if (m_currentScope) + { + m_currentScope->m_reflectionOverride = ReflectionType::Reflected; + } + break; + } + case IdentifierStyle::Unreflected: + { + m_reader.advanceToken(); + if (m_currentScope) + { + m_currentScope->m_reflectionOverride = ReflectionType::NotReflected; + } + break; + } + case IdentifierStyle::Access: + { + m_reader.advanceToken(); + SLANG_RETURN_ON_FAIL(expect(TokenType::Colon)); + break; + } + default: + { + IdentifierFlags flags = getFlags(style); + + if (flags & IdentifierFlag::StartScope) + { + Node::Type type = _toNodeType(style); + SLANG_RETURN_ON_FAIL(_maybeParseNode(type)); + } + else + { + // Special case the node that's the root of the hierarchy (as far as reflection is concerned) + // This could be a field + if (m_currentScope->acceptsFields()) + { + SLANG_RETURN_ON_FAIL(_maybeParseField()); + } + else + { + m_reader.advanceToken(); + } + } + break; + } + } + break; + } + case TokenType::LBrace: + { + SLANG_RETURN_ON_FAIL(consumeToClosingBrace()); + break; + } + case TokenType::RBrace: + { + SLANG_RETURN_ON_FAIL(popScope()); + m_reader.advanceToken(); + break; + } + case TokenType::EndOfFile: + { + // Okay we need to confirm that we are in the root node, and with no open braces + if (m_currentScope != m_nodeTree->getRootNode()) + { + m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::braceOpenAtEndOfFile); + return SLANG_FAIL; + } + + return SLANG_OK; + } + case TokenType::Pound: + { + Token token = m_reader.peekToken(); + if (token.flags & TokenFlag::AtStartOfLine) + { + // We are just going to ignore all of these for now.... + m_reader.advanceToken(); + while (m_reader.peekTokenType() != TokenType::EndOfDirective && m_reader.peekTokenType() != TokenType::EndOfFile) + { + m_reader.advanceToken(); + } + break; + } + // Skip it then + m_reader.advanceToken(); + break; + } + default: + { + // Skip it then + m_reader.advanceToken(); + break; + } + } + } +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!! ParseSharedState !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +NodeTree::NodeTree(StringSlicePool* typePool, NamePool* namePool, IdentifierLookup* identifierLookup): + m_typePool(typePool), + m_namePool(namePool), + m_identifierLookup(identifierLookup), + m_typeSetPool(StringSlicePool::Style::Empty) +{ + m_rootNode = new ScopeNode(Node::Type::Namespace); + m_rootNode->m_reflectionType = ReflectionType::Reflected; +} + +TypeSet* NodeTree::getTypeSet(const UnownedStringSlice& slice) +{ + Index index = m_typeSetPool.findIndex(slice); + if (index < 0) + { + return nullptr; + } + return m_typeSets[index]; +} + +TypeSet* NodeTree::getOrAddTypeSet(const UnownedStringSlice& slice) +{ + const Index index = Index(m_typeSetPool.add(slice)); + if (index >= m_typeSets.getCount()) + { + SLANG_ASSERT(m_typeSets.getCount() == index); + TypeSet* typeSet = new TypeSet; + + m_typeSets.add(typeSet); + typeSet->m_macroName = m_typeSetPool.getSlice(StringSlicePool::Handle(index)); + return typeSet; + } + else + { + return m_typeSets[index]; + } +} + +SourceOrigin* NodeTree::addSourceOrigin(SourceFile* sourceFile, const Options& options) +{ + // Calculate from the path, a 'macro origin' name. + const String macroOrigin = calcMacroOrigin(sourceFile->getPathInfo().foundPath, options); + + SourceOrigin* origin = new SourceOrigin(sourceFile, macroOrigin); + m_sourceOrigins.add(origin); + return origin; +} + +/* static */String NodeTree::calcMacroOrigin(const String& filePath, const Options& options) +{ + // Get the filename without extension + String fileName = Path::getFileNameWithoutExt(filePath); + + // We can work on just the slice + UnownedStringSlice slice = fileName.getUnownedSlice(); + + // Filename prefix + if (options.m_stripFilePrefix.getLength() && slice.startsWith(options.m_stripFilePrefix.getUnownedSlice())) + { + const Index len = options.m_stripFilePrefix.getLength(); + slice = UnownedStringSlice(slice.begin() + len, slice.end()); + } + + // Trim - + slice = slice.trim('-'); + + StringBuilder out; + NameConventionUtil::convert(slice, CharCase::Upper, NameConvention::Snake, out); + return out; +} + +SlangResult NodeTree::_calcDerivedTypesRec(ScopeNode* inScopeNode, DiagnosticSink* sink) +{ + if (inScopeNode->isClassLike()) + { + ClassLikeNode* classLikeNode = static_cast<ClassLikeNode*>(inScopeNode); + + if (classLikeNode->m_super.hasContent()) + { + ScopeNode* parentScope = classLikeNode->m_parentScope; + if (parentScope == nullptr) + { + sink->diagnoseRaw(Severity::Error, UnownedStringSlice::fromLiteral("Can't lookup in scope if there is none!")); + return SLANG_FAIL; + } + + Node* superNode = Node::findNode(parentScope, classLikeNode->m_super.getContent()); + + if (!superNode) + { + if (classLikeNode->isReflected()) + { + sink->diagnose(classLikeNode->m_name, CPPDiagnostics::superTypeNotFound, classLikeNode->getAbsoluteName()); + return SLANG_FAIL; + } + } + else + { + ClassLikeNode* superType = as<ClassLikeNode>(superNode); + + if (!superType) + { + sink->diagnose(classLikeNode->m_name, CPPDiagnostics::superTypeNotAType, classLikeNode->getAbsoluteName()); + return SLANG_FAIL; + } + + if (superType->m_typeSet != classLikeNode->m_typeSet) + { + sink->diagnose(classLikeNode->m_name, CPPDiagnostics::typeInDifferentTypeSet, classLikeNode->m_name.getContent(), classLikeNode->m_typeSet->m_macroName, superType->m_typeSet->m_macroName); + return SLANG_FAIL; + } + + // The base class must be defined in same scope (as we didn't allow different scopes for base classes) + superType->addDerived(classLikeNode); + } + } + else + { + // Add to it's own typeset + if (classLikeNode->isReflected()) + { + classLikeNode->m_typeSet->m_baseTypes.add(classLikeNode); + } + } + } + + for (Node* child : inScopeNode->m_children) + { + ScopeNode* childScope = as<ScopeNode>(child); + if (childScope) + { + SLANG_RETURN_ON_FAIL(_calcDerivedTypesRec(childScope, sink)); + } + } + + return SLANG_OK; +} + +SlangResult NodeTree::calcDerivedTypes(DiagnosticSink* sink) +{ + return _calcDerivedTypesRec(m_rootNode, sink); +} + + +} // namespace CppExtract diff --git a/tools/slang-cpp-extractor/parser.h b/tools/slang-cpp-extractor/parser.h new file mode 100644 index 000000000..58a13c19f --- /dev/null +++ b/tools/slang-cpp-extractor/parser.h @@ -0,0 +1,162 @@ +#ifndef CPP_EXTRACT_PARSER_H +#define CPP_EXTRACT_PARSER_H + +#include "diagnostics.h" +#include "node.h" +#include "identifier-lookup.h" + +#include "../../source/compiler-core/slang-lexer.h" + +namespace CppExtract { +using namespace Slang; + +class TypeSet : public RefObject +{ +public: + /// This is the looked up name. + UnownedStringSlice m_macroName; ///< The name extracted from the macro SLANG_ABSTRACT_AST_CLASS -> AST + + String m_typeName; ///< The enum type name associated with this type for AST it is ASTNode + String m_fileMark; ///< This 'mark' becomes of the output filename + + List<ClassLikeNode*> m_baseTypes; ///< The base types for this type set +}; + +class SourceOrigin : public RefObject +{ +public: + + void addNode(Node* node) + { + if (auto classLike = as<ClassLikeNode>(node)) + { + SLANG_ASSERT(classLike->m_origin == nullptr); + classLike->m_origin = this; + } + + m_nodes.add(node); + } + + SourceOrigin(SourceFile* sourceFile, const String& macroOrigin) : + m_sourceFile(sourceFile), + m_macroOrigin(macroOrigin) + {} + + String m_macroOrigin; ///< The macro text is inserted into the macro to identify the origin. It is based on the filename + SourceFile* m_sourceFile; ///< The source file - also holds the path information + + /// All of the nodes defined in this file in the order they were defined + /// Note that the same namespace may be listed multiple times. + List<RefPtr<Node> > m_nodes; +}; + +struct Options; +class IdentifierLookup; + +/* NodeTree holds nodes that have been parsed into a tree rooted on the 'rootNode'. +Also contains other state associated with or useful to a node tree */ +class NodeTree +{ +public: + friend class Parser; + /// Get all of the parsed source origins + const List<RefPtr<SourceOrigin> >& getSourceOrigins() const { return m_sourceOrigins; } + + TypeSet* getTypeSet(const UnownedStringSlice& slice); + TypeSet* getOrAddTypeSet(const UnownedStringSlice& slice); + + SourceOrigin* addSourceOrigin(SourceFile* sourceFile, const Options& options); + + /// Get all of the type sets + const List<RefPtr<TypeSet>>& getTypeSets() const { return m_typeSets; } + + /// Get the root node + Node* getRootNode() const { return m_rootNode; } + + /// When parsing we don't lookup all up super types/add derived types. This is because + /// we allow files to be processed in any order, so we have to do the type lookup as a separate operation + SlangResult calcDerivedTypes(DiagnosticSink* sink); + + NodeTree(StringSlicePool* typePool, NamePool* namePool, IdentifierLookup* identifierLookup); + + static String calcMacroOrigin(const String& filePath, const Options& options); + +protected: + SlangResult _calcDerivedTypesRec(ScopeNode* node, DiagnosticSink* sink); + + StringSlicePool m_typeSetPool; ///< Pool for type set names + List<RefPtr<TypeSet> > m_typeSets; ///< The type sets + + IdentifierLookup* m_identifierLookup; + StringSlicePool* m_typePool; ///< Pool for just types + + NamePool* m_namePool; + + RefPtr<ScopeNode> m_rootNode; ///< The root scope + + List<RefPtr<SourceOrigin>> m_sourceOrigins; +}; + +class Parser +{ +public: + + SlangResult expect(TokenType type, Token* outToken = nullptr); + + bool advanceIfMarker(Token* outToken = nullptr); + bool advanceIfToken(TokenType type, Token* outToken = nullptr); + bool advanceIfStyle(IdentifierStyle style, Token* outToken = nullptr); + + SlangResult pushAnonymousNamespace(); + SlangResult pushScope(ScopeNode* node); + SlangResult consumeToClosingBrace(const Token* openBraceToken = nullptr); + SlangResult popScope(); + + /// Parse the contents of the source file + SlangResult parse(SourceOrigin* sourceOrigin, const Options* options); + + Parser(NodeTree* nodeTree, DiagnosticSink* sink); + +protected: + static Node::Type _toNodeType(IdentifierStyle style); + + bool _isMarker(const UnownedStringSlice& name); + + SlangResult _parsePreDeclare(); + SlangResult _parseTypeSet(); + + SlangResult _maybeParseNode(Node::Type type); + SlangResult _maybeParseField(); + + SlangResult _maybeParseType(UnownedStringSlice& outType); + + SlangResult _maybeParseType(UnownedStringSlice& outType, Index& ioTemplateDepth); + SlangResult _maybeParseTemplateArgs(Index& ioTemplateDepth); + SlangResult _maybeParseTemplateArg(Index& ioTemplateDepth); + + /// Parse balanced - if a sink is set will report to that sink + SlangResult _parseBalanced(DiagnosticSink* sink); + + /// Concatenate all tokens from start to the current position + UnownedStringSlice _concatTokens(TokenReader::ParsingCursor start); + + void _consumeTypeModifiers(); + + SlangResult _consumeToSync(); + + TokenList m_tokenList; + TokenReader m_reader; + + ScopeNode* m_currentScope; ///< The current scope being processed + SourceOrigin* m_sourceOrigin; ///< The source origin that all tokens are in + + DiagnosticSink* m_sink; ///< Diagnostic sink + + NodeTree* m_nodeTree; ///< Shared state between parses. Nodes will be added to this + + const Options* m_options; +}; + +} // CppExtract + +#endif diff --git a/tools/slang-cpp-extractor/slang-cpp-extractor-main.cpp b/tools/slang-cpp-extractor/slang-cpp-extractor-main.cpp index 2aa8446ad..7ec53e29f 100644 --- a/tools/slang-cpp-extractor/slang-cpp-extractor-main.cpp +++ b/tools/slang-cpp-extractor/slang-cpp-extractor-main.cpp @@ -15,13 +15,16 @@ #include "../../source/core/slang-writer.h" #include "../../source/core/slang-file-system.h" -#include "../../source/compiler-core/slang-name-convention-util.h" #include "../../source/compiler-core/slang-source-loc.h" #include "../../source/compiler-core/slang-lexer.h" #include "../../source/compiler-core/slang-diagnostic-sink.h" #include "../../source/compiler-core/slang-name.h" +#include "../../source/compiler-core/slang-name-convention-util.h" -#include "slang-cpp-extractor-diagnostics.h" +#include "node.h" +#include "diagnostics.h" +#include "options.h" +#include "parser.h" /* Some command lines: @@ -29,2128 +32,22 @@ Some command lines: -d source/slang slang-ast-support-types.h slang-ast-base.h slang-ast-decl.h slang-ast-expr.h slang-ast-modifier.h slang-ast-stmt.h slang-ast-type.h slang-ast-val.h -strip-prefix slang- -o slang-generated -output-fields -mark-suffix _CLASS */ -namespace SlangExperimental +namespace CppExtract { using namespace Slang; -enum class IdentifierStyle -{ - None, ///< It's not an identifier - - Identifier, ///< Just an identifier - - PreDeclare, ///< Declare a type (not visible in C++ code) - TypeSet, ///< TypeSet - - TypeModifier, ///< const, volatile etc - Keyword, ///< A keyword C/C++ keyword that is not another type - Class, ///< class - Struct, ///< struct - Namespace, ///< namespace - Access, ///< public, protected, private - - Reflected, - Unreflected, - - CountOf, -}; - -typedef uint32_t IdentifierFlags; -struct IdentifierFlag -{ - enum Enum : IdentifierFlags - { - StartScope = 0x1, ///< namespace, struct or class - ClassLike = 0x2, ///< Struct or class - Keyword = 0x4, - Reflection = 0x8, - }; -}; - -static const IdentifierFlags kIdentifierFlags[Index(IdentifierStyle::CountOf)] = -{ - 0, /// None - 0, /// Identifier - 0, /// Declare type - 0, /// Type set - IdentifierFlag::Keyword, /// TypeModifier - IdentifierFlag::Keyword, /// Keyword - IdentifierFlag::Keyword | IdentifierFlag::StartScope | IdentifierFlag::ClassLike, /// Class - IdentifierFlag::Keyword | IdentifierFlag::StartScope | IdentifierFlag::ClassLike, /// Struct - IdentifierFlag::Keyword | IdentifierFlag::StartScope, /// Namespace - IdentifierFlag::Keyword, /// Access - IdentifierFlag::Reflection, /// Reflected - IdentifierFlag::Reflection, /// Unreflected -}; - -SLANG_FORCE_INLINE IdentifierFlags getFlags(IdentifierStyle style) -{ - return kIdentifierFlags[Index(style)]; -} - -SLANG_FORCE_INLINE bool hasFlag(IdentifierStyle style, IdentifierFlag::Enum flag) -{ - return (getFlags(style) & flag) != 0; -} - -class IdentifierLookup -{ -public: - - IdentifierStyle get(const UnownedStringSlice& slice) const - { - Index index = m_pool.findIndex(slice); - return (index >= 0) ? m_styles[index] : IdentifierStyle::None; - } - - void set(const char* name, IdentifierStyle style) - { - set(UnownedStringSlice(name), style); - } - - void set(const UnownedStringSlice& name, IdentifierStyle style) - { - StringSlicePool::Handle handle; - if (m_pool.findOrAdd(name, handle)) - { - // Add the extra flags - m_styles[Index(handle)] = style; - } - else - { - Index index = Index(handle); - SLANG_ASSERT(index == m_styles.getCount()); - m_styles.add(style); - } - } - - void set(const char*const* names, size_t namesCount, IdentifierStyle style) - { - for (size_t i = 0; i < namesCount; ++i) - { - set(UnownedStringSlice(names[i]), style); - } - } - void reset() - { - m_styles.clear(); - m_pool.clear(); - } - - IdentifierLookup(): - m_pool(StringSlicePool::Style::Empty) - { - SLANG_ASSERT(m_pool.getSlicesCount() == 0); - } -protected: - List<IdentifierStyle> m_styles; - StringSlicePool m_pool; -}; - -enum class ReflectionType -{ - NotReflected, - Reflected, -}; - -// Pre-declare -class TypeSet; -class SourceOrigin; - -struct ScopeNode; - -class Node : public RefObject -{ -public: - enum class Type - { - Invalid, - - StructType, - ClassType, - - Namespace, - AnonymousNamespace, - - Field, - }; - - static bool isScopeType(Type type) { return int(type) >= int(Type::StructType) && int(type) <= int(Type::AnonymousNamespace); } - static bool isClassLikeType(Type type) { return type == Type::StructType || type == Type::ClassType; } - - enum class TypeRange - { - ScopeStart = int(Type::StructType), - ScopeEnd = int(Type::AnonymousNamespace), - - ClassLikeStart = int(Type::StructType), - ClassLikeEnd = int(Type::ClassType), - }; - - static bool isType(Type type) { return true; } - - bool isClassLike() const { return isClassLikeType(m_type); } - - virtual void dump(int indent, StringBuilder& out) = 0; - - /// Do depth first traversal of nodes in scopes - virtual void calcScopeDepthFirst(List<Node*>& outNodes); - - /// Calculate the absolute name for this namespace/type - void calcAbsoluteName(StringBuilder& outName) const; - - /// Get the absolute name - String getAbsoluteName() const { StringBuilder buf; calcAbsoluteName(buf); return buf.ProduceString(); } - - /// Calculate the scope path to this node, from the root - void calcScopePath(List<Node*>& outPath) { calcScopePath(this, outPath); } - - /// True if reflected - bool isReflected() const { return m_reflectionType == ReflectionType::Reflected; } - - typedef bool (*Filter)(Node* node); - - static bool isClassLikeAndReflected(Node* node) { return node->isClassLike() && node->isReflected(); } - static bool isClassLike(Node* node) { return isClassLikeType(node->m_type); } - - template <typename T> - static void filter(Filter filter, List<T*>& io) { const Node* _isNodeDerived = (T*)nullptr; SLANG_UNUSED(_isNodeDerived); filterImpl(filter, reinterpret_cast<List<Node*>&>(io)); } - - static void filterImpl(Filter filter, List<Node*>& io); - - static void calcScopePath(Node* node, List<Node*>& outPath); - - Node(Type type): - m_type(type), - m_parentScope(nullptr), - m_reflectionType(ReflectionType::NotReflected) - { - } - - Type m_type; ///< The type of node this is - - ReflectionType m_reflectionType; /// Classes can be traversed, but not reflected. To be reflected they have to contain the marker - - Token m_name; ///< The name of this scope/type - - ScopeNode* m_parentScope; ///< The scope this type/scope is defined in -}; - -struct ScopeNode : public Node -{ - typedef Node Super; - - static bool isType(Type type) { return isScopeType(type); } - - virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; - virtual void calcScopeDepthFirst(List<Node*>& outNodes) SLANG_OVERRIDE; - - /// True if can accept fields (class like types can) - bool acceptsFields() const { return isClassLike(); } - - /// Gets the reflection for any contained types - ReflectionType getContainedReflectionType() const { return m_reflectionType == ReflectionType::NotReflected ? ReflectionType::NotReflected : m_reflectionOverride; } - - /// Add a child node to this nodes scope - void addChild(Node* child); - - /// Find a child node in this scope with the specified name. Return nullptr if not found - Node* findChild(const UnownedStringSlice& name) const; - - /// Gets the anonymous namespace associated with this scope - ScopeNode* getAnonymousNamespace(); - - ScopeNode(Type type) : - Super(type), - m_reflectionOverride(ReflectionType::Reflected), - m_anonymousNamespace(nullptr) - { - } - - /// For child types, fields, how reflection is handled. If this type is not reflected - ReflectionType m_reflectionOverride; - - /// All of the types and namespaces in this *scope* - List<RefPtr<Node>> m_children; - - /// Map from a name (in this scope) to the Node - Dictionary<UnownedStringSlice, Node*> m_childMap; - - /// There can only be one anonymousNamespace for a scope. If there is one it's held here - ScopeNode* m_anonymousNamespace; -}; - -struct FieldNode : public Node -{ - typedef Node Super; - - static bool isType(Type type) { return type == Type::Field; } - - virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; - - FieldNode(): - Super(Type::Field) - { - } - - UnownedStringSlice m_fieldType; - - // We may want to add initializer tokens -}; - -struct ClassLikeNode : public ScopeNode -{ - typedef ScopeNode Super; - - static bool isType(Type type) { return isClassLikeType(type); } - - /// Add a node that is derived from this - void addDerived(ClassLikeNode* derived); - void dumpDerived(int indentCount, StringBuilder& out); - - /// Calculates the derived depth - Index calcDerivedDepth() const; - - /// Find the last (reflected) derived type - ClassLikeNode* findLastDerived(); - - /// Traverse the hierarchy of derived nodes, in depth first order - void calcDerivedDepthFirst(List<ClassLikeNode*>& outNodes); - - /// True if has a derived type that is reflected - bool hasReflectedDerivedType() const; - /// Stores in out any reflected derived types - void getReflectedDerivedTypes(List<ClassLikeNode*>& out) const; - - virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; - - ClassLikeNode(Type type): - Super(type), - m_origin(nullptr), - m_typeSet(nullptr), - m_superNode(nullptr) - { - SLANG_ASSERT(type == Type::ClassType || type == Type::StructType); - } - - SourceOrigin* m_origin; ///< Defines where this was uniquely defined. - - Token m_marker; ///< The marker associated with this scope (typically the marker is SLANG_CLASS etc, that is used to identify reflectedType) - - List<RefPtr<ClassLikeNode>> m_derivedTypes; ///< All of the types derived from this type - - TypeSet* m_typeSet; ///< The typeset this type belongs to. - - Token m_super; ///< Super class name - ClassLikeNode* m_superNode; ///< If this is a class/struct, the type it is derived from (or nullptr if base) -}; - -template <typename T> -T* as(Node* node) { return (node && T::isType(node->m_type)) ? static_cast<T*>(node) : nullptr; } - -class SourceOrigin : public RefObject -{ -public: - - void addNode(Node* node) - { - if (auto classLike = as<ClassLikeNode>(node)) - { - SLANG_ASSERT(classLike->m_origin == nullptr); - classLike->m_origin = this; - } - - m_nodes.add(node); - } - - SourceOrigin(SourceFile* sourceFile, const String& macroOrigin) : - m_sourceFile(sourceFile), - m_macroOrigin(macroOrigin) - {} - - ///< The macro text is inserted into the macro to identify the origin. It is based on the filename - String m_macroOrigin; - /// The source file - also holds the path information - SourceFile* m_sourceFile; - - /// All of the nodes defined in this file in the order they were defined - /// Note that the same namespace may be listed multiple times. - List<RefPtr<Node> > m_nodes; -}; - -class TypeSet : public RefObject -{ -public: - - /// This is the looked up name. - UnownedStringSlice m_macroName; ///< The name extracted from the macro SLANG_ABSTRACT_AST_CLASS -> AST - - String m_typeName; ///< The enum type name associated with this type for AST it is ASTNode - String m_fileMark; ///< This 'mark' becomes of the output filename - - List<ClassLikeNode*> m_baseTypes; ///< The base types for this type set -}; - -struct Options; - -class CPPExtractor -{ -public: - - SlangResult expect(TokenType type, Token* outToken = nullptr); - - bool advanceIfMarker(Token* outToken = nullptr); - bool advanceIfToken(TokenType type, Token* outToken = nullptr); - bool advanceIfStyle(IdentifierStyle style, Token* outToken = nullptr); - - SlangResult pushAnonymousNamespace(); - SlangResult pushScope(ScopeNode* node); - SlangResult consumeToClosingBrace(const Token* openBraceToken = nullptr); - SlangResult popScope(); - - /// Parse the contents of the source file - SlangResult parse(SourceFile* sourceFile, const Options* options); - - /// When parsing we don't lookup all up super types/add derived types. This is because - /// we allow files to be processed in any order, so we have to do the type lookup as a separate operation - SlangResult calcDerivedTypes(); - - /// Find the name starting in specified scope - Node* findNode(ScopeNode* scope, const UnownedStringSlice& name); - - /// Get all of the parsed source origins - const List<RefPtr<SourceOrigin> >& getSourceOrigins() const { return m_origins; } - - TypeSet* getTypeSet(const UnownedStringSlice& slice); - TypeSet* getOrAddTypeSet(const UnownedStringSlice& slice); - - /// Get all of the type sets - const List<RefPtr<TypeSet>>& getTypeSets() const { return m_typeSets; } - - /// Get the root node - Node* getRootNode() const { return m_rootNode; } - - CPPExtractor(StringSlicePool* typePool, NamePool* namePool, DiagnosticSink* sink, IdentifierLookup* identifierLookup); - -protected: - static Node::Type _toNodeType(IdentifierStyle style); - - bool _isMarker(const UnownedStringSlice& name); - - SlangResult _parsePreDeclare(); - SlangResult _parseTypeSet(); - - SlangResult _maybeParseNode(Node::Type type); - SlangResult _maybeParseField(); - - SlangResult _maybeParseType(UnownedStringSlice& outType); - - SlangResult _maybeParseType(UnownedStringSlice& outType, Index& ioTemplateDepth); - SlangResult _maybeParseTemplateArgs(Index& ioTemplateDepth); - SlangResult _maybeParseTemplateArg(Index& ioTemplateDepth); - - /// Parse balanced - if a sink is set will report to that sink - SlangResult _parseBalanced(DiagnosticSink* sink); - - SlangResult _calcDerivedTypesRec(ScopeNode* node); - static String _calcMacroOrigin(const String& filePath, const Options& options); - - /// Concatenate all tokens from start to the current position - UnownedStringSlice _concatTokens(TokenReader::ParsingCursor start); - - void _consumeTypeModifiers(); - - SlangResult _consumeToSync(); - - TokenList m_tokenList; - TokenReader m_reader; - - ScopeNode* m_currentScope; ///< The current scope being processed - - RefPtr<ScopeNode> m_rootNode; ///< The root scope - - SourceOrigin* m_origin; - - DiagnosticSink* m_sink; - - NamePool* m_namePool; - - List<RefPtr<SourceOrigin>> m_origins; - - const Options* m_options; - - StringSlicePool m_typeSetPool; ///< Pool for type set names - List<RefPtr<TypeSet> > m_typeSets; ///< The type sets - - IdentifierLookup* m_identifierLookup; - StringSlicePool* m_typePool; ///< Pool for just types -}; - - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Node Impl !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - - static void _indent(Index indentCount, StringBuilder& out) { for (Index i = 0; i < indentCount; ++i) { - out << " "; - } -} - -void Node::calcScopeDepthFirst(List<Node*>& outNodes) -{ - outNodes.add(this); -} - -void Node::calcAbsoluteName(StringBuilder& outName) const -{ - List<Node*> path; - calcScopePath(const_cast<Node*>(this), path); - - // 1 so we skip the global scope - for (Index i = 1; i < path.getCount(); ++i) - { - Node* node = path[i]; - - if (i > 1) - { - outName << "::"; - } - - if (node->m_type == Type::AnonymousNamespace) - { - outName << "{Anonymous}"; - } - else - { - outName << node->m_name.getContent(); - } - } -} - -/* static */void Node::calcScopePath(Node* node, List<Node*>& outPath) -{ - outPath.clear(); - - while (node) - { - outPath.add(node); - node = node->m_parentScope; - } - - // reverse the order, so we go from root to the node - outPath.reverse(); -} - -/* static */void Node::filterImpl(Filter inFilter, List<Node*>& ioNodes) -{ - // Filter out all the unreflected nodes - Index count = ioNodes.getCount(); - for (Index j = 0; j < count; ) - { - Node* node = ioNodes[j]; - - if (!inFilter(node)) - { - ioNodes.removeAt(j); - count--; - } - else - { - j++; - } - } -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ScopeNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -ScopeNode* ScopeNode::getAnonymousNamespace() -{ - if (!m_anonymousNamespace) - { - m_anonymousNamespace = new ScopeNode(Type::AnonymousNamespace); - m_anonymousNamespace->m_parentScope = this; - m_children.add(m_anonymousNamespace); - } - - return m_anonymousNamespace; -} - -void ScopeNode::addChild(Node* child) -{ - SLANG_ASSERT(child->m_parentScope == nullptr); - // Can't add anonymous namespace this way - should be added via getAnonymousNamespace - SLANG_ASSERT(child->m_type != Type::AnonymousNamespace); - - child->m_parentScope = this; - m_children.add(child); - - if (child->m_name.hasContent()) - { - m_childMap.Add(child->m_name.getContent(), child); - } -} - -Node* ScopeNode::findChild(const UnownedStringSlice& name) const -{ - Node** nodePtr = m_childMap.TryGetValue(name); - return (nodePtr) ? *nodePtr : nullptr; -} - -void ScopeNode::calcScopeDepthFirst(List<Node*>& outNodes) -{ - outNodes.add(this); - for (Node* child : m_children) - { - child->calcScopeDepthFirst(outNodes); - } -} - -void ScopeNode::dump(int indentCount, StringBuilder& out) -{ - _indent(indentCount, out); - - switch (m_type) - { - case Type::AnonymousNamespace: - { - out << "namespace {\n"; - } - case Type::Namespace: - { - if (m_name.hasContent()) - { - out << "namespace " << m_name.getContent() << " {\n"; - } - else - { - out << "{\n"; - } - break; - } - } - - for (Node* child : m_children) - { - child->dump(indentCount + 1, out); - } - - _indent(indentCount, out); - out << "}\n"; -} - -/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! FieldNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ - -void FieldNode::dump(int indent, StringBuilder& out) -{ - if (isReflected()) - { - _indent(indent, out); - out << m_fieldType << " " << m_name.getContent() << "\n"; - } -} - -/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ClassLikeNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ - -/// Add a node that is derived from this -void ClassLikeNode::addDerived(ClassLikeNode* derived) -{ - SLANG_ASSERT(derived->m_superNode == nullptr); - derived->m_superNode = this; - m_derivedTypes.add(derived); -} - -void ClassLikeNode::calcDerivedDepthFirst(List<ClassLikeNode*>& outNodes) -{ - outNodes.add(this); - for (ClassLikeNode* derivedType : m_derivedTypes) - { - derivedType->calcDerivedDepthFirst(outNodes); - } -} - -void ClassLikeNode::dumpDerived(int indentCount, StringBuilder& out) -{ - if (isClassLike() && isReflected() && m_name.hasContent()) - { - _indent(indentCount, out); - out << m_name.getContent() << "\n"; - } - - for (ClassLikeNode* derivedType : m_derivedTypes) - { - derivedType->dumpDerived(indentCount + 1, out); - } -} - -Index ClassLikeNode::calcDerivedDepth() const -{ - const ClassLikeNode* node = this; - Index count = 0; - - while (node) - { - count++; - node = node->m_superNode; - } - - return count; -} - -ClassLikeNode* ClassLikeNode::findLastDerived() -{ - for (Index i = m_derivedTypes.getCount() - 1; i >= 0; --i) - { - ClassLikeNode* derivedType = m_derivedTypes[i]; - ClassLikeNode* found = derivedType->findLastDerived(); - if (found) - { - return found; - } - } - return this; -} - -bool ClassLikeNode::hasReflectedDerivedType() const -{ - for (ClassLikeNode* type : m_derivedTypes) - { - if (type->isReflected()) - { - return true; - } - } - return false; -} - -void ClassLikeNode::getReflectedDerivedTypes(List<ClassLikeNode*>& out) const -{ - out.clear(); - for (ClassLikeNode* type : m_derivedTypes) - { - if (type->isReflected()) - { - out.add(type); - } - } -} - -void ClassLikeNode::dump(int indentCount, StringBuilder& out) -{ - _indent(indentCount, out); - - const char* typeName = (m_type == Type::StructType) ? "struct" : "class"; - - out << typeName << " "; - - if (!isReflected()) - { - out << " ("; - } - out << m_name.getContent(); - if (!isReflected()) - { - out << ") "; - } - - if (m_super.hasContent()) - { - out << " : " << m_super.getContent(); - } - - out << " {\n"; - - for (Node* child : m_children) - { - child->dump(indentCount + 1, out); - } - - _indent(indentCount, out); - out << "}\n"; -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Options !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -struct Options -{ - void reset() - { - *this = Options(); - } - - Options() - { - m_markPrefix = "SLANG_"; - m_markSuffix = "_CLASS"; - } - - bool m_defs = false; ///< If set will output a '-defs.h' file for each of the input files, that corresponds to previous defs files (although doesn't have fields/RAW) - bool m_dump = false; ///< If true will dump to stderr the types/fields and hierarchy it extracted - - bool m_outputFields = false; ///< When dumping macros also dump field definitions - - List<String> m_inputPaths; ///< The input paths to the files to be processed - - String m_outputPath; ///< The output path. Note that the extractor can generate multiple output files, and this will actually be the 'stem' of several files - - String m_inputDirectory; ///< The input directory that is by default used for reading m_inputPaths from. - String m_markPrefix; ///< The prefix of the 'marker' used to identify a reflected type - String m_markSuffix; ///< The postfix of the 'marker' used to identify a reflected type - String m_stripFilePrefix; ///< Used for the 'origin' information, this is stripped from the source filename, and the remainder of the filename (without extension) is 'macroized' -}; - -struct OptionsParser -{ - /// Parse the parameters. NOTE! Must have the program path removed - SlangResult parse(int argc, const char*const* argv, DiagnosticSink* sink, Options& outOptions); - - SlangResult _parseArgWithValue(const char* option, String& outValue); - SlangResult _parseArgReplaceValue(const char* option, String& outValue); - SlangResult _parseArgFlag(const char* option, bool& outFlag); - - String m_reflectType; - - Index m_index; - Int m_argCount; - const char*const* m_args; - DiagnosticSink* m_sink; -}; - -SlangResult OptionsParser::_parseArgFlag(const char* option, bool& outFlag) -{ - SLANG_ASSERT(UnownedStringSlice(m_args[m_index]) == option); - SLANG_ASSERT(m_index < m_argCount); - - m_index ++; - outFlag = true; - return SLANG_OK; -} - -SlangResult OptionsParser::_parseArgWithValue(const char* option, String& ioValue) -{ - SLANG_ASSERT(UnownedStringSlice(m_args[m_index]) == option); - if (m_index + 1 < m_argCount) - { - // Next parameter is the output path, there can only be one - if (ioValue.getLength()) - { - // There already is output - m_sink->diagnose(SourceLoc(), CPPDiagnostics::optionAlreadyDefined, option, ioValue); - return SLANG_FAIL; - } - } - else - { - m_sink->diagnose(SourceLoc(), CPPDiagnostics::requireValueAfterOption, option); - return SLANG_FAIL; - } - - ioValue = m_args[m_index + 1]; - m_index += 2; - return SLANG_OK; -} - -SlangResult OptionsParser::_parseArgReplaceValue(const char* option, String& ioValue) -{ - SLANG_ASSERT(UnownedStringSlice(m_args[m_index]) == option); - if (m_index + 1 >= m_argCount) - { - m_sink->diagnose(SourceLoc(), CPPDiagnostics::requireValueAfterOption, option); - return SLANG_FAIL; - } - - ioValue = m_args[m_index + 1]; - m_index += 2; - return SLANG_OK; -} - -SlangResult OptionsParser::parse(int argc, const char*const* argv, DiagnosticSink* sink, Options& outOptions) -{ - outOptions.reset(); - - m_index = 0; - m_argCount = argc; - m_args = argv; - m_sink = sink; - - outOptions.reset(); - - while (m_index < m_argCount) - { - const UnownedStringSlice arg = UnownedStringSlice(argv[m_index]); - - if (arg.getLength() > 0 && arg[0] == '-') - { - if (arg == "-d") - { - SLANG_RETURN_ON_FAIL(_parseArgWithValue("-d", outOptions.m_inputDirectory)); - continue; - } - else if (arg == "-o") - { - SLANG_RETURN_ON_FAIL(_parseArgWithValue("-o", outOptions.m_outputPath)); - continue; - } - else if (arg == "-dump") - { - outOptions.m_dump = true; - m_index++; - continue; - } - else if (arg == "-mark-prefix") - { - SLANG_RETURN_ON_FAIL(_parseArgReplaceValue("-mark-prefix", outOptions.m_markPrefix)); - continue; - } - else if (arg == "-mark-suffix") - { - SLANG_RETURN_ON_FAIL(_parseArgReplaceValue("-mark-suffix", outOptions.m_markSuffix)); - continue; - } - else if (arg == "-defs") - { - SLANG_RETURN_ON_FAIL(_parseArgFlag("-defs", outOptions.m_defs)); - continue; - } - else if (arg == "-output-fields") - { - SLANG_RETURN_ON_FAIL(_parseArgFlag("-output-fields", outOptions.m_outputFields)); - continue; - } - else if (arg == "-strip-prefix") - { - SLANG_RETURN_ON_FAIL(_parseArgWithValue("-strip-prefix", outOptions.m_stripFilePrefix)); - continue; - } - - m_sink->diagnose(SourceLoc(), CPPDiagnostics::unknownOption, arg); - return SLANG_FAIL; - } - else - { - // If it starts with - then it an unknown option - outOptions.m_inputPaths.add(arg); - m_index++; - } - } - - if (outOptions.m_inputPaths.getCount() < 0) - { - m_sink->diagnose(SourceLoc(), CPPDiagnostics::noInputPathsSpecified); - return SLANG_FAIL; - } - - return SLANG_OK; -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! CPPExtractor !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -CPPExtractor::CPPExtractor(StringSlicePool* typePool, NamePool* namePool, DiagnosticSink* sink, IdentifierLookup* identifierLookup): - m_typePool(typePool), - m_sink(sink), - m_namePool(namePool), - m_identifierLookup(identifierLookup), - m_typeSetPool(StringSlicePool::Style::Empty) -{ - m_rootNode = new ScopeNode(Node::Type::Namespace); - m_rootNode->m_reflectionType = ReflectionType::Reflected; -} - -TypeSet* CPPExtractor::getTypeSet(const UnownedStringSlice& slice) -{ - Index index = m_typeSetPool.findIndex(slice); - if (index < 0) - { - return nullptr; - } - return m_typeSets[index]; -} - -TypeSet* CPPExtractor::getOrAddTypeSet(const UnownedStringSlice& slice) -{ - const Index index = Index(m_typeSetPool.add(slice)); - if (index >= m_typeSets.getCount()) - { - SLANG_ASSERT(m_typeSets.getCount() == index); - TypeSet* typeSet = new TypeSet; - - m_typeSets.add(typeSet); - typeSet->m_macroName = m_typeSetPool.getSlice(StringSlicePool::Handle(index)); - return typeSet; - } - else - { - return m_typeSets[index]; - } -} - -bool CPPExtractor::_isMarker(const UnownedStringSlice& name) -{ - return name.startsWith(m_options->m_markPrefix.getUnownedSlice()) && name.endsWith(m_options->m_markSuffix.getUnownedSlice()); -} - -SlangResult CPPExtractor::expect(TokenType type, Token* outToken) -{ - if (m_reader.peekTokenType() != type) - { - m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::expectingToken, type); - return SLANG_FAIL; - } - - if (outToken) - { - *outToken = m_reader.advanceToken(); - } - else - { - m_reader.advanceToken(); - } - return SLANG_OK; -} - -bool CPPExtractor::advanceIfToken(TokenType type, Token* outToken) -{ - if (m_reader.peekTokenType() == type) - { - Token token = m_reader.advanceToken(); - if (outToken) - { - *outToken = token; - } - return true; - } - return false; -} - -bool CPPExtractor::advanceIfMarker(Token* outToken) -{ - const Token peekToken = m_reader.peekToken(); - if (peekToken.type == TokenType::Identifier && _isMarker(peekToken.getContent())) - { - m_reader.advanceToken(); - if (outToken) - { - *outToken = peekToken; - } - return true; - } - return false; -} - -bool CPPExtractor::advanceIfStyle(IdentifierStyle style, Token* outToken) -{ - if (m_reader.peekTokenType() == TokenType::Identifier) - { - IdentifierStyle readStyle = m_identifierLookup->get(m_reader.peekToken().getContent()); - if (readStyle == style) - { - Token token = m_reader.advanceToken(); - if (outToken) - { - *outToken = token; - } - return true; - } - } - return false; -} - - -SlangResult CPPExtractor::pushAnonymousNamespace() -{ - m_currentScope = m_currentScope->getAnonymousNamespace(); - - if (m_origin) - { - m_origin->addNode(m_currentScope); - } - - return SLANG_OK; -} - -SlangResult CPPExtractor::pushScope(ScopeNode* scopeNode) -{ - if (m_origin) - { - m_origin->addNode(scopeNode); - } - - if (scopeNode->m_name.hasContent()) - { - // For anonymous namespace, we should look if we already have one and just reopen that. Doing so will mean will - // find anonymous namespace clashes - - if (Node* foundNode = m_currentScope->findChild(scopeNode->m_name.getContent())) - { - if (scopeNode->isClassLike()) - { - m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::typeAlreadyDeclared, scopeNode->m_name.getContent()); - m_sink->diagnose(foundNode->m_name, CPPDiagnostics::seeDeclarationOf, scopeNode->m_name.getContent()); - return SLANG_FAIL; - } - - if (foundNode->m_type == Node::Type::Namespace) - { - if (foundNode->m_type != scopeNode->m_type) - { - // Different types can't work - m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::typeAlreadyDeclared, scopeNode->m_name.getContent()); - return SLANG_FAIL; - } - - ScopeNode* foundScopeNode = as<ScopeNode>(foundNode); - SLANG_ASSERT(foundScopeNode); - - // Make sure the node is empty, as we are *not* going to add it, we are just going to use - // the pre-existing namespace - SLANG_ASSERT(scopeNode->m_children.getCount() == 0); - - // We can just use the pre-existing namespace - m_currentScope = foundScopeNode; - return SLANG_OK; - } - } - } - - m_currentScope->addChild(scopeNode); - m_currentScope = scopeNode; - return SLANG_OK; -} - -SlangResult CPPExtractor::popScope() -{ - if (m_currentScope->m_parentScope == nullptr) - { - m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::scopeNotClosed); - return SLANG_FAIL; - } - - m_currentScope = m_currentScope->m_parentScope; - return SLANG_OK; -} - -SlangResult CPPExtractor::consumeToClosingBrace(const Token* inOpenBraceToken) -{ - Token openToken; - if (inOpenBraceToken) - { - openToken = *inOpenBraceToken; - } - else - { - openToken = m_reader.advanceToken(); - } - - while (true) - { - switch (m_reader.peekTokenType()) - { - case TokenType::EndOfFile: - { - m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::didntFindMatchingBrace); - m_sink->diagnose(openToken, CPPDiagnostics::seeOpen); - return SLANG_FAIL; - } - case TokenType::LBrace: - { - SLANG_RETURN_ON_FAIL(consumeToClosingBrace()); - break; - } - case TokenType::RBrace: - { - m_reader.advanceToken(); - return SLANG_OK; - } - default: - { - m_reader.advanceToken(); - break; - } - } - } -} - - -SlangResult CPPExtractor::_maybeParseNode(Node::Type type) -{ - // We are looking for - // struct/class identifier [: [public|private|protected] Identifier ] { [public|private|proctected:]* marker ( identifier ); - - if (type == Node::Type::Namespace) - { - // consume namespace - SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier)); - - Token name; - if (advanceIfToken(TokenType::LBrace)) - { - return pushAnonymousNamespace(); - } - else if (advanceIfToken(TokenType::Identifier, &name)) - { - if (advanceIfToken(TokenType::LBrace)) - { - // Okay looks like we are opening a namespace - RefPtr<ScopeNode> node(new ScopeNode(Node::Type::Namespace)); - node->m_name = name; - // Push the node - return pushScope(node); - } - } - - // Just ignore it then - return SLANG_OK; - } - - // Must be class | struct - - SLANG_ASSERT(type == Node::Type::ClassType || type == Node::Type::StructType); - - Token name; - - // consume class | struct - SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier)); - // Next is the class name - SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &name)); - - if (m_reader.peekTokenType() == TokenType::Semicolon) - { - // pre declaration; - return SLANG_OK; - } - - RefPtr<ClassLikeNode> node(new ClassLikeNode(type)); - node->m_name = name; - - // Defaults to not reflected - SLANG_ASSERT(!node->isReflected()); - - if (advanceIfToken(TokenType::Colon)) - { - // Could have public - advanceIfStyle(IdentifierStyle::Access); - - if (!advanceIfToken(TokenType::Identifier, &node->m_super)) - { - return SLANG_OK; - } - } - - if (m_reader.peekTokenType() != TokenType::LBrace) - { - // Consume up until we see a brace else it's an error - while (true) - { - const TokenType peekTokenType = m_reader.peekTokenType(); - if (peekTokenType == TokenType::EndOfFile) - { - // Expecting brace - m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::expectingToken, TokenType::LBrace); - return SLANG_FAIL; - } - else if (peekTokenType == TokenType::LBrace) - { - break; - } - m_reader.advanceToken(); - } - - return pushScope(node); - } - - Token braceToken = m_reader.advanceToken(); - - while (true) - { - // Okay now we are looking for the markers, or visibility qualifiers - if (advanceIfStyle(IdentifierStyle::Access)) - { - // Consume it and a colon - if (SLANG_FAILED(expect(TokenType::Colon))) - { - consumeToClosingBrace(&braceToken); - return SLANG_OK; - } - continue; - } - - switch (m_reader.peekTokenType()) - { - case TokenType::Identifier: break; - case TokenType::RBrace: - { - SLANG_RETURN_ON_FAIL(pushScope(node)); - SLANG_RETURN_ON_FAIL(popScope()); - m_reader.advanceToken(); - return SLANG_OK; - } - default: - { - SLANG_RETURN_ON_FAIL(pushScope(node)); - return SLANG_OK; - } - } - - // If it's one of the markers, then we continue to extract parameter - if (advanceIfMarker(&node->m_marker)) - { - break; - } - - // We still need to add the node, - SLANG_RETURN_ON_FAIL(pushScope(node)); - return SLANG_OK; - } - - // Let's extract the type set - { - UnownedStringSlice slice(node->m_marker.getContent()); - - SLANG_ASSERT(_isMarker(slice)); - - // Strip the prefix and suffix - slice = UnownedStringSlice(slice.begin() + m_options->m_markPrefix.getLength(), slice.end() - m_options->m_markSuffix.getLength()); - - // Strip ABSTRACT_ if it's there - UnownedStringSlice abstractSlice("ABSTRACT_"); - if (slice.startsWith(abstractSlice)) - { - slice = UnownedStringSlice(slice.begin() + abstractSlice.getLength(), slice.end()); - } - - // TODO: We could strip other stuff or have other heuristics there, but this is - // probably okay for now - - // Set the typeSet - node->m_typeSet = getOrAddTypeSet(slice); - } - - // Okay now looking for ( identifier) - Token typeNameToken; - - SLANG_RETURN_ON_FAIL(expect(TokenType::LParent)); - SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &typeNameToken)); - SLANG_RETURN_ON_FAIL(expect(TokenType::RParent)); - - if (typeNameToken.getContent() != node->m_name.getContent()) - { - m_sink->diagnose(typeNameToken, CPPDiagnostics::typeNameDoesntMatch, node->m_name.getContent()); - return SLANG_FAIL; - } - - node->m_reflectionType = ReflectionType::Reflected; - return pushScope(node); -} - -SlangResult CPPExtractor::_consumeToSync() -{ - while (true) - { - TokenType type = m_reader.peekTokenType(); - - switch (type) - { - case TokenType::Semicolon: - { - m_reader.advanceToken(); - return SLANG_OK; - } - case TokenType::Pound: - case TokenType::EndOfFile: - case TokenType::LBrace: - case TokenType::RBrace: - { - return SLANG_OK; - } - } - - m_reader.advanceToken(); - } -} - -SlangResult CPPExtractor::_maybeParseTemplateArg(Index& ioTemplateDepth) -{ - switch (m_reader.peekTokenType()) - { - case TokenType::Identifier: - { - UnownedStringSlice name; - SLANG_RETURN_ON_FAIL(_maybeParseType(name, ioTemplateDepth)); - return SLANG_OK; - } - case TokenType::IntegerLiteral: - { - m_reader.advanceToken(); - return SLANG_OK; - } - default: break; - } - return SLANG_FAIL; -} - -SlangResult CPPExtractor::_maybeParseTemplateArgs(Index& ioTemplateDepth) -{ - if (!advanceIfToken(TokenType::OpLess)) - { - return SLANG_FAIL; - } - - ioTemplateDepth++; - - while (true) - { - if (ioTemplateDepth == 0) - { - return SLANG_OK; - } - - switch (m_reader.peekTokenType()) - { - case TokenType::OpGreater: - { - if (ioTemplateDepth <= 0) - { - m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::unexpectedTemplateClose); - return SLANG_FAIL; - } - ioTemplateDepth--; - m_reader.advanceToken(); - return SLANG_OK; - } - case TokenType::OpRsh: - { - if (ioTemplateDepth <= 1) - { - m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::unexpectedTemplateClose); - return SLANG_FAIL; - } - ioTemplateDepth -= 2; - m_reader.advanceToken(); - return SLANG_OK; - } - default: - { - while (true) - { - SLANG_RETURN_ON_FAIL(_maybeParseTemplateArg(ioTemplateDepth)); - - if (m_reader.peekTokenType() == TokenType::Comma) - { - m_reader.advanceToken(); - // If there is a comma parse another arg - continue; - } - break; - } - break; - } - } - } -} - -void CPPExtractor::_consumeTypeModifiers() -{ - while (advanceIfStyle(IdentifierStyle::TypeModifier)); -} - -// True if two of these token types of the same type placed immediately after one another -// produce a different token. Can be conservative, as if not strictly required -// it will just mean more spacing in the output -static bool _canRepeatTokenType(TokenType type) -{ - switch (type) - { - case TokenType::OpAdd: - case TokenType::OpSub: - case TokenType::OpAnd: - case TokenType::OpOr: - case TokenType::OpGreater: - case TokenType::OpLess: - case TokenType::Identifier: - case TokenType::OpAssign: - case TokenType::Colon: - { - return false; - } - default: break; - } - return true; -} - -// Returns true if there needs to be a space between the previous token type, and the current token -// type for correct output. It is assumed that the token stream is appropriate. -// The implementation might need more sophistication, but this at least avoids Blah const * -> Blahconst* -static bool _tokenConcatNeedsSpace(TokenType prev, TokenType cur) -{ - if ((cur == TokenType::OpAssign) || - (prev == cur && !_canRepeatTokenType(cur))) - { - return true; - } - return false; -} - -UnownedStringSlice CPPExtractor::_concatTokens(TokenReader::ParsingCursor start) -{ - auto endCursor = m_reader.getCursor(); - m_reader.setCursor(start); - - TokenType prevTokenType = TokenType::Unknown; - - StringBuilder buf; - while (!m_reader.isAtCursor(endCursor)) - { - const Token token = m_reader.advanceToken(); - // Check if we need a space between tokens - if (_tokenConcatNeedsSpace(prevTokenType, token.type)) - { - buf << " "; - } - buf << token.getContent(); - - prevTokenType = token.type; - } - - return m_typePool->getSlice(m_typePool->add(buf)); -} - - -SlangResult CPPExtractor::_maybeParseType(UnownedStringSlice& outType, Index& ioTemplateDepth) -{ - auto startCursor = m_reader.getCursor(); - - _consumeTypeModifiers(); - - advanceIfToken(TokenType::Scope); - while (true) - { - Token identifierToken; - if (!advanceIfToken(TokenType::Identifier, &identifierToken)) - { - return SLANG_FAIL; - } - - const IdentifierStyle style = m_identifierLookup->get(identifierToken.getContent()); - if (hasFlag(style, IdentifierFlag::Keyword)) - { - return SLANG_FAIL; - } - - if (advanceIfToken(TokenType::Scope)) - { - continue; - } - break; - } - - if (m_reader.peekTokenType() == TokenType::OpLess) - { - SLANG_RETURN_ON_FAIL(_maybeParseTemplateArgs(ioTemplateDepth)); - } - - // Strip all the consts etc modifiers - _consumeTypeModifiers(); - - // It's a reference and we are done - if (advanceIfToken(TokenType::OpBitAnd)) - { - return SLANG_OK; - } - - while (true) - { - if (advanceIfToken(TokenType::OpMul)) - { - // Strip all the consts - _consumeTypeModifiers(); - continue; - } - break; - } - - // We can build up the out type, from the tokens we found - outType = _concatTokens(startCursor); - return SLANG_OK; -} - -SlangResult CPPExtractor::_maybeParseType(UnownedStringSlice& outType) -{ - Index templateDepth = 0; - SlangResult res = _maybeParseType(outType, templateDepth); - if (SLANG_FAILED(res) && m_sink->getErrorCount()) - { - return res; - } - - if (templateDepth != 0) - { - m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::unexpectedTemplateClose); - return SLANG_FAIL; - } - return SLANG_OK; -} - -static bool _isBalancedOpen(TokenType tokenType) -{ - return tokenType == TokenType::LBrace || - tokenType == TokenType::LParent || - tokenType == TokenType::LBracket; -} - -static bool _isBalancedClose(TokenType tokenType) -{ - return tokenType == TokenType::RBrace || - tokenType == TokenType::RParent || - tokenType == TokenType::RBracket; -} - -static TokenType _getBalancedClose(TokenType tokenType) -{ - SLANG_ASSERT(_isBalancedOpen(tokenType)); - switch (tokenType) - { - case TokenType::LBrace: return TokenType::RBrace; - case TokenType::LParent: return TokenType::RParent; - case TokenType::LBracket: return TokenType::RBracket; - default: return TokenType::Unknown; - } -} - -SlangResult CPPExtractor::_parseBalanced(DiagnosticSink* sink) -{ - const TokenType openTokenType = m_reader.peekTokenType(); - if (!_isBalancedOpen(openTokenType)) - { - return SLANG_FAIL; - } - - // Save the start token - const Token startToken = m_reader.advanceToken(); - // Get the token type that would close the open - const TokenType closeTokenType = _getBalancedClose(openTokenType); - - while (true) - { - const TokenType tokenType = m_reader.peekTokenType(); - - // If we hit the closing token, we are done - if (tokenType == closeTokenType) - { - m_reader.advanceToken(); - return SLANG_OK; - } - - // If we hit a balanced open, recurse - if (_isBalancedOpen(tokenType)) - { - SLANG_RETURN_ON_FAIL(_parseBalanced(sink)); - continue; - } - - // If we hit a close token that doesn't match, then the balancing has gone wrong - if (_isBalancedClose(tokenType)) - { - // Only diagnose if required - if (sink) - { - sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::unexpectedUnbalancedToken); - sink->diagnose(startToken, CPPDiagnostics::seeOpen); - } - return SLANG_FAIL; - } - - // If we hit the end of the file and have not hit the closing token, then - // somethings gone wrong - if (tokenType == TokenType::EndOfFile) - { - if (sink) - { - sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::unexpectedEndOfFile); - sink->diagnose(startToken, CPPDiagnostics::seeOpen); - } - - return SLANG_FAIL; - } - - // Skip the token - m_reader.advanceToken(); - } -} - -SlangResult CPPExtractor::_maybeParseField() -{ - // Can only add a field if we are in a class - SLANG_ASSERT(m_currentScope->isClassLike()); - - UnownedStringSlice typeName; - if (SLANG_FAILED(_maybeParseType(typeName))) - { - if (m_sink->getErrorCount()) - { - return SLANG_FAIL; - } - - _consumeToSync(); - return SLANG_OK; - } - - if (m_reader.peekTokenType() != TokenType::Identifier) - { - _consumeToSync(); - return SLANG_OK; - } - - Token fieldName = m_reader.advanceToken(); - - if (m_reader.peekTokenType() == TokenType::LBracket) - { - auto startCursor = m_reader.getCursor(); - - // If it's not balanced we just assume it's not correct - and ignore - if (SLANG_FAILED(_parseBalanced(nullptr))) - { - _consumeToSync(); - return SLANG_OK; - } - - UnownedStringSlice arraySuffix = _concatTokens(startCursor); - - // The overall type is the typename concated with the arraySuffix - StringBuilder buf; - buf << typeName << arraySuffix; - - typeName = m_typePool->getSlice( m_typePool->add(buf)); - } - - switch (m_reader.peekTokenType()) - { - case TokenType::OpAssign: - { - // Special case to handle - // Type operator=(... - - m_reader.advanceToken(); - if (m_reader.peekTokenType() == TokenType::LParent) - { - // Not a field - break; - } - } - case TokenType::Semicolon: - { - FieldNode* fieldNode = new FieldNode; - - fieldNode->m_fieldType = typeName; - fieldNode->m_name = fieldName; - fieldNode->m_reflectionType = m_currentScope->getContainedReflectionType(); - - m_currentScope->addChild(fieldNode); - break; - } - default: break; - } - - _consumeToSync(); - return SLANG_OK; -} - -/* static */Node::Type CPPExtractor::_toNodeType(IdentifierStyle style) -{ - switch (style) - { - case IdentifierStyle::Class: return Node::Type::ClassType; - case IdentifierStyle::Struct: return Node::Type::StructType; - case IdentifierStyle::Namespace: return Node::Type::Namespace; - default: return Node::Type::Invalid; - } -} - -static UnownedStringSlice _trimUnderscorePrefix(const UnownedStringSlice& slice) -{ - if (slice.getLength() && slice[0] == '_') - { - return UnownedStringSlice(slice.begin() + 1, slice.end()); - } - else - { - return slice; - } -} - - -SlangResult CPPExtractor::_parsePreDeclare() -{ - // Skip the declare type token - m_reader.advanceToken(); - - SLANG_RETURN_ON_FAIL(expect(TokenType::LParent)); - - // Get the typeSet - Token typeSetToken; - SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &typeSetToken)); - TypeSet* typeSet = getOrAddTypeSet(typeSetToken.getContent()); - - SLANG_RETURN_ON_FAIL(expect(TokenType::Comma)); - - // Get the type of type - Node::Type nodeType; - { - Token typeToken; - SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &typeToken)); - - const IdentifierStyle style = m_identifierLookup->get(typeToken.getContent()); - - if (style != IdentifierStyle::Struct && style != IdentifierStyle::Class) - { - m_sink->diagnose(typeToken, CPPDiagnostics::expectingTypeKeyword, typeToken.getContent()); - return SLANG_FAIL; - } - nodeType = _toNodeType(style); - } - - Token name; - Token super; - - SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &name)); - - if (advanceIfToken(TokenType::Colon)) - { - SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &super)); - } - - SLANG_RETURN_ON_FAIL(expect(TokenType::RParent)); - - switch (nodeType) - { - case Node::Type::ClassType: - case Node::Type::StructType: - { - RefPtr<ClassLikeNode> node(new ClassLikeNode(nodeType)); - - node->m_name = name; - node->m_super = super; - node->m_typeSet = typeSet; - - // Assume it is reflected - node->m_reflectionType = ReflectionType::Reflected; - - SLANG_RETURN_ON_FAIL(pushScope(node)); - // Pop out of the node - popScope(); - break; - } - default: - { - return SLANG_FAIL; - } - } - - - return SLANG_OK; -} - -SlangResult CPPExtractor::_parseTypeSet() -{ - // Skip the declare type token - m_reader.advanceToken(); - - SLANG_RETURN_ON_FAIL(expect(TokenType::LParent)); - - Token typeSetToken; - SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &typeSetToken)); - - TypeSet* typeSet = getOrAddTypeSet(typeSetToken.getContent()); - - SLANG_RETURN_ON_FAIL(expect(TokenType::Comma)); - - // Get the type of type - Token typeToken; - SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &typeToken)); - - SLANG_RETURN_ON_FAIL(expect(TokenType::RParent)); - - // Set the typename - typeSet->m_typeName = typeToken.getContent(); - - return SLANG_OK; -} - -SlangResult CPPExtractor::parse(SourceFile* sourceFile, const Options* options) -{ - SLANG_ASSERT(options); - m_options = options; - - // Calculate from the path, a 'macro origin' name. - const String macroOrigin = _calcMacroOrigin(sourceFile->getPathInfo().foundPath, *options); - - RefPtr<SourceOrigin> origin = new SourceOrigin(sourceFile, macroOrigin); - m_origins.add(origin); - - // Set the current origin - m_origin = origin; - - SourceManager* manager = sourceFile->getSourceManager(); - - SourceView* sourceView = manager->createSourceView(sourceFile, nullptr, SourceLoc::fromRaw(0)); - - Lexer lexer; - - m_currentScope = m_rootNode; - - lexer.initialize(sourceView, m_sink, m_namePool, manager->getMemoryArena()); - m_tokenList = lexer.lexAllTokens(); - // See if there were any errors - if (m_sink->getErrorCount()) - { - return SLANG_FAIL; - } - - m_reader = TokenReader(m_tokenList); - - while (true) - { - switch (m_reader.peekTokenType()) - { - case TokenType::Identifier: - { - const IdentifierStyle style = m_identifierLookup->get(m_reader.peekToken().getContent()); - - switch (style) - { - case IdentifierStyle::PreDeclare: - { - SLANG_RETURN_ON_FAIL(_parsePreDeclare()); - break; - } - case IdentifierStyle::TypeSet: - { - SLANG_RETURN_ON_FAIL(_parseTypeSet()); - break; - } - case IdentifierStyle::Reflected: - { - m_reader.advanceToken(); - if (m_currentScope) - { - m_currentScope->m_reflectionOverride = ReflectionType::Reflected; - } - break; - } - case IdentifierStyle::Unreflected: - { - m_reader.advanceToken(); - if (m_currentScope) - { - m_currentScope->m_reflectionOverride = ReflectionType::NotReflected; - } - break; - } - case IdentifierStyle::Access: - { - m_reader.advanceToken(); - SLANG_RETURN_ON_FAIL(expect(TokenType::Colon)); - break; - } - default: - { - IdentifierFlags flags = getFlags(style); - - if (flags & IdentifierFlag::StartScope) - { - Node::Type type = _toNodeType(style); - SLANG_RETURN_ON_FAIL(_maybeParseNode(type)); - } - else - { - // Special case the node that's the root of the hierarchy (as far as reflection is concerned) - // This could be a field - if (m_currentScope->acceptsFields()) - { - SLANG_RETURN_ON_FAIL(_maybeParseField()); - } - else - { - m_reader.advanceToken(); - } - } - break; - } - } - break; - } - case TokenType::LBrace: - { - SLANG_RETURN_ON_FAIL(consumeToClosingBrace()); - break; - } - case TokenType::RBrace: - { - SLANG_RETURN_ON_FAIL(popScope()); - m_reader.advanceToken(); - break; - } - case TokenType::EndOfFile: - { - // Okay we need to confirm that we are in the root node, and with no open braces - if (m_currentScope != m_rootNode) - { - m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::braceOpenAtEndOfFile); - return SLANG_FAIL; - } - - return SLANG_OK; - } - case TokenType::Pound: - { - Token token = m_reader.peekToken(); - if (token.flags & TokenFlag::AtStartOfLine) - { - // We are just going to ignore all of these for now.... - m_reader.advanceToken(); - while (m_reader.peekTokenType() != TokenType::EndOfDirective && m_reader.peekTokenType() != TokenType::EndOfFile) - { - m_reader.advanceToken(); - } - break; - } - // Skip it then - m_reader.advanceToken(); - break; - } - default: - { - // Skip it then - m_reader.advanceToken(); - break; - } - } - } -} - -Node* CPPExtractor::findNode(ScopeNode* scope, const UnownedStringSlice& name) -{ - // TODO(JS): We may want to lookup based on the path. - // If the name is qualified, we give up for not - if (String(name).indexOf("::") >= 0) - { - return nullptr; - } - - // Okay try in all scopes up to the root - while (scope) - { - if (Node* node = scope->findChild(name)) - { - return node; - } - - scope = scope->m_parentScope; + out << CPP_EXTRACT_INDENT_STRING; } - - return nullptr; -} - -SlangResult CPPExtractor::_calcDerivedTypesRec(ScopeNode* inScopeNode) -{ - if (inScopeNode->isClassLike()) - { - ClassLikeNode* classLikeNode = static_cast<ClassLikeNode*>(inScopeNode); - - if (classLikeNode->m_super.hasContent()) - { - ScopeNode* parentScope = classLikeNode->m_parentScope; - if (parentScope == nullptr) - { - m_sink->diagnoseRaw(Severity::Error, UnownedStringSlice::fromLiteral("Can't lookup in scope if there is none!")); - return SLANG_FAIL; - } - - Node* superNode = findNode(parentScope, classLikeNode->m_super.getContent()); - - if (!superNode) - { - if (classLikeNode->isReflected()) - { - m_sink->diagnose(classLikeNode->m_name, CPPDiagnostics::superTypeNotFound, classLikeNode->getAbsoluteName()); - return SLANG_FAIL; - } - } - else - { - ClassLikeNode* superType = as<ClassLikeNode>(superNode); - - if (!superType) - { - m_sink->diagnose(classLikeNode->m_name, CPPDiagnostics::superTypeNotAType, classLikeNode->getAbsoluteName()); - return SLANG_FAIL; - } - - if (superType->m_typeSet != classLikeNode->m_typeSet) - { - m_sink->diagnose(classLikeNode->m_name, CPPDiagnostics::typeInDifferentTypeSet, classLikeNode->m_name.getContent(), classLikeNode->m_typeSet->m_macroName, superType->m_typeSet->m_macroName); - return SLANG_FAIL; - } - - // The base class must be defined in same scope (as we didn't allow different scopes for base classes) - superType->addDerived(classLikeNode); - } - } - else - { - // Add to it's own typeset - if (classLikeNode->isReflected()) - { - classLikeNode->m_typeSet->m_baseTypes.add(classLikeNode); - } - } - } - - for (Node* child : inScopeNode->m_children) - { - ScopeNode* childScope = as<ScopeNode>(child); - if (childScope) - { - SLANG_RETURN_ON_FAIL(_calcDerivedTypesRec(childScope)); - } - } - - return SLANG_OK; -} - -SlangResult CPPExtractor::calcDerivedTypes() -{ - return _calcDerivedTypesRec(m_rootNode); -} - -/* static */String CPPExtractor::_calcMacroOrigin(const String& filePath, const Options& options) -{ - // Get the filename without extension - String fileName = Path::getFileNameWithoutExt(filePath); - - // We can work on just the slice - UnownedStringSlice slice = fileName.getUnownedSlice(); - - // Filename prefix - if (options.m_stripFilePrefix.getLength() && slice.startsWith(options.m_stripFilePrefix.getUnownedSlice())) - { - const Index len = options.m_stripFilePrefix.getLength(); - slice = UnownedStringSlice(slice.begin() + len, slice.end()); - } - - // Trim - - slice = slice.trim('-'); - - StringBuilder out; - NameConventionUtil::convert(slice, CharCase::Upper, NameConvention::Snake, out); - return out; } // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! CPPExtractorApp !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -class CPPExtractorApp +class App { public: @@ -2163,21 +60,21 @@ public: SlangResult executeWithArgs(int argc, const char*const* argv); /// Write output - SlangResult writeOutput(CPPExtractor& extractor); + SlangResult writeOutput(NodeTree* tree); /// Write def files - SlangResult writeDefs(CPPExtractor& extractor); + SlangResult writeDefs(NodeTree* tree); /// Calculate the header - SlangResult calcTypeHeader(CPPExtractor& extractor, TypeSet* typeSet, StringBuilder& out); - SlangResult calcChildrenHeader(CPPExtractor& exctractor, TypeSet* typeSet, StringBuilder& out); - SlangResult calcOriginHeader(CPPExtractor& extractor, StringBuilder& out); + SlangResult calcTypeHeader(NodeTree* tree, TypeSet* typeSet, StringBuilder& out); + SlangResult calcChildrenHeader(NodeTree* tree, TypeSet* typeSet, StringBuilder& out); + SlangResult calcOriginHeader(NodeTree* tree, StringBuilder& out); - SlangResult calcDef(CPPExtractor& extractor, SourceOrigin* origin, StringBuilder& out); + SlangResult calcDef(NodeTree* tree, SourceOrigin* origin, StringBuilder& out); const Options& getOptions() const { return m_options; } - CPPExtractorApp(DiagnosticSink* sink, SourceManager* sourceManager, RootNamePool* rootNamePool): + App(DiagnosticSink* sink, SourceManager* sourceManager, RootNamePool* rootNamePool): m_sink(sink), m_sourceManager(sourceManager), m_slicePool(StringSlicePool::Style::Default) @@ -2199,7 +96,7 @@ protected: StringSlicePool m_slicePool; }; -SlangResult CPPExtractorApp::readAllText(const Slang::String& fileName, String& outRead) +SlangResult App::readAllText(const Slang::String& fileName, String& outRead) { try { @@ -2220,7 +117,7 @@ SlangResult CPPExtractorApp::readAllText(const Slang::String& fileName, String& return SLANG_OK; } -SlangResult CPPExtractorApp::writeAllText(const Slang::String& fileName, const UnownedStringSlice& text) +SlangResult App::writeAllText(const Slang::String& fileName, const UnownedStringSlice& text) { try { @@ -2245,7 +142,7 @@ SlangResult CPPExtractorApp::writeAllText(const Slang::String& fileName, const U return SLANG_OK; } -SlangResult CPPExtractorApp::calcDef(CPPExtractor& extractor, SourceOrigin* origin, StringBuilder& out) +SlangResult App::calcDef(NodeTree* tree, SourceOrigin* origin, StringBuilder& out) { Node* currentScope = nullptr; @@ -2268,7 +165,7 @@ SlangResult CPPExtractorApp::calcDef(CPPExtractor& extractor, SourceOrigin* orig return SLANG_OK; } -SlangResult CPPExtractorApp::calcChildrenHeader(CPPExtractor& extractor, TypeSet* typeSet, StringBuilder& out) +SlangResult App::calcChildrenHeader(NodeTree* tree, TypeSet* typeSet, StringBuilder& out) { const List<ClassLikeNode*>& baseTypes = typeSet->m_baseTypes; const String& reflectTypeName = typeSet->m_typeName; @@ -2386,13 +283,13 @@ SlangResult CPPExtractorApp::calcChildrenHeader(CPPExtractor& extractor, TypeSet return SLANG_OK; } -SlangResult CPPExtractorApp::calcOriginHeader(CPPExtractor& extractor, StringBuilder& out) +SlangResult App::calcOriginHeader(NodeTree* tree, StringBuilder& out) { // Do macros by origin out << "// Origin macros\n\n"; - for (SourceOrigin* origin : extractor.getSourceOrigins()) + for (SourceOrigin* origin : tree->getSourceOrigins()) { out << "#define " << m_options.m_markPrefix << "ORIGIN_" << origin->m_macroOrigin << "(x, param) \\\n"; @@ -2412,7 +309,7 @@ SlangResult CPPExtractorApp::calcOriginHeader(CPPExtractor& extractor, StringBui return SLANG_OK; } -SlangResult CPPExtractorApp::calcTypeHeader(CPPExtractor& extractor, TypeSet* typeSet, StringBuilder& out) +SlangResult App::calcTypeHeader(NodeTree* tree, TypeSet* typeSet, StringBuilder& out) { const List<ClassLikeNode*>& baseTypes = typeSet->m_baseTypes; const String& reflectTypeName = typeSet->m_typeName; @@ -2576,9 +473,9 @@ SlangResult CPPExtractorApp::calcTypeHeader(CPPExtractor& extractor, TypeSet* ty return SLANG_OK; } -SlangResult CPPExtractorApp::writeDefs(CPPExtractor& extractor) +SlangResult App::writeDefs(NodeTree* tree) { - const auto& origins = extractor.getSourceOrigins(); + const auto& origins = tree->getSourceOrigins(); for (SourceOrigin* origin : origins) { @@ -2595,7 +492,7 @@ SlangResult CPPExtractorApp::writeDefs(CPPExtractor& extractor) outPath << pathWithoutExt << "-defs." << ext; StringBuilder content; - SLANG_RETURN_ON_FAIL(calcDef(extractor, origin, content)); + SLANG_RETURN_ON_FAIL(calcDef(tree, origin, content)); // Write the defs file SLANG_RETURN_ON_FAIL(writeAllText(outPath, content.getUnownedSlice())); @@ -2604,7 +501,7 @@ SlangResult CPPExtractorApp::writeDefs(CPPExtractor& extractor) return SLANG_OK; } -SlangResult CPPExtractorApp::writeOutput(CPPExtractor& extractor) +SlangResult App::writeOutput(NodeTree* tree) { String path; if (m_options.m_inputDirectory.getLength()) @@ -2627,12 +524,12 @@ SlangResult CPPExtractorApp::writeOutput(CPPExtractor& extractor) // Strip the extension if set path = Path::getPathWithoutExt(path); - for (TypeSet* typeSet : extractor.getTypeSets()) + for (TypeSet* typeSet : tree->getTypeSets()) { { /// Calculate the header StringBuilder header; - SLANG_RETURN_ON_FAIL(calcTypeHeader(extractor, typeSet, header)); + SLANG_RETURN_ON_FAIL(calcTypeHeader(tree, typeSet, header)); // Write it out @@ -2643,7 +540,7 @@ SlangResult CPPExtractorApp::writeOutput(CPPExtractor& extractor) { StringBuilder childrenHeader; - SLANG_RETURN_ON_FAIL(calcChildrenHeader(extractor, typeSet, childrenHeader)); + SLANG_RETURN_ON_FAIL(calcChildrenHeader(tree, typeSet, childrenHeader)); StringBuilder headerPath; headerPath << path << "-" << typeSet->m_fileMark << "-macro." + ext; @@ -2654,7 +551,7 @@ SlangResult CPPExtractorApp::writeOutput(CPPExtractor& extractor) return SLANG_OK; } -/* static */void CPPExtractorApp::_initIdentifierLookup(const Options& options, IdentifierLookup& outLookup) +/* static */void App::_initIdentifierLookup(const Options& options, IdentifierLookup& outLookup) { outLookup.reset(); @@ -2699,15 +596,16 @@ SlangResult CPPExtractorApp::writeOutput(CPPExtractor& extractor) } } -SlangResult CPPExtractorApp::execute(const Options& options) +SlangResult App::execute(const Options& options) { m_options = options; IdentifierLookup identifierLookup; _initIdentifierLookup(options, identifierLookup); - CPPExtractor extractor(&m_slicePool, &m_namePool, m_sink, &identifierLookup); + NodeTree tree(&m_slicePool, &m_namePool, &identifierLookup); + // Read in each of the input files for (Index i = 0; i < m_options.m_inputPaths.getCount(); ++i) { @@ -2730,14 +628,17 @@ SlangResult CPPExtractorApp::execute(const Options& options) SourceFile* sourceFile = m_sourceManager->createSourceFileWithString(pathInfo, contents); - SLANG_RETURN_ON_FAIL(extractor.parse(sourceFile, &m_options)); + SourceOrigin* sourceOrigin = tree.addSourceOrigin(sourceFile, options); + + Parser parser(&tree, m_sink); + SLANG_RETURN_ON_FAIL(parser.parse(sourceOrigin, &m_options)); } - SLANG_RETURN_ON_FAIL(extractor.calcDerivedTypes()); + SLANG_RETURN_ON_FAIL(tree.calcDerivedTypes(m_sink)); // Okay let's check out the typeSets { - for (TypeSet* typeSet : extractor.getTypeSets()) + for (TypeSet* typeSet : tree.getTypeSets()) { // The macro name is in upper snake, so split it List<UnownedStringSlice> slices; @@ -2766,11 +667,11 @@ SlangResult CPPExtractorApp::execute(const Options& options) { { StringBuilder buf; - extractor.getRootNode()->dump(0, buf); + tree.getRootNode()->dump(0, buf); m_sink->writer->write(buf.getBuffer(), buf.getLength()); } - for (TypeSet* typeSet : extractor.getTypeSets()) + for (TypeSet* typeSet : tree.getTypeSets()) { const List<ClassLikeNode*>& baseTypes = typeSet->m_baseTypes; @@ -2785,19 +686,19 @@ SlangResult CPPExtractorApp::execute(const Options& options) if (options.m_defs) { - SLANG_RETURN_ON_FAIL(writeDefs(extractor)); + SLANG_RETURN_ON_FAIL(writeDefs(&tree)); } if (options.m_outputPath.getLength()) { - SLANG_RETURN_ON_FAIL(writeOutput(extractor)); + SLANG_RETURN_ON_FAIL(writeOutput(&tree)); } return SLANG_OK; } /// Execute -SlangResult CPPExtractorApp::executeWithArgs(int argc, const char*const* argv) +SlangResult App::executeWithArgs(int argc, const char*const* argv) { Options options; OptionsParser optionsParser; @@ -2806,14 +707,13 @@ SlangResult CPPExtractorApp::executeWithArgs(int argc, const char*const* argv) return SLANG_OK; } -} // namespace SlangExperimental +} // namespace CppExtract int main(int argc, const char*const* argv) { - using namespace SlangExperimental; + using namespace CppExtract; using namespace Slang; - { ComPtr<ISlangWriter> writer(new FileWriter(stderr, WriterFlag::AutoFlush)); @@ -2841,7 +741,7 @@ int main(int argc, const char*const* argv) sink.diagnose(SourceLoc(), CPPDiagnostics::commandLine, builder); } - CPPExtractorApp app(&sink, &sourceManager, &rootNamePool); + App app(&sink, &sourceManager, &rootNamePool); try { |