C++ Extractor (#1337)

* WIP: Doing texing using slangs lexer for cpp-extractor

* Node tree for C++ extraction.

* Bug fixing.
Add dump of hierarchy.

* First pass at extracting fields.

* Parse template types.

* Use diagnostics defs for C++ extractor.

* Simplify Diagnostic Defs.

* Remove the brace stack.

* Added IdentifierLookup.

* Add handling for >> style template close.

* Improved identifier handling/keywords.

* Added ability to check if reader is at cursor position.

* Handling of an unspecified root type.

* Parsing code comments.
Tidy up some parsing - to use advanceIf functions more.

* Improve path handling.

* Fixes around changes to Path interface.

* Working Range, Type and Scope header.

* Extract the middle part of marker and put in output. Gives more flexibility at macro injection, and in class definitions.

* Split DERIVED types into it's own macro, to provide way to generate for derived types.

* Fix clang/g++ compile issue.

* Tabs -> spaces.

* Fix small bug in getFileNameWithoutExt

* Small improvement around naming.

Co-authored-by: Tim Foley <tfoleyNV@users.noreply.github.com>

10 files changed, 168 insertions, 133 deletions

diff --git a/source/core/slang-gcc-compiler-util.cpp b/source/core/slang-gcc-compiler-util.cpp
index 3f045aa80..68f89d9e1 100644
--- a/source/core/slang-gcc-compiler-util.cpp
+++ b/source/core/slang-gcc-compiler-util.cpp
@@ -265,7 +265,7 @@ static SlangResult _parseGCCFamilyLine(const UnownedStringSlice& line, LineParse
     else if (split.getCount() == 4)
     {
         // Probably a link error, give the source line
-        String ext = Path::getFileExt(split[0]);
+        String ext = Path::getPathExt(split[0]);
 
         // Maybe a bit fragile -> but probably okay for now
         if (ext != "o" && ext != "obj")
diff --git a/source/core/slang-io.cpp b/source/core/slang-io.cpp
index e02bbb89f..1717c8354 100644
--- a/source/core/slang-io.cpp
+++ b/source/core/slang-io.cpp
@@ -144,56 +144,64 @@ namespace Slang
     }
 
 
-	bool File::exists(const String& fileName)
-	{
+    bool File::exists(const String& fileName)
+    {
 #ifdef _WIN32
-		struct _stat32 statVar;
-		return ::_wstat32(((String)fileName).toWString(), &statVar) != -1;
+        struct _stat32 statVar;
+        return ::_wstat32(((String)fileName).toWString(), &statVar) != -1;
 #else
-		struct stat statVar;
-		return ::stat(fileName.getBuffer(), &statVar) == 0;
+        struct stat statVar;
+        return ::stat(fileName.getBuffer(), &statVar) == 0;
 #endif
-	}
-
-	String Path::truncateExt(const String& path)
-	{
-		UInt dotPos = path.lastIndexOf('.');
-		if (dotPos != -1)
-			return path.subString(0, dotPos);
-		else
-			return path;
-	}
-	String Path::replaceExt(const String& path, const char* newExt)
-	{
-		StringBuilder sb(path.getLength()+10);
-		UInt dotPos = path.lastIndexOf('.');
-		if (dotPos == -1)
-			dotPos = path.getLength();
-		sb.Append(path.getBuffer(), dotPos);
-		sb.Append('.');
-		sb.Append(newExt);
-		return sb.ProduceString();
-	}
-
-    static UInt findLastSeparator(String const& path)
-    {
-		UInt slashPos = path.lastIndexOf('/');
-        UInt backslashPos = path.lastIndexOf('\\');
-
-        if (slashPos == -1) return backslashPos;
-        if (backslashPos == -1) return slashPos;
-
-        UInt pos = slashPos;
-        if (backslashPos > slashPos)
-            pos = backslashPos;
-
-        return pos;
-    }
-
-	String Path::getFileName(const String& path)
-	{
-        UInt pos = findLastSeparator(path);
-        if (pos != -1)
+    }
+
+    String Path::replaceExt(const String& path, const char* newExt)
+    {
+        StringBuilder sb(path.getLength() + 10);
+        Index dotPos = findExtIndex(path);
+
+        if (dotPos < 0)
+            dotPos = path.getLength();
+        sb.Append(path.getBuffer(), dotPos);
+        sb.Append('.');
+        sb.Append(newExt);
+        return sb.ProduceString();
+    }
+
+    /* static */ Index Path::findLastSeparatorIndex(String const& path)
+    {
+        const char* chars = path.getBuffer();
+        for (Index i = path.getLength() - 1; i >= 0; --i)
+        {
+            const char c = chars[i];
+            if (c == '/' || c == '\\')
+            {
+                return i;
+            }
+        }
+        return -1;
+    }
+
+    /* static */Index Path::findExtIndex(String const& path)
+    {
+        const Index sepIndex = findLastSeparatorIndex(path);
+
+        const Index dotIndex = path.lastIndexOf('.');
+        if (sepIndex >= 0)
+        {
+            // Index has to be in the last part of the path
+            return (dotIndex > sepIndex) ? dotIndex : -1;
+        }
+        else
+        {
+            return dotIndex;
+        }
+    }
+
+    String Path::getFileName(const String& path)
+    {
+        Index pos = findLastSeparatorIndex(path);
+        if (pos >= 0)
         {
             pos = pos + 1;
             return path.subString(pos, path.getLength() - pos);
@@ -202,32 +210,45 @@ namespace Slang
         {
             return path;
         }
-	}
-	String Path::getFileNameWithoutExt(const String& path)
-	{
-        String fileName = getFileName(path);
-		UInt dotPos = fileName.lastIndexOf('.');
-		if (dotPos == -1)
-            return fileName;
-		return fileName.subString(0, dotPos);
-	}
-	String Path::getFileExt(const String& path)
-	{
-		UInt dotPos = path.lastIndexOf('.');
-		if (dotPos != -1)
-			return path.subString(dotPos+1, path.getLength()-dotPos-1);
-		else
-			return "";
-	}
-	String Path::getParentDirectory(const String& path)
-	{
-        UInt pos = findLastSeparator(path);
-		if (pos != -1)
-			return path.subString(0, pos);
-		else
-			return "";
-	}
-	
+    }
+
+    /* static */String Path::getFileNameWithoutExt(const String& path)
+    {
+        Index sepIndex = findLastSeparatorIndex(path);
+        sepIndex = (sepIndex < 0) ? 0 : (sepIndex + 1);
+        Index dotIndex = findExtIndex(path);
+        dotIndex = (dotIndex < 0) ? path.getLength() : dotIndex;
+
+        return path.subString(sepIndex, dotIndex - sepIndex);
+    }
+
+    /* static*/ String Path::getPathWithoutExt(const String& path)
+    {
+        Index dotPos = findExtIndex(path);
+        if (dotPos >= 0)
+            return path.subString(0, dotPos);
+        else
+            return path;
+    }
+
+    String Path::getPathExt(const String& path)
+    {
+        const Index dotPos = findExtIndex(path);
+        if (dotPos >= 0)
+            return path.subString(dotPos + 1, path.getLength() - dotPos - 1);
+        else
+            return "";
+    }
+
+    String Path::getParentDirectory(const String& path)
+    {
+        Index pos = findLastSeparatorIndex(path);
+        if (pos >= 0)
+            return path.subString(0, pos);
+        else
+            return "";
+    }
+    
     /* static */void Path::append(StringBuilder& ioBuilder, const UnownedStringSlice& path)
     {
         if (ioBuilder.getLength() == 0)
@@ -267,14 +288,14 @@ namespace Slang
         combineIntoBuilder(path1.getUnownedSlice(), path2.getUnownedSlice(), sb);
         return sb.ProduceString();
     }
-	String Path::combine(const String& path1, const String& path2, const String& path3)
-	{
-		StringBuilder sb;
+    String Path::combine(const String& path1, const String& path2, const String& path3)
+    {
+        StringBuilder sb;
         sb.append(path1);
         append(sb, path2.getUnownedSlice());
         append(sb, path3.getUnownedSlice());
-		return sb.ProduceString();
-	}
+        return sb.ProduceString();
+    }
 
     /* static */ bool Path::isDriveSpecification(const UnownedStringSlice& element)
     {
@@ -423,14 +444,14 @@ namespace Slang
         return builder.ToString();
     }
 
-	bool Path::createDirectory(const String& path)
-	{
+    bool Path::createDirectory(const String& path)
+    {
 #if defined(_WIN32)
-		return _wmkdir(path.toWString()) == 0;
+        return _wmkdir(path.toWString()) == 0;
 #else 
-		return mkdir(path.getBuffer(), 0777) == 0;
+        return mkdir(path.getBuffer(), 0777) == 0;
 #endif
-	}
+    }
 
     /* static */SlangResult Path::getPathType(const String& path, SlangPathType* pathTypeOut)
     {
@@ -650,33 +671,33 @@ namespace Slang
         return _getExecutablePath();
     }
 
-	Slang::String File::readAllText(const Slang::String& fileName)
-	{
-		StreamReader reader(new FileStream(fileName, FileMode::Open, FileAccess::Read, FileShare::ReadWrite));
-		return reader.ReadToEnd();
-	}
-
-	Slang::List<unsigned char> File::readAllBytes(const Slang::String& fileName)
-	{
-		RefPtr<FileStream> fs = new FileStream(fileName, FileMode::Open, FileAccess::Read, FileShare::ReadWrite);
-		List<unsigned char> buffer;
-		while (!fs->isEnd())
-		{
-			unsigned char ch;
-			int read = (int)fs->read(&ch, 1);
-			if (read)
-				buffer.add(ch);
-			else
-				break;
-		}
-		return _Move(buffer);
-	}
-
-	void File::writeAllText(const Slang::String& fileName, const Slang::String& text)
-	{
-		StreamWriter writer(new FileStream(fileName, FileMode::Create));
-		writer.Write(text);
-	}
+    Slang::String File::readAllText(const Slang::String& fileName)
+    {
+        StreamReader reader(new FileStream(fileName, FileMode::Open, FileAccess::Read, FileShare::ReadWrite));
+        return reader.ReadToEnd();
+    }
+
+    Slang::List<unsigned char> File::readAllBytes(const Slang::String& fileName)
+    {
+        RefPtr<FileStream> fs = new FileStream(fileName, FileMode::Open, FileAccess::Read, FileShare::ReadWrite);
+        List<unsigned char> buffer;
+        while (!fs->isEnd())
+        {
+            unsigned char ch;
+            int read = (int)fs->read(&ch, 1);
+            if (read)
+                buffer.add(ch);
+            else
+                break;
+        }
+        return _Move(buffer);
+    }
+
+    void File::writeAllText(const Slang::String& fileName, const Slang::String& text)
+    {
+        StreamWriter writer(new FileStream(fileName, FileMode::Create));
+        writer.Write(text);
+    }
 
 
 }
diff --git a/source/core/slang-io.h b/source/core/slang-io.h
index a42a6991b..b3d03d9a4 100644
--- a/source/core/slang-io.h
+++ b/source/core/slang-io.h
@@ -11,15 +11,15 @@ namespace Slang
 	class File
 	{
 	public:
-		static bool exists(const Slang::String& fileName);
-		static Slang::String readAllText(const Slang::String& fileName);
-		static Slang::List<unsigned char> readAllBytes(const Slang::String& fileName);
-		static void writeAllText(const Slang::String& fileName, const Slang::String& text);
+		static bool exists(const String& fileName);
+		static String readAllText(const String& fileName);
+		static List<unsigned char> readAllBytes(const String& fileName);
+		static void writeAllText(const String& fileName, const String& text);
         static SlangResult remove(const String& fileName);
 
         static SlangResult makeExecutable(const String& fileName);
 
-        static SlangResult generateTemporary(const UnownedStringSlice& prefix, Slang::String& outFileName);
+        static SlangResult generateTemporary(const UnownedStringSlice& prefix, String& outFileName);
 	};
 
 	class Path
@@ -27,13 +27,20 @@ namespace Slang
 	public:
 		static const char kPathDelimiter = '/';
 
-		static String truncateExt(const String& path);
+
+            /// Returns -1 if no separator is found
+        static Index findLastSeparatorIndex(String const& path);
+            /// Finds the index of the last dot in a path, else returns -1
+        static Index findExtIndex(String const& path);
+
 		static String replaceExt(const String& path, const char* newExt);
 		static String getFileName(const String& path);
-		static String getFileNameWithoutExt(const String& path);
-		static String getFileExt(const String& path);
+		static String getPathWithoutExt(const String& path);
+		static String getPathExt(const String& path);
 		static String getParentDirectory(const String& path);
 
+        static String getFileNameWithoutExt(const String& path);
+
 		static String combine(const String& path1, const String& path2);
 		static String combine(const String& path1, const String& path2, const String& path3);
 
diff --git a/source/slang/slang-diagnostics.cpp b/source/slang/slang-diagnostics.cpp
index 99cae490e..bb8b63540 100644
--- a/source/slang/slang-diagnostics.cpp
+++ b/source/slang/slang-diagnostics.cpp
@@ -124,15 +124,6 @@ void printDiagnosticArg(StringBuilder& sb, CodeGenTarget val)
     }
 }
 
-void printDiagnosticArg(StringBuilder& sb, Stage val)
-{
-    sb << getStageName(val);
-}
-
-void printDiagnosticArg(StringBuilder& sb, ProfileVersion val)
-{
-    sb << Profile(val).getName();
-}
 
 
 SourceLoc const& getDiagnosticPos(SyntaxNode const* syntax)
diff --git a/source/slang/slang-diagnostics.h b/source/slang/slang-diagnostics.h
index 385a40167..8ab860708 100644
--- a/source/slang/slang-diagnostics.h
+++ b/source/slang/slang-diagnostics.h
@@ -99,8 +99,6 @@ namespace Slang
     void printDiagnosticArg(StringBuilder& sb, TokenType tokenType);
     void printDiagnosticArg(StringBuilder& sb, Token const& token);
     void printDiagnosticArg(StringBuilder& sb, CodeGenTarget val);
-    void printDiagnosticArg(StringBuilder& sb, Stage val);
-    void printDiagnosticArg(StringBuilder& sb, ProfileVersion val);
     void printDiagnosticArg(StringBuilder& sb, Val* val);
 
     template<typename T>
diff --git a/source/slang/slang-lexer.h b/source/slang/slang-lexer.h
index d1a1ba844..d404296ff 100644
--- a/source/slang/slang-lexer.h
+++ b/source/slang/slang-lexer.h
@@ -69,6 +69,10 @@ namespace Slang
             m_cursor = cursor.tokenReaderCursor;
             m_nextToken = cursor.nextToken;
         }
+        bool isAtCursor(const ParsingCursor& cursor) const
+        {
+            return cursor.tokenReaderCursor == m_cursor;
+        }
         bool isAtEnd() const { return m_cursor == m_end; }
         Token& peekToken();
         TokenType peekTokenType() const;
diff --git a/source/slang/slang-options.cpp b/source/slang/slang-options.cpp
index 92ae637b1..c642c196a 100644
--- a/source/slang/slang-options.cpp
+++ b/source/slang/slang-options.cpp
@@ -341,7 +341,7 @@ struct OptionsParser
     void addOutputPath(char const* inPath)
     {
         String path = String(inPath);
-        String ext = Path::getFileExt(path);
+        String ext = Path::getPathExt(path);
 
         if (ext == "slang-module" || ext == "slang-lib")
         {
diff --git a/source/slang/slang-profile.cpp b/source/slang/slang-profile.cpp
index 204b467f9..e5ce6fde3 100644
--- a/source/slang/slang-profile.cpp
+++ b/source/slang/slang-profile.cpp
@@ -29,6 +29,15 @@ const char* getStageName(Stage stage)
 
 }
 
+void printDiagnosticArg(StringBuilder& sb, Stage val)
+{
+    sb << getStageName(val);
+}
+
+void printDiagnosticArg(StringBuilder& sb, ProfileVersion val)
+{
+    sb << Profile(val).getName();
+}
 
 
 }
diff --git a/source/slang/slang-profile.h b/source/slang/slang-profile.h
index 1ae752d5a..c90ca66d0 100644
--- a/source/slang/slang-profile.h
+++ b/source/slang/slang-profile.h
@@ -41,6 +41,9 @@ namespace Slang
 #include "slang-profile-defs.h"
     };
 
+
+    void printDiagnosticArg(StringBuilder& sb, ProfileVersion val);
+    
     enum class Stage : SlangStage
     {
         Unknown = SLANG_STAGE_NONE,
@@ -51,6 +54,8 @@ namespace Slang
 
     const char* getStageName(Stage stage);
 
+    void printDiagnosticArg(StringBuilder& sb, Stage val);
+
     ProfileFamily getProfileFamily(ProfileVersion version);
 
     struct Profile
diff --git a/source/slang/slang-state-serialize.cpp b/source/slang/slang-state-serialize.cpp
index 9122b80fd..abdaae7c6 100644
--- a/source/slang/slang-state-serialize.cpp
+++ b/source/slang/slang-state-serialize.cpp
@@ -571,7 +571,7 @@ static bool _isStorable(const PathInfo::Type type)
             }
 
             String filename = Path::getFileNameWithoutExt(path);
-            String ext = Path::getFileExt(path);
+            String ext = Path::getPathExt(path);
 
             StringBuilder uniqueName;
             for (Index j = 0; j < 0x10000; j++)
@@ -1121,7 +1121,7 @@ struct LoadContext
     String parentDir = Path::getParentDirectory(absPath);
 
     String baseName = Path::getFileNameWithoutExt(filename);
-    String ext = Path::getFileExt(filename);
+    String ext = Path::getPathExt(filename);
 
     if (ext.getLength() == 0)
     {