From 02e44bade6370309c0292e84178095c2bae299be Mon Sep 17 00:00:00 2001
From: jsmall-nvidia <jsmall@nvidia.com>
Date: Thu, 20 Dec 2018 13:23:58 -0500
Subject: Feature/lex memory reduction (#762)

* Only do scrubbing if needed. When allocating content try to limit size (with scrubbing each token takes up 1k), now it's 16 bytes min size.

* Don't allocate for every call to write on the CallbackWriter - use the m_appendBuffer.

* Don't allocate memory for CallbackWriter use m_appendBuffer.

* Use UnownedStringSlice for suffix output for parsing float/int literals.
Fix typo in invalidFloatingPointLiteralSuffix

* Using memory arena to hold tokens that are not in SourceManager.

* Improve comment on lexing.

* Make UnownedStringSlice allocation simpler on SourceManager.

* Fix error on gcc around UnownedStringSlice - because VC converted string + UnownedStringSlice automatically into a String.

* Fix generateName needing concat string for gcc.

* When constructing a Token in parseAttributeName - because it's a Identifier, we have to set the Name.

* Remove translation through String on getIntrinsicOp

* Make func-cbuffer-param disablable with -exclude compatibility-issue

* Move memory leak in render-test.

* From review - can just use "?:" instead of performing a concat.
---
 source/core/slang-string.h   |  4 ++--
 source/core/slang-writer.cpp | 30 +++++++++++++++++++++++++-----
 source/core/slang-writer.h   |  1 +
 3 files changed, 28 insertions(+), 7 deletions(-)

(limited to 'source/core')

diff --git a/source/core/slang-string.h b/source/core/slang-string.h
index 5a20d8b75..4c87ce81e 100644
--- a/source/core/slang-string.h
+++ b/source/core/slang-string.h
@@ -111,9 +111,9 @@ namespace Slang
             return -1;
         }
 
-        const char& operator[](int i) const
+        const char& operator[](UInt i) const
         {
-            assert(i >= 0 && i < int(endData - beginData));
+            assert(i < UInt(endData - beginData));
             return beginData[i];
         }
 
diff --git a/source/core/slang-writer.cpp b/source/core/slang-writer.cpp
index 21c8c6209..433cc6992 100644
--- a/source/core/slang-writer.cpp
+++ b/source/core/slang-writer.cpp
@@ -80,15 +80,35 @@ SLANG_NO_THROW SlangResult SLANG_MCALL AppendBufferWriter::endAppendBuffer(char*
 
 /* !!!!!!!!!!!!!!!!!!!!!!!!! CallbackWriter !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!*/
 
+SLANG_NO_THROW char* SLANG_MCALL CallbackWriter::beginAppendBuffer(size_t maxNumChars)
+{
+    // Add one so there is always space for end termination, we need for the callback.
+    m_appendBuffer.SetSize(maxNumChars + 1);
+    return m_appendBuffer.Buffer();
+}
+
 SlangResult CallbackWriter::write(const char* chars, size_t numChars)
 {
     if (numChars > 0)
     {
-        // Make sure zero terminated
-        StringBuilder builder;
-        builder.Append(chars, numChars);
-
-        m_callback(builder.Buffer(), (void*)m_data);
+        char* appendBuffer = m_appendBuffer.Buffer();
+        // See if it's from an append buffer
+        if (chars >= appendBuffer && (chars + numChars) < (appendBuffer + m_appendBuffer.Count()))
+        {
+            // Set terminating 0
+            appendBuffer[(chars + numChars) - appendBuffer] = 0;
+
+            m_callback(chars, (void*)m_data);
+        }
+        else
+        {
+            // Use the append buffer to add the terminating 0
+            m_appendBuffer.SetSize(numChars + 1);
+            ::memcpy(m_appendBuffer.Buffer(), chars, numChars);
+            m_appendBuffer[numChars] = 0;
+
+            m_callback(m_appendBuffer.Buffer(), (void*)m_data);
+        }
     }
 
     return SLANG_OK;
diff --git a/source/core/slang-writer.h b/source/core/slang-writer.h
index 0e67684ec..b18af373f 100644
--- a/source/core/slang-writer.h
+++ b/source/core/slang-writer.h
@@ -85,6 +85,7 @@ class CallbackWriter : public AppendBufferWriter
 public:
     typedef AppendBufferWriter Parent;
     // ISlangWriter
+    SLANG_NO_THROW char* SLANG_MCALL beginAppendBuffer(size_t maxNumChars) SLANG_OVERRIDE;
     SLANG_NO_THROW virtual SlangResult SLANG_MCALL write(const char* chars, size_t numChars) SLANG_OVERRIDE;
     
     CallbackWriter(SlangDiagnosticCallback callback, const void* data, WriterFlags flags) :
-- 
cgit v1.2.3