From 8c4603c73675958efc960fbd4bb599a2909d106a Mon Sep 17 00:00:00 2001 From: Konstantin Date: Mon, 16 Jan 2023 14:52:43 +0100 Subject: Source codes --- Tools/CompressShaders/Cabinet.cs | 60 +++++++ Tools/CompressShaders/CompressShaders.cs | 244 +++++++++++++++++++++++++++ Tools/CompressShaders/CompressShaders.csproj | 10 ++ Tools/CompressShaders/DetectFp64.cs | 43 +++++ Tools/CompressShaders/LanguageCodes.cs | 103 +++++++++++ Tools/CompressShaders/Readme.txt | 10 ++ Tools/CompressShaders/ShaderNames.cs | 27 +++ 7 files changed, 497 insertions(+) create mode 100644 Tools/CompressShaders/Cabinet.cs create mode 100644 Tools/CompressShaders/CompressShaders.cs create mode 100644 Tools/CompressShaders/CompressShaders.csproj create mode 100644 Tools/CompressShaders/DetectFp64.cs create mode 100644 Tools/CompressShaders/LanguageCodes.cs create mode 100644 Tools/CompressShaders/Readme.txt create mode 100644 Tools/CompressShaders/ShaderNames.cs (limited to 'Tools/CompressShaders') diff --git a/Tools/CompressShaders/Cabinet.cs b/Tools/CompressShaders/Cabinet.cs new file mode 100644 index 0000000..b53fd18 --- /dev/null +++ b/Tools/CompressShaders/Cabinet.cs @@ -0,0 +1,60 @@ +using System.ComponentModel; +using System.Runtime.InteropServices; + +namespace CompressShaders +{ + /// Lossless data compressor implemented by Cabinet.dll Windows component + /// + /// Whisper.dll consumes that component in runtime, to decompress these shader binaries + /// If you wonder why not gzip — because the OS doesn’t include an API for that, at least not an API usable from C or C++.
+ /// .NET standard library includes gzip algorithm, but we don't want Whisper.dll to depend on .NET. + /// + static class Cabinet + { + /// Compression algorithm + /// + enum eCompressionAlgorithm: uint + { + MSZIP = 2, + XPRESS = 3, + XPRESS_HUFF = 4, + LZMS = 5, + } + /// The value should match constexpr DWORD compressionAlgorithm constant,
in Whisper/D3D/shaders.cpp source file + const eCompressionAlgorithm algo = eCompressionAlgorithm.MSZIP; + + [DllImport( "Cabinet.dll", SetLastError = true )] + static extern bool CreateCompressor( eCompressionAlgorithm Algorithm, IntPtr AllocationRoutines, out IntPtr CompressorHandle ); + + [DllImport( "Cabinet.dll", SetLastError = true )] + static extern bool CloseCompressor( IntPtr CompressorHandle ); + + [DllImport( "Cabinet.dll", SetLastError = true )] + static extern bool Compress( IntPtr CompressorHandle, [In] byte[] UncompressedData, IntPtr UncompressedDataSize, [Out] byte[] CompressedBuffer, IntPtr CompressedBufferSize, out IntPtr CompressedDataSize ); + + /// Compress an array of bytes into another, smaller array of bytes + /// In practice, the compression ratio is about 7.1 for the shader binaries in Release configuration. + public static byte[] compressBuffer( byte[] src ) + { + if( src.Length <= 0 ) + throw new ArgumentException( "The source buffer is empty" ); + IntPtr hCompressor; + if( !CreateCompressor( algo, IntPtr.Zero, out hCompressor ) ) + throw new Win32Exception( "Unable to create the compressor" ); + try + { + byte[] dest = new byte[ src.Length * 2 ]; + IntPtr srcSize = new IntPtr( src.Length ); + IntPtr destSize = new IntPtr( src.Length * 2 ); + if( !Compress( hCompressor, src, srcSize, dest, destSize, out destSize ) ) + throw new Win32Exception( "Compress failed" ); + Array.Resize( ref dest, (int)destSize ); + return dest; + } + finally + { + CloseCompressor( hCompressor ); + } + } + } +} \ No newline at end of file diff --git a/Tools/CompressShaders/CompressShaders.cs b/Tools/CompressShaders/CompressShaders.cs new file mode 100644 index 0000000..814f966 --- /dev/null +++ b/Tools/CompressShaders/CompressShaders.cs @@ -0,0 +1,244 @@ +using System.Runtime.CompilerServices; +namespace CompressShaders; + +record struct sShaderBinary +{ + public string name; + public byte[] data; + + public sShaderBinary( string path ) + { + name = Path.GetFileNameWithoutExtension( path ); + data = File.ReadAllBytes( path ); + } + + public bool wave64 => name.EndsWith( "64" ); + public string uniqueName => wave64 ? name.Substring( 0, name.Length - 2 ) : name; +} + +sealed class FoundShaders +{ + public readonly sShaderBinary[] binaries; + public readonly string[] names; + public readonly int[] wave32, wave64; + + public FoundShaders( IEnumerable found ) + { + binaries = found + .OrderBy( b => b.name ) + .ToArray(); + + names = binaries + .Select( b => b.uniqueName ) + .Distinct() + .ToArray(); + + wave32 = new int[ names.Length ]; + wave64 = new int[ names.Length ]; + for( int i = 0; i < names.Length; i++ ) + { + int i32 = findIndex( names[ i ], false ); + int i64 = findIndex( names[ i ], true ); + if( i32 >= 0 && i64 >= 0 ) + { + wave32[ i ] = i32; + wave64[ i ] = i64; + continue; + } + if( i32 >= 0 ) + { + wave32[ i ] = wave64[ i ] = i32; + continue; + } + throw new ApplicationException( $"Wave64 shader {names[ i ]} doesn't have the corresponding Wave32 one" ); + } + } + + int findIndex( string name, bool wave64 ) + { + for( int i = 0; i < binaries.Length; i++ ) + { + sShaderBinary sb = binaries[ i ]; + if( sb.uniqueName != name ) + continue; + if( sb.wave64 == wave64 ) + return i; + } + return -1; + } +} + +class Program +{ + static string getSolutionRoot( [CallerFilePath] string? path = null ) + { + string? dir = Path.GetDirectoryName( path ); + dir = Path.GetDirectoryName( dir ); + dir = Path.GetDirectoryName( dir ); + return dir ?? throw new ApplicationException(); + } + +#if DEBUG + const string config = "Debug"; +#else + const string config = "Release"; +#endif + + static string shadersBinDir( string root ) + { + return Path.Combine( root, "ComputeShaders", "x64", config ); + } + + static IEnumerable readShaders( string root ) + { + string dir = shadersBinDir( root ); + foreach( string path in Directory.EnumerateFiles( dir, "*.cso" ) ) + yield return new sShaderBinary( path ); + } + + static void writeHeader( string root, IEnumerable names ) + { + string path = Path.Combine( root, "Whisper", "D3D", "shaderNames.h" ); + using var stream = File.CreateText( path ); + stream.WriteLine( @"// This header is generated by a tool +#pragma once +#include + +namespace DirectCompute +{ + enum struct eComputeShader: uint16_t + {" ); + + int id = 0; + foreach( string name in names ) + { + stream.WriteLine( "\t\t{0} = {1},", name, id ); + id++; + } + stream.Write( @" }; + + const char* computeShaderName( eComputeShader cs ); +}" ); + } + + static void writeCpp( string root, IEnumerable names ) + { + string path = Path.Combine( root, "Whisper", "D3D", "shaderNames.cpp" ); + ShaderNames.write( path, names ); + } + + static void writePayloadIDs( StreamWriter stream, string varName, int[] ids ) + { + stream.Write( @" +static const std::array {1} = {{", ids.Length, varName ); + + for( int i = 0; i < ids.Length; i++ ) + { + if( 0 == i % 16 ) + stream.Write( "\r\n\t" ); + else + stream.Write( ' ' ); + stream.Write( "{0},", ids[ i ] ); + } + stream.Write( @" +};" ); + } + + static void writePayload( string root, FoundShaders shaders, out int cbSource, out int cbCompressed ) + { + MemoryStream ms = new MemoryStream(); + List offsets = new List(); + foreach( var bin in shaders.binaries ) + { + offsets.Add( (int)ms.Length ); + ms.Write( bin.data ); + } + offsets.Add( (int)ms.Length ); + + byte[] dxbc = ms.ToArray(); + byte[] compressed = Cabinet.compressBuffer( dxbc ); + cbSource = dxbc.Length; + cbCompressed = compressed.Length; + + string path = Path.Combine( root, "Whisper", "D3D", $"shaderData-{config}.inl" ); + using var stream = File.CreateText( path ); + stream.Write( @"// This source file is generated by a tool + +// This array contains concatenated and compressed DXBC binaries for all compiled compute shaders +static const std::array s_compressedShaders = +{{", compressed.Length ); + + for( int i = 0; i < compressed.Length; i++ ) + { + if( 0 == i % 16 ) + stream.Write( "\r\n\t" ); + else + stream.Write( ' ' ); + stream.Write( "0x{0:X02},", compressed[ i ] ); + } + + stream.Write( @" +}}; + +// This array contains start offsets of shader binaries in the decompressed DXBC blob. +// It includes one more entry for the end of the complete decompressed blob. +static const std::array s_shaderOffsets = {{", offsets.Count ); + + for( int i = 0; i < offsets.Count; i++ ) + { + if( 0 == i % 16 ) + stream.Write( "\r\n\t" ); + else + stream.Write( ' ' ); + stream.Write( "{0},", offsets[ i ] ); + } + stream.Write( @" +};" ); + + stream.Write( @" +// Index = eComputeShader enum value, value = index of the shader binary to use on nVidia and Intel GPUs" ); + writePayloadIDs( stream, "s_shaderBlobs32", shaders.wave32 ); + stream.Write( @" +// Index = eComputeShader enum value, value = index of the shader binary to use on AMD GPUs" ); + writePayloadIDs( stream, "s_shaderBlobs64", shaders.wave64 ); + + ulong fp64Flags = 0; + for( int i = 0; i < shaders.binaries.Length; i++ ) + { + bool fp64 = DetectFp64.usesFp64( shaders.binaries[ i ].data ); + if( fp64 ) + fp64Flags |= (ulong)1 << i; + } + + stream.Write( @" +// Bitmap of the shader binaries which use FP64 arithmetic instructions +constexpr uint64_t fp64ShadersBitmap = 0x{0:X}ull;", fp64Flags ); + } + + static void mainImpl() + { + string root = getSolutionRoot(); + LanguageCodes.produce( root ); + + FoundShaders shaders = new FoundShaders( readShaders( root ) ); + + writeHeader( root, shaders.names ); + writeCpp( root, shaders.names ); + writePayload( root, shaders, out int cbIn, out int cbOut ); + Console.WriteLine( "Compressed {0} compute shaders, {1:F1} kb -> {2:F1} kb", shaders.binaries.Length, cbIn / 1024.0, cbOut / 1024.0 ); + } + + static int Main( string[] args ) + { + try + { + mainImpl(); + return 0; + } + catch( Exception ex ) + { + Console.WriteLine( ex.Message ); + return ex.HResult; + } + } +} \ No newline at end of file diff --git a/Tools/CompressShaders/CompressShaders.csproj b/Tools/CompressShaders/CompressShaders.csproj new file mode 100644 index 0000000..dee1710 --- /dev/null +++ b/Tools/CompressShaders/CompressShaders.csproj @@ -0,0 +1,10 @@ + + + Exe + net6.0 + enable + enable + true + false + + \ No newline at end of file diff --git a/Tools/CompressShaders/DetectFp64.cs b/Tools/CompressShaders/DetectFp64.cs new file mode 100644 index 0000000..1d75126 --- /dev/null +++ b/Tools/CompressShaders/DetectFp64.cs @@ -0,0 +1,43 @@ +#pragma warning disable CS0649 +using System.Runtime.InteropServices; + +namespace CompressShaders +{ + static class DetectFp64 + { + struct DXBCHeader + { + public uint FourCC; // Four character code "DXBC" + public uint Hash0; // 32-bit hash of the DXBC file + public uint Hash1; // 32-bit hash of the DXBC file + public uint Hash2; // 32-bit hash of the DXBC file + public uint Hash3; // 32-bit hash of the DXBC file + public uint unknownOne; + public uint TotalSize; // Total size of the DXBC file in bytes + public int NumChunks; // Number of chunks in the DXBC file + }; + + public static bool usesFp64( ReadOnlySpan dxbc ) + { + ReadOnlySpan dxbcHeaderSpan = MemoryMarshal.Cast( dxbc ); + DXBCHeader dxbcHeader = dxbcHeaderSpan[ 0 ]; + + int cbHeader = Marshal.SizeOf(); + int nChunks = dxbcHeader.NumChunks; + ReadOnlySpan chunkOffsets = MemoryMarshal.Cast( dxbc.Slice( cbHeader, nChunks * 4 ) ); + foreach( int off in chunkOffsets ) + { + uint id = MemoryMarshal.Cast( dxbc.Slice( off, 4 ) )[ 0 ]; + const uint SFI0 = 0x30494653; + if( id != SFI0 ) + continue; + int size = MemoryMarshal.Cast( dxbc.Slice( off + 4, 4 ) )[ 0 ]; + if( size < 4 ) + throw new ApplicationException(); + uint data = MemoryMarshal.Cast( dxbc.Slice( off + 8, 4 ) )[ 0 ]; + return 0 != ( data & 1u ); + } + return false; + } + } +} \ No newline at end of file diff --git a/Tools/CompressShaders/LanguageCodes.cs b/Tools/CompressShaders/LanguageCodes.cs new file mode 100644 index 0000000..71a9909 --- /dev/null +++ b/Tools/CompressShaders/LanguageCodes.cs @@ -0,0 +1,103 @@ +using System.Globalization; +using System.Text.RegularExpressions; + +namespace CompressShaders +{ + static class LanguageCodes + { + record struct Row + { + public string keySource; + public uint keyValue; + public int code; + public string name; + } + + static uint makeKey( string str ) + { + if( str.Length > 4 ) + throw new ArgumentException(); + uint k = 0; + int shift = 0; + foreach( char c in str ) + { + if( c >= 0x80 ) + throw new ArgumentException(); + uint u = (uint)c; + k |= ( u << shift ); + shift += 8; + } + return k; + } + + static IEnumerable load( string path ) + { + using var stm = File.OpenText( path ); + while( true ) + { + string? line = stm.ReadLine(); + if( null == line ) + break; + if( string.IsNullOrWhiteSpace( line ) ) + continue; + string[] fields = line.Split( '\t' ); + yield return new Row() + { + keySource = fields[ 0 ], + keyValue = makeKey( fields[ 0 ] ), + code = int.Parse( fields[ 1 ] ), + name = fields[ 2 ] + }; + } + } + + static void writeCpp( string inl, Row[] data ) + { + // TODO [very low]: sort them by the key here, then in C++ use binary search instead of the hash map + using var stm = File.CreateText( inl ); + stm.WriteLine( "// This file is generated by a tool, from the `languageCodez.tsv` file in this repository" ); + foreach( Row row in data ) + stm.WriteLine( "Lang{{ 0x{0:X}, {1}, \"{2}\" }},", row.keyValue, row.code, row.name ); + } + + static readonly CultureInfo ci = new CultureInfo( "en-US", false ); + static string titleCase( this string name ) => + ci.TextInfo.ToTitleCase( name.ToLower( ci ) ); + + static void writeCs( string cs, Row[] data ) + { + using var stm = File.CreateText( cs ); + stm.WriteLine( @"// This file is generated by a tool, from the `languageCodez.tsv` file in this repository +namespace Whisper +{ + /// Supported languages + public enum eLanguage: uint + {" ); + + foreach( Row row in data ) + { + string tc = row.name.titleCase(); + stm.WriteLine( " /// {0}", tc ); + tc = Regex.Replace( tc, @"\s+", string.Empty ); + stm.WriteLine( " {0} = 0x{1:X},", tc, row.keyValue ); + } + stm.Write( @" } +}" ); + } + + static void produce( string tsv, string inl, string cs ) + { + Row[] data = load( tsv ).OrderBy( r => r.name ).ToArray(); + writeCpp( inl, data ); + writeCs( cs, data ); + } + + public static void produce( string solutionRoot ) + { + string tsv = Path.Combine( solutionRoot, "Whisper\\Whisper\\languageCodez.tsv" ); + string inl = Path.Combine( solutionRoot, "Whisper\\Whisper\\languageCodez.inl" ); + string cs = Path.Combine( solutionRoot, "WhisperNet\\API\\eLanguage.cs" ); + produce( tsv, inl, cs ); + } + } +} \ No newline at end of file diff --git a/Tools/CompressShaders/Readme.txt b/Tools/CompressShaders/Readme.txt new file mode 100644 index 0000000..69ef35a --- /dev/null +++ b/Tools/CompressShaders/Readme.txt @@ -0,0 +1,10 @@ +This project builds a C# console app which serves as a code generator for a few pieces of Whisper.dll and WhisperNet.dll. + +Specifically, it generates two things. + +1. It compresses the compiled DXBC shaders into a blob of bytes, and prints std::array with these bytes into shaderData-Release.inl and shaderData-Debug.inl C++ files. + +2. It parses the `languageCodez.tsv`, and generates both C++ and C# code with the data from that table. + +The tool uses relative paths across source files. +These paths will break if you move the source of the tool, or the source data of the tool. \ No newline at end of file diff --git a/Tools/CompressShaders/ShaderNames.cs b/Tools/CompressShaders/ShaderNames.cs new file mode 100644 index 0000000..81ba46e --- /dev/null +++ b/Tools/CompressShaders/ShaderNames.cs @@ -0,0 +1,27 @@ +static class ShaderNames +{ + public static void write( string path, IEnumerable names ) + { + string[] arr = names.ToArray(); + using var stream = File.CreateText( path ); + stream.WriteLine( @"// This source file is generated by a tool +#include ""stdafx.h"" +#include ""shaderNames.h"" +" ); + + stream.WriteLine( "static const std::array s_shaderNames = ", arr.Length ); + stream.WriteLine( "{" ); + foreach( string name in arr ) + stream.WriteLine( @" ""{0}"",", name ); + + stream.Write( @"}; + +const char* DirectCompute::computeShaderName( eComputeShader cs ) +{ + const uint16_t i = (uint16_t)cs; + if( i < s_shaderNames.size() ) + return s_shaderNames[ i ]; + return nullptr; +}" ); + } +} \ No newline at end of file -- cgit v1.2.3