diff options
Diffstat (limited to 'Tools/CompressShaders')
| -rw-r--r-- | Tools/CompressShaders/Cabinet.cs | 60 | ||||
| -rw-r--r-- | Tools/CompressShaders/CompressShaders.cs | 244 | ||||
| -rw-r--r-- | Tools/CompressShaders/CompressShaders.csproj | 10 | ||||
| -rw-r--r-- | Tools/CompressShaders/DetectFp64.cs | 43 | ||||
| -rw-r--r-- | Tools/CompressShaders/LanguageCodes.cs | 103 | ||||
| -rw-r--r-- | Tools/CompressShaders/Readme.txt | 10 | ||||
| -rw-r--r-- | Tools/CompressShaders/ShaderNames.cs | 27 |
7 files changed, 497 insertions, 0 deletions
diff --git a/Tools/CompressShaders/Cabinet.cs b/Tools/CompressShaders/Cabinet.cs new file mode 100644 index 0000000..b53fd18 --- /dev/null +++ b/Tools/CompressShaders/Cabinet.cs @@ -0,0 +1,60 @@ +using System.ComponentModel; +using System.Runtime.InteropServices; + +namespace CompressShaders +{ + /// <summary>Lossless data compressor implemented by <c>Cabinet.dll</c> Windows component</summary> + /// <remarks> + /// <para>Whisper.dll consumes that component in runtime, to decompress these shader binaries</para> + /// <para>If you wonder why not gzip — because the OS doesn’t include an API for that, at least not an API usable from C or C++.<br/> + /// .NET standard library includes gzip algorithm, but we don't want Whisper.dll to depend on .NET.</para> + /// </remarks> + static class Cabinet + { + /// <summary>Compression algorithm</summary> + /// <seealso href="https://learn.microsoft.com/en-us/windows/win32/cmpapi/using-the-compression-api#selecting-the-compression-algorithm" /> + enum eCompressionAlgorithm: uint + { + MSZIP = 2, + XPRESS = 3, + XPRESS_HUFF = 4, + LZMS = 5, + } + /// <summary>The value should match <c>constexpr DWORD compressionAlgorithm</c> constant,<br/>in <c>Whisper/D3D/shaders.cpp</c> source file</summary> + const eCompressionAlgorithm algo = eCompressionAlgorithm.MSZIP; + + [DllImport( "Cabinet.dll", SetLastError = true )] + static extern bool CreateCompressor( eCompressionAlgorithm Algorithm, IntPtr AllocationRoutines, out IntPtr CompressorHandle ); + + [DllImport( "Cabinet.dll", SetLastError = true )] + static extern bool CloseCompressor( IntPtr CompressorHandle ); + + [DllImport( "Cabinet.dll", SetLastError = true )] + static extern bool Compress( IntPtr CompressorHandle, [In] byte[] UncompressedData, IntPtr UncompressedDataSize, [Out] byte[] CompressedBuffer, IntPtr CompressedBufferSize, out IntPtr CompressedDataSize ); + + /// <summary>Compress an array of bytes into another, smaller array of bytes</summary> + /// <remarks>In practice, the compression ratio is about 7.1 for the shader binaries in Release configuration.</remarks> + public static byte[] compressBuffer( byte[] src ) + { + if( src.Length <= 0 ) + throw new ArgumentException( "The source buffer is empty" ); + IntPtr hCompressor; + if( !CreateCompressor( algo, IntPtr.Zero, out hCompressor ) ) + throw new Win32Exception( "Unable to create the compressor" ); + try + { + byte[] dest = new byte[ src.Length * 2 ]; + IntPtr srcSize = new IntPtr( src.Length ); + IntPtr destSize = new IntPtr( src.Length * 2 ); + if( !Compress( hCompressor, src, srcSize, dest, destSize, out destSize ) ) + throw new Win32Exception( "Compress failed" ); + Array.Resize( ref dest, (int)destSize ); + return dest; + } + finally + { + CloseCompressor( hCompressor ); + } + } + } +}
\ No newline at end of file diff --git a/Tools/CompressShaders/CompressShaders.cs b/Tools/CompressShaders/CompressShaders.cs new file mode 100644 index 0000000..814f966 --- /dev/null +++ b/Tools/CompressShaders/CompressShaders.cs @@ -0,0 +1,244 @@ +using System.Runtime.CompilerServices; +namespace CompressShaders; + +record struct sShaderBinary +{ + public string name; + public byte[] data; + + public sShaderBinary( string path ) + { + name = Path.GetFileNameWithoutExtension( path ); + data = File.ReadAllBytes( path ); + } + + public bool wave64 => name.EndsWith( "64" ); + public string uniqueName => wave64 ? name.Substring( 0, name.Length - 2 ) : name; +} + +sealed class FoundShaders +{ + public readonly sShaderBinary[] binaries; + public readonly string[] names; + public readonly int[] wave32, wave64; + + public FoundShaders( IEnumerable<sShaderBinary> found ) + { + binaries = found + .OrderBy( b => b.name ) + .ToArray(); + + names = binaries + .Select( b => b.uniqueName ) + .Distinct() + .ToArray(); + + wave32 = new int[ names.Length ]; + wave64 = new int[ names.Length ]; + for( int i = 0; i < names.Length; i++ ) + { + int i32 = findIndex( names[ i ], false ); + int i64 = findIndex( names[ i ], true ); + if( i32 >= 0 && i64 >= 0 ) + { + wave32[ i ] = i32; + wave64[ i ] = i64; + continue; + } + if( i32 >= 0 ) + { + wave32[ i ] = wave64[ i ] = i32; + continue; + } + throw new ApplicationException( $"Wave64 shader {names[ i ]} doesn't have the corresponding Wave32 one" ); + } + } + + int findIndex( string name, bool wave64 ) + { + for( int i = 0; i < binaries.Length; i++ ) + { + sShaderBinary sb = binaries[ i ]; + if( sb.uniqueName != name ) + continue; + if( sb.wave64 == wave64 ) + return i; + } + return -1; + } +} + +class Program +{ + static string getSolutionRoot( [CallerFilePath] string? path = null ) + { + string? dir = Path.GetDirectoryName( path ); + dir = Path.GetDirectoryName( dir ); + dir = Path.GetDirectoryName( dir ); + return dir ?? throw new ApplicationException(); + } + +#if DEBUG + const string config = "Debug"; +#else + const string config = "Release"; +#endif + + static string shadersBinDir( string root ) + { + return Path.Combine( root, "ComputeShaders", "x64", config ); + } + + static IEnumerable<sShaderBinary> readShaders( string root ) + { + string dir = shadersBinDir( root ); + foreach( string path in Directory.EnumerateFiles( dir, "*.cso" ) ) + yield return new sShaderBinary( path ); + } + + static void writeHeader( string root, IEnumerable<string> names ) + { + string path = Path.Combine( root, "Whisper", "D3D", "shaderNames.h" ); + using var stream = File.CreateText( path ); + stream.WriteLine( @"// This header is generated by a tool +#pragma once +#include <stdint.h> + +namespace DirectCompute +{ + enum struct eComputeShader: uint16_t + {" ); + + int id = 0; + foreach( string name in names ) + { + stream.WriteLine( "\t\t{0} = {1},", name, id ); + id++; + } + stream.Write( @" }; + + const char* computeShaderName( eComputeShader cs ); +}" ); + } + + static void writeCpp( string root, IEnumerable<string> names ) + { + string path = Path.Combine( root, "Whisper", "D3D", "shaderNames.cpp" ); + ShaderNames.write( path, names ); + } + + static void writePayloadIDs( StreamWriter stream, string varName, int[] ids ) + { + stream.Write( @" +static const std::array<uint8_t, {0}> {1} = {{", ids.Length, varName ); + + for( int i = 0; i < ids.Length; i++ ) + { + if( 0 == i % 16 ) + stream.Write( "\r\n\t" ); + else + stream.Write( ' ' ); + stream.Write( "{0},", ids[ i ] ); + } + stream.Write( @" +};" ); + } + + static void writePayload( string root, FoundShaders shaders, out int cbSource, out int cbCompressed ) + { + MemoryStream ms = new MemoryStream(); + List<int> offsets = new List<int>(); + foreach( var bin in shaders.binaries ) + { + offsets.Add( (int)ms.Length ); + ms.Write( bin.data ); + } + offsets.Add( (int)ms.Length ); + + byte[] dxbc = ms.ToArray(); + byte[] compressed = Cabinet.compressBuffer( dxbc ); + cbSource = dxbc.Length; + cbCompressed = compressed.Length; + + string path = Path.Combine( root, "Whisper", "D3D", $"shaderData-{config}.inl" ); + using var stream = File.CreateText( path ); + stream.Write( @"// This source file is generated by a tool + +// This array contains concatenated and compressed DXBC binaries for all compiled compute shaders +static const std::array<uint8_t, {0}> s_compressedShaders = +{{", compressed.Length ); + + for( int i = 0; i < compressed.Length; i++ ) + { + if( 0 == i % 16 ) + stream.Write( "\r\n\t" ); + else + stream.Write( ' ' ); + stream.Write( "0x{0:X02},", compressed[ i ] ); + } + + stream.Write( @" +}}; + +// This array contains start offsets of shader binaries in the decompressed DXBC blob. +// It includes one more entry for the end of the complete decompressed blob. +static const std::array<uint32_t, {0}> s_shaderOffsets = {{", offsets.Count ); + + for( int i = 0; i < offsets.Count; i++ ) + { + if( 0 == i % 16 ) + stream.Write( "\r\n\t" ); + else + stream.Write( ' ' ); + stream.Write( "{0},", offsets[ i ] ); + } + stream.Write( @" +};" ); + + stream.Write( @" +// Index = eComputeShader enum value, value = index of the shader binary to use on nVidia and Intel GPUs" ); + writePayloadIDs( stream, "s_shaderBlobs32", shaders.wave32 ); + stream.Write( @" +// Index = eComputeShader enum value, value = index of the shader binary to use on AMD GPUs" ); + writePayloadIDs( stream, "s_shaderBlobs64", shaders.wave64 ); + + ulong fp64Flags = 0; + for( int i = 0; i < shaders.binaries.Length; i++ ) + { + bool fp64 = DetectFp64.usesFp64( shaders.binaries[ i ].data ); + if( fp64 ) + fp64Flags |= (ulong)1 << i; + } + + stream.Write( @" +// Bitmap of the shader binaries which use FP64 arithmetic instructions +constexpr uint64_t fp64ShadersBitmap = 0x{0:X}ull;", fp64Flags ); + } + + static void mainImpl() + { + string root = getSolutionRoot(); + LanguageCodes.produce( root ); + + FoundShaders shaders = new FoundShaders( readShaders( root ) ); + + writeHeader( root, shaders.names ); + writeCpp( root, shaders.names ); + writePayload( root, shaders, out int cbIn, out int cbOut ); + Console.WriteLine( "Compressed {0} compute shaders, {1:F1} kb -> {2:F1} kb", shaders.binaries.Length, cbIn / 1024.0, cbOut / 1024.0 ); + } + + static int Main( string[] args ) + { + try + { + mainImpl(); + return 0; + } + catch( Exception ex ) + { + Console.WriteLine( ex.Message ); + return ex.HResult; + } + } +}
\ No newline at end of file diff --git a/Tools/CompressShaders/CompressShaders.csproj b/Tools/CompressShaders/CompressShaders.csproj new file mode 100644 index 0000000..dee1710 --- /dev/null +++ b/Tools/CompressShaders/CompressShaders.csproj @@ -0,0 +1,10 @@ +<Project Sdk="Microsoft.NET.Sdk"> + <PropertyGroup> + <OutputType>Exe</OutputType> + <TargetFramework>net6.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + <CheckForOverflowUnderflow>true</CheckForOverflowUnderflow> + <AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath> + </PropertyGroup> +</Project>
\ No newline at end of file diff --git a/Tools/CompressShaders/DetectFp64.cs b/Tools/CompressShaders/DetectFp64.cs new file mode 100644 index 0000000..1d75126 --- /dev/null +++ b/Tools/CompressShaders/DetectFp64.cs @@ -0,0 +1,43 @@ +#pragma warning disable CS0649 +using System.Runtime.InteropServices; + +namespace CompressShaders +{ + static class DetectFp64 + { + struct DXBCHeader + { + public uint FourCC; // Four character code "DXBC" + public uint Hash0; // 32-bit hash of the DXBC file + public uint Hash1; // 32-bit hash of the DXBC file + public uint Hash2; // 32-bit hash of the DXBC file + public uint Hash3; // 32-bit hash of the DXBC file + public uint unknownOne; + public uint TotalSize; // Total size of the DXBC file in bytes + public int NumChunks; // Number of chunks in the DXBC file + }; + + public static bool usesFp64( ReadOnlySpan<byte> dxbc ) + { + ReadOnlySpan<DXBCHeader> dxbcHeaderSpan = MemoryMarshal.Cast<byte, DXBCHeader>( dxbc ); + DXBCHeader dxbcHeader = dxbcHeaderSpan[ 0 ]; + + int cbHeader = Marshal.SizeOf<DXBCHeader>(); + int nChunks = dxbcHeader.NumChunks; + ReadOnlySpan<int> chunkOffsets = MemoryMarshal.Cast<byte, int>( dxbc.Slice( cbHeader, nChunks * 4 ) ); + foreach( int off in chunkOffsets ) + { + uint id = MemoryMarshal.Cast<byte, uint>( dxbc.Slice( off, 4 ) )[ 0 ]; + const uint SFI0 = 0x30494653; + if( id != SFI0 ) + continue; + int size = MemoryMarshal.Cast<byte, int>( dxbc.Slice( off + 4, 4 ) )[ 0 ]; + if( size < 4 ) + throw new ApplicationException(); + uint data = MemoryMarshal.Cast<byte, uint>( dxbc.Slice( off + 8, 4 ) )[ 0 ]; + return 0 != ( data & 1u ); + } + return false; + } + } +}
\ No newline at end of file diff --git a/Tools/CompressShaders/LanguageCodes.cs b/Tools/CompressShaders/LanguageCodes.cs new file mode 100644 index 0000000..71a9909 --- /dev/null +++ b/Tools/CompressShaders/LanguageCodes.cs @@ -0,0 +1,103 @@ +using System.Globalization; +using System.Text.RegularExpressions; + +namespace CompressShaders +{ + static class LanguageCodes + { + record struct Row + { + public string keySource; + public uint keyValue; + public int code; + public string name; + } + + static uint makeKey( string str ) + { + if( str.Length > 4 ) + throw new ArgumentException(); + uint k = 0; + int shift = 0; + foreach( char c in str ) + { + if( c >= 0x80 ) + throw new ArgumentException(); + uint u = (uint)c; + k |= ( u << shift ); + shift += 8; + } + return k; + } + + static IEnumerable<Row> load( string path ) + { + using var stm = File.OpenText( path ); + while( true ) + { + string? line = stm.ReadLine(); + if( null == line ) + break; + if( string.IsNullOrWhiteSpace( line ) ) + continue; + string[] fields = line.Split( '\t' ); + yield return new Row() + { + keySource = fields[ 0 ], + keyValue = makeKey( fields[ 0 ] ), + code = int.Parse( fields[ 1 ] ), + name = fields[ 2 ] + }; + } + } + + static void writeCpp( string inl, Row[] data ) + { + // TODO [very low]: sort them by the key here, then in C++ use binary search instead of the hash map + using var stm = File.CreateText( inl ); + stm.WriteLine( "// This file is generated by a tool, from the `languageCodez.tsv` file in this repository" ); + foreach( Row row in data ) + stm.WriteLine( "Lang{{ 0x{0:X}, {1}, \"{2}\" }},", row.keyValue, row.code, row.name ); + } + + static readonly CultureInfo ci = new CultureInfo( "en-US", false ); + static string titleCase( this string name ) => + ci.TextInfo.ToTitleCase( name.ToLower( ci ) ); + + static void writeCs( string cs, Row[] data ) + { + using var stm = File.CreateText( cs ); + stm.WriteLine( @"// This file is generated by a tool, from the `languageCodez.tsv` file in this repository +namespace Whisper +{ + /// <summary>Supported languages</summary> + public enum eLanguage: uint + {" ); + + foreach( Row row in data ) + { + string tc = row.name.titleCase(); + stm.WriteLine( " /// <summary>{0}</summary>", tc ); + tc = Regex.Replace( tc, @"\s+", string.Empty ); + stm.WriteLine( " {0} = 0x{1:X},", tc, row.keyValue ); + } + stm.Write( @" } +}" ); + } + + static void produce( string tsv, string inl, string cs ) + { + Row[] data = load( tsv ).OrderBy( r => r.name ).ToArray(); + writeCpp( inl, data ); + writeCs( cs, data ); + } + + public static void produce( string solutionRoot ) + { + string tsv = Path.Combine( solutionRoot, "Whisper\\Whisper\\languageCodez.tsv" ); + string inl = Path.Combine( solutionRoot, "Whisper\\Whisper\\languageCodez.inl" ); + string cs = Path.Combine( solutionRoot, "WhisperNet\\API\\eLanguage.cs" ); + produce( tsv, inl, cs ); + } + } +}
\ No newline at end of file diff --git a/Tools/CompressShaders/Readme.txt b/Tools/CompressShaders/Readme.txt new file mode 100644 index 0000000..69ef35a --- /dev/null +++ b/Tools/CompressShaders/Readme.txt @@ -0,0 +1,10 @@ +This project builds a C# console app which serves as a code generator for a few pieces of Whisper.dll and WhisperNet.dll. + +Specifically, it generates two things. + +1. It compresses the compiled DXBC shaders into a blob of bytes, and prints std::array with these bytes into shaderData-Release.inl and shaderData-Debug.inl C++ files. + +2. It parses the `languageCodez.tsv`, and generates both C++ and C# code with the data from that table. + +The tool uses relative paths across source files. +These paths will break if you move the source of the tool, or the source data of the tool.
\ No newline at end of file diff --git a/Tools/CompressShaders/ShaderNames.cs b/Tools/CompressShaders/ShaderNames.cs new file mode 100644 index 0000000..81ba46e --- /dev/null +++ b/Tools/CompressShaders/ShaderNames.cs @@ -0,0 +1,27 @@ +static class ShaderNames +{ + public static void write( string path, IEnumerable<string> names ) + { + string[] arr = names.ToArray(); + using var stream = File.CreateText( path ); + stream.WriteLine( @"// This source file is generated by a tool +#include ""stdafx.h"" +#include ""shaderNames.h"" +" ); + + stream.WriteLine( "static const std::array<const char*, {0}> s_shaderNames = ", arr.Length ); + stream.WriteLine( "{" ); + foreach( string name in arr ) + stream.WriteLine( @" ""{0}"",", name ); + + stream.Write( @"}; + +const char* DirectCompute::computeShaderName( eComputeShader cs ) +{ + const uint16_t i = (uint16_t)cs; + if( i < s_shaderNames.size() ) + return s_shaderNames[ i ]; + return nullptr; +}" ); + } +}
\ No newline at end of file |