yum-archive/TaSTT-Whisper

High-performance GPGPU inference of OpenAI's Whisper automatic speech recognition (ASR) model

git clone https://git.yummers.dev/yum-archive/TaSTT-Whisper

KonstantinComments7ef711a

master
branchesmasterall refs →
2.9 KiB105 linesraw
1using System.Globalization;
2using System.Text.RegularExpressions;
3
4namespace CompressShaders
5{
6	static class LanguageCodes
7	{
8		record struct Row
9		{
10			public string keySource;
11			public uint keyValue;
12			public int code;
13			public string name;
14		}
15
16		static uint makeKey( string str )
17		{
18			if( str.Length > 4 )
19				throw new ArgumentException();
20			uint k = 0;
21			int shift = 0;
22			foreach( char c in str )
23			{
24				if( c >= 0x80 )
25					throw new ArgumentException();
26				uint u = (uint)c;
27				k |= ( u << shift );
28				shift += 8;
29			}
30			return k;
31		}
32
33		static IEnumerable<Row> load( string path )
34		{
35			using var stm = File.OpenText( path );
36			while( true )
37			{
38				string? line = stm.ReadLine();
39				if( null == line )
40					break;
41				if( string.IsNullOrWhiteSpace( line ) )
42					continue;
43				string[] fields = line.Split( '\t' );
44				yield return new Row()
45				{
46					keySource = fields[ 0 ],
47					keyValue = makeKey( fields[ 0 ] ),
48					code = int.Parse( fields[ 1 ] ),
49					name = fields[ 2 ]
50				};
51			}
52		}
53
54		static void writeCpp( string inl, Row[] data )
55		{
56			// TODO [very low]: sort them by the key here, then in C++ use binary search instead of the hash map
57			using var stm = File.CreateText( inl );
58			stm.WriteLine( "// This file is generated by a tool, from the `languageCodez.tsv` file in this repository" );
59			foreach( Row row in data )
60				stm.WriteLine( "Lang{{ 0x{0:X}, {1}, \"{2}\" }},", row.keyValue, row.code, row.name );
61		}
62
63		static readonly CultureInfo ci = new CultureInfo( "en-US", false );
64		static string titleCase( this string name ) =>
65			ci.TextInfo.ToTitleCase( name.ToLower( ci ) );
66
67		static void writeCs( string cs, Row[] data )
68		{
69			using var stm = File.CreateText( cs );
70			stm.WriteLine( @"// This file is generated by a tool, from the `languageCodez.tsv` file in this repository
71namespace Whisper
72{
73	/// <summary>Supported languages</summary>
74	/// <remarks>The values of this enum are zero-padded ASCII strings.<br/>
75	/// It seems OpenAI tried to implement ISO 639-1, but they used the version of the standard from 1988.</remarks>
76	public enum eLanguage: uint
77	{" );
78
79			foreach( Row row in data )
80			{
81				string tc = row.name.titleCase();
82				stm.WriteLine( "		/// <summary>{0}</summary>", tc );
83				tc = Regex.Replace( tc, @"\s+", string.Empty );
84				stm.WriteLine( "		{0} = 0x{1:X},  // \"{2}\"", tc, row.keyValue, row.keySource );
85			}
86			stm.Write( @"	}
87}" );
88		}
89
90		static void produce( string tsv, string inl, string cs )
91		{
92			Row[] data = load( tsv ).OrderBy( r => r.name ).ToArray();
93			writeCpp( inl, data );
94			writeCs( cs, data );
95		}
96
97		public static void produce( string solutionRoot )
98		{
99			string tsv = Path.Combine( solutionRoot, "Whisper\\Whisper\\languageCodez.tsv" );
100			string inl = Path.Combine( solutionRoot, "Whisper\\Whisper\\languageCodez.inl" );
101			string cs = Path.Combine( solutionRoot, "WhisperNet\\API\\eLanguage.cs" );
102			produce( tsv, inl, cs );
103		}
104	}
105}