A tool to summarize performance data into one table

7 files changed, 361 insertions, 1 deletions

diff --git a/.gitignore b/.gitignore
index 1cbaff1..fa7e167 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,4 +14,6 @@ Examples/TranscribeCS/obj/
 *.json
 *.user
 Examples/MicrophoneCS/obj/
-Examples/MicrophoneCS/bin/
\ No newline at end of file
+Examples/MicrophoneCS/bin/
+Tools/PerfSummary/bin/
+Tools/PerfSummary/obj/
\ No newline at end of file
diff --git a/SampleClips/summary.tsv b/SampleClips/summary.tsv
new file mode 100644
index 0000000..c591dce
--- /dev/null
+++ b/SampleClips/summary.tsv
@@ -0,0 +1,17 @@
+Audio Clip	Model	GPU	Total, sec	Relative speed	Encode, sec	Decode, sec	RAM, MB	VRAM, MB
+jfk.wav	medium	GeForce 1080Ti	1.46731	7.496711669653993	0.754654	0.587324	2.84085	2185.85088
+jfk.wav	medium	GeForce 1650	4.59853	2.3920687698025236	3.46286	0.926677	2.84085	2185.85088
+jfk.wav	medium	Ryzen 5 5600U	9.9723	1.1030554636342669	8.28303	1.45228	2.84099	2233.47712
+jfk.wav	medium	Ryzen 7 5700G	6.09091	1.805969879705988	4.76175	1.20028	2.84099	2233.47712
+jfk.wav	large	GeForce 1080Ti	2.62992	4.182636734197239	1.54197	0.966163	2.8558	4050.03264
+jfk.wav	large	GeForce 1650	8.71063	1.2628248473416963	6.83046	1.62184	2.8558	4050.03264
+jfk.wav	large	Ryzen 5 5600U	19.41	0.5667181865018032	16.6535	2.4954	2.85594	4109.568
+jfk.wav	large	Ryzen 7 5700G	11.1008	0.9909195733640819	8.85998	2.10771	2.85594	4109.568
+columbia.wma	medium	GeForce 1080Ti	19.7043	10.087239841049923	7.70488	11.6985	91.9293	2303.75424
+columbia.wma	medium	GeForce 1650	55.336	3.591911233193581	34.044	20.9967	91.9293	2303.75424
+columbia.wma	medium	Ryzen 5 5600U	97.4095	2.0404785980833493	62.3057	34.644	91.9295	2355.75296
+columbia.wma	medium	Ryzen 7 5700G	76.7069	2.5911880156804665	46.1492	30.199	91.9295	2355.75296
+columbia.wma	large	GeForce 1080Ti	33.7046	5.897177239901972	13.6283	19.8563	93.1333	4186.81856
+columbia.wma	large	GeForce 1650	98.7705	2.0123619906753536	61.0298	37.395	93.1333	4186.81856
+columbia.wma	large	Ryzen 5 5600U	174.677	1.1378830641698678	111.531	62.3642	93.1334	4255.55968
+columbia.wma	large	Ryzen 7 5700G	133.87	1.4847389258235602	79.3112	54.0997	93.1334	4255.55968
diff --git a/Tools/PerfSummary/LogParser.cs b/Tools/PerfSummary/LogParser.cs
new file mode 100644
index 0000000..c89d695
--- /dev/null
+++ b/Tools/PerfSummary/LogParser.cs
@@ -0,0 +1,207 @@
+using System.Globalization;
+using System.Text.RegularExpressions;
+
+namespace PerfSummary
+{
+	enum eInputClip: byte
+	{
+		jfk,
+		columbia,
+	}
+	enum eWhisperModel: byte
+	{
+		medium,
+		large
+	}
+
+	struct LogName
+	{
+		public readonly eInputClip clip;
+		public readonly eWhisperModel model;
+		public readonly string gpu;
+
+		public override string ToString() => $"{clip}-{model}-{gpu}";
+
+		public static LogName? tryParse( string path )
+		{
+			string? ext = Path.GetExtension( path );
+			if( ext == null || !ext.Equals( ".txt", StringComparison.InvariantCultureIgnoreCase ) )
+				return null;
+
+			string name = Path.GetFileNameWithoutExtension( path );
+			string[] fields = name.Split( '-' );
+			if( fields.Length != 3 )
+				return null;
+
+			return new LogName( fields );
+		}
+
+		LogName( string[] fields )
+		{
+			clip = Enum.Parse<eInputClip>( fields[ 0 ] );
+			model = Enum.Parse<eWhisperModel>( fields[ 1 ] );
+			gpu = fields[ 2 ];
+		}
+	}
+
+	record class LogData
+	{
+		public LogName name { get; init; }
+		// The numbers are seconds
+		public double runComplete { get; init; }
+		public double encode { get; init; }
+		public double decode { get; init; }
+		// The numbers are megabytes
+		public double ram { get; init; }
+		public double vram { get; init; }
+	}
+
+	static class LogParser
+	{
+		public static IEnumerable<LogData> parse( string folder )
+		{
+			foreach( string path in Directory.EnumerateFiles( folder, "*.txt" ) )
+			{
+				LogName? name = LogName.tryParse( path );
+				if( name == null )
+					continue;
+				yield return parseFile( name.Value, path );
+			}
+		}
+
+		enum eSection: byte
+		{
+			CPU, GPU, Shaders, Memory
+		}
+		static readonly (string, eSection)[] sectionMarkers = new (string, eSection)[]
+		{
+			("CPU Tasks", eSection.CPU),
+			("GPU Tasks", eSection.GPU),
+			("Compute Shaders", eSection.Shaders),
+			("Memory Usage", eSection.Memory),
+		};
+
+		static bool tryParseSection( ref eSection? section, string line )
+		{
+			foreach( (string marker, eSection s) in sectionMarkers )
+			{
+				if( line.Contains( marker ) )
+				{
+					section = s;
+					return true;
+				}
+			}
+			return false;
+		}
+
+		static bool tryParseTime( ref double? val, string key, string line )
+		{
+			if( !line.StartsWith( key ) )
+				return false;
+			if( !char.IsWhiteSpace( line[ key.Length ] ) )
+				return false;
+
+			line = line.Substring( key.Length ).TrimStart();
+			int comma = line.IndexOf( ',' );
+			if( comma > 0 )
+				line = line.Substring( 0, comma );
+			string[] fields = line.Split( ' ', StringSplitOptions.RemoveEmptyEntries );
+			if( fields.Length != 2 )
+				throw new ArgumentException();
+
+			double v = double.Parse( fields[ 0 ], CultureInfo.InvariantCulture );
+			switch( fields[ 1 ] )
+			{
+				case "seconds":
+					val = v;
+					return true;
+				case "milliseconds":
+					val = v / 1E3;
+					return true;
+				case "microseconds":
+					val = v / 1E6;
+					return true;
+			}
+			throw new ArgumentException();
+		}
+
+		static readonly Regex reMemory = new Regex( @"^Total\s+([0-9\.]+)\s+(\S+)\s+RAM, ([0-9\.]+)\s+(\S+)\s+VRAM$" );
+
+		static double parseMemory( Match m, int iv )
+		{
+			double v = double.Parse( m.Groups[ iv ].Value, CultureInfo.InvariantCulture );
+			string u = m.Groups[ iv + 1 ].Value;
+			switch( u )
+			{
+				case "bytes":
+					return v / ( 1 << 20 );
+				case "KB":
+					return v / ( 1 << 10 );
+				case "MB":
+					return v;
+				case "GB":
+					return v * ( 1 << 10 );
+			}
+			throw new ArgumentException();
+		}
+
+		static bool tryParseMemory( ref double? ram, ref double? vram, string line )
+		{
+			Match m = reMemory.Match( line );
+			if( !m.Success )
+				return false;
+			ram = parseMemory( m, 1 );
+			vram = parseMemory( m, 3 );
+			return true;
+		}
+
+		static LogData parseFile( in LogName name, string path )
+		{
+			using var reader = File.OpenText( path );
+			double? runComplete = null;
+			double? encode = null;
+			double? decode = null;
+			double? ram = null;
+			double? vram = null;
+			eSection? section = null;
+
+			while( true )
+			{
+				string? line = reader.ReadLine();
+				if( line == null )
+					break;
+				if( string.IsNullOrEmpty( line ) )
+					continue;
+				if( tryParseSection( ref section, line ) )
+					continue;
+
+				switch( section )
+				{
+					case eSection.CPU:
+						tryParseTime( ref runComplete, "RunComplete", line );
+						break;
+					case eSection.GPU:
+						tryParseTime( ref encode, "Encode", line );
+						tryParseTime( ref decode, "Decode", line );
+						break;
+					case eSection.Memory:
+						tryParseMemory( ref ram, ref vram, line );
+						break;
+				}
+			}
+
+			if( null == runComplete || null == encode || null == decode || null == ram || null == vram )
+				throw new ArgumentException();
+
+			return new LogData()
+			{
+				name = name,
+				runComplete = runComplete.Value,
+				encode = encode.Value,
+				decode = decode.Value,
+				ram = ram.Value,
+				vram = vram.Value,
+			};
+		}
+	}
+}
\ No newline at end of file
diff --git a/Tools/PerfSummary/PerfSummary.cs b/Tools/PerfSummary/PerfSummary.cs
new file mode 100644
index 0000000..d957a96
--- /dev/null
+++ b/Tools/PerfSummary/PerfSummary.cs
@@ -0,0 +1,29 @@
+using System.Runtime.CompilerServices;
+
+namespace PerfSummary
+{
+	internal class Program
+	{
+		static string getSolutionRoot( [CallerFilePath] string? path = null )
+		{
+			string? dir = Path.GetDirectoryName( path );
+			dir = Path.GetDirectoryName( dir );
+			dir = Path.GetDirectoryName( dir );
+			return dir ?? throw new ApplicationException();
+		}
+
+		static void Main( string[] args )
+		{
+			string root = getSolutionRoot();
+			root = Path.Combine( root, "SampleClips" );
+
+			LogData[] logs = LogParser.parse( root )
+				.OrderBy( x => x.name.clip )
+				.ThenBy( x => x.name.model )
+				.ThenBy( x => x.name.gpu )
+				.ToArray();
+
+			Summary.print( logs, root );
+		}
+	}
+}
\ No newline at end of file
diff --git a/Tools/PerfSummary/PerfSummary.csproj b/Tools/PerfSummary/PerfSummary.csproj
new file mode 100644
index 0000000..dee1710
--- /dev/null
+++ b/Tools/PerfSummary/PerfSummary.csproj
@@ -0,0 +1,10 @@
+<Project Sdk="Microsoft.NET.Sdk">
+	<PropertyGroup>
+		<OutputType>Exe</OutputType>
+		<TargetFramework>net6.0</TargetFramework>
+		<ImplicitUsings>enable</ImplicitUsings>
+		<Nullable>enable</Nullable>
+		<CheckForOverflowUnderflow>true</CheckForOverflowUnderflow>
+		<AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
+	</PropertyGroup>
+</Project>
\ No newline at end of file
diff --git a/Tools/PerfSummary/Summary.cs b/Tools/PerfSummary/Summary.cs
new file mode 100644
index 0000000..8cea958
--- /dev/null
+++ b/Tools/PerfSummary/Summary.cs
@@ -0,0 +1,88 @@
+using System.Globalization;
+
+namespace PerfSummary
+{
+	static class Summary
+	{
+		public static void print( LogData[] logs, string folder )
+		{
+			string path = Path.Combine( folder, "summary.tsv" );
+			using var writer = File.CreateText( path );
+
+			string[] header = new string[]
+			{
+				"Audio Clip", "Model", "GPU",
+				"Total, sec", "Relative speed",
+				"Encode, sec", "Decode, sec",
+				"RAM, MB", "VRAM, MB"
+			};
+			writer.fields( header );
+
+			foreach( var item in logs )
+				writer.fields( item.makeFields() );
+		}
+
+		static IEnumerable<string> makeFields( this LogData log )
+		{
+			yield return log.clip();
+			yield return log.name.model.ToString();
+			yield return log.gpu();
+			yield return log.runComplete.print();
+			yield return log.relative();
+			yield return log.encode.print();
+			yield return log.decode.print();
+			yield return log.ram.print();
+			yield return log.vram.print();
+		}
+
+		static string print( this double v ) =>
+			v.ToString( CultureInfo.InvariantCulture );
+
+		static string relative( this LogData log )
+		{
+			double duration;
+			switch( log.name.clip )
+			{
+				case eInputClip.jfk:
+					duration = 11;
+					break;
+				case eInputClip.columbia:
+					duration = TimeSpan.FromTicks( 1987620000 ).TotalSeconds;
+					break;
+				default:
+					throw new NotImplementedException();
+			}
+
+			double rel = duration / log.runComplete;
+			return rel.print();
+		}
+
+		static string clip( this LogData log )
+		{
+			return log.name.clip switch
+			{
+				eInputClip.jfk => "jfk.wav",
+				eInputClip.columbia => "columbia.wma",
+				 _ => throw new ArgumentException()
+			};
+		}
+
+		static string gpu( this LogData log )
+		{
+			return log.name.gpu switch
+			{
+				"1080ti" => "GeForce 1080Ti",
+				"1650" => "GeForce 1650",
+				"vega7" => "Ryzen 5 5600U",
+				"vega8" => "Ryzen 7 5700G",
+				_ => log.name.gpu
+			};
+		}
+
+		static void fields( this StreamWriter writer, IEnumerable<string> fields )
+		{
+			string line = string.Join( "\t", fields );
+			writer.WriteLine( line );
+		}
+	}
+}
\ No newline at end of file
diff --git a/WhisperCpp.sln b/WhisperCpp.sln
index 4c2ae51..84d9a2f 100644
--- a/WhisperCpp.sln
+++ b/WhisperCpp.sln
@@ -43,6 +43,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		Readme.md = Readme.md
 	EndProjectSection
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PerfSummary", "Tools\PerfSummary\PerfSummary.csproj", "{8AC301F0-FEC9-4F26-83DD-DB32969CD510}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -91,6 +93,10 @@ Global
 		{CD9E49F0-75A3-4F91-AC71-336109EE39C6}.Debug|x64.Build.0 = Debug|x64
 		{CD9E49F0-75A3-4F91-AC71-336109EE39C6}.Release|x64.ActiveCfg = Release|x64
 		{CD9E49F0-75A3-4F91-AC71-336109EE39C6}.Release|x64.Build.0 = Release|x64
+		{8AC301F0-FEC9-4F26-83DD-DB32969CD510}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{8AC301F0-FEC9-4F26-83DD-DB32969CD510}.Debug|x64.Build.0 = Debug|Any CPU
+		{8AC301F0-FEC9-4F26-83DD-DB32969CD510}.Release|x64.ActiveCfg = Release|Any CPU
+		{8AC301F0-FEC9-4F26-83DD-DB32969CD510}.Release|x64.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -103,6 +109,7 @@ Global
 		{A49305C0-7022-45A6-89B4-4BD33138C98A} = {B988C132-115D-4157-99FE-0D891CE45A82}
 		{8478A77C-D851-4C63-9511-1770CC82D33E} = {90D16EBB-08A4-4C9B-9991-B1B2E036838C}
 		{CD9E49F0-75A3-4F91-AC71-336109EE39C6} = {B988C132-115D-4157-99FE-0D891CE45A82}
+		{8AC301F0-FEC9-4F26-83DD-DB32969CD510} = {90D16EBB-08A4-4C9B-9991-B1B2E036838C}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {07D5F1CF-1FAD-4F40-806A-B148CD609961}