yum-archive/TaSTT-Whisper

High-performance GPGPU inference of OpenAI's Whisper automatic speech recognition (ASR) model

git clone https://git.yummers.dev/yum-archive/TaSTT-Whisper

KonstantinBugfix: when processing files, “Run” CPU block was erroneously measured twiceaaeab77

master
branchesmasterall refs →
2.2 KiB125 linesraw
1#pragma once
2#include <atlcoll.h>
3#include "CpuProfiler.h"
4
5namespace DirectCompute
6{
7	enum struct eComputeShader : uint16_t;
8	enum struct eProfilerBlock : uint16_t;
9}
10
11namespace Whisper
12{
13	struct WhisperModel;
14
15	enum struct eCpuBlock : uint8_t
16	{
17		LoadModel,
18		RunComplete,
19		Run,
20		Callbacks,
21		Spectrogram,
22		Sample,
23		VAD,
24		Encode,
25		Decode,
26		DecodeStep,
27		DecodeLayer,
28	};
29
30	class ProfileCollection
31	{
32	public:
33		ProfileCollection( const WhisperModel& model );
34
35		struct Measure
36		{
37			size_t count = 0;
38			// 100-nanosecond ticks
39			uint64_t totalTicks = 0;
40
41			void reset()
42			{
43				count = 0;
44				totalTicks = 0;
45			}
46
47			void print( const char* name ) const;
48
49			void add( uint64_t val )
50			{
51				count++;
52				totalTicks += val;
53			}
54		};
55
56		Measure& measure( DirectCompute::eProfilerBlock which );
57		Measure& measure( DirectCompute::eComputeShader which );
58		Measure& measure( eCpuBlock which );
59#if PROFILER_COLLECT_TAGS
60		Measure& measure( DirectCompute::eComputeShader which, uint16_t tag );
61#endif
62		void print();
63
64		void reset();
65
66		class CpuRaii
67		{
68			Measure* dest;
69			const int64_t tsc;
70
71		public:
72			CpuRaii( Measure& m ) : dest( &m ), tsc( tscNow() )
73			{ }
74			CpuRaii( const CpuRaii& ) = delete;
75			CpuRaii( CpuRaii&& that ) noexcept :
76				tsc( that.tsc )
77			{
78				dest = that.dest;
79				that.dest = nullptr;
80			}
81
82			~CpuRaii()
83			{
84				if( nullptr != dest )
85				{
86					const int64_t elapsed = tscNow() - tsc;
87					dest->add( ticksFromTsc( elapsed ) );
88				}
89			}
90		};
91
92		decltype( auto ) cpuBlock( eCpuBlock which )
93		{
94			return CpuRaii{ measure( which ) };
95		}
96
97		uint16_t makeTagId( const char* tag );
98
99	private:
100		CAtlMap<uint32_t, Measure> measures;
101		CComAutoCriticalSection critSec;
102#if PROFILER_COLLECT_TAGS
103		CAtlMap<const char*, uint16_t> tagIDs;
104		std::vector<const char*> tagNames;
105		CAtlMap<uint32_t, Measure> taggedShaders;
106		std::vector<uint32_t> taggedKeysTemp;
107		struct TaggedTemp
108		{
109			uint64_t ticks;
110			size_t count;
111			const char* name;
112
113			bool operator<( const TaggedTemp& that ) const
114			{
115				// Flipping the comparison to sort in descending order
116				return ticks > that.ticks;
117			}
118
119			void print() const;
120		};
121		std::vector<TaggedTemp> taggedTimes;
122#endif
123		std::vector<uint32_t> keysTemp;
124	};
125}