yum-archive/TaSTT-Whisper

High-performance GPGPU inference of OpenAI's Whisper automatic speech recognition (ASR) model

git clone https://git.yummers.dev/yum-archive/TaSTT-Whisper

KonstantinExperimental, alternative busy wait implementationcacec67

master
branchesmasterall refs →
4.2 KiB189 linesraw
1#pragma once
2#include "../D3D/device.h"
3#include "ProfileCollection.h"
4#include "DelayExecution.h"
5
6namespace DirectCompute
7{
8	enum struct eProfilerBlock : uint16_t
9	{
10		LoadModel = 0x1000,
11		Run = 0x2000,
12		Encode = 0x3000,
13		EncodeLayer = 0x4000,
14		Decode = 0x5000,
15		DecodeStep = 0x6000,
16		DecodeLayer = 0x7000,
17	};
18
19	enum struct eComputeShader : uint16_t;
20
21	class GpuProfiler
22	{
23		DelayExecution delay;
24		CComPtr<ID3D11Query> disjoint;
25
26		enum struct eEvent
27		{
28			None = 0,
29			BlockStart,
30			BlockEnd,
31			Shader
32		};
33
34		struct BlockState;
35		static constexpr uint16_t EmptyShader = ~(uint16_t)0;
36
37		// A circular buffer with in-flight queries which feeds timestamps into the iTimestampSink interface
38		class Queue
39		{
40			static constexpr size_t queueLength = 32;
41
42			// Ring buffer for individual measures
43			struct Entry
44			{
45				CComPtr<ID3D11Query> query;
46				BlockState* block;
47				eEvent event;
48				uint16_t shader;
49#if PROFILER_COLLECT_TAGS
50				uint16_t tag = 0;
51#endif
52				void join( GpuProfiler& owner );
53			};
54
55			GpuProfiler& owner;
56			std::array<Entry, queueLength> queue;
57			size_t nextEntry = 0;
58
59		public:
60			Queue( GpuProfiler& gp ) : owner( gp ) {}
61
62			HRESULT create();
63
64			// Begin a next query. Eventually, this will result in the BlockState.haveTimestamp callback
65			void submit( BlockState* block, eEvent evt, uint16_t shader = EmptyShader, uint16_t tag = 0 );
66
67			// Wait for all the pending queries, and call their callbacks
68			void join();
69		};
70		Queue queries;
71
72		struct sProfilerData;
73		struct BlockState
74		{
75			int64_t timeStart = -1;
76			sProfilerData* destBlock = nullptr;
77			int64_t shaderStart = -1;
78			uint16_t prevShader = EmptyShader;
79			uint16_t prevShaderTag = 0;
80			BlockState* parentBlock = nullptr;
81			void haveTimestamp( eEvent evt, uint16_t cs, uint16_t tag, uint64_t time, GpuProfiler& profiler );
82		private:
83			void completePrevShader( uint64_t time, GpuProfiler& profiler );
84		};
85		CAtlMap<eProfilerBlock, BlockState> blockStates;
86		std::vector<BlockState*> stack;
87
88		struct sProfilerData
89		{
90			// Count of accumulated measures
91			size_t callsPending;
92			// Total time spent running all instances of that measure, expressed in GPU ticks
93			uint64_t timePending;
94
95			Whisper::ProfileCollection::Measure* dest;
96
97			inline void makeTime( uint64_t freq );
98			inline void addPending( int64_t time );
99			inline void reset();
100			inline void dropPending();
101
102			sProfilerData()
103			{
104				reset();
105			}
106		};
107
108		CAtlMap<uint16_t, sProfilerData> results;
109#if PROFILER_COLLECT_TAGS
110		CAtlMap<uint32_t, sProfilerData> resultsTagged;
111#endif
112		void resultsMakeTime( uint64_t freq );
113		void resultsDropPending();
114		void resultsReset();
115
116		void blockStart( eProfilerBlock which );
117		void blockEnd();
118
119		Whisper::ProfileCollection& dest;
120#if PROFILER_COLLECT_TAGS
121		uint16_t m_nextTag = 0;
122#endif
123	public:
124
125		GpuProfiler( Whisper::ProfileCollection& pc ) :
126			dest( pc ), queries( *this ) { }
127
128		HRESULT create( size_t maxDepth = 3 );
129
130		class BlockRaii
131		{
132			GpuProfiler* profiler;
133
134		public:
135			BlockRaii( GpuProfiler& owner, eProfilerBlock which )
136			{
137				owner.blockStart( which );
138				profiler = &owner;
139			}
140			~BlockRaii()
141			{
142				if( nullptr != profiler )
143				{
144					profiler->blockEnd();
145					profiler = nullptr;
146				}
147			}
148			BlockRaii( BlockRaii&& that ) noexcept :
149				profiler( that.profiler )
150			{
151				that.profiler = nullptr;
152			}
153			BlockRaii( const BlockRaii& ) = delete;
154			void operator=( const BlockRaii& ) = delete;
155			void operator=( BlockRaii&& ) = delete;
156		};
157
158		BlockRaii block( eProfilerBlock which )
159		{
160			return BlockRaii{ *this, which };
161		}
162
163		void computeShader( eComputeShader cs );
164
165		bool profileShaders = false;
166		// bool profileShaders = true;
167
168		decltype( auto ) cpuBlock( Whisper::eCpuBlock block )
169		{
170			return dest.cpuBlock( block );
171		}
172		Whisper::ProfileCollection& profiler() { return dest; }
173
174		// Set tag string for the next compute shader
175		// The string should be readonly: for performance reason the implementation doesn�t copy nor compare any strings, it only keeps the pointer
176#if PROFILER_COLLECT_TAGS
177		uint16_t setNextTag( const char* name );
178#else
179		inline uint16_t setNextTag( const char* name ) { return 0; }
180#endif
181
182		void setNextTag( uint16_t tag )
183		{
184#if PROFILER_COLLECT_TAGS
185			m_nextTag = tag;
186#endif
187		}
188	};
189}