yum-archive/TaSTT-Whisper

High-performance GPGPU inference of OpenAI's Whisper automatic speech recognition (ASR) model

git clone https://git.yummers.dev/yum-archive/TaSTT-Whisper

KonstantinSource codes8c4603c

master
branchesmasterall refs →
2.5 KiB88 linesraw
1// Compute tensor = GELU( tensor + repeat( pattern, tensor ) ) in 1 shot, without VRAM allocations
2// Dispatch [ nb[ 1 ], nb[ 2 ], nb[ 3 ] ] thread groups of this shader, where nb is size of the destination tensor
3RWBuffer<float> tensor: register( u0 );
4Buffer<float> pattern: register( t0 );
5Buffer<uint> lookupTable: register( t1 );
6
7cbuffer Constants: register( b0 )
8{
9	uint4 tensorSize: packoffset( c0 );
10	uint4 tensorStrides: packoffset( c1 );
11	uint4 patternSize: packoffset( c2 );
12	uint4 patternStrides: packoffset( c3 );
13}
14
15#ifndef THREADS
16#define THREADS 1024
17#endif
18
19#include "repeatUtils.hlsli"
20#include "miscUtils.hlsli"
21
22inline float gelu( float x )
23{
24#if 1
25	const uint index = fp16Rounded( x );
26	const uint res16 = lookupTable[ index ];
27	return f16tof32( res16 );
28#else
29	// This version is much slower, at least on AMD, despite saving these VRAM loads.
30	const float GELU_COEF_A = 0.044715;
31	const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876;
32	return 0.5 * x * ( 1.0 + tanh( SQRT_2_OVER_PI * x * ( 1.0 + GELU_COEF_A * x * x ) ) );
33#endif
34}
35
36inline void computeSimple( uint idx, float add )
37{
38	float f = tensor[ idx ];
39	f += add;
40	f = gelu( f );
41	tensor[ idx ] = f;
42}
43
44[ numthreads( THREADS, 1, 1 ) ]
45void main( uint3 group: SV_GroupID, uint thread : SV_GroupIndex )
46{
47	uint3 it = tensorIteratorState( group, thread, tensorSize, tensorStrides );
48	uint rsi = rowOffset( group % patternSize.yzw, patternStrides );
49
50	if( patternSize[ 0 ] == 1 )
51	{
52		// The pattern only has 1 column - broadcasting over the row
53		const float p = pattern[ rsi ];
54		ROW_LOOP( it )
55			computeSimple( it.x, p );
56	}
57	else if( patternSize[ 0 ] <= THREADS )
58	{
59		// pattern size doesn't exceed thread group size: load pattern value outside of the loop
60		const uint threadsPerGroup = THREADS - ( THREADS % patternSize[ 0 ] );
61		if( thread >= threadsPerGroup )
62			return;
63
64		const float p = pattern[ rsi + ( thread % patternSize[ 0 ] ) * patternStrides[ 0 ] ];
65		ROW_LOOP_EX( it, threadsPerGroup, tensorStrides )
66			computeSimple( it.x, p );
67	}
68	else
69	{
70		// Pattern rows are larger than the thread group, need to stream from both buffers
71		const uint rsiInc = THREADS * patternStrides[ 0 ];
72		const uint rsiDec = patternSize[ 0 ] * patternStrides[ 0 ];
73		const uint rsiEnd = rsi + rsiDec;
74		rsi += thread * patternStrides[ 0 ];
75
76		ROW_LOOP( it )
77		{
78			float f = tensor[ it.x ];
79			float p = pattern[ rsi ];
80			rsi += rsiInc;
81			if( rsi >= rsiEnd )
82				rsi -= rsiDec;
83			f += p;
84			f = gelu( f );
85			tensor[ it.x ] = f;
86		}
87	}
88}