yum-archive/TaSTT-Whisper

High-performance GPGPU inference of OpenAI's Whisper automatic speech recognition (ASR) model

git clone https://git.yummers.dev/yum-archive/TaSTT-Whisper

KonstantinSource codes8c4603c

master
branchesmasterall refs →
2.1 KiB79 linesraw
1// Special softMax shader for matrices with rows of 1500 elements.
2// Uses group shared buffer of that length to save global memory bandwidth, more than 2x faster than the original.
3// Dispatch [ nr, 1, 1 ] thread groups of this shader
4RWBuffer<float> result: register( u0 );
5
6cbuffer Constants: register( b0 )
7{
8	uint4 elements: packoffset( c0 );
9	uint4 strides: packoffset( c1 );
10	uint nr: packoffset( c2.x );
11	float inputScale: packoffset( c2.y );
12}
13
14#include "miscUtils.hlsli"
15#include "groupReduce64.hlsli"
16
17static const uint THREADS = 64;
18static const uint ROW_LENGTH = 1500;
19groupshared float rowBuffer[ ROW_LENGTH ];
20
21static const float negativeInfinity = asfloat( 0xff800000 );
22
23[ numthreads( THREADS, 1, 1 ) ]
24void main( uint3 group: SV_GroupID, uint thread : SV_GroupIndex )
25{
26	const uint p = group.x * strides[ 1 ];
27	const uint nc = ROW_LENGTH;
28	uint i;
29
30	float m = negativeInfinity;
31	// First pass: compute maximum, and copy the row into the group shared buffer
32	for( i = thread; i < nc; i += THREADS )
33	{
34		float f = result[ p + i ];
35		m = max( m, f );
36		rowBuffer[ i ] = f;
37	}
38	horizontalMaxBroadcast( thread, m );
39
40	// Second pass: apply initial scale, compute the exponent, and compute total sum over the row
41	float sum = 0;
42	for( i = thread; i < nc; i += THREADS )
43	{
44		float f = rowBuffer[ i ];
45
46		[branch]
47		if( f != negativeInfinity )
48		{
49			f = ( f - m ) * inputScale;
50#if 1
51			// At least on Radeon Graphics GPU inside Ryzen 7 5700G, computing exponent instead of loading from the buffer improves the performance
52			f = exp( f );
53#else
54			uint s = fp16Rounded( f );
55			s = lookupTable[ s ];
56			f = f16tof32( s );
57#endif
58			sum += f;
59		}
60		else
61			f = 0;
62
63		rowBuffer[ i ] = f;
64	}
65
66	horizontalSum( thread, sum );
67	if( 0 == thread )
68		sharedAccumulators[ 0 ] = 1.0 / sum;
69	GroupMemoryBarrierWithGroupSync();
70	const float scale = sharedAccumulators[ 0 ];
71
72	// Final pass: apply the final scale, and copy the row from the group shared buffer back into the global memory
73	for( i = thread; i < nc; i += THREADS )
74	{
75		float f = rowBuffer[ i ];
76		f *= scale;
77		result[ p + i ] = f;
78	}
79}