yum-archive/TaSTT-Whisper

High-performance GPGPU inference of OpenAI's Whisper automatic speech recognition (ASR) model

git clone https://git.yummers.dev/yum-archive/TaSTT-Whisper

KonstantinSource codes8c4603c

master
branchesmasterall refs →
3.2 KiB119 linesraw
1// Optimized version of convolutionMain2.hlsl for kernel size = 3
2// Dispatch [ ( ( ne10 / 2 ) + TILE_Y - 1 ) / TILE_Y, ne02, 1 ] thread groups of this shader
3#ifndef TILE_Y
4static const uint TILE_Y = 8;
5#endif
6#ifndef THREADS
7static const uint THREADS = 64;
8#endif
9
10Buffer<float> arg0: register( t0 );
11Buffer<float> arg1: register( t1 );
12RWBuffer<float> result: register( u0 );
13
14cbuffer Constants: register( b0 )
15{
16	uint4 src0_elements: packoffset( c0 );
17	uint4 src0_strides: packoffset( c1 );
18	uint4 src1_elements: packoffset( c2 );
19	uint4 result_elements: packoffset( c4 );
20	uint4 result_strides: packoffset( c5 );
21}
22
23// The accumulators we're after
24groupshared float resTemp[ TILE_Y ][ THREADS ];
25
26// Multiply + accumulate the specified row
27inline void accumulate( float a0, float a1, const uint resultRow, const uint thread )
28{
29	float acc = resTemp[ resultRow ][ thread ];
30	acc = mad( a0, a1, acc );
31	resTemp[ resultRow ][ thread ] = acc;
32}
33
34inline void convolutionTile( const uint s0, uint s1, const uint thread, const uint stride, const uint height )
35{
36	// Load 3 rows from arg0
37	const float3 a0 = float3( arg0[ s0 ], arg0[ s0 + stride ], arg0[ s0 + stride * 2 ] );
38
39	// Row 0
40	float a1 = arg1[ s1 ];
41	accumulate( a0[ 0 ], a1, 0, thread );
42	s1 += stride;
43
44	for( uint i = 1; i < height; i++ )
45	{
46		// Row i*2-1
47		// Even-indexed rows only contribute to a single output rows, after muiltiplied by kernel row #1
48		a1 = arg1[ s1 ];
49		accumulate( a0[ 1 ], a1, i - 1, thread );
50		s1 += stride;
51
52		// Row i*2, contributes to 2 output rows corresponding to kernel rows #0 and #2
53		a1 = arg1[ s1 ];
54		accumulate( a0[ 2 ], a1, i - 1, thread );
55		accumulate( a0[ 0 ], a1, i, thread );
56		s1 += stride;
57	}
58
59	// Row height*2 - 1
60	a1 = arg1[ s1 ];
61	accumulate( a0[ 1 ], a1, height - 1, thread );
62	s1 += stride;
63
64	// Row height*2
65	a1 = arg1[ s1 ];
66	accumulate( a0[ 2 ], a1, height - 1, thread );
67}
68
69#include "miscUtils.hlsli"
70
71[ numthreads( THREADS, 1, 1 ) ]
72void main( uint3 group: SV_GroupID, uint thread : SV_GroupIndex )
73{
74	uint i;
75	// Zero out the accumulators
76	for( i = 0; i < TILE_Y; i++ )
77		resTemp[ i ][ thread ] = 0.0;
78	GroupMemoryBarrierWithGroupSync();
79
80	const uint i1 = group.y;
81	const uint i0 = group.x * TILE_Y * 2;
82	const uint height = min( TILE_Y, ( src1_elements.x / 2 ) - group.x * TILE_Y );
83
84	const uint ne00 = src0_elements[ 0 ];
85	const uint ne01 = src0_elements[ 1 ];
86	const int ew0 = roundUp32( ne01 );
87
88	uint s0 = i1 * ew0 * ne00;
89	const uint s0End = s0 + ew0;
90	uint s1 = i0 * ew0;
91	s0 += thread;
92	s1 += thread;
93	for( ; s0 < s0End; s0 += THREADS, s1 += THREADS )
94		convolutionTile( s0, s1, thread, ew0, height );
95
96	GroupMemoryBarrierWithGroupSync();
97
98	// Now we need horizontal sums of these shared accumulators, i.e. reduce [height][THREADS] shared array into [height][1] column
99	for( i = THREADS / 2; i > 0; i /= 2 )
100	{
101		if( thread < i )
102		{
103			for( uint j = 0; j < height; j++ )
104			{
105				float sum = resTemp[ j ][ thread ];
106				sum += resTemp[ j ][ thread + i ];
107				resTemp[ j ][ thread ] = sum;
108			}
109		}
110		GroupMemoryBarrierWithGroupSync();
111	}
112
113	// And finally, store that column to global memory
114	if( thread >= height )
115		return;
116	const uint nb1 = result_strides[ 1 ];
117	const uint rdi = i1 * nb1 + group.x * TILE_Y + thread;
118	result[ rdi ] = resTemp[ thread ][ 0 ];
119}