yum-archive/TaSTT-Whisper

High-performance GPGPU inference of OpenAI's Whisper automatic speech recognition (ASR) model

git clone https://git.yummers.dev/yum-archive/TaSTT-Whisper

KonstantinSource codes8c4603c

master
branchesmasterall refs →
2.0 KiB73 linesraw
1// Compute tensor = ( tensor + repeat( pattern, tensor ) ) * scale in 1 shot, without VRAM allocations
2// Dispatch [ nb[ 1 ], nb[ 2 ], nb[ 3 ] ] thread groups of this shader, where nb is size of the destination tensor
3RWBuffer<float> tensor: register( u0 );
4Buffer<float> pattern: register( t0 );
5
6cbuffer Constants: register( b0 )
7{
8	uint4 tensorSize: packoffset( c0 );
9	uint4 tensorStrides: packoffset( c1 );
10	uint4 patternSize: packoffset( c2 );
11	uint4 patternStrides: packoffset( c3 );
12	float scalingMul : packoffset( c4.x );
13}
14
15#ifndef THREADS
16#define THREADS 512
17#endif
18
19#include "repeatUtils.hlsli"
20
21inline void computeSimple( uint idx, float add )
22{
23	float f = tensor[ idx ];
24	f += add;
25	f *= scalingMul;
26	tensor[ idx ] = f;
27}
28
29[ numthreads( THREADS, 1, 1 ) ]
30void main( uint3 group: SV_GroupID, uint thread : SV_GroupIndex )
31{
32	uint3 it = tensorIteratorState( group, thread, tensorSize, tensorStrides );
33	uint rsi = rowOffset( group % patternSize.yzw, patternStrides );
34
35	if( patternSize[ 0 ] == 1 )
36	{
37		// The pattern only has 1 column - broadcasting over the row
38		const float p = pattern[ rsi ];
39		ROW_LOOP( it )
40			computeSimple( it.x, p );
41	}
42	else if( patternSize[ 0 ] <= THREADS )
43	{
44		// pattern size doesn't exceed thread group size: load pattern value outside of the loop
45		const uint threadsPerGroup = THREADS - ( THREADS % patternSize[ 0 ] );
46		if( thread >= threadsPerGroup )
47			return;
48
49		const float p = pattern[ rsi + ( thread % patternSize[ 0 ] ) * patternStrides[ 0 ] ];
50		ROW_LOOP_EX( it, threadsPerGroup, tensorStrides )
51			computeSimple( it.x, p );
52	}
53	else
54	{
55		// Pattern rows are larger than the thread group, need to stream from both buffers
56		const uint rsiInc = THREADS * patternStrides[ 0 ];
57		const uint rsiDec = patternSize[ 0 ] * patternStrides[ 0 ];
58		const uint rsiEnd = rsi + rsiDec;
59		rsi += thread * patternStrides[ 0 ];
60
61		ROW_LOOP( it )
62		{
63			float f = tensor[ it.x ];
64			float p = pattern[ rsi ];
65			rsi += rsiInc;
66			if( rsi >= rsiEnd )
67				rsi -= rsiDec;
68			f += p;
69			f *= scalingMul;
70			tensor[ it.x ] = f;
71		}
72	}
73}