yum-archive/TaSTT-Whisper

High-performance GPGPU inference of OpenAI's Whisper automatic speech recognition (ASR) model

git clone https://git.yummers.dev/yum-archive/TaSTT-Whisper

KonstantinSource codes8c4603c

master
branchesmasterall refs →
5.3 KiB183 linesraw
1#include "stdafx.h"
2#include "tensorOpsTests.h"
3#include "MlContext.h"
4#include "TensorEx.h"
5#include "../D3D/shaders.h"
6#include "../D3D/Binder.h"
7#include "testUtils.h"
8#include "../Whisper/WhisperContext.h"
9
10void DirectCompute::testMulMat( const ggml_tensor* src0, const ggml_tensor* src1, const ggml_tensor* dst, const void* tempBuffer )
11{
12	return;
13	CaptureRaii capture;
14	const size_t nb00 = src0->nb[ 0 ];
15	const size_t nb01 = src0->nb[ 1 ];
16
17	if( src0->type != GGML_TYPE_F16 )
18		return; // TODO
19
20	if( nb01 < nb00 )
21		return;	// TODO
22
23	WhisperContext& ctx = WhisperContext::current();
24
25	Tensor arg0, arg1;
26	check( arg0.create( *src0, eBufferUse::Immutable, true ) );
27	check( arg1.create( *src1, eBufferUse::Immutable, true ) );
28	TensorEx res;
29	check( res.create( *dst, eBufferUse::ReadWriteDownload, false ) );
30
31	ctx.mulMat( arg0, arg1, res );
32
33	std::vector<float> tv;
34	check( res.download( tv ) );
35
36	const size_t len = tv.size();
37	computeDiff( tv.data(), (const float*)dst->data, len ).print( "testMulMat-product" );
38
39#if 0
40	dbgWriteBinaryFile( L"product-orig.bin", dst->data, len * 4 );
41	dbgWriteBinaryFile( L"product-gpu.bin", tv.data(), len * 4 );
42	__debugbreak();
43#endif
44}
45
46#if 0
47void DirectCompute::testMulMatReshape( const ggml_tensor* src1, const void* tempBuffer )
48{
49	Tensor src;
50	check( src.create( *src1, eBufferUse::Immutable, true ) );
51
52	const size_t ne10 = (uint32_t)src1->ne[ 0 ];
53	const size_t ne11 = (uint32_t)src1->ne[ 1 ];
54	const size_t ne12 = (uint32_t)src1->ne[ 2 ];
55	const size_t ne13 = (uint32_t)src1->ne[ 3 ];
56	if( 1 != ne13 )
57		throw E_UNEXPECTED;
58	const size_t tempLength = ne10 * ne11 * ne12 * ne13;
59
60	Context& ctx = Context::current();
61	const ReadWriteViews& temp = ctx.temp.fp16( tempLength );
62
63	{
64		Binder bind;
65		ctx.cb.bind();
66
67		bindShader( eComputeShader::mulMatDotReshape );
68
69		ctx.cb.update( src );
70		bind.bind( src, temp );
71		context()->Dispatch( (UINT)ne11, (UINT)ne12, 1 );
72	}
73
74	std::vector<uint16_t> reshaped;
75	check( downloadBuffer( temp, reshaped ) );
76	computeDiff( reshaped.data(), (const uint16_t*)tempBuffer, reshaped.size() ).print( "testMulMatReshape" );
77
78#if 0
79	dbgWriteBinaryFile( L"fp32.bin", src1->data, ggml_nbytes( src1 ) );
80	dbgWriteBinaryFile( L"fp16-cpu.bin", tempBuffer, reshaped.size() * 2 );
81	dbgWriteBinaryFile( L"fp16-gpu.bin", reshaped.data(), reshaped.size() * 2 );
82	__debugbreak();
83#endif
84}
85#endif
86
87void DirectCompute::computeMulMat( const ggml_tensor* src0, const ggml_tensor* src1, ggml_tensor* dst )
88{
89	CaptureRaii capture;
90	const size_t nb00 = src0->nb[ 0 ];
91	const size_t nb01 = src0->nb[ 1 ];
92
93	if( src0->type != GGML_TYPE_F16 )
94		throw E_INVALIDARG;
95	if( nb01 < nb00 )
96		throw E_INVALIDARG;
97
98	WhisperContext& ctx = WhisperContext::current();
99
100	Tensor arg0, arg1;
101	check( arg0.create( *src0, eBufferUse::Immutable, true ) );
102	check( arg1.create( *src1, eBufferUse::Immutable, true ) );
103	TensorEx res;
104	check( res.create( *dst, eBufferUse::ReadWriteDownload, false ) );
105
106	ctx.mulMat( arg0, arg1, res );
107
108	check( res.download( dst->data ) );
109}
110
111void DirectCompute::testFlashAttention( const ggml_tensor* q, const ggml_tensor* k, const ggml_tensor* v, bool masked, const ggml_tensor* dst )
112{
113	CaptureRaii capture;
114
115	Tensor Q, K, V;
116	TensorEx res;
117	check( Q.create( *q, eBufferUse::Immutable, true ) );
118	check( K.create( *k, eBufferUse::Immutable, true ) );
119	check( V.create( *v, eBufferUse::Immutable, true ) );
120	check( res.create( *dst, eBufferUse::ReadWriteDownload, false ) );
121
122	WhisperContext& ctx = WhisperContext::current();
123	ctx.flashAttention( Q, K, V, res, masked );
124
125	std::vector<float> tv;
126	check( res.download( tv ) );
127
128	const size_t len = tv.size();
129	computeDiff( tv.data(), (const float*)dst->data, len ).print( "testFlashAttention" );
130}
131
132void DirectCompute::computeFlashAttention( const ggml_tensor* q, const ggml_tensor* k, const ggml_tensor* v, bool masked, ggml_tensor* dst )
133{
134	CaptureRaii capture;
135
136	Tensor Q, K, V;
137	TensorEx res;
138	check( Q.create( *q, eBufferUse::Immutable, true ) );
139	check( K.create( *k, eBufferUse::Immutable, true ) );
140	check( V.create( *v, eBufferUse::Immutable, true ) );
141	check( res.create( *dst, eBufferUse::ReadWriteDownload, false ) );
142
143	WhisperContext& ctx = WhisperContext::current();
144	ctx.flashAttention( Q, K, V, res, masked );
145
146	check( res.download( dst->data ) );
147}
148
149void DirectCompute::testConvolution( const ggml_tensor* src0, const ggml_tensor* src1, const ggml_tensor* dst )
150{
151	CaptureRaii capture;
152
153	Tensor arg0, arg1;
154	check( arg0.create( *src0, eBufferUse::Immutable, true ) );
155	check( arg1.create( *src1, eBufferUse::Immutable, true ) );
156	TensorEx res;
157	check( res.create( *dst, eBufferUse::ReadWriteDownload, false ) );
158
159	WhisperContext& ctx = WhisperContext::current();
160	ctx.convolution( arg0, arg1, res );
161
162	std::vector<float> tv;
163	check( res.download( tv ) );
164
165	const size_t len = tv.size();
166	computeDiff( tv.data(), (const float*)dst->data, len ).print( "testConvolution" );
167}
168
169void DirectCompute::computeConvolution( const ggml_tensor* src0, const ggml_tensor* src1, ggml_tensor* dst )
170{
171	CaptureRaii capture;
172
173	Tensor arg0, arg1;
174	check( arg0.create( *src0, eBufferUse::Immutable, true ) );
175	check( arg1.create( *src1, eBufferUse::Immutable, true ) );
176	TensorEx res;
177	check( res.create( *dst, eBufferUse::ReadWriteDownload, false ) );
178
179	WhisperContext& ctx = WhisperContext::current();
180	ctx.convolution( arg0, arg1, res );
181
182	res.download( dst->data );
183}