yum-archive/TaSTT-Whisper

High-performance GPGPU inference of OpenAI's Whisper automatic speech recognition (ASR) model

git clone https://git.yummers.dev/yum-archive/TaSTT-Whisper

KonstantinSource codes8c4603c

master
branchesmasterall refs →
2.7 KiB131 linesraw
1#pragma once
2#include <vector>
3#include "Tensor.h"
4#include "LargeBuffer.h"
5#if TENSOR_GGML_COMPAT
6#include "../source/ggml.h"
7#endif
8
9namespace CpuCompute
10{
11	// A set of tensors for one decoder's layer
12	struct LayerDecoder
13	{
14		// decoder.blocks.*.attn_ln
15		TensorPair attnLn0;
16		// decoder.blocks.*.attn.out
17		TensorPair attnLn1;
18		// decoder.blocks.*.attn.query
19		TensorPair attnQuery;
20		// decoder.blocks.*.attn.key
21		Tensor attnKey;
22		// decoder.blocks.*.attn.value
23		TensorPair attnValue;
24		// decoder.blocks.*.cross_attn_ln
25		TensorPair crossAttnLn0;
26		// decoder.blocks.*.cross_attn.out
27		TensorPair crossAttnLn1;
28		// decoder.blocks.*.cross_attn.query
29		TensorPair crossAttnQuery;
30
31		// decoder.blocks.*.cross_attn.key
32		// Tensor crossAttnKey;
33		// decoder.blocks.*.cross_attn.value
34		// TensorPair crossAttnValue;
35
36		// decoder.blocks.*.mlp_ln
37		TensorPair mlpLn;
38		// decoder.blocks.*.mlp.0
39		TensorPair mlp0;
40		// decoder.blocks.*.mlp.2
41		TensorPair mlp1;
42
43#if TENSOR_GGML_COMPAT
44		// decoder.blocks.*.attn_ln
45		ggml_tensor* attn_ln_0_w;
46		ggml_tensor* attn_ln_0_b;
47
48		// decoder.blocks.*.attn.out
49		ggml_tensor* attn_ln_1_w;
50		ggml_tensor* attn_ln_1_b;
51
52		// decoder.blocks.*.attn.query
53		ggml_tensor* attn_q_w;
54		ggml_tensor* attn_q_b;
55
56		// decoder.blocks.*.attn.key
57		ggml_tensor* attn_k_w;
58
59		// decoder.blocks.*.attn.value
60		ggml_tensor* attn_v_w;
61		ggml_tensor* attn_v_b;
62
63		// decoder.blocks.*.cross_attn_ln
64		ggml_tensor* cross_attn_ln_0_w;
65		ggml_tensor* cross_attn_ln_0_b;
66
67		// decoder.blocks.*.cross_attn.out
68		ggml_tensor* cross_attn_ln_1_w;
69		ggml_tensor* cross_attn_ln_1_b;
70
71		// decoder.blocks.*.cross_attn.query
72		ggml_tensor* cross_attn_q_w;
73		ggml_tensor* cross_attn_q_b;
74
75		// decoder.blocks.*.mlp_ln
76		ggml_tensor* mlp_ln_w;
77		ggml_tensor* mlp_ln_b;
78
79		// decoder.blocks.*.mlp.0
80		ggml_tensor* mlp_0_w;
81		ggml_tensor* mlp_0_b;
82
83		// decoder.blocks.*.mlp.2
84		ggml_tensor* mlp_1_w;
85		ggml_tensor* mlp_1_b;
86#endif
87	};
88
89	struct DecoderTensors
90	{
91		// decoder.positional_embedding
92		Tensor positionalEmbedding;
93
94		// decoder.token_embedding
95		Tensor tokenEmbedding;
96
97		// decoder.ln
98		TensorPair ln;
99		// A vector of layers
100		std::vector<LayerDecoder> layers;
101
102		void setMemoryBuffer( LargeBuffer&& mem ) noexcept
103		{
104			memory = std::move( mem );
105#if TENSOR_GGML_COMPAT
106			makeCompatTensors();
107#endif
108		}
109
110#if TENSOR_GGML_COMPAT
111		void makeCompatTensors();
112
113		// decoder.positional_embedding
114		ggml_tensor* d_pe; // DD
115
116		// decoder.token_embedding
117		ggml_tensor* d_te; // DD
118
119		// decoder.ln
120		ggml_tensor* d_ln_w; // DD
121		ggml_tensor* d_ln_b; // DD
122#endif
123
124	private:
125		// A smart pointer which owns the memory for all the above tensors
126		LargeBuffer memory;
127#if TENSOR_GGML_COMPAT
128		std::vector<ggml_tensor> ggml;
129#endif
130	};
131}