yum-archive/TaSTT-Whisper

High-performance GPGPU inference of OpenAI's Whisper automatic speech recognition (ASR) model

git clone https://git.yummers.dev/yum-archive/TaSTT-Whisper

KonstantinSource codes8c4603c

master
branchesmasterall refs →
3.7 KiB139 linesraw
1#pragma once
2#include "../D3D/enums.h"
3#include "../ML/TensorShape.h"
4// 1 = new tensors can be allocated with `nullptr` iMemoryAllocator, by allocating memory internally and counting these references
5// 0 = memory allocator is mandatory, create() methods will fail with E_POINTER if the allocator is `nullptr`
6#define TENSOR_INTERNAL_ALLOC 0
7
8// 1 = expose compatibility API for GGML interop
9#define TENSOR_GGML_COMPAT 0
10
11#if TENSOR_GGML_COMPAT
12#include "../source/ggml.h"
13#endif
14
15namespace CpuCompute
16{
17	using DirectCompute::TensorShape;
18	using DirectCompute::eDataType;
19
20	__interface iMemoryAllocator
21	{
22		void* allocate( size_t cb, size_t align );
23	};
24	__interface iArenaAllocator : public iMemoryAllocator
25	{
26		void resetArena();
27	};
28
29#if TENSOR_GGML_COMPAT
30	class Tensor;
31	class GgmlTensorView
32	{
33		ggml_tensor tensor;
34	public:
35		GgmlTensorView( const Tensor& t );
36		operator ggml_tensor* ( ) { return &tensor; }
37	};
38#endif
39
40	// A functional equivalent of ggml_tensor structure, designed for use from C++
41	class Tensor : public TensorShape
42	{
43		void* m_data = nullptr;
44
45		eDataType m_type = (eDataType)0xFF;
46
47#if TENSOR_INTERNAL_ALLOC
48		// True when the memory block was allocated internally by this class
49		// In this case, this class does reference counting to support cheap copies.
50		// False when it's owned by someone else, such as iMemoryAllocator object, or a GGML's tensor
51		bool ownsMemory = false;
52		void deallocate();
53#endif
54
55		// Private constructors for fromData() methods
56		Tensor( void* pointer, eDataType type, std::initializer_list<uint32_t> size );
57		Tensor( void* pointer, eDataType type, uint32_t length ) noexcept;
58	public:
59		// Trivial constructors
60		Tensor() = default;
61#if TENSOR_INTERNAL_ALLOC
62		~Tensor()
63		{
64			deallocate();
65		}
66#else
67		~Tensor() = default;
68#endif
69		Tensor( Tensor&& that ) noexcept;
70		void operator=( Tensor&& that ) noexcept;
71		Tensor( const Tensor& that );
72		void operator=( const Tensor& that );
73
74		// Allocate a new tensor
75		HRESULT create( eDataType type, const std::array<uint32_t, 4>& sizeElements, iMemoryAllocator* alloc = nullptr );
76		// Allocate a new tensor
77		HRESULT create( eDataType type, std::initializer_list<uint32_t> sizeElements, iMemoryAllocator* alloc = nullptr );
78		// Attach to pre-existing block of memory, interpreting the data as a dense tensor of the specified type and size
79		HRESULT attach( void* pointer, eDataType type, std::initializer_list<uint32_t> sizeElements );
80		// Attach to pre-existing block of memory, interpret the data as a dense vector of the specified type and length
81		static Tensor fromData( void* pointer, eDataType type, uint32_t length );
82
83		eDataType type() const { return m_type; }
84		void* data() const { return m_data; }
85
86		uint16_t* fp16()
87		{
88			assert( m_type == eDataType::FP16 );
89			assert( nullptr != m_data );
90			return (uint16_t*)m_data;
91		}
92		const uint16_t* fp16() const
93		{
94			assert( m_type == eDataType::FP16 );
95			assert( nullptr != m_data );
96			return (uint16_t*)m_data;
97		}
98		float* fp32()
99		{
100			assert( m_type == eDataType::FP32 );
101			assert( nullptr != m_data );
102			return (float*)m_data;
103		}
104		const float* fp32() const
105		{
106			assert( m_type == eDataType::FP32 );
107			assert( nullptr != m_data );
108			return (float*)m_data;
109		}
110
111		Tensor reshape3d( uint32_t ne0, uint32_t ne1, uint32_t ne2 ) const;
112
113		void setType( eDataType dt )
114		{
115			m_type = dt;
116		}
117		void setDataPointer( void* pv )
118		{
119			m_data = pv;
120		}
121
122#if TENSOR_GGML_COMPAT
123		// Compatibility with GGML's tensors, for testing and lulz
124		Tensor( const ggml_tensor* ggml );
125		ggml_tensor ggml() const;
126
127		operator GgmlTensorView() const
128		{
129			return GgmlTensorView( *this );
130		}
131#endif
132	};
133
134	// A pair of tensors containing weights and biases; apparently, both tensors are of the same shape
135	struct TensorPair
136	{
137		Tensor w, b;
138	};
139}