yum-archive/TaSTT-Whisper

High-performance GPGPU inference of OpenAI's Whisper automatic speech recognition (ASR) model

git clone https://git.yummers.dev/yum-archive/TaSTT-Whisper

KonstantinSource codes8c4603c

master
branchesmasterall refs →
813 B36 linesraw
1#pragma once
2#include "Tensor.h"
3#include "LargeBuffer.h"
4#include "../Whisper/sModelParams.h"
5
6namespace CpuCompute
7{
8	class KvTensors
9	{
10		uint16_t* keys = nullptr;
11		uint16_t* values = nullptr;
12		uint32_t size = 0;
13
14		CpuCompute::LargeBuffer memory;
15
16	public:
17		// Create these two large tensors, FP16 precision
18		HRESULT create( const Whisper::sModelParams& mp );
19
20		// A slice of model.memory_cross_k tensor
21		Tensor keysView( uint32_t len, uint32_t off ) const
22		{
23			if( len + off <= size )
24				return Tensor::fromData( keys + off, eDataType::FP16, len );
25			throw E_BOUNDS;
26		}
27
28		// A slice of model.memory_cross_v tensor
29		Tensor valuesView( uint32_t len, uint32_t off ) const
30		{
31			if( len + off <= size )
32				return Tensor::fromData( values + off, eDataType::FP16, len );
33			throw E_BOUNDS;
34		}
35	};
36}