summaryrefslogtreecommitdiffstats
path: root/WhisperNet
diff options
context:
space:
mode:
authorKonstantin <const@const.me>2023-01-16 14:52:43 +0100
committerKonstantin <const@const.me>2023-01-16 14:52:43 +0100
commit8c4603c73675958efc960fbd4bb599a2909d106a (patch)
tree714dc6fc9a1672d5fd7f89676b97e10959662abc /WhisperNet
parent990a8d0dbaefc996244097397259e92758b15cce (diff)
Source codes
Diffstat (limited to 'WhisperNet')
-rw-r--r--WhisperNet/API/CaptureDeviceId.cs24
-rw-r--r--WhisperNet/API/Parameters.cs95
-rw-r--r--WhisperNet/API/SpecialTokens.cs23
-rw-r--r--WhisperNet/API/eCaptureStatus.cs19
-rw-r--r--WhisperNet/API/eLanguage.cs206
-rw-r--r--WhisperNet/API/eLogLevel.cs34
-rw-r--r--WhisperNet/API/eModelImplementation.cs25
-rw-r--r--WhisperNet/API/eResultFlags.cs21
-rw-r--r--WhisperNet/API/iAudioBuffer.cs27
-rw-r--r--WhisperNet/API/iAudioReader.cs23
-rw-r--r--WhisperNet/API/iMediaFoundation.cs36
-rw-r--r--WhisperNet/API/iModel.cs27
-rw-r--r--WhisperNet/API/sCaptureParams.cs37
-rw-r--r--WhisperNet/Callbacks.cs44
-rw-r--r--WhisperNet/CaptureCallbacks.cs49
-rw-r--r--WhisperNet/Context.cs201
-rw-r--r--WhisperNet/ExtensionMethods.cs69
-rw-r--r--WhisperNet/Internal/AssemblyInfo.cs8
-rw-r--r--WhisperNet/Internal/NativeLogger.cs138
-rw-r--r--WhisperNet/Internal/iContext.cs37
-rw-r--r--WhisperNet/Internal/iTranscribeResult.cs122
-rw-r--r--WhisperNet/Internal/sCaptureCallbacks.cs23
-rw-r--r--WhisperNet/Internal/sCaptureDevice.cs22
-rw-r--r--WhisperNet/Internal/sFullParams.cs40
-rw-r--r--WhisperNet/Internal/sLoadModelCallbacks.cs64
-rw-r--r--WhisperNet/Internal/sLoggerSetup.cs16
-rw-r--r--WhisperNet/Internal/sProgressSink.cs20
-rw-r--r--WhisperNet/Library.cs110
-rw-r--r--WhisperNet/Readme.txt1
-rw-r--r--WhisperNet/WhisperNet.csproj24
-rw-r--r--WhisperNet/WhisperNet.nuspec27
31 files changed, 1612 insertions, 0 deletions
diff --git a/WhisperNet/API/CaptureDeviceId.cs b/WhisperNet/API/CaptureDeviceId.cs
new file mode 100644
index 0000000..9636e53
--- /dev/null
+++ b/WhisperNet/API/CaptureDeviceId.cs
@@ -0,0 +1,24 @@
+using Whisper.Internal;
+
+namespace Whisper
+{
+ /// <summary>Identifiers for an audio capture device</summary>
+ public record struct CaptureDeviceId
+ {
+ /// <summary>The display name is suitable for showing to the user, but might not be unique.</summary>
+ public string displayName;
+
+ /// <summary>Endpoint ID for an audio capture device.<br/>
+ /// It uniquely identifies the device on the system, but is not a readable string.</summary>
+ public string endpoint;
+
+ internal CaptureDeviceId( in sCaptureDevice rsi )
+ {
+ displayName = rsi.displayName ?? "<display name unavailable>";
+ endpoint = rsi.endpoint ?? throw new ApplicationException( "The device has no endpoint ID" );
+ }
+
+ /// <summary>Returns a String which represents the object instance</summary>
+ public override string ToString() => $"Capture device: \"{displayName}\"";
+ }
+} \ No newline at end of file
diff --git a/WhisperNet/API/Parameters.cs b/WhisperNet/API/Parameters.cs
new file mode 100644
index 0000000..d2b53f9
--- /dev/null
+++ b/WhisperNet/API/Parameters.cs
@@ -0,0 +1,95 @@
+// Missing XML comment for publicly visible type or member
+// TODO: remove this line and document them.
+#pragma warning disable CS1591
+
+namespace Whisper
+{
+ /// <summary>Available sampling strategies</summary>
+ public enum eSamplingStrategy: int
+ {
+ /// <summary>Always select the most probable token</summary>
+ Greedy,
+ /// <summary>TODO: not implemented yet!</summary>
+ BeamSearch,
+ };
+
+ [Flags]
+ public enum eFullParamsFlags: uint
+ {
+ None = 0,
+ Translate = 1,
+ NoContext = 2,
+ SingleSegment = 4,
+ PrintSpecial = 8,
+ PrintProgress = 0x10,
+ PrintRealtime = 0x20,
+ PrintTimestamps = 0x40,
+
+ // Experimental
+ TokenTimestamps = 0x100,
+ SpeedupAudio = 0x200,
+ };
+
+ /// <summary>Transcribe parameters</summary>
+ public struct Parameters
+ {
+ /// <summary>Sampling strategy</summary>
+ public eSamplingStrategy strategy;
+
+ /// <summary>Count of CPU worker threads to use</summary>
+ /// <remarks>So far, the GPU model only uses CPU threads for MEL spectrograms</remarks>
+ public int cpuThreads;
+
+ public int n_max_text_ctx;
+ /// <summary>start offset in ms</summary>
+ public int offset_ms;
+ /// <summary>audio duration to process in ms</summary>
+ public int duration_ms;
+ public eFullParamsFlags flags;
+
+ /// <summary>Set or clear the specified flag in the <see cref="flags" /> field of this structure</summary>
+ public void setFlag( eFullParamsFlags flag, bool set )
+ {
+ if( flag != eFullParamsFlags.None )
+ {
+ if( set )
+ flags |= flag;
+ else
+ flags &= ~flag;
+ return;
+ }
+ throw new ArgumentException();
+ }
+
+ /// <summary>Language</summary>
+ public eLanguage language;
+
+ // [EXPERIMENTAL] token-level timestamps
+ /// <summary>timestamp token probability threshold (~0.01)</summary>
+ public float thold_pt;
+ /// <summary>timestamp token sum probability threshold (~0.01)</summary>
+ public float thold_ptsum;
+ /// <summary>max segment length in characters</summary>
+ public int max_len;
+ /// <summary>max tokens per segment (0 = no limit)</summary>
+ public int max_tokens;
+
+ public struct sGreedy
+ {
+ public int n_past;
+ }
+ public sGreedy greedy;
+
+ public struct sBeamSearch
+ {
+ public int n_past;
+ public int beam_width;
+ public int n_best;
+ }
+ public sBeamSearch beamSearch;
+
+ // [EXPERIMENTAL] speed-up techniques
+ /// <summary>overwrite the audio context size (0 = use default)</summary>
+ public int audioContextSize;
+ }
+} \ No newline at end of file
diff --git a/WhisperNet/API/SpecialTokens.cs b/WhisperNet/API/SpecialTokens.cs
new file mode 100644
index 0000000..d672369
--- /dev/null
+++ b/WhisperNet/API/SpecialTokens.cs
@@ -0,0 +1,23 @@
+namespace Whisper
+{
+ /// <summary>Special tokens defined in the model</summary>
+ public readonly struct SpecialTokens
+ {
+ /// <summary>The end of a transcription</summary>
+ public readonly int TranscriptionEnd; // token_eot
+ /// <summary>Start of a transcription</summary>
+ public readonly int TranscriptionStart; // token_sot
+ /// <summary>Represents the previous word in the transcription. It is used to help the model predict the current word based on the context of the words that came before it.</summary>
+ public readonly int PreviousWord; // token_prev
+ /// <summary>Start of a sentence</summary>
+ public readonly int SentenceStart; // token_solm
+ /// <summary>Represents the word "not" in the transcription</summary>
+ public readonly int Not; // token_not
+ /// <summary>New transcription</summary>
+ public readonly int TranscriptionBegin; // token_beg
+ /// <summary>token_translate</summary>
+ public readonly int TaskTranslate;
+ /// <summary>token_transcribe</summary>
+ public readonly int TaskTranscribe;
+ }
+} \ No newline at end of file
diff --git a/WhisperNet/API/eCaptureStatus.cs b/WhisperNet/API/eCaptureStatus.cs
new file mode 100644
index 0000000..41f05fb
--- /dev/null
+++ b/WhisperNet/API/eCaptureStatus.cs
@@ -0,0 +1,19 @@
+namespace Whisper
+{
+ /// <summary>Status of the voice capture</summary>
+ [Flags]
+ public enum eCaptureStatus: byte
+ {
+ /// <summary>Doing nothing</summary>
+ None = 0,
+ /// <summary>Capturing the audio</summary>
+ Listening = 1,
+ /// <summary>A voice is detected in the captured audio, recording</summary>
+ Voice = 2,
+ /// <summary>Transcribing a recorded piece of the audio</summary>
+ Transcribing = 4,
+ /// <summary>The computer is unable to transcribe the audio quickly enough,<br/>
+ /// and the capture is dropping the incoming audio samples.</summary>
+ Stalled = 0x80,
+ }
+} \ No newline at end of file
diff --git a/WhisperNet/API/eLanguage.cs b/WhisperNet/API/eLanguage.cs
new file mode 100644
index 0000000..1241077
--- /dev/null
+++ b/WhisperNet/API/eLanguage.cs
@@ -0,0 +1,206 @@
+// This file is generated by a tool, from the `languageCodez.tsv` file in this repository
+namespace Whisper
+{
+ /// <summary>Supported languages</summary>
+ public enum eLanguage: uint
+ {
+ /// <summary>Afrikaans</summary>
+ Afrikaans = 0x6661,
+ /// <summary>Albanian</summary>
+ Albanian = 0x7173,
+ /// <summary>Amharic</summary>
+ Amharic = 0x6D61,
+ /// <summary>Arabic</summary>
+ Arabic = 0x7261,
+ /// <summary>Armenian</summary>
+ Armenian = 0x7968,
+ /// <summary>Assamese</summary>
+ Assamese = 0x7361,
+ /// <summary>Azerbaijani</summary>
+ Azerbaijani = 0x7A61,
+ /// <summary>Bashkir</summary>
+ Bashkir = 0x6162,
+ /// <summary>Basque</summary>
+ Basque = 0x7565,
+ /// <summary>Belarusian</summary>
+ Belarusian = 0x6562,
+ /// <summary>Bengali</summary>
+ Bengali = 0x6E62,
+ /// <summary>Bosnian</summary>
+ Bosnian = 0x7362,
+ /// <summary>Breton</summary>
+ Breton = 0x7262,
+ /// <summary>Bulgarian</summary>
+ Bulgarian = 0x6762,
+ /// <summary>Catalan</summary>
+ Catalan = 0x6163,
+ /// <summary>Chinese</summary>
+ Chinese = 0x687A,
+ /// <summary>Croatian</summary>
+ Croatian = 0x7268,
+ /// <summary>Czech</summary>
+ Czech = 0x7363,
+ /// <summary>Danish</summary>
+ Danish = 0x6164,
+ /// <summary>Dutch</summary>
+ Dutch = 0x6C6E,
+ /// <summary>English</summary>
+ English = 0x6E65,
+ /// <summary>Estonian</summary>
+ Estonian = 0x7465,
+ /// <summary>Faroese</summary>
+ Faroese = 0x6F66,
+ /// <summary>Finnish</summary>
+ Finnish = 0x6966,
+ /// <summary>French</summary>
+ French = 0x7266,
+ /// <summary>Galician</summary>
+ Galician = 0x6C67,
+ /// <summary>Georgian</summary>
+ Georgian = 0x616B,
+ /// <summary>German</summary>
+ German = 0x6564,
+ /// <summary>Greek</summary>
+ Greek = 0x6C65,
+ /// <summary>Gujarati</summary>
+ Gujarati = 0x7567,
+ /// <summary>Haitian Creole</summary>
+ HaitianCreole = 0x7468,
+ /// <summary>Hausa</summary>
+ Hausa = 0x6168,
+ /// <summary>Hawaiian</summary>
+ Hawaiian = 0x776168,
+ /// <summary>Hebrew</summary>
+ Hebrew = 0x7769,
+ /// <summary>Hindi</summary>
+ Hindi = 0x6968,
+ /// <summary>Hungarian</summary>
+ Hungarian = 0x7568,
+ /// <summary>Icelandic</summary>
+ Icelandic = 0x7369,
+ /// <summary>Indonesian</summary>
+ Indonesian = 0x6469,
+ /// <summary>Italian</summary>
+ Italian = 0x7469,
+ /// <summary>Japanese</summary>
+ Japanese = 0x616A,
+ /// <summary>Javanese</summary>
+ Javanese = 0x776A,
+ /// <summary>Kannada</summary>
+ Kannada = 0x6E6B,
+ /// <summary>Kazakh</summary>
+ Kazakh = 0x6B6B,
+ /// <summary>Khmer</summary>
+ Khmer = 0x6D6B,
+ /// <summary>Korean</summary>
+ Korean = 0x6F6B,
+ /// <summary>Lao</summary>
+ Lao = 0x6F6C,
+ /// <summary>Latin</summary>
+ Latin = 0x616C,
+ /// <summary>Latvian</summary>
+ Latvian = 0x766C,
+ /// <summary>Lingala</summary>
+ Lingala = 0x6E6C,
+ /// <summary>Lithuanian</summary>
+ Lithuanian = 0x746C,
+ /// <summary>Luxembourgish</summary>
+ Luxembourgish = 0x626C,
+ /// <summary>Macedonian</summary>
+ Macedonian = 0x6B6D,
+ /// <summary>Malagasy</summary>
+ Malagasy = 0x676D,
+ /// <summary>Malay</summary>
+ Malay = 0x736D,
+ /// <summary>Malayalam</summary>
+ Malayalam = 0x6C6D,
+ /// <summary>Maltese</summary>
+ Maltese = 0x746D,
+ /// <summary>Maori</summary>
+ Maori = 0x696D,
+ /// <summary>Marathi</summary>
+ Marathi = 0x726D,
+ /// <summary>Mongolian</summary>
+ Mongolian = 0x6E6D,
+ /// <summary>Myanmar</summary>
+ Myanmar = 0x796D,
+ /// <summary>Nepali</summary>
+ Nepali = 0x656E,
+ /// <summary>Norwegian</summary>
+ Norwegian = 0x6F6E,
+ /// <summary>Nynorsk</summary>
+ Nynorsk = 0x6E6E,
+ /// <summary>Occitan</summary>
+ Occitan = 0x636F,
+ /// <summary>Pashto</summary>
+ Pashto = 0x7370,
+ /// <summary>Persian</summary>
+ Persian = 0x6166,
+ /// <summary>Polish</summary>
+ Polish = 0x6C70,
+ /// <summary>Portuguese</summary>
+ Portuguese = 0x7470,
+ /// <summary>Punjabi</summary>
+ Punjabi = 0x6170,
+ /// <summary>Romanian</summary>
+ Romanian = 0x6F72,
+ /// <summary>Russian</summary>
+ Russian = 0x7572,
+ /// <summary>Sanskrit</summary>
+ Sanskrit = 0x6173,
+ /// <summary>Serbian</summary>
+ Serbian = 0x7273,
+ /// <summary>Shona</summary>
+ Shona = 0x6E73,
+ /// <summary>Sindhi</summary>
+ Sindhi = 0x6473,
+ /// <summary>Sinhala</summary>
+ Sinhala = 0x6973,
+ /// <summary>Slovak</summary>
+ Slovak = 0x6B73,
+ /// <summary>Slovenian</summary>
+ Slovenian = 0x6C73,
+ /// <summary>Somali</summary>
+ Somali = 0x6F73,
+ /// <summary>Spanish</summary>
+ Spanish = 0x7365,
+ /// <summary>Sundanese</summary>
+ Sundanese = 0x7573,
+ /// <summary>Swahili</summary>
+ Swahili = 0x7773,
+ /// <summary>Swedish</summary>
+ Swedish = 0x7673,
+ /// <summary>Tagalog</summary>
+ Tagalog = 0x6C74,
+ /// <summary>Tajik</summary>
+ Tajik = 0x6774,
+ /// <summary>Tamil</summary>
+ Tamil = 0x6174,
+ /// <summary>Tatar</summary>
+ Tatar = 0x7474,
+ /// <summary>Telugu</summary>
+ Telugu = 0x6574,
+ /// <summary>Thai</summary>
+ Thai = 0x6874,
+ /// <summary>Tibetan</summary>
+ Tibetan = 0x6F62,
+ /// <summary>Turkish</summary>
+ Turkish = 0x7274,
+ /// <summary>Turkmen</summary>
+ Turkmen = 0x6B74,
+ /// <summary>Ukrainian</summary>
+ Ukrainian = 0x6B75,
+ /// <summary>Urdu</summary>
+ Urdu = 0x7275,
+ /// <summary>Uzbek</summary>
+ Uzbek = 0x7A75,
+ /// <summary>Vietnamese</summary>
+ Vietnamese = 0x6976,
+ /// <summary>Welsh</summary>
+ Welsh = 0x7963,
+ /// <summary>Yiddish</summary>
+ Yiddish = 0x6979,
+ /// <summary>Yoruba</summary>
+ Yoruba = 0x6F79,
+ }
+} \ No newline at end of file
diff --git a/WhisperNet/API/eLogLevel.cs b/WhisperNet/API/eLogLevel.cs
new file mode 100644
index 0000000..ae494d4
--- /dev/null
+++ b/WhisperNet/API/eLogLevel.cs
@@ -0,0 +1,34 @@
+namespace Whisper
+{
+ /// <summary>Message log level</summary>
+ public enum eLogLevel: byte
+ {
+ /// <summary>Error message</summary>
+ Error = 0,
+ /// <summary>Warning message</summary>
+ Warning = 1,
+ /// <summary>Informational message</summary>
+ Info = 2,
+ /// <summary>Debug message</summary>
+ Debug = 3
+ }
+
+ /// <summary>A delegate to receive log messages from the library</summary>
+ public delegate void pfnLogMessage( eLogLevel level, string message );
+
+ /// <summary>Log destination flags</summary>
+ [Flags]
+ public enum eLoggerFlags: byte
+ {
+ /// <summary>No special flags</summary>
+ None = 0,
+
+ /// <summary>In addition to calling the delegate, print messaged to standard error</summary>
+ UseStandardError = 1,
+
+ /// <summary>Don’t format error codes into messages</summary>
+ /// <remarks>It’s recommended to use this flag in .NET.<br/>
+ /// The standard library already formats these messages automatically, as needed.</remarks>
+ SkipFormatMessage = 2,
+ }
+} \ No newline at end of file
diff --git a/WhisperNet/API/eModelImplementation.cs b/WhisperNet/API/eModelImplementation.cs
new file mode 100644
index 0000000..1b0a079
--- /dev/null
+++ b/WhisperNet/API/eModelImplementation.cs
@@ -0,0 +1,25 @@
+namespace Whisper
+{
+ /// <summary>Implementation value for the <see cref="Library.loadModel(string, eModelImplementation)" /> factory function</summary>
+ public enum eModelImplementation: uint
+ {
+ /// <summary>GPGPU implementation based on Direct3D 11.0 compute shaders</summary>
+ GPU = 1,
+
+ /// <summary>A hybrid implementation which uses DirectCompute for encode, and decodes on CPU</summary>
+ /// <remarks>
+ /// <para>The build of the native DLL included into this nuget package doesn’t implement this version.<br/>
+ /// To enable, edit <c>stdafx.h</c> in Whisper project, change the value of <c>BUILD_HYBRID_VERSION</c> macro from zero to one, and build.</para>
+ /// <para>This implementation requires a CPU with AVX1, FMA3, F16C and BMI1 instruction set extensions.</para>
+ /// </remarks>
+ Hybrid = 2,
+
+ /// <summary>A reference implementation which uses the original GGML CPU-running code.</summary>
+ /// <remarks>
+ /// <para>The build of the native DLL included into this nuget package doesn’t implement this version either.<br/>
+ /// To enable, edit <c>stdafx.h</c> in Whisper project, change the value of <c>BUILD_BOTH_VERSIONS</c> macro from zero to one, and build the project.</para>
+ /// <para>This implementation requires a CPU with AVX1, FMA3, and F16C instruction set extensions.</para>
+ /// </remarks>
+ Reference = 3,
+ }
+} \ No newline at end of file
diff --git a/WhisperNet/API/eResultFlags.cs b/WhisperNet/API/eResultFlags.cs
new file mode 100644
index 0000000..1de61ab
--- /dev/null
+++ b/WhisperNet/API/eResultFlags.cs
@@ -0,0 +1,21 @@
+namespace Whisper
+{
+ /// <summary>Flags for <see cref="Context.results(eResultFlags)" /> method</summary>
+ [Flags]
+ public enum eResultFlags: uint
+ {
+ /// <summary>No flags</summary>
+ None = 0,
+
+ /// <summary>Return individual tokens in addition to the segments</summary>
+ Tokens = 1,
+
+ /// <summary>Return timestamps</summary>
+ Timestamps = 2,
+
+ /// <summary>Create a new COM object for the results.</summary>
+ /// <remarks>Without this flag, the context returns a pointer to the COM object stored in the context.<br/>
+ /// The content of that object is replaced every time you call <see cref="Internal.iContext.getResults(eResultFlags)" /> method.</remarks>
+ NewObject = 0x100,
+ }
+} \ No newline at end of file
diff --git a/WhisperNet/API/iAudioBuffer.cs b/WhisperNet/API/iAudioBuffer.cs
new file mode 100644
index 0000000..1b35621
--- /dev/null
+++ b/WhisperNet/API/iAudioBuffer.cs
@@ -0,0 +1,27 @@
+using ComLight;
+using System.Runtime.InteropServices;
+
+namespace Whisper
+{
+ /// <summary>A buffer with a chunk of audio.</summary>
+ /// <remarks>Note the interface supports both marshaling directions.<br/>
+ /// I have not tested, but you should be able to implement this interface in C#, to supply PCM audio data to the native code</remarks>
+ [ComInterface( "013583aa-c9eb-42bc-83db-633c2c317051", eMarshalDirection.BothWays )]
+ public interface iAudioBuffer: IDisposable
+ {
+ /// <summary>Count of samples in the buffer</summary>
+ int countSamples();
+
+ /// <summary>Unmanaged pointer to the internal buffer containing single-channel FP32 samples.</summary>
+ /// <remarks>If you implementing this interface in C# and your audio data is on the managed heap, use <see cref="GCHandle" /> to make sure it doesn't move.<br/>
+ /// Or better yet, move the data to unmanaged buffer allocated with <see cref="Marshal.AllocHGlobal(int)" /> or <see cref="Marshal.AllocCoTaskMem(int)" /> method.</remarks>
+ IntPtr getPcmMono();
+
+ /// <summary>Unmanaged pointer to the internal buffer containing stereo FP32 samples.</summary>
+ /// <remarks>When the buffer doesn’t have stereo data, the method gonna return <see cref="IntPtr.Zero" />.</remarks>
+ IntPtr getPcmStereo();
+
+ /// <summary>Start time of the buffer, relative to the start of the media</summary>
+ void getTime( out TimeSpan time );
+ }
+} \ No newline at end of file
diff --git a/WhisperNet/API/iAudioReader.cs b/WhisperNet/API/iAudioReader.cs
new file mode 100644
index 0000000..68cf916
--- /dev/null
+++ b/WhisperNet/API/iAudioReader.cs
@@ -0,0 +1,23 @@
+using ComLight;
+
+namespace Whisper
+{
+ /// <summary>Audio stream reader object</summary>
+ /// <remarks>The implementation is forward-only, and these objects ain’t reusable.<br/>
+ /// To read a source file multiple time, dispose and re-create the reader.</remarks>
+ [ComInterface( "35b988da-04a6-476a-a193-d8891d5dc390", eMarshalDirection.ToManaged )]
+ public interface iAudioReader: IDisposable
+ {
+ /// <summary>Get duration of the media file</summary>
+ [RetValIndex]
+ TimeSpan getDuration();
+ }
+
+ /// <summary>Audio capture reader object</summary>
+ /// <remarks>This interface has no public methods callable from C#.<br/>
+ /// It’s only here to pass data between different functions implemented in C++.</remarks>
+ [ComInterface( "747752c2-d9fd-40df-8847-583c781bf013", eMarshalDirection.ToManaged )]
+ public interface iAudioCapture: IDisposable
+ {
+ }
+} \ No newline at end of file
diff --git a/WhisperNet/API/iMediaFoundation.cs b/WhisperNet/API/iMediaFoundation.cs
new file mode 100644
index 0000000..535f904
--- /dev/null
+++ b/WhisperNet/API/iMediaFoundation.cs
@@ -0,0 +1,36 @@
+using ComLight;
+using System.Runtime.InteropServices;
+using Whisper.Internal;
+
+namespace Whisper
+{
+ /// <summary>Exposes a small subset of MS Media Foundation framework.</summary>
+ /// <remarks>That framework is a part of Windows OS, since Vista.</remarks>
+ /// <seealso href="https://learn.microsoft.com/en-us/windows/win32/medfound/microsoft-media-foundation-sdk" />
+ [ComInterface( "fb9763a5-d77d-4b6e-aff8-f494813cebd8", eMarshalDirection.ToManaged ), CustomConventions( typeof( NativeLogger ) )]
+ public interface iMediaFoundation: IDisposable
+ {
+ /// <summary>Decode complete audio file into a new memory buffer.</summary>
+ /// <returns>
+ /// Under the hood, the method asks MF to resample and convert audio into the suitable type for the Whisper model.<br/>
+ /// If the path is a video file, the method will decode the first audio track.
+ /// </returns>
+ [RetValIndex( 2 )]
+ iAudioBuffer loadAudioFile( [MarshalAs( UnmanagedType.LPWStr )] string path, [MarshalAs( UnmanagedType.U1 )] bool stereo = false );
+
+ /// <summary>Create a reader to stream the audio file from disk</summary>
+ /// <returns>
+ /// Under the hood, the method asks MF to resample and convert audio into the suitable type for the Whisper model.<br/>
+ /// If the path is a video file, the method will decode the first audio track.
+ /// </returns>
+ [RetValIndex( 2 )]
+ iAudioReader openAudioFile( [MarshalAs( UnmanagedType.LPWStr )] string path, [MarshalAs( UnmanagedType.U1 )] bool stereo = false );
+
+ /// <summary>List capture devices</summary>
+ void listCaptureDevices( [MarshalAs( UnmanagedType.FunctionPtr )] pfnFoundCaptureDevices pfn, IntPtr pv );
+
+ /// <summary>Open audio capture device</summary>
+ [RetValIndex( 2 )]
+ iAudioCapture openCaptureDevice( [MarshalAs( UnmanagedType.LPWStr )] string endpoint, [In] ref sCaptureParams captureParams );
+ }
+} \ No newline at end of file
diff --git a/WhisperNet/API/iModel.cs b/WhisperNet/API/iModel.cs
new file mode 100644
index 0000000..8ec6d17
--- /dev/null
+++ b/WhisperNet/API/iModel.cs
@@ -0,0 +1,27 @@
+using ComLight;
+using System.ComponentModel;
+
+namespace Whisper
+{
+ /// <summary>A model in VRAM, loaded from GGML file.</summary>
+ /// <remarks>This objetc doesn't keep any mutable state, and can be safely used from multiple threads concurrently</remarks>
+ [ComInterface( "abefb4c9-e8d8-46a3-8747-5afbadef1adb", eMarshalDirection.ToManaged ), CustomConventions( typeof( Internal.NativeLogger ) )]
+ public interface iModel: IDisposable
+ {
+ /// <summary>Create a context to transcribe audio with this model</summary>
+ /// <remarks>Don't call this method, use <see cref="ExtensionMethods.createContext(iModel)" /> instead.</remarks>
+ [RetValIndex, EditorBrowsable( EditorBrowsableState.Never )]
+ Internal.iContext createContextInternal();
+
+ /// <summary>True if this model is multi-lingual</summary>
+ bool isMultilingual();
+
+ /// <summary>Retrieve integer IDs of the special tokens defined by the model</summary>
+ [RetValIndex]
+ SpecialTokens getSpecialTokens();
+
+ /// <summary>Try to resolve integer token ID into string.</summary>
+ /// <remarks>Don't call this method, use <see cref="ExtensionMethods.stringFromToken(iModel, int)" /> instead.</remarks>
+ IntPtr stringFromTokenInternal( int id );
+ }
+} \ No newline at end of file
diff --git a/WhisperNet/API/sCaptureParams.cs b/WhisperNet/API/sCaptureParams.cs
new file mode 100644
index 0000000..7595a69
--- /dev/null
+++ b/WhisperNet/API/sCaptureParams.cs
@@ -0,0 +1,37 @@
+namespace Whisper
+{
+ /// <summary>Flags for the audio capture</summary>
+ [Flags]
+ public enum eCaptureFlags: uint
+ {
+ /// <summary>No special flags</summary>
+ None = 0,
+ /// <summary>When the capture device supports stereo, keep stereo PCM samples in addition to mono</summary>
+ Stereo = 1,
+ }
+
+ /// <summary>Parameters for audio capture</summary>
+ public struct sCaptureParams
+ {
+ /// <summary>Minimum transcribe duration in seconds</summary>
+ public float minDuration;
+ /// <summary>Maximum transcribe duration in seconds</summary>
+ public float maxDuration;
+ /// <summary></summary>
+ public float dropStartSilence;
+ /// <summary></summary>
+ public float pauseDuration;
+ /// <summary>Flags for the audio capture</summary>
+ public eCaptureFlags flags;
+
+ /// <summary>Initialize the structure with some reasonable default values</summary>
+ public sCaptureParams()
+ {
+ minDuration = 7.0f; // 7 seconds
+ maxDuration = 11.0f; // 11 seconds
+ dropStartSilence = 0.25f; // 250 ms
+ pauseDuration = 0.333f; // 333 ms
+ flags = eCaptureFlags.None;
+ }
+ }
+} \ No newline at end of file
diff --git a/WhisperNet/Callbacks.cs b/WhisperNet/Callbacks.cs
new file mode 100644
index 0000000..db38718
--- /dev/null
+++ b/WhisperNet/Callbacks.cs
@@ -0,0 +1,44 @@
+using Whisper.Internal;
+
+namespace Whisper
+{
+ /// <summary>Implement this abstract class to receive callbacks from the native code</summary>
+ public abstract class Callbacks
+ {
+ /// <summary>The callback is called before every encoder run.</summary>
+ /// <remarks>If it returns false, the processing is aborted.</remarks>
+ protected virtual bool onEncoderBegin( Context sender ) { return true; }
+
+ /// <summary>This callback is called on each new segment</summary>
+ protected virtual void onNewSegment( Context sender, int countNew ) { }
+
+ const int S_OK = 0;
+ const int S_FALSE = 1;
+ internal int encoderBegin( Context sender )
+ {
+ try
+ {
+ return onEncoderBegin( sender ) ? S_OK : S_FALSE;
+ }
+ catch( Exception ex )
+ {
+ NativeLogger.captureException( ex );
+ return ex.HResult;
+ }
+ }
+
+ internal int newSegment( Context sender, int countNew )
+ {
+ try
+ {
+ onNewSegment( sender, countNew );
+ return S_OK;
+ }
+ catch( Exception ex )
+ {
+ NativeLogger.captureException( ex );
+ return ex.HResult;
+ }
+ }
+ }
+} \ No newline at end of file
diff --git a/WhisperNet/CaptureCallbacks.cs b/WhisperNet/CaptureCallbacks.cs
new file mode 100644
index 0000000..26013f9
--- /dev/null
+++ b/WhisperNet/CaptureCallbacks.cs
@@ -0,0 +1,49 @@
+using Whisper.Internal;
+
+namespace Whisper
+{
+ /// <summary>Implement this abstract class to provide callbacks for audio capture method</summary>
+ public abstract class CaptureCallbacks
+ {
+ /// <summary>Override this method to support cancellation</summary>
+ protected virtual bool shouldCancel( Context sender ) { return false; }
+
+ /// <summary>Override this method to get notified about status changes</summary>
+ protected virtual void captureStatusChanged( Context sender, eCaptureStatus status ) { }
+
+ internal pfnShouldCancel cancel( Context sender )
+ {
+ const int S_OK = 0;
+ const int S_FALSE = 1;
+ return delegate ( IntPtr pv )
+ {
+ try
+ {
+ return shouldCancel( sender ) ? S_OK : S_FALSE;
+ }
+ catch( Exception ex )
+ {
+ NativeLogger.captureException( ex );
+ return ex.HResult;
+ }
+ };
+ }
+
+ internal pfnCaptureStatus status( Context sender )
+ {
+ return delegate ( IntPtr pv, eCaptureStatus status )
+ {
+ try
+ {
+ captureStatusChanged( sender, status );
+ return 0;
+ }
+ catch( Exception ex )
+ {
+ NativeLogger.captureException( ex );
+ return ex.HResult;
+ }
+ };
+ }
+ }
+} \ No newline at end of file
diff --git a/WhisperNet/Context.cs b/WhisperNet/Context.cs
new file mode 100644
index 0000000..6c6a737
--- /dev/null
+++ b/WhisperNet/Context.cs
@@ -0,0 +1,201 @@
+using System.Diagnostics;
+using Whisper.Internal;
+using Whisper.Internals;
+
+namespace Whisper
+{
+ /// <summary>Stateful context, contains methods to transcribe audio</summary>
+ public sealed class Context: IDisposable
+ {
+ iContext context;
+ // Caching the results object here saves time spent in ComLight library creating these callable proxies over and over again for the same underlying C++ object
+ readonly iTranscribeResult transcribeResult;
+ sFullParams fullParams;
+ sProgressSink progressSink;
+ bool disposed = false;
+ readonly Action<object> pfnBuffer, pfnStream;
+
+ internal Context( Internal.iContext context )
+ {
+ this.context = context;
+ transcribeResult = context.getResults( eResultFlags.None );
+ fullParams = context.fullDefaultParams( eSamplingStrategy.Greedy );
+ pfnBuffer = processBuffer;
+ pfnStream = processStream;
+ progressSink = default;
+ }
+
+ void IDisposable.Dispose()
+ {
+ if( disposed )
+ return;
+ disposed = true;
+ context?.Dispose();
+ GC.SuppressFinalize( this );
+ }
+
+ /// <summary>Adjustable parameters</summary>
+ public ref Parameters parameters => ref fullParams.publicParams;
+
+ void processBuffer( object buffer )
+ {
+ context.runFull( ref fullParams, (iAudioBuffer)buffer );
+ }
+ void processStream( object reader )
+ {
+ context.runStreamed( ref fullParams, ref progressSink, (iAudioReader)reader );
+ }
+
+ void runImpl( object source, Callbacks? callbacks, ReadOnlySpan<int> promptTokens, Action<object> pfn )
+ {
+ if( null != callbacks )
+ {
+ // TODO [very low, performance]: the following code creates 2 new GC-allocated objects on each call.
+ // Possible to optimize by caching these function pointers in static readonly fields, and use another [ThreadStatic] field for the callbacks object
+ fullParams.newSegmentCallback = delegate ( IntPtr ctx, int countNew, IntPtr userData )
+ {
+ return callbacks.newSegment( this, countNew );
+ };
+
+ fullParams.encoderBeginCallback = delegate ( IntPtr ctx, IntPtr userData )
+ {
+ return callbacks.encoderBegin( this );
+ };
+ }
+
+ try
+ {
+ if( promptTokens.IsEmpty )
+ {
+ pfn( source );
+ return;
+ }
+ unsafe
+ {
+ fixed( int* tokens = promptTokens )
+ {
+ fullParams.prompt_tokens = (IntPtr)tokens;
+ fullParams.prompt_n_tokens = promptTokens.Length;
+ pfn( source );
+ }
+ }
+ }
+ finally
+ {
+ // Reset these delegates.
+ // Otherwise, this class will retain the callbacks object preventing it from being garbage collected.
+ fullParams.newSegmentCallback = null;
+ fullParams.encoderBeginCallback = null;
+
+ fullParams.prompt_tokens = IntPtr.Zero;
+ fullParams.prompt_n_tokens = 0;
+ }
+ }
+
+ /// <summary>Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text</summary>
+ public void runFull( iAudioBuffer buffer, Callbacks? callbacks, ReadOnlySpan<int> promptTokens )
+ {
+ runImpl( buffer, callbacks, promptTokens, pfnBuffer );
+ }
+ /// <summary>Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text</summary>
+ public void runFull( iAudioBuffer buffer, Callbacks? callbacks = null ) =>
+ runFull( buffer, callbacks, ReadOnlySpan<int>.Empty );
+ /// <summary>Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text</summary>
+ public void runFull( iAudioBuffer buffer, Callbacks? callbacks, int[]? promptTokens ) =>
+ runFull( buffer, callbacks, promptTokens ?? ReadOnlySpan<int>.Empty );
+
+ /// <summary>Run the entire model, streaming audio from the provided reader object</summary>
+ public void runFull( iAudioReader reader, Callbacks? callbacks, Action<double>? pfnProgress, ReadOnlySpan<int> promptTokens )
+ {
+ if( null != pfnProgress )
+ {
+ progressSink.pfn = delegate ( double value, IntPtr context, IntPtr pv )
+ {
+ try
+ {
+ pfnProgress.Invoke( value );
+ return 0;
+ }
+ catch( Exception ex )
+ {
+ return ex.HResult;
+ }
+ };
+ }
+ try
+ {
+ runImpl( reader, callbacks, promptTokens, pfnStream );
+ }
+ finally
+ {
+ progressSink.pfn = null;
+ }
+ }
+
+ /// <summary>Run the entire model, streaming audio from the provided reader object</summary>
+ public void runFull( iAudioReader reader, Action<double>? pfnProgress = null, Callbacks? callbacks = null ) =>
+ runFull( reader, callbacks, pfnProgress, ReadOnlySpan<int>.Empty );
+
+ /// <summary>Run the entire model, streaming audio from the provided reader object</summary>
+ public void runFull( iAudioReader reader, Callbacks? callbacks, Action<double>? pfnProgress, int[]? promptTokens ) =>
+ runFull( reader, callbacks, pfnProgress, promptTokens ?? ReadOnlySpan<int>.Empty );
+
+ /// <summary>Get text results out of the context</summary>
+ public TranscribeResult results( eResultFlags flags = eResultFlags.None )
+ {
+ if( flags.HasFlag( eResultFlags.NewObject ) )
+ throw new ArgumentException();
+
+ iTranscribeResult res = context.getResults( flags );
+ Debug.Assert( ReferenceEquals( res, transcribeResult ) );
+ return new TranscribeResult( res );
+ }
+
+ /// <summary>Print timing data</summary>
+ public void timingsPrint() => context.timingsPrint();
+
+ /// <summary>Reset timing data</summary>
+ public void timingsReset() => context.timingsReset();
+
+ /// <summary>Continuously process audio from microphone or a similar capture device</summary>
+ /// <remarks>It’s recommended to call this method on a background thread.</remarks>
+ public void runCapture( iAudioCapture capture, Callbacks? callbacks, CaptureCallbacks? captureCallbacks )
+ {
+ if( null != callbacks )
+ {
+ // TODO [very low, performance]: the following code creates 2 new GC-allocated objects on each call.
+ // Possible to optimize by caching these function pointers in static readonly fields, and use another [ThreadStatic] field for the callbacks object
+ fullParams.newSegmentCallback = delegate ( IntPtr ctx, int countNew, IntPtr userData )
+ {
+ return callbacks.newSegment( this, countNew );
+ };
+
+ fullParams.encoderBeginCallback = delegate ( IntPtr ctx, IntPtr userData )
+ {
+ return callbacks.encoderBegin( this );
+ };
+ }
+
+ try
+ {
+ sCaptureCallbacks cc = default;
+ if( captureCallbacks != null )
+ {
+ cc.shouldCancel = captureCallbacks.cancel( this );
+ cc.captureStatus = captureCallbacks.status( this );
+ }
+ context.runCapture( ref fullParams, ref cc, capture );
+ }
+ finally
+ {
+ // Reset these delegates.
+ // Otherwise, this class will retain the callbacks object preventing it from being garbage collected.
+ fullParams.newSegmentCallback = null;
+ fullParams.encoderBeginCallback = null;
+
+ fullParams.prompt_tokens = IntPtr.Zero;
+ fullParams.prompt_n_tokens = 0;
+ }
+ }
+ }
+} \ No newline at end of file
diff --git a/WhisperNet/ExtensionMethods.cs b/WhisperNet/ExtensionMethods.cs
new file mode 100644
index 0000000..4380ece
--- /dev/null
+++ b/WhisperNet/ExtensionMethods.cs
@@ -0,0 +1,69 @@
+using System.Runtime.InteropServices;
+using Whisper.Internal;
+
+namespace Whisper
+{
+ /// <summary>Extension methods of these COM interfaces</summary>
+ public static class ExtensionMethods
+ {
+ /// <summary>Create a context to transcribe audio with this model</summary>
+ public static Context createContext( this iModel model )
+ {
+ iContext ctx = model.createContextInternal();
+ return new Context( ctx );
+ }
+
+ /// <summary>Convert language into a short ID string, like <c>"en"</c></summary>
+ public static string getCode( this eLanguage lang )
+ {
+ unsafe
+ {
+ sbyte* ptr = stackalloc sbyte[ 5 ];
+ *(uint*)ptr = (uint)lang;
+ ptr[ 4 ] = 0;
+ return new string( ptr );
+ }
+ }
+
+ /// <summary>Resolve integer token ID into string.</summary>
+ /// <remarks>If the token ID was not found in the model, the method returns null without raising exceptions.</remarks>
+ public static string? stringFromToken( this iModel model, int idToken ) =>
+ Marshal.PtrToStringUTF8( model.stringFromTokenInternal( idToken ) );
+
+ /// <summary>List capture devices</summary>
+ public static CaptureDeviceId[]? listCaptureDevices( this iMediaFoundation mf )
+ {
+ List<CaptureDeviceId>? list = null;
+
+ pfnFoundCaptureDevices pfn = delegate ( int len, sCaptureDevice[]? arr, IntPtr pv )
+ {
+ try
+ {
+ if( len == 0 || arr == null )
+ return 1;
+
+ list = new List<CaptureDeviceId>( len );
+ foreach( var i in arr )
+ list.Add( new CaptureDeviceId( i ) );
+ return 0;
+ }
+ catch( Exception ex )
+ {
+ NativeLogger.captureException( ex );
+ return ex.HResult;
+ }
+ };
+
+ mf.listCaptureDevices( pfn, IntPtr.Zero );
+
+ return list?.ToArray();
+ }
+
+ /// <summary>Open audio capture device</summary>
+ public static iAudioCapture openCaptureDevice( this iMediaFoundation mf, in CaptureDeviceId id, sCaptureParams? cp = null )
+ {
+ sCaptureParams captureParams = cp ?? new sCaptureParams();
+ return mf.openCaptureDevice( id.endpoint, ref captureParams );
+ }
+ }
+} \ No newline at end of file
diff --git a/WhisperNet/Internal/AssemblyInfo.cs b/WhisperNet/Internal/AssemblyInfo.cs
new file mode 100644
index 0000000..29ec638
--- /dev/null
+++ b/WhisperNet/Internal/AssemblyInfo.cs
@@ -0,0 +1,8 @@
+using System.Reflection;
+using System.Runtime.InteropServices;
+[assembly: AssemblyTitle( "WhisperNet" )]
+[assembly: AssemblyCopyright( "Copyright © const.me, 2022" )]
+[assembly: ComVisible( false )]
+[assembly: Guid( "ced6cdb7-e040-4398-bae8-3417e5fa35f1" )]
+[assembly: AssemblyVersion( "1.0.0.0" )]
+[assembly: AssemblyDescription( "DirectCompute port of whisper.cpp library, C# bindings" )] \ No newline at end of file
diff --git a/WhisperNet/Internal/NativeLogger.cs b/WhisperNet/Internal/NativeLogger.cs
new file mode 100644
index 0000000..b4b4eb2
--- /dev/null
+++ b/WhisperNet/Internal/NativeLogger.cs
@@ -0,0 +1,138 @@
+using System.Runtime.CompilerServices;
+using System.Runtime.ExceptionServices;
+using System.Runtime.InteropServices;
+
+namespace Whisper.Internal
+{
+ /// <summary>Utility class to supply logging function pointer to the C++ library,<br/>
+ /// and provide custom calling conventions to ComLight runtime to convert error messages printed in C++ into .NET exception messages</summary>
+ public static class NativeLogger
+ {
+ internal static void startup() { }
+
+ static NativeLogger()
+ {
+ sink = logSink;
+ sLoggerSetup setup = default;
+ setup.sink = sink;
+ setup.level = eLogLevel.Warning;
+ Library.setupLogger( ref setup );
+ }
+
+ internal static void setup( eLogLevel lvl, eLoggerFlags flags, pfnLogMessage? pfn )
+ {
+ logMessage = pfn;
+
+ sLoggerSetup setup = default;
+ setup.sink = sink;
+ setup.level = lvl;
+ setup.flags = flags;
+ Library.setupLogger( ref setup );
+ }
+
+ // This field is here to protect the function pointer from being collected by the GC
+ static readonly pfnLoggerSink sink;
+
+ static void logSink( IntPtr context, eLogLevel lvl, string message )
+ {
+ if( lvl == eLogLevel.Error )
+ state.setText( message );
+ logMessage?.Invoke( lvl, message );
+ }
+
+ sealed class ThreadState
+ {
+ string? errorText = null;
+ ExceptionDispatchInfo? dispatchInfo = null;
+
+ public void setText( string text ) => errorText = text;
+ public void capture( Exception ex ) => dispatchInfo = ExceptionDispatchInfo.Capture( ex );
+
+ public void clear()
+ {
+ errorText = null;
+ dispatchInfo = null;
+ }
+
+ public void Deconstruct( out string? text, out ExceptionDispatchInfo? edi )
+ {
+ text = errorText;
+ edi = dispatchInfo;
+ errorText = null;
+ dispatchInfo = null;
+ }
+ }
+
+ [ThreadStatic]
+ static ThreadState state = new ThreadState();
+
+ internal static void captureException( Exception ex ) =>
+ state.capture( ex );
+
+ static pfnLogMessage? logMessage = null;
+
+ /// <summary>Called internally by ComLight runtime</summary>
+ [MethodImpl( MethodImplOptions.AggressiveInlining )]
+ public static void prologue()
+ {
+ // https://stackoverflow.com/a/2043505/126995
+ if( null != state )
+ state.clear();
+ else
+ createState();
+ }
+
+ [MethodImpl( MethodImplOptions.NoInlining )]
+ static void createState()
+ {
+ state = new ThreadState();
+ }
+
+ /// <summary>Epilogue implementation for unsuccessful status codes</summary>
+ [MethodImpl( MethodImplOptions.NoInlining )]
+ static void throwException( int hr )
+ {
+ // Move state from the thread local object into local variables, and clear that object
+ (string? text, ExceptionDispatchInfo? edi) = state;
+
+ if( null != edi && edi.SourceException.HResult == hr )
+ {
+ // The error comes from a callback, and we have original context of that exception.
+ // Re-throw the original exception.
+ // This uses the original error message, and even correctly deals with the stack trace.
+ edi.Throw();
+ }
+
+ if( null != text )
+ {
+ // C++ code has printed an error on the current thread, between prologue and epilogue.
+ // Use that text for the exception message.
+ Exception? ex = Marshal.GetExceptionForHR( hr );
+ throw new ApplicationException( text, ex );
+ }
+
+ // We don’t have any additional info about the exception.
+ // Throw an exception from just the HRESULT code.
+ Marshal.ThrowExceptionForHR( hr );
+ }
+
+ /// <summary>Called internally by ComLight runtime</summary>
+ [MethodImpl( MethodImplOptions.AggressiveInlining )]
+ public static void throwForHR( int hr )
+ {
+ if( hr >= 0 )
+ return; // SUCCEEDED
+ throwException( hr );
+ }
+
+ /// <summary>Called internally by ComLight runtime</summary>
+ [MethodImpl( MethodImplOptions.AggressiveInlining )]
+ public static bool throwAndReturnBool( int hr )
+ {
+ if( hr >= 0 )
+ return 0 == hr;
+ throwException( hr );
+ return false;
+ }
+ }
+} \ No newline at end of file
diff --git a/WhisperNet/Internal/iContext.cs b/WhisperNet/Internal/iContext.cs
new file mode 100644
index 0000000..6adf8c5
--- /dev/null
+++ b/WhisperNet/Internal/iContext.cs
@@ -0,0 +1,37 @@
+using ComLight;
+using System.Runtime.InteropServices;
+using Whisper.Internals;
+
+namespace Whisper.Internal
+{
+ /// <summary>Stateful context, contains methods to transcribe audio</summary>
+ [ComInterface( "b9956374-3b18-4943-90f2-2ab18a404537", eMarshalDirection.ToManaged ), CustomConventions( typeof( NativeLogger ) )]
+ public interface iContext: IDisposable
+ {
+ /// <summary>Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text</summary>
+ void runFull( [In] ref sFullParams @params, iAudioBuffer buffer );
+
+ /// <summary>Run the entire model, streaming audio from the provided reader object</summary>
+ void runStreamed( [In] ref sFullParams @params, [In] ref sProgressSink progressSink, iAudioReader reader );
+
+ /// <summary>Continuously process audio from microphone or a similar capture device</summary>
+ void runCapture( [In] ref sFullParams @params, [In] ref sCaptureCallbacks callbacks, iAudioCapture reader );
+
+ /// <summary>Get text results out of the context</summary>
+ [RetValIndex( 1 )]
+ iTranscribeResult getResults( eResultFlags flags );
+
+ /// <summary>Get the model which was used to create this context</summary>
+ [RetValIndex]
+ iModel getModel();
+
+ /// <summary>Full the default parameters of the model, for the specified sampling strategy</summary>
+ [RetValIndex( 1 )]
+ sFullParams fullDefaultParams( eSamplingStrategy strategy );
+
+ /// <summary>Print timing data</summary>
+ void timingsPrint();
+ /// <summary>Reset timing data</summary>
+ void timingsReset();
+ }
+} \ No newline at end of file
diff --git a/WhisperNet/Internal/iTranscribeResult.cs b/WhisperNet/Internal/iTranscribeResult.cs
new file mode 100644
index 0000000..cbf49dd
--- /dev/null
+++ b/WhisperNet/Internal/iTranscribeResult.cs
@@ -0,0 +1,122 @@
+#pragma warning disable CS0649 // Field is never assigned to
+using ComLight;
+using System.ComponentModel;
+using System.Runtime.InteropServices;
+
+namespace Whisper.Internal
+{
+ /// <summary>Size of the buffers owned by the <see cref="iTranscribeResult" /> object</summary>
+ public readonly struct sTranscribeLength
+ {
+ /// <summary>Count of segments</summary>
+ public readonly int countSegments;
+ /// <summary>Total count of tokens, for all segments combined</summary>
+ public readonly int countTokens;
+ }
+
+ /// <summary>Output data from the model</summary>
+ [ComInterface( "2871a73f-5ce3-48f8-8779-6582ee11935e", eMarshalDirection.ToManaged ), CustomConventions( typeof( NativeLogger ) )]
+ public interface iTranscribeResult
+ {
+ /// <summary>Get size of the buffers</summary>
+ [RetValIndex, EditorBrowsable( EditorBrowsableState.Never )]
+ public sTranscribeLength getSize();
+
+ /// <summary>Pointer to segment data, a vector of <see cref="sSegment" /> structures</summary>
+ [EditorBrowsable( EditorBrowsableState.Never )]
+ public IntPtr getSegments();
+
+ /// <summary>Pointer to tokens data, a vector of <see cref="sToken" /> structures</summary>
+ [EditorBrowsable( EditorBrowsableState.Never )]
+ public IntPtr getTokens();
+ }
+}
+
+namespace Whisper
+{
+ /// <summary>Start and end times of a segment or token</summary>
+ /// <remarks>The times are relative to the start of the media</remarks>
+ public readonly struct sTimeInterval
+ {
+ /// <summary>Start time</summary>
+ public readonly TimeSpan begin;
+ /// <summary>End time</summary>
+ public readonly TimeSpan end;
+ }
+
+ /// <summary>Segment data</summary>
+ public readonly struct sSegment
+ {
+ internal readonly IntPtr m_text;
+ /// <summary>Segment text</summary>
+ public string? text => Marshal.PtrToStringUTF8( m_text );
+ /// <summary>Start and end times of the segment</summary>
+ public readonly sTimeInterval time;
+ /// <summary>Slice of the tokens</summary>
+ public readonly int firstToken, countTokens;
+ }
+
+ /// <summary>Token flags</summary>
+ [Flags]
+ public enum eTokenFlags: uint
+ {
+ /// <summary>The token is special</summary>
+ Special = 1,
+ }
+
+ /// <summary>Token data</summary>
+ public readonly struct sToken
+ {
+ internal readonly IntPtr m_text;
+ /// <summary>Token text</summary>
+ public string? text => Marshal.PtrToStringUTF8( m_text );
+ /// <summary>Start and end times of the token</summary>
+ public readonly sTimeInterval time;
+ /// <summary>Probability of the token</summary>
+ public readonly float probability;
+ /// <summary>Probability of the timestamp token</summary>
+ public readonly float probabilityTimestamp;
+ /// <summary>Sum of probabilities of all timestamp tokens</summary>
+ public readonly float ptsum;
+ /// <summary>Voice length of the token</summary>
+ public readonly float vlen;
+ /// <summary>Token id</summary>
+ public readonly int id;
+ /// <summary>Token flags</summary>
+ readonly eTokenFlags flags;
+ /// <summary>True if the token flags has the specified bit set</summary>
+ public bool hasFlag( eTokenFlags bit ) => flags.HasFlag( bit );
+ }
+
+ /// <summary>Output data from the model</summary>
+ public readonly ref struct TranscribeResult
+ {
+ /// <summary>Segments in the results</summary>
+ public readonly ReadOnlySpan<sSegment> segments;
+ /// <summary>Tokens in the results, for all segments</summary>
+ public readonly ReadOnlySpan<sToken> tokens;
+
+ internal TranscribeResult( Internal.iTranscribeResult i )
+ {
+ Internal.sTranscribeLength len = i.getSize();
+ unsafe
+ {
+ // This does not copy the buffers to managed memory.
+ // Instead, the C# spans directly reference the native memory stored in these std::vectors
+ if( len.countSegments > 0 )
+ segments = new ReadOnlySpan<sSegment>( (void*)i.getSegments(), len.countSegments );
+ else
+ segments = ReadOnlySpan<sSegment>.Empty;
+
+ if( len.countTokens > 0 )
+ tokens = new ReadOnlySpan<sToken>( (void*)i.getTokens(), len.countTokens );
+ else
+ tokens = ReadOnlySpan<sToken>.Empty;
+ }
+ }
+
+ /// <summary>Get tokens for the specified segment</summary>
+ public ReadOnlySpan<sToken> getTokens( in sSegment seg ) =>
+ tokens.Slice( seg.firstToken, seg.countTokens );
+ }
+} \ No newline at end of file
diff --git a/WhisperNet/Internal/sCaptureCallbacks.cs b/WhisperNet/Internal/sCaptureCallbacks.cs
new file mode 100644
index 0000000..483c2f2
--- /dev/null
+++ b/WhisperNet/Internal/sCaptureCallbacks.cs
@@ -0,0 +1,23 @@
+using System.Runtime.InteropServices;
+
+namespace Whisper.Internal
+{
+ /// <summary>Unmanaged code calls this to check for cancellation</summary>
+ [UnmanagedFunctionPointer( CallingConvention.StdCall )]
+ public delegate int pfnShouldCancel( IntPtr pv );
+
+ /// <summary>Unmanaged code calls this to notify about the status</summary>
+ [UnmanagedFunctionPointer( CallingConvention.StdCall )]
+ public delegate int pfnCaptureStatus( IntPtr pv, eCaptureStatus status );
+
+ /// <summary>Capture callbacks for unmanaged code</summary>
+ public struct sCaptureCallbacks
+ {
+ /// <summary>Cancellation function pointer</summary>
+ public pfnShouldCancel shouldCancel;
+ /// <summary>Capture status function pointer</summary>
+ public pfnCaptureStatus captureStatus;
+ /// <summary>COntext pointer, only needed for C++ compatibility</summary>
+ public IntPtr pv;
+ }
+} \ No newline at end of file
diff --git a/WhisperNet/Internal/sCaptureDevice.cs b/WhisperNet/Internal/sCaptureDevice.cs
new file mode 100644
index 0000000..e2d524d
--- /dev/null
+++ b/WhisperNet/Internal/sCaptureDevice.cs
@@ -0,0 +1,22 @@
+#pragma warning disable CS0649 // Field is never assigned to
+using System.Runtime.InteropServices;
+
+namespace Whisper.Internal
+{
+ /// <summary>Identifiers for an audio capture device</summary>
+ public struct sCaptureDevice
+ {
+ readonly IntPtr m_displayName;
+ /// <summary>The display name is suitable for showing to the user, but might not be unique.</summary>
+ public string? displayName => Marshal.PtrToStringUni( m_displayName );
+
+ readonly IntPtr m_endpoint;
+ /// <summary>Endpoint ID for an audio capture device.<br/>
+ /// It uniquely identifies the device on the system, but is not a readable string.</summary>
+ public string? endpoint => Marshal.PtrToStringUni( m_endpoint );
+ }
+
+ /// <summary>Function pointer to consume a list of audio capture device IDs</summary>
+ [UnmanagedFunctionPointer( CallingConvention.StdCall )]
+ public delegate int pfnFoundCaptureDevices( int len, [In, MarshalAs( UnmanagedType.LPArray, SizeParamIndex = 0 )] sCaptureDevice[]? arr, IntPtr pv );
+} \ No newline at end of file
diff --git a/WhisperNet/Internal/sFullParams.cs b/WhisperNet/Internal/sFullParams.cs
new file mode 100644
index 0000000..7347afe
--- /dev/null
+++ b/WhisperNet/Internal/sFullParams.cs
@@ -0,0 +1,40 @@
+#pragma warning disable CS0649 // Field is never assigned to
+
+// Missing XML comment for publicly visible type or member
+// TODO: remove this line and document them.
+#pragma warning disable CS1591
+
+using System.Runtime.InteropServices;
+
+namespace Whisper.Internals
+{
+ /// <summary>This callback is called on each new segment</summary>
+ [UnmanagedFunctionPointer( CallingConvention.Cdecl )]
+ delegate int pfnNewSegment( IntPtr ctx, int countNew, IntPtr userData );
+
+ /// <summary>The callback is called before every encoder run. If it returns S_FALSE, the processing is aborted.</summary>
+ [UnmanagedFunctionPointer( CallingConvention.Cdecl )]
+ delegate int pfnEncoderBegin( IntPtr ctx, IntPtr userData );
+
+ /// <summary>Transcribe parameters</summary>
+ public struct sFullParams
+ {
+ internal Parameters publicParams;
+ // The rest of these parameters are not exposed to the user-friendly public API of this DLL
+
+ internal IntPtr prompt_tokens;
+ internal int prompt_n_tokens;
+
+ /// <summary>This callback is called on each new segment</summary>
+ [MarshalAs( UnmanagedType.FunctionPtr )]
+ internal pfnNewSegment? newSegmentCallback;
+ /// <summary>Parameter for the above, not needed in C#</summary>
+ internal IntPtr newSegmentCallbackData;
+
+ /// <summary>The callback is called before every encoder run. If it returns false, the processing is aborted</summary>
+ [MarshalAs( UnmanagedType.FunctionPtr )]
+ internal pfnEncoderBegin? encoderBeginCallback;
+ /// <summary>Parameter for the above, not needed in C#</summary>
+ internal IntPtr encoderBeginCallbackData;
+ }
+} \ No newline at end of file
diff --git a/WhisperNet/Internal/sLoadModelCallbacks.cs b/WhisperNet/Internal/sLoadModelCallbacks.cs
new file mode 100644
index 0000000..07f5199
--- /dev/null
+++ b/WhisperNet/Internal/sLoadModelCallbacks.cs
@@ -0,0 +1,64 @@
+using System.Runtime.InteropServices;
+
+namespace Whisper.Internal
+{
+ /// <summary>Function pointer to report model loading progress</summary>
+ [UnmanagedFunctionPointer( CallingConvention.StdCall )]
+ delegate int pfnLoadProgress( double progress, IntPtr pv );
+
+ /// <summary>Function pointer to implement cooperative cancellation</summary>
+ [UnmanagedFunctionPointer( CallingConvention.StdCall )]
+ delegate int pfnCancel( IntPtr pv );
+
+ /// <summary>Callback functions for loading models</summary>
+ public struct sLoadModelCallbacks
+ {
+ /// <summary>Function pointer to report model loading progress</summary>
+ [MarshalAs( UnmanagedType.FunctionPtr )]
+ pfnLoadProgress? progress;
+
+ /// <summary>Function pointer to implement cooperative cancellation</summary>
+ [MarshalAs( UnmanagedType.FunctionPtr )]
+ pfnCancel? cancel;
+
+ // Not needed in C#, delegates can capture things
+ IntPtr pv;
+
+ /// <summary>Wrap idiomatic C# things into these low-level C callbacks</summary>
+ internal sLoadModelCallbacks( CancellationToken cancelToken, Action<double>? pfnProgress )
+ {
+ if( cancelToken != CancellationToken.None )
+ {
+ cancel = delegate ( IntPtr pv )
+ {
+ if( cancelToken.IsCancellationRequested )
+ return 1; // S_FALSE
+ return 0; // S_OK
+ };
+ }
+ else
+ cancel = null;
+
+ if( null != pfnProgress )
+ {
+ progress = delegate ( double val, IntPtr pv )
+ {
+ try
+ {
+ pfnProgress( val );
+ return 0; // S_OK
+ }
+ catch( Exception ex )
+ {
+ NativeLogger.captureException( ex );
+ return ex.HResult;
+ }
+ };
+ }
+ else
+ progress = null;
+
+ pv = IntPtr.Zero;
+ }
+ }
+} \ No newline at end of file
diff --git a/WhisperNet/Internal/sLoggerSetup.cs b/WhisperNet/Internal/sLoggerSetup.cs
new file mode 100644
index 0000000..ed1baa4
--- /dev/null
+++ b/WhisperNet/Internal/sLoggerSetup.cs
@@ -0,0 +1,16 @@
+using System.Runtime.InteropServices;
+
+namespace Whisper.Internal
+{
+ [UnmanagedFunctionPointer( CallingConvention.StdCall )]
+ delegate void pfnLoggerSink( IntPtr context, eLogLevel lvl, [MarshalAs( UnmanagedType.LPUTF8Str )] string message );
+
+ struct sLoggerSetup
+ {
+ [MarshalAs( UnmanagedType.FunctionPtr )]
+ public pfnLoggerSink sink;
+ IntPtr context;
+ public eLogLevel level;
+ public eLoggerFlags flags;
+ }
+} \ No newline at end of file
diff --git a/WhisperNet/Internal/sProgressSink.cs b/WhisperNet/Internal/sProgressSink.cs
new file mode 100644
index 0000000..9155677
--- /dev/null
+++ b/WhisperNet/Internal/sProgressSink.cs
@@ -0,0 +1,20 @@
+#pragma warning disable CS0649 // Field is never assigned to
+using System.Runtime.InteropServices;
+
+namespace Whisper.Internal
+{
+ /// <summary>A callback to get notified about the progress</summary>
+ [UnmanagedFunctionPointer( CallingConvention.StdCall )]
+ delegate int pfnReportProgress( double value, IntPtr context, IntPtr pv );
+
+ /// <summary>C structure with a progress reporting function pointer</summary>
+ public struct sProgressSink
+ {
+ /// <summary>A callback to get notified about the progress</summary>
+ [MarshalAs( UnmanagedType.FunctionPtr )]
+ internal pfnReportProgress? pfn;
+
+ /// <summary>Last parameter to the callback</summary>
+ internal IntPtr pv;
+ }
+} \ No newline at end of file
diff --git a/WhisperNet/Library.cs b/WhisperNet/Library.cs
new file mode 100644
index 0000000..72ecb6e
--- /dev/null
+++ b/WhisperNet/Library.cs
@@ -0,0 +1,110 @@
+using ComLight;
+using System.Runtime.InteropServices;
+using System.Runtime.Intrinsics.X86;
+using Whisper.Internal;
+
+namespace Whisper
+{
+ /// <summary>Factory methods implemented by the C++ DLL</summary>
+ public static class Library
+ {
+ static Library()
+ {
+ if( Environment.OSVersion.Platform != PlatformID.Win32NT )
+ throw new ApplicationException( "This library requires Windows OS" );
+ if( !Environment.Is64BitProcess )
+ throw new ApplicationException( "This library only works in 64-bit processes" );
+ if( RuntimeInformation.ProcessArchitecture != Architecture.X64 )
+ throw new ApplicationException( "This library requires a processor with AMD64 instruction set" );
+ if( !Sse41.IsSupported )
+ throw new ApplicationException( "This library requires a CPU with SSE 4.1 support" );
+ NativeLogger.startup();
+ }
+
+ const string dll = "Whisper.dll";
+
+ [DllImport( dll, CallingConvention = RuntimeClass.defaultCallingConvention, PreserveSig = false )]
+ internal static extern void setupLogger( [In] ref sLoggerSetup setup );
+
+ [DllImport( dll, CallingConvention = RuntimeClass.defaultCallingConvention, PreserveSig = true )]
+ static extern int loadModel( [MarshalAs( UnmanagedType.LPWStr )] string path, eModelImplementation impl,
+ [In] ref sLoadModelCallbacks callbacks,
+ [MarshalAs( UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof( Marshaler<iModel> ) )] out iModel model );
+
+ /// <summary>Load Whisper model from GGML file on disk</summary>
+ /// <remarks>Models are large, depending on user’s disk speed this might take a while, and this function blocks the calling thread.<br/>
+ /// Consider <see cref="loadModelAsync" /> instead.</remarks>
+ /// <seealso href="https://huggingface.co/datasets/ggerganov/whisper.cpp" />
+ public static iModel loadModel( string path, eModelImplementation impl = eModelImplementation.GPU )
+ {
+ iModel model;
+ sLoadModelCallbacks callbacks = default;
+ NativeLogger.prologue();
+ int hr = loadModel( path, impl, ref callbacks, out model );
+ NativeLogger.throwForHR( hr );
+ return model;
+ }
+
+ /// <summary>Load Whisper model on a background thread, with optional progress reporting and cancellation</summary>
+ public static Task<iModel> loadModelAsync( string path, CancellationToken cancelToken, Action<double>? pfnProgress = null, eModelImplementation impl = eModelImplementation.GPU )
+ {
+ TaskCompletionSource<iModel> tcs = new TaskCompletionSource<iModel>();
+
+ WaitCallback wcb = delegate ( object? state )
+ {
+ try
+ {
+ sLoadModelCallbacks callbacks = new sLoadModelCallbacks( cancelToken, pfnProgress );
+
+ iModel model;
+ NativeLogger.prologue();
+ int hr = loadModel( path, impl, ref callbacks, out model );
+ NativeLogger.throwForHR( hr );
+
+ tcs.SetResult( model );
+ }
+ catch( Exception ex )
+ {
+ tcs.SetException( ex );
+ }
+ };
+
+ ThreadPool.QueueUserWorkItem( wcb );
+ return tcs.Task;
+ }
+
+ [DllImport( dll, CallingConvention = RuntimeClass.defaultCallingConvention, PreserveSig = true )]
+ static extern int initMediaFoundation( [MarshalAs( UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof( Marshaler<iMediaFoundation> ) )] out iMediaFoundation mf );
+
+ /// <summary>Initialize Media Foundation runtime</summary>
+ public static iMediaFoundation initMediaFoundation()
+ {
+ iMediaFoundation mf;
+ NativeLogger.prologue();
+ int hr = initMediaFoundation( out mf );
+ NativeLogger.throwForHR( hr );
+ return mf;
+ }
+
+ // The .NET runtime uses UTF-16 for the strings, so we only need the Unicode version of this function.
+ // The native DLL exports both Unicode and ASCII versions.
+ [DllImport( dll, CallingConvention = RuntimeClass.defaultCallingConvention, PreserveSig = true )]
+ static extern uint findLanguageKeyW( [MarshalAs( UnmanagedType.LPWStr )] string lang );
+
+ /// <summary>Try to resolve language code string like <c>"en"</c>, <c>"pl"</c> or <c>"uk"</c> into the strongly-typed enum.</summary>
+ /// <remarks>The function is case-sensitive, <c>"EN"</c> or <c>"UK"</c> gonna fail.</remarks>
+ public static eLanguage? languageFromCode( string lang )
+ {
+ uint key = findLanguageKeyW( lang );
+ if( key != uint.MaxValue )
+ return (eLanguage)key;
+ return null;
+ }
+
+ /// <summary>Set up delegate to receive log messages from the C++ library</summary>
+ public static void setLogSink( eLogLevel lvl, eLoggerFlags flags = eLoggerFlags.SkipFormatMessage, pfnLogMessage? pfn = null )
+ {
+ NativeLogger.setup( lvl, flags, pfn );
+ }
+ }
+} \ No newline at end of file
diff --git a/WhisperNet/Readme.txt b/WhisperNet/Readme.txt
new file mode 100644
index 0000000..50b3bb3
--- /dev/null
+++ b/WhisperNet/Readme.txt
@@ -0,0 +1 @@
+This project builds .NET DLL which wraps Whisper.dll into idiomatic C# API. \ No newline at end of file
diff --git a/WhisperNet/WhisperNet.csproj b/WhisperNet/WhisperNet.csproj
new file mode 100644
index 0000000..105aa44
--- /dev/null
+++ b/WhisperNet/WhisperNet.csproj
@@ -0,0 +1,24 @@
+<Project Sdk="Microsoft.NET.Sdk">
+ <PropertyGroup>
+ <TargetFramework>net6.0-windows</TargetFramework>
+ <ImplicitUsings>enable</ImplicitUsings>
+ <Nullable>enable</Nullable>
+ <CheckForOverflowUnderflow>true</CheckForOverflowUnderflow>
+ <AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
+ <GenerateDocumentationFile>True</GenerateDocumentationFile>
+ <AllowUnsafeBlocks>True</AllowUnsafeBlocks>
+ <RootNamespace>Whisper</RootNamespace>
+ <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
+ <PlatformTarget>x64</PlatformTarget>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)'=='Release'">
+ <GeneratePackageOnBuild>True</GeneratePackageOnBuild>
+ <NuspecFile>WhisperNet.nuspec</NuspecFile>
+ </PropertyGroup>
+ <ItemGroup>
+ <Content Include="..\x64\Release\Whisper.dll" Link="Whisper.dll" />
+ </ItemGroup>
+ <ItemGroup>
+ <PackageReference Include="ComLightInterop" Version="1.3.7" />
+ </ItemGroup>
+</Project> \ No newline at end of file
diff --git a/WhisperNet/WhisperNet.nuspec b/WhisperNet/WhisperNet.nuspec
new file mode 100644
index 0000000..d0a61f7
--- /dev/null
+++ b/WhisperNet/WhisperNet.nuspec
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="utf-8"?>
+<package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd">
+ <metadata>
+ <id>WhisperNet</id>
+ <version>1.0</version>
+ <authors>Konstantin, const.me</authors>
+ <license type="expression">MPL-2.0</license>
+ <projectUrl>https://github.com/Const-me/Whisper</projectUrl>
+ <description>High-performance GPGPU inference of OpenAI's Whisper automatic speech recognition (ASR) model</description>
+ <releaseNotes>Initial public version</releaseNotes>
+ <copyright>Copyright © const.me, 2022-2023</copyright>
+ <tags>whisper, gpgpu, speech recognition</tags>
+ <repository type="git" url="https://github.com/Const-me/Whisper.git" />
+ <dependencies>
+ <group targetFramework="net6.0">
+ <dependency id="ComLightInterop" version="1.3.7" />
+ </group>
+ </dependencies>
+ </metadata>
+ <files>
+ <!-- Managed DLL with XML documentation -->
+ <file src="bin/Release/WhisperNet.dll" target="lib/net6.0/" />
+ <file src="bin/Release/WhisperNet.xml" target="lib/net6.0/" />
+ <!-- The C++ DLL -->
+ <file src="../x64/Release/Whisper.dll" target="runtimes/win-x64/native/" />
+ </files>
+</package> \ No newline at end of file