diff options
| author | Konstantin <const@const.me> | 2023-01-16 14:52:43 +0100 |
|---|---|---|
| committer | Konstantin <const@const.me> | 2023-01-16 14:52:43 +0100 |
| commit | 8c4603c73675958efc960fbd4bb599a2909d106a (patch) | |
| tree | 714dc6fc9a1672d5fd7f89676b97e10959662abc /WhisperNet | |
| parent | 990a8d0dbaefc996244097397259e92758b15cce (diff) | |
Source codes
Diffstat (limited to 'WhisperNet')
31 files changed, 1612 insertions, 0 deletions
diff --git a/WhisperNet/API/CaptureDeviceId.cs b/WhisperNet/API/CaptureDeviceId.cs new file mode 100644 index 0000000..9636e53 --- /dev/null +++ b/WhisperNet/API/CaptureDeviceId.cs @@ -0,0 +1,24 @@ +using Whisper.Internal; + +namespace Whisper +{ + /// <summary>Identifiers for an audio capture device</summary> + public record struct CaptureDeviceId + { + /// <summary>The display name is suitable for showing to the user, but might not be unique.</summary> + public string displayName; + + /// <summary>Endpoint ID for an audio capture device.<br/> + /// It uniquely identifies the device on the system, but is not a readable string.</summary> + public string endpoint; + + internal CaptureDeviceId( in sCaptureDevice rsi ) + { + displayName = rsi.displayName ?? "<display name unavailable>"; + endpoint = rsi.endpoint ?? throw new ApplicationException( "The device has no endpoint ID" ); + } + + /// <summary>Returns a String which represents the object instance</summary> + public override string ToString() => $"Capture device: \"{displayName}\""; + } +}
\ No newline at end of file diff --git a/WhisperNet/API/Parameters.cs b/WhisperNet/API/Parameters.cs new file mode 100644 index 0000000..d2b53f9 --- /dev/null +++ b/WhisperNet/API/Parameters.cs @@ -0,0 +1,95 @@ +// Missing XML comment for publicly visible type or member +// TODO: remove this line and document them. +#pragma warning disable CS1591 + +namespace Whisper +{ + /// <summary>Available sampling strategies</summary> + public enum eSamplingStrategy: int + { + /// <summary>Always select the most probable token</summary> + Greedy, + /// <summary>TODO: not implemented yet!</summary> + BeamSearch, + }; + + [Flags] + public enum eFullParamsFlags: uint + { + None = 0, + Translate = 1, + NoContext = 2, + SingleSegment = 4, + PrintSpecial = 8, + PrintProgress = 0x10, + PrintRealtime = 0x20, + PrintTimestamps = 0x40, + + // Experimental + TokenTimestamps = 0x100, + SpeedupAudio = 0x200, + }; + + /// <summary>Transcribe parameters</summary> + public struct Parameters + { + /// <summary>Sampling strategy</summary> + public eSamplingStrategy strategy; + + /// <summary>Count of CPU worker threads to use</summary> + /// <remarks>So far, the GPU model only uses CPU threads for MEL spectrograms</remarks> + public int cpuThreads; + + public int n_max_text_ctx; + /// <summary>start offset in ms</summary> + public int offset_ms; + /// <summary>audio duration to process in ms</summary> + public int duration_ms; + public eFullParamsFlags flags; + + /// <summary>Set or clear the specified flag in the <see cref="flags" /> field of this structure</summary> + public void setFlag( eFullParamsFlags flag, bool set ) + { + if( flag != eFullParamsFlags.None ) + { + if( set ) + flags |= flag; + else + flags &= ~flag; + return; + } + throw new ArgumentException(); + } + + /// <summary>Language</summary> + public eLanguage language; + + // [EXPERIMENTAL] token-level timestamps + /// <summary>timestamp token probability threshold (~0.01)</summary> + public float thold_pt; + /// <summary>timestamp token sum probability threshold (~0.01)</summary> + public float thold_ptsum; + /// <summary>max segment length in characters</summary> + public int max_len; + /// <summary>max tokens per segment (0 = no limit)</summary> + public int max_tokens; + + public struct sGreedy + { + public int n_past; + } + public sGreedy greedy; + + public struct sBeamSearch + { + public int n_past; + public int beam_width; + public int n_best; + } + public sBeamSearch beamSearch; + + // [EXPERIMENTAL] speed-up techniques + /// <summary>overwrite the audio context size (0 = use default)</summary> + public int audioContextSize; + } +}
\ No newline at end of file diff --git a/WhisperNet/API/SpecialTokens.cs b/WhisperNet/API/SpecialTokens.cs new file mode 100644 index 0000000..d672369 --- /dev/null +++ b/WhisperNet/API/SpecialTokens.cs @@ -0,0 +1,23 @@ +namespace Whisper +{ + /// <summary>Special tokens defined in the model</summary> + public readonly struct SpecialTokens + { + /// <summary>The end of a transcription</summary> + public readonly int TranscriptionEnd; // token_eot + /// <summary>Start of a transcription</summary> + public readonly int TranscriptionStart; // token_sot + /// <summary>Represents the previous word in the transcription. It is used to help the model predict the current word based on the context of the words that came before it.</summary> + public readonly int PreviousWord; // token_prev + /// <summary>Start of a sentence</summary> + public readonly int SentenceStart; // token_solm + /// <summary>Represents the word "not" in the transcription</summary> + public readonly int Not; // token_not + /// <summary>New transcription</summary> + public readonly int TranscriptionBegin; // token_beg + /// <summary>token_translate</summary> + public readonly int TaskTranslate; + /// <summary>token_transcribe</summary> + public readonly int TaskTranscribe; + } +}
\ No newline at end of file diff --git a/WhisperNet/API/eCaptureStatus.cs b/WhisperNet/API/eCaptureStatus.cs new file mode 100644 index 0000000..41f05fb --- /dev/null +++ b/WhisperNet/API/eCaptureStatus.cs @@ -0,0 +1,19 @@ +namespace Whisper +{ + /// <summary>Status of the voice capture</summary> + [Flags] + public enum eCaptureStatus: byte + { + /// <summary>Doing nothing</summary> + None = 0, + /// <summary>Capturing the audio</summary> + Listening = 1, + /// <summary>A voice is detected in the captured audio, recording</summary> + Voice = 2, + /// <summary>Transcribing a recorded piece of the audio</summary> + Transcribing = 4, + /// <summary>The computer is unable to transcribe the audio quickly enough,<br/> + /// and the capture is dropping the incoming audio samples.</summary> + Stalled = 0x80, + } +}
\ No newline at end of file diff --git a/WhisperNet/API/eLanguage.cs b/WhisperNet/API/eLanguage.cs new file mode 100644 index 0000000..1241077 --- /dev/null +++ b/WhisperNet/API/eLanguage.cs @@ -0,0 +1,206 @@ +// This file is generated by a tool, from the `languageCodez.tsv` file in this repository +namespace Whisper +{ + /// <summary>Supported languages</summary> + public enum eLanguage: uint + { + /// <summary>Afrikaans</summary> + Afrikaans = 0x6661, + /// <summary>Albanian</summary> + Albanian = 0x7173, + /// <summary>Amharic</summary> + Amharic = 0x6D61, + /// <summary>Arabic</summary> + Arabic = 0x7261, + /// <summary>Armenian</summary> + Armenian = 0x7968, + /// <summary>Assamese</summary> + Assamese = 0x7361, + /// <summary>Azerbaijani</summary> + Azerbaijani = 0x7A61, + /// <summary>Bashkir</summary> + Bashkir = 0x6162, + /// <summary>Basque</summary> + Basque = 0x7565, + /// <summary>Belarusian</summary> + Belarusian = 0x6562, + /// <summary>Bengali</summary> + Bengali = 0x6E62, + /// <summary>Bosnian</summary> + Bosnian = 0x7362, + /// <summary>Breton</summary> + Breton = 0x7262, + /// <summary>Bulgarian</summary> + Bulgarian = 0x6762, + /// <summary>Catalan</summary> + Catalan = 0x6163, + /// <summary>Chinese</summary> + Chinese = 0x687A, + /// <summary>Croatian</summary> + Croatian = 0x7268, + /// <summary>Czech</summary> + Czech = 0x7363, + /// <summary>Danish</summary> + Danish = 0x6164, + /// <summary>Dutch</summary> + Dutch = 0x6C6E, + /// <summary>English</summary> + English = 0x6E65, + /// <summary>Estonian</summary> + Estonian = 0x7465, + /// <summary>Faroese</summary> + Faroese = 0x6F66, + /// <summary>Finnish</summary> + Finnish = 0x6966, + /// <summary>French</summary> + French = 0x7266, + /// <summary>Galician</summary> + Galician = 0x6C67, + /// <summary>Georgian</summary> + Georgian = 0x616B, + /// <summary>German</summary> + German = 0x6564, + /// <summary>Greek</summary> + Greek = 0x6C65, + /// <summary>Gujarati</summary> + Gujarati = 0x7567, + /// <summary>Haitian Creole</summary> + HaitianCreole = 0x7468, + /// <summary>Hausa</summary> + Hausa = 0x6168, + /// <summary>Hawaiian</summary> + Hawaiian = 0x776168, + /// <summary>Hebrew</summary> + Hebrew = 0x7769, + /// <summary>Hindi</summary> + Hindi = 0x6968, + /// <summary>Hungarian</summary> + Hungarian = 0x7568, + /// <summary>Icelandic</summary> + Icelandic = 0x7369, + /// <summary>Indonesian</summary> + Indonesian = 0x6469, + /// <summary>Italian</summary> + Italian = 0x7469, + /// <summary>Japanese</summary> + Japanese = 0x616A, + /// <summary>Javanese</summary> + Javanese = 0x776A, + /// <summary>Kannada</summary> + Kannada = 0x6E6B, + /// <summary>Kazakh</summary> + Kazakh = 0x6B6B, + /// <summary>Khmer</summary> + Khmer = 0x6D6B, + /// <summary>Korean</summary> + Korean = 0x6F6B, + /// <summary>Lao</summary> + Lao = 0x6F6C, + /// <summary>Latin</summary> + Latin = 0x616C, + /// <summary>Latvian</summary> + Latvian = 0x766C, + /// <summary>Lingala</summary> + Lingala = 0x6E6C, + /// <summary>Lithuanian</summary> + Lithuanian = 0x746C, + /// <summary>Luxembourgish</summary> + Luxembourgish = 0x626C, + /// <summary>Macedonian</summary> + Macedonian = 0x6B6D, + /// <summary>Malagasy</summary> + Malagasy = 0x676D, + /// <summary>Malay</summary> + Malay = 0x736D, + /// <summary>Malayalam</summary> + Malayalam = 0x6C6D, + /// <summary>Maltese</summary> + Maltese = 0x746D, + /// <summary>Maori</summary> + Maori = 0x696D, + /// <summary>Marathi</summary> + Marathi = 0x726D, + /// <summary>Mongolian</summary> + Mongolian = 0x6E6D, + /// <summary>Myanmar</summary> + Myanmar = 0x796D, + /// <summary>Nepali</summary> + Nepali = 0x656E, + /// <summary>Norwegian</summary> + Norwegian = 0x6F6E, + /// <summary>Nynorsk</summary> + Nynorsk = 0x6E6E, + /// <summary>Occitan</summary> + Occitan = 0x636F, + /// <summary>Pashto</summary> + Pashto = 0x7370, + /// <summary>Persian</summary> + Persian = 0x6166, + /// <summary>Polish</summary> + Polish = 0x6C70, + /// <summary>Portuguese</summary> + Portuguese = 0x7470, + /// <summary>Punjabi</summary> + Punjabi = 0x6170, + /// <summary>Romanian</summary> + Romanian = 0x6F72, + /// <summary>Russian</summary> + Russian = 0x7572, + /// <summary>Sanskrit</summary> + Sanskrit = 0x6173, + /// <summary>Serbian</summary> + Serbian = 0x7273, + /// <summary>Shona</summary> + Shona = 0x6E73, + /// <summary>Sindhi</summary> + Sindhi = 0x6473, + /// <summary>Sinhala</summary> + Sinhala = 0x6973, + /// <summary>Slovak</summary> + Slovak = 0x6B73, + /// <summary>Slovenian</summary> + Slovenian = 0x6C73, + /// <summary>Somali</summary> + Somali = 0x6F73, + /// <summary>Spanish</summary> + Spanish = 0x7365, + /// <summary>Sundanese</summary> + Sundanese = 0x7573, + /// <summary>Swahili</summary> + Swahili = 0x7773, + /// <summary>Swedish</summary> + Swedish = 0x7673, + /// <summary>Tagalog</summary> + Tagalog = 0x6C74, + /// <summary>Tajik</summary> + Tajik = 0x6774, + /// <summary>Tamil</summary> + Tamil = 0x6174, + /// <summary>Tatar</summary> + Tatar = 0x7474, + /// <summary>Telugu</summary> + Telugu = 0x6574, + /// <summary>Thai</summary> + Thai = 0x6874, + /// <summary>Tibetan</summary> + Tibetan = 0x6F62, + /// <summary>Turkish</summary> + Turkish = 0x7274, + /// <summary>Turkmen</summary> + Turkmen = 0x6B74, + /// <summary>Ukrainian</summary> + Ukrainian = 0x6B75, + /// <summary>Urdu</summary> + Urdu = 0x7275, + /// <summary>Uzbek</summary> + Uzbek = 0x7A75, + /// <summary>Vietnamese</summary> + Vietnamese = 0x6976, + /// <summary>Welsh</summary> + Welsh = 0x7963, + /// <summary>Yiddish</summary> + Yiddish = 0x6979, + /// <summary>Yoruba</summary> + Yoruba = 0x6F79, + } +}
\ No newline at end of file diff --git a/WhisperNet/API/eLogLevel.cs b/WhisperNet/API/eLogLevel.cs new file mode 100644 index 0000000..ae494d4 --- /dev/null +++ b/WhisperNet/API/eLogLevel.cs @@ -0,0 +1,34 @@ +namespace Whisper +{ + /// <summary>Message log level</summary> + public enum eLogLevel: byte + { + /// <summary>Error message</summary> + Error = 0, + /// <summary>Warning message</summary> + Warning = 1, + /// <summary>Informational message</summary> + Info = 2, + /// <summary>Debug message</summary> + Debug = 3 + } + + /// <summary>A delegate to receive log messages from the library</summary> + public delegate void pfnLogMessage( eLogLevel level, string message ); + + /// <summary>Log destination flags</summary> + [Flags] + public enum eLoggerFlags: byte + { + /// <summary>No special flags</summary> + None = 0, + + /// <summary>In addition to calling the delegate, print messaged to standard error</summary> + UseStandardError = 1, + + /// <summary>Don’t format error codes into messages</summary> + /// <remarks>It’s recommended to use this flag in .NET.<br/> + /// The standard library already formats these messages automatically, as needed.</remarks> + SkipFormatMessage = 2, + } +}
\ No newline at end of file diff --git a/WhisperNet/API/eModelImplementation.cs b/WhisperNet/API/eModelImplementation.cs new file mode 100644 index 0000000..1b0a079 --- /dev/null +++ b/WhisperNet/API/eModelImplementation.cs @@ -0,0 +1,25 @@ +namespace Whisper +{ + /// <summary>Implementation value for the <see cref="Library.loadModel(string, eModelImplementation)" /> factory function</summary> + public enum eModelImplementation: uint + { + /// <summary>GPGPU implementation based on Direct3D 11.0 compute shaders</summary> + GPU = 1, + + /// <summary>A hybrid implementation which uses DirectCompute for encode, and decodes on CPU</summary> + /// <remarks> + /// <para>The build of the native DLL included into this nuget package doesn’t implement this version.<br/> + /// To enable, edit <c>stdafx.h</c> in Whisper project, change the value of <c>BUILD_HYBRID_VERSION</c> macro from zero to one, and build.</para> + /// <para>This implementation requires a CPU with AVX1, FMA3, F16C and BMI1 instruction set extensions.</para> + /// </remarks> + Hybrid = 2, + + /// <summary>A reference implementation which uses the original GGML CPU-running code.</summary> + /// <remarks> + /// <para>The build of the native DLL included into this nuget package doesn’t implement this version either.<br/> + /// To enable, edit <c>stdafx.h</c> in Whisper project, change the value of <c>BUILD_BOTH_VERSIONS</c> macro from zero to one, and build the project.</para> + /// <para>This implementation requires a CPU with AVX1, FMA3, and F16C instruction set extensions.</para> + /// </remarks> + Reference = 3, + } +}
\ No newline at end of file diff --git a/WhisperNet/API/eResultFlags.cs b/WhisperNet/API/eResultFlags.cs new file mode 100644 index 0000000..1de61ab --- /dev/null +++ b/WhisperNet/API/eResultFlags.cs @@ -0,0 +1,21 @@ +namespace Whisper +{ + /// <summary>Flags for <see cref="Context.results(eResultFlags)" /> method</summary> + [Flags] + public enum eResultFlags: uint + { + /// <summary>No flags</summary> + None = 0, + + /// <summary>Return individual tokens in addition to the segments</summary> + Tokens = 1, + + /// <summary>Return timestamps</summary> + Timestamps = 2, + + /// <summary>Create a new COM object for the results.</summary> + /// <remarks>Without this flag, the context returns a pointer to the COM object stored in the context.<br/> + /// The content of that object is replaced every time you call <see cref="Internal.iContext.getResults(eResultFlags)" /> method.</remarks> + NewObject = 0x100, + } +}
\ No newline at end of file diff --git a/WhisperNet/API/iAudioBuffer.cs b/WhisperNet/API/iAudioBuffer.cs new file mode 100644 index 0000000..1b35621 --- /dev/null +++ b/WhisperNet/API/iAudioBuffer.cs @@ -0,0 +1,27 @@ +using ComLight; +using System.Runtime.InteropServices; + +namespace Whisper +{ + /// <summary>A buffer with a chunk of audio.</summary> + /// <remarks>Note the interface supports both marshaling directions.<br/> + /// I have not tested, but you should be able to implement this interface in C#, to supply PCM audio data to the native code</remarks> + [ComInterface( "013583aa-c9eb-42bc-83db-633c2c317051", eMarshalDirection.BothWays )] + public interface iAudioBuffer: IDisposable + { + /// <summary>Count of samples in the buffer</summary> + int countSamples(); + + /// <summary>Unmanaged pointer to the internal buffer containing single-channel FP32 samples.</summary> + /// <remarks>If you implementing this interface in C# and your audio data is on the managed heap, use <see cref="GCHandle" /> to make sure it doesn't move.<br/> + /// Or better yet, move the data to unmanaged buffer allocated with <see cref="Marshal.AllocHGlobal(int)" /> or <see cref="Marshal.AllocCoTaskMem(int)" /> method.</remarks> + IntPtr getPcmMono(); + + /// <summary>Unmanaged pointer to the internal buffer containing stereo FP32 samples.</summary> + /// <remarks>When the buffer doesn’t have stereo data, the method gonna return <see cref="IntPtr.Zero" />.</remarks> + IntPtr getPcmStereo(); + + /// <summary>Start time of the buffer, relative to the start of the media</summary> + void getTime( out TimeSpan time ); + } +}
\ No newline at end of file diff --git a/WhisperNet/API/iAudioReader.cs b/WhisperNet/API/iAudioReader.cs new file mode 100644 index 0000000..68cf916 --- /dev/null +++ b/WhisperNet/API/iAudioReader.cs @@ -0,0 +1,23 @@ +using ComLight; + +namespace Whisper +{ + /// <summary>Audio stream reader object</summary> + /// <remarks>The implementation is forward-only, and these objects ain’t reusable.<br/> + /// To read a source file multiple time, dispose and re-create the reader.</remarks> + [ComInterface( "35b988da-04a6-476a-a193-d8891d5dc390", eMarshalDirection.ToManaged )] + public interface iAudioReader: IDisposable + { + /// <summary>Get duration of the media file</summary> + [RetValIndex] + TimeSpan getDuration(); + } + + /// <summary>Audio capture reader object</summary> + /// <remarks>This interface has no public methods callable from C#.<br/> + /// It’s only here to pass data between different functions implemented in C++.</remarks> + [ComInterface( "747752c2-d9fd-40df-8847-583c781bf013", eMarshalDirection.ToManaged )] + public interface iAudioCapture: IDisposable + { + } +}
\ No newline at end of file diff --git a/WhisperNet/API/iMediaFoundation.cs b/WhisperNet/API/iMediaFoundation.cs new file mode 100644 index 0000000..535f904 --- /dev/null +++ b/WhisperNet/API/iMediaFoundation.cs @@ -0,0 +1,36 @@ +using ComLight; +using System.Runtime.InteropServices; +using Whisper.Internal; + +namespace Whisper +{ + /// <summary>Exposes a small subset of MS Media Foundation framework.</summary> + /// <remarks>That framework is a part of Windows OS, since Vista.</remarks> + /// <seealso href="https://learn.microsoft.com/en-us/windows/win32/medfound/microsoft-media-foundation-sdk" /> + [ComInterface( "fb9763a5-d77d-4b6e-aff8-f494813cebd8", eMarshalDirection.ToManaged ), CustomConventions( typeof( NativeLogger ) )] + public interface iMediaFoundation: IDisposable + { + /// <summary>Decode complete audio file into a new memory buffer.</summary> + /// <returns> + /// Under the hood, the method asks MF to resample and convert audio into the suitable type for the Whisper model.<br/> + /// If the path is a video file, the method will decode the first audio track. + /// </returns> + [RetValIndex( 2 )] + iAudioBuffer loadAudioFile( [MarshalAs( UnmanagedType.LPWStr )] string path, [MarshalAs( UnmanagedType.U1 )] bool stereo = false ); + + /// <summary>Create a reader to stream the audio file from disk</summary> + /// <returns> + /// Under the hood, the method asks MF to resample and convert audio into the suitable type for the Whisper model.<br/> + /// If the path is a video file, the method will decode the first audio track. + /// </returns> + [RetValIndex( 2 )] + iAudioReader openAudioFile( [MarshalAs( UnmanagedType.LPWStr )] string path, [MarshalAs( UnmanagedType.U1 )] bool stereo = false ); + + /// <summary>List capture devices</summary> + void listCaptureDevices( [MarshalAs( UnmanagedType.FunctionPtr )] pfnFoundCaptureDevices pfn, IntPtr pv ); + + /// <summary>Open audio capture device</summary> + [RetValIndex( 2 )] + iAudioCapture openCaptureDevice( [MarshalAs( UnmanagedType.LPWStr )] string endpoint, [In] ref sCaptureParams captureParams ); + } +}
\ No newline at end of file diff --git a/WhisperNet/API/iModel.cs b/WhisperNet/API/iModel.cs new file mode 100644 index 0000000..8ec6d17 --- /dev/null +++ b/WhisperNet/API/iModel.cs @@ -0,0 +1,27 @@ +using ComLight; +using System.ComponentModel; + +namespace Whisper +{ + /// <summary>A model in VRAM, loaded from GGML file.</summary> + /// <remarks>This objetc doesn't keep any mutable state, and can be safely used from multiple threads concurrently</remarks> + [ComInterface( "abefb4c9-e8d8-46a3-8747-5afbadef1adb", eMarshalDirection.ToManaged ), CustomConventions( typeof( Internal.NativeLogger ) )] + public interface iModel: IDisposable + { + /// <summary>Create a context to transcribe audio with this model</summary> + /// <remarks>Don't call this method, use <see cref="ExtensionMethods.createContext(iModel)" /> instead.</remarks> + [RetValIndex, EditorBrowsable( EditorBrowsableState.Never )] + Internal.iContext createContextInternal(); + + /// <summary>True if this model is multi-lingual</summary> + bool isMultilingual(); + + /// <summary>Retrieve integer IDs of the special tokens defined by the model</summary> + [RetValIndex] + SpecialTokens getSpecialTokens(); + + /// <summary>Try to resolve integer token ID into string.</summary> + /// <remarks>Don't call this method, use <see cref="ExtensionMethods.stringFromToken(iModel, int)" /> instead.</remarks> + IntPtr stringFromTokenInternal( int id ); + } +}
\ No newline at end of file diff --git a/WhisperNet/API/sCaptureParams.cs b/WhisperNet/API/sCaptureParams.cs new file mode 100644 index 0000000..7595a69 --- /dev/null +++ b/WhisperNet/API/sCaptureParams.cs @@ -0,0 +1,37 @@ +namespace Whisper +{ + /// <summary>Flags for the audio capture</summary> + [Flags] + public enum eCaptureFlags: uint + { + /// <summary>No special flags</summary> + None = 0, + /// <summary>When the capture device supports stereo, keep stereo PCM samples in addition to mono</summary> + Stereo = 1, + } + + /// <summary>Parameters for audio capture</summary> + public struct sCaptureParams + { + /// <summary>Minimum transcribe duration in seconds</summary> + public float minDuration; + /// <summary>Maximum transcribe duration in seconds</summary> + public float maxDuration; + /// <summary></summary> + public float dropStartSilence; + /// <summary></summary> + public float pauseDuration; + /// <summary>Flags for the audio capture</summary> + public eCaptureFlags flags; + + /// <summary>Initialize the structure with some reasonable default values</summary> + public sCaptureParams() + { + minDuration = 7.0f; // 7 seconds + maxDuration = 11.0f; // 11 seconds + dropStartSilence = 0.25f; // 250 ms + pauseDuration = 0.333f; // 333 ms + flags = eCaptureFlags.None; + } + } +}
\ No newline at end of file diff --git a/WhisperNet/Callbacks.cs b/WhisperNet/Callbacks.cs new file mode 100644 index 0000000..db38718 --- /dev/null +++ b/WhisperNet/Callbacks.cs @@ -0,0 +1,44 @@ +using Whisper.Internal; + +namespace Whisper +{ + /// <summary>Implement this abstract class to receive callbacks from the native code</summary> + public abstract class Callbacks + { + /// <summary>The callback is called before every encoder run.</summary> + /// <remarks>If it returns false, the processing is aborted.</remarks> + protected virtual bool onEncoderBegin( Context sender ) { return true; } + + /// <summary>This callback is called on each new segment</summary> + protected virtual void onNewSegment( Context sender, int countNew ) { } + + const int S_OK = 0; + const int S_FALSE = 1; + internal int encoderBegin( Context sender ) + { + try + { + return onEncoderBegin( sender ) ? S_OK : S_FALSE; + } + catch( Exception ex ) + { + NativeLogger.captureException( ex ); + return ex.HResult; + } + } + + internal int newSegment( Context sender, int countNew ) + { + try + { + onNewSegment( sender, countNew ); + return S_OK; + } + catch( Exception ex ) + { + NativeLogger.captureException( ex ); + return ex.HResult; + } + } + } +}
\ No newline at end of file diff --git a/WhisperNet/CaptureCallbacks.cs b/WhisperNet/CaptureCallbacks.cs new file mode 100644 index 0000000..26013f9 --- /dev/null +++ b/WhisperNet/CaptureCallbacks.cs @@ -0,0 +1,49 @@ +using Whisper.Internal; + +namespace Whisper +{ + /// <summary>Implement this abstract class to provide callbacks for audio capture method</summary> + public abstract class CaptureCallbacks + { + /// <summary>Override this method to support cancellation</summary> + protected virtual bool shouldCancel( Context sender ) { return false; } + + /// <summary>Override this method to get notified about status changes</summary> + protected virtual void captureStatusChanged( Context sender, eCaptureStatus status ) { } + + internal pfnShouldCancel cancel( Context sender ) + { + const int S_OK = 0; + const int S_FALSE = 1; + return delegate ( IntPtr pv ) + { + try + { + return shouldCancel( sender ) ? S_OK : S_FALSE; + } + catch( Exception ex ) + { + NativeLogger.captureException( ex ); + return ex.HResult; + } + }; + } + + internal pfnCaptureStatus status( Context sender ) + { + return delegate ( IntPtr pv, eCaptureStatus status ) + { + try + { + captureStatusChanged( sender, status ); + return 0; + } + catch( Exception ex ) + { + NativeLogger.captureException( ex ); + return ex.HResult; + } + }; + } + } +}
\ No newline at end of file diff --git a/WhisperNet/Context.cs b/WhisperNet/Context.cs new file mode 100644 index 0000000..6c6a737 --- /dev/null +++ b/WhisperNet/Context.cs @@ -0,0 +1,201 @@ +using System.Diagnostics; +using Whisper.Internal; +using Whisper.Internals; + +namespace Whisper +{ + /// <summary>Stateful context, contains methods to transcribe audio</summary> + public sealed class Context: IDisposable + { + iContext context; + // Caching the results object here saves time spent in ComLight library creating these callable proxies over and over again for the same underlying C++ object + readonly iTranscribeResult transcribeResult; + sFullParams fullParams; + sProgressSink progressSink; + bool disposed = false; + readonly Action<object> pfnBuffer, pfnStream; + + internal Context( Internal.iContext context ) + { + this.context = context; + transcribeResult = context.getResults( eResultFlags.None ); + fullParams = context.fullDefaultParams( eSamplingStrategy.Greedy ); + pfnBuffer = processBuffer; + pfnStream = processStream; + progressSink = default; + } + + void IDisposable.Dispose() + { + if( disposed ) + return; + disposed = true; + context?.Dispose(); + GC.SuppressFinalize( this ); + } + + /// <summary>Adjustable parameters</summary> + public ref Parameters parameters => ref fullParams.publicParams; + + void processBuffer( object buffer ) + { + context.runFull( ref fullParams, (iAudioBuffer)buffer ); + } + void processStream( object reader ) + { + context.runStreamed( ref fullParams, ref progressSink, (iAudioReader)reader ); + } + + void runImpl( object source, Callbacks? callbacks, ReadOnlySpan<int> promptTokens, Action<object> pfn ) + { + if( null != callbacks ) + { + // TODO [very low, performance]: the following code creates 2 new GC-allocated objects on each call. + // Possible to optimize by caching these function pointers in static readonly fields, and use another [ThreadStatic] field for the callbacks object + fullParams.newSegmentCallback = delegate ( IntPtr ctx, int countNew, IntPtr userData ) + { + return callbacks.newSegment( this, countNew ); + }; + + fullParams.encoderBeginCallback = delegate ( IntPtr ctx, IntPtr userData ) + { + return callbacks.encoderBegin( this ); + }; + } + + try + { + if( promptTokens.IsEmpty ) + { + pfn( source ); + return; + } + unsafe + { + fixed( int* tokens = promptTokens ) + { + fullParams.prompt_tokens = (IntPtr)tokens; + fullParams.prompt_n_tokens = promptTokens.Length; + pfn( source ); + } + } + } + finally + { + // Reset these delegates. + // Otherwise, this class will retain the callbacks object preventing it from being garbage collected. + fullParams.newSegmentCallback = null; + fullParams.encoderBeginCallback = null; + + fullParams.prompt_tokens = IntPtr.Zero; + fullParams.prompt_n_tokens = 0; + } + } + + /// <summary>Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text</summary> + public void runFull( iAudioBuffer buffer, Callbacks? callbacks, ReadOnlySpan<int> promptTokens ) + { + runImpl( buffer, callbacks, promptTokens, pfnBuffer ); + } + /// <summary>Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text</summary> + public void runFull( iAudioBuffer buffer, Callbacks? callbacks = null ) => + runFull( buffer, callbacks, ReadOnlySpan<int>.Empty ); + /// <summary>Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text</summary> + public void runFull( iAudioBuffer buffer, Callbacks? callbacks, int[]? promptTokens ) => + runFull( buffer, callbacks, promptTokens ?? ReadOnlySpan<int>.Empty ); + + /// <summary>Run the entire model, streaming audio from the provided reader object</summary> + public void runFull( iAudioReader reader, Callbacks? callbacks, Action<double>? pfnProgress, ReadOnlySpan<int> promptTokens ) + { + if( null != pfnProgress ) + { + progressSink.pfn = delegate ( double value, IntPtr context, IntPtr pv ) + { + try + { + pfnProgress.Invoke( value ); + return 0; + } + catch( Exception ex ) + { + return ex.HResult; + } + }; + } + try + { + runImpl( reader, callbacks, promptTokens, pfnStream ); + } + finally + { + progressSink.pfn = null; + } + } + + /// <summary>Run the entire model, streaming audio from the provided reader object</summary> + public void runFull( iAudioReader reader, Action<double>? pfnProgress = null, Callbacks? callbacks = null ) => + runFull( reader, callbacks, pfnProgress, ReadOnlySpan<int>.Empty ); + + /// <summary>Run the entire model, streaming audio from the provided reader object</summary> + public void runFull( iAudioReader reader, Callbacks? callbacks, Action<double>? pfnProgress, int[]? promptTokens ) => + runFull( reader, callbacks, pfnProgress, promptTokens ?? ReadOnlySpan<int>.Empty ); + + /// <summary>Get text results out of the context</summary> + public TranscribeResult results( eResultFlags flags = eResultFlags.None ) + { + if( flags.HasFlag( eResultFlags.NewObject ) ) + throw new ArgumentException(); + + iTranscribeResult res = context.getResults( flags ); + Debug.Assert( ReferenceEquals( res, transcribeResult ) ); + return new TranscribeResult( res ); + } + + /// <summary>Print timing data</summary> + public void timingsPrint() => context.timingsPrint(); + + /// <summary>Reset timing data</summary> + public void timingsReset() => context.timingsReset(); + + /// <summary>Continuously process audio from microphone or a similar capture device</summary> + /// <remarks>It’s recommended to call this method on a background thread.</remarks> + public void runCapture( iAudioCapture capture, Callbacks? callbacks, CaptureCallbacks? captureCallbacks ) + { + if( null != callbacks ) + { + // TODO [very low, performance]: the following code creates 2 new GC-allocated objects on each call. + // Possible to optimize by caching these function pointers in static readonly fields, and use another [ThreadStatic] field for the callbacks object + fullParams.newSegmentCallback = delegate ( IntPtr ctx, int countNew, IntPtr userData ) + { + return callbacks.newSegment( this, countNew ); + }; + + fullParams.encoderBeginCallback = delegate ( IntPtr ctx, IntPtr userData ) + { + return callbacks.encoderBegin( this ); + }; + } + + try + { + sCaptureCallbacks cc = default; + if( captureCallbacks != null ) + { + cc.shouldCancel = captureCallbacks.cancel( this ); + cc.captureStatus = captureCallbacks.status( this ); + } + context.runCapture( ref fullParams, ref cc, capture ); + } + finally + { + // Reset these delegates. + // Otherwise, this class will retain the callbacks object preventing it from being garbage collected. + fullParams.newSegmentCallback = null; + fullParams.encoderBeginCallback = null; + + fullParams.prompt_tokens = IntPtr.Zero; + fullParams.prompt_n_tokens = 0; + } + } + } +}
\ No newline at end of file diff --git a/WhisperNet/ExtensionMethods.cs b/WhisperNet/ExtensionMethods.cs new file mode 100644 index 0000000..4380ece --- /dev/null +++ b/WhisperNet/ExtensionMethods.cs @@ -0,0 +1,69 @@ +using System.Runtime.InteropServices; +using Whisper.Internal; + +namespace Whisper +{ + /// <summary>Extension methods of these COM interfaces</summary> + public static class ExtensionMethods + { + /// <summary>Create a context to transcribe audio with this model</summary> + public static Context createContext( this iModel model ) + { + iContext ctx = model.createContextInternal(); + return new Context( ctx ); + } + + /// <summary>Convert language into a short ID string, like <c>"en"</c></summary> + public static string getCode( this eLanguage lang ) + { + unsafe + { + sbyte* ptr = stackalloc sbyte[ 5 ]; + *(uint*)ptr = (uint)lang; + ptr[ 4 ] = 0; + return new string( ptr ); + } + } + + /// <summary>Resolve integer token ID into string.</summary> + /// <remarks>If the token ID was not found in the model, the method returns null without raising exceptions.</remarks> + public static string? stringFromToken( this iModel model, int idToken ) => + Marshal.PtrToStringUTF8( model.stringFromTokenInternal( idToken ) ); + + /// <summary>List capture devices</summary> + public static CaptureDeviceId[]? listCaptureDevices( this iMediaFoundation mf ) + { + List<CaptureDeviceId>? list = null; + + pfnFoundCaptureDevices pfn = delegate ( int len, sCaptureDevice[]? arr, IntPtr pv ) + { + try + { + if( len == 0 || arr == null ) + return 1; + + list = new List<CaptureDeviceId>( len ); + foreach( var i in arr ) + list.Add( new CaptureDeviceId( i ) ); + return 0; + } + catch( Exception ex ) + { + NativeLogger.captureException( ex ); + return ex.HResult; + } + }; + + mf.listCaptureDevices( pfn, IntPtr.Zero ); + + return list?.ToArray(); + } + + /// <summary>Open audio capture device</summary> + public static iAudioCapture openCaptureDevice( this iMediaFoundation mf, in CaptureDeviceId id, sCaptureParams? cp = null ) + { + sCaptureParams captureParams = cp ?? new sCaptureParams(); + return mf.openCaptureDevice( id.endpoint, ref captureParams ); + } + } +}
\ No newline at end of file diff --git a/WhisperNet/Internal/AssemblyInfo.cs b/WhisperNet/Internal/AssemblyInfo.cs new file mode 100644 index 0000000..29ec638 --- /dev/null +++ b/WhisperNet/Internal/AssemblyInfo.cs @@ -0,0 +1,8 @@ +using System.Reflection; +using System.Runtime.InteropServices; +[assembly: AssemblyTitle( "WhisperNet" )] +[assembly: AssemblyCopyright( "Copyright © const.me, 2022" )] +[assembly: ComVisible( false )] +[assembly: Guid( "ced6cdb7-e040-4398-bae8-3417e5fa35f1" )] +[assembly: AssemblyVersion( "1.0.0.0" )] +[assembly: AssemblyDescription( "DirectCompute port of whisper.cpp library, C# bindings" )]
\ No newline at end of file diff --git a/WhisperNet/Internal/NativeLogger.cs b/WhisperNet/Internal/NativeLogger.cs new file mode 100644 index 0000000..b4b4eb2 --- /dev/null +++ b/WhisperNet/Internal/NativeLogger.cs @@ -0,0 +1,138 @@ +using System.Runtime.CompilerServices; +using System.Runtime.ExceptionServices; +using System.Runtime.InteropServices; + +namespace Whisper.Internal +{ + /// <summary>Utility class to supply logging function pointer to the C++ library,<br/> + /// and provide custom calling conventions to ComLight runtime to convert error messages printed in C++ into .NET exception messages</summary> + public static class NativeLogger + { + internal static void startup() { } + + static NativeLogger() + { + sink = logSink; + sLoggerSetup setup = default; + setup.sink = sink; + setup.level = eLogLevel.Warning; + Library.setupLogger( ref setup ); + } + + internal static void setup( eLogLevel lvl, eLoggerFlags flags, pfnLogMessage? pfn ) + { + logMessage = pfn; + + sLoggerSetup setup = default; + setup.sink = sink; + setup.level = lvl; + setup.flags = flags; + Library.setupLogger( ref setup ); + } + + // This field is here to protect the function pointer from being collected by the GC + static readonly pfnLoggerSink sink; + + static void logSink( IntPtr context, eLogLevel lvl, string message ) + { + if( lvl == eLogLevel.Error ) + state.setText( message ); + logMessage?.Invoke( lvl, message ); + } + + sealed class ThreadState + { + string? errorText = null; + ExceptionDispatchInfo? dispatchInfo = null; + + public void setText( string text ) => errorText = text; + public void capture( Exception ex ) => dispatchInfo = ExceptionDispatchInfo.Capture( ex ); + + public void clear() + { + errorText = null; + dispatchInfo = null; + } + + public void Deconstruct( out string? text, out ExceptionDispatchInfo? edi ) + { + text = errorText; + edi = dispatchInfo; + errorText = null; + dispatchInfo = null; + } + } + + [ThreadStatic] + static ThreadState state = new ThreadState(); + + internal static void captureException( Exception ex ) => + state.capture( ex ); + + static pfnLogMessage? logMessage = null; + + /// <summary>Called internally by ComLight runtime</summary> + [MethodImpl( MethodImplOptions.AggressiveInlining )] + public static void prologue() + { + // https://stackoverflow.com/a/2043505/126995 + if( null != state ) + state.clear(); + else + createState(); + } + + [MethodImpl( MethodImplOptions.NoInlining )] + static void createState() + { + state = new ThreadState(); + } + + /// <summary>Epilogue implementation for unsuccessful status codes</summary> + [MethodImpl( MethodImplOptions.NoInlining )] + static void throwException( int hr ) + { + // Move state from the thread local object into local variables, and clear that object + (string? text, ExceptionDispatchInfo? edi) = state; + + if( null != edi && edi.SourceException.HResult == hr ) + { + // The error comes from a callback, and we have original context of that exception. + // Re-throw the original exception. + // This uses the original error message, and even correctly deals with the stack trace. + edi.Throw(); + } + + if( null != text ) + { + // C++ code has printed an error on the current thread, between prologue and epilogue. + // Use that text for the exception message. + Exception? ex = Marshal.GetExceptionForHR( hr ); + throw new ApplicationException( text, ex ); + } + + // We don’t have any additional info about the exception. + // Throw an exception from just the HRESULT code. + Marshal.ThrowExceptionForHR( hr ); + } + + /// <summary>Called internally by ComLight runtime</summary> + [MethodImpl( MethodImplOptions.AggressiveInlining )] + public static void throwForHR( int hr ) + { + if( hr >= 0 ) + return; // SUCCEEDED + throwException( hr ); + } + + /// <summary>Called internally by ComLight runtime</summary> + [MethodImpl( MethodImplOptions.AggressiveInlining )] + public static bool throwAndReturnBool( int hr ) + { + if( hr >= 0 ) + return 0 == hr; + throwException( hr ); + return false; + } + } +}
\ No newline at end of file diff --git a/WhisperNet/Internal/iContext.cs b/WhisperNet/Internal/iContext.cs new file mode 100644 index 0000000..6adf8c5 --- /dev/null +++ b/WhisperNet/Internal/iContext.cs @@ -0,0 +1,37 @@ +using ComLight; +using System.Runtime.InteropServices; +using Whisper.Internals; + +namespace Whisper.Internal +{ + /// <summary>Stateful context, contains methods to transcribe audio</summary> + [ComInterface( "b9956374-3b18-4943-90f2-2ab18a404537", eMarshalDirection.ToManaged ), CustomConventions( typeof( NativeLogger ) )] + public interface iContext: IDisposable + { + /// <summary>Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text</summary> + void runFull( [In] ref sFullParams @params, iAudioBuffer buffer ); + + /// <summary>Run the entire model, streaming audio from the provided reader object</summary> + void runStreamed( [In] ref sFullParams @params, [In] ref sProgressSink progressSink, iAudioReader reader ); + + /// <summary>Continuously process audio from microphone or a similar capture device</summary> + void runCapture( [In] ref sFullParams @params, [In] ref sCaptureCallbacks callbacks, iAudioCapture reader ); + + /// <summary>Get text results out of the context</summary> + [RetValIndex( 1 )] + iTranscribeResult getResults( eResultFlags flags ); + + /// <summary>Get the model which was used to create this context</summary> + [RetValIndex] + iModel getModel(); + + /// <summary>Full the default parameters of the model, for the specified sampling strategy</summary> + [RetValIndex( 1 )] + sFullParams fullDefaultParams( eSamplingStrategy strategy ); + + /// <summary>Print timing data</summary> + void timingsPrint(); + /// <summary>Reset timing data</summary> + void timingsReset(); + } +}
\ No newline at end of file diff --git a/WhisperNet/Internal/iTranscribeResult.cs b/WhisperNet/Internal/iTranscribeResult.cs new file mode 100644 index 0000000..cbf49dd --- /dev/null +++ b/WhisperNet/Internal/iTranscribeResult.cs @@ -0,0 +1,122 @@ +#pragma warning disable CS0649 // Field is never assigned to +using ComLight; +using System.ComponentModel; +using System.Runtime.InteropServices; + +namespace Whisper.Internal +{ + /// <summary>Size of the buffers owned by the <see cref="iTranscribeResult" /> object</summary> + public readonly struct sTranscribeLength + { + /// <summary>Count of segments</summary> + public readonly int countSegments; + /// <summary>Total count of tokens, for all segments combined</summary> + public readonly int countTokens; + } + + /// <summary>Output data from the model</summary> + [ComInterface( "2871a73f-5ce3-48f8-8779-6582ee11935e", eMarshalDirection.ToManaged ), CustomConventions( typeof( NativeLogger ) )] + public interface iTranscribeResult + { + /// <summary>Get size of the buffers</summary> + [RetValIndex, EditorBrowsable( EditorBrowsableState.Never )] + public sTranscribeLength getSize(); + + /// <summary>Pointer to segment data, a vector of <see cref="sSegment" /> structures</summary> + [EditorBrowsable( EditorBrowsableState.Never )] + public IntPtr getSegments(); + + /// <summary>Pointer to tokens data, a vector of <see cref="sToken" /> structures</summary> + [EditorBrowsable( EditorBrowsableState.Never )] + public IntPtr getTokens(); + } +} + +namespace Whisper +{ + /// <summary>Start and end times of a segment or token</summary> + /// <remarks>The times are relative to the start of the media</remarks> + public readonly struct sTimeInterval + { + /// <summary>Start time</summary> + public readonly TimeSpan begin; + /// <summary>End time</summary> + public readonly TimeSpan end; + } + + /// <summary>Segment data</summary> + public readonly struct sSegment + { + internal readonly IntPtr m_text; + /// <summary>Segment text</summary> + public string? text => Marshal.PtrToStringUTF8( m_text ); + /// <summary>Start and end times of the segment</summary> + public readonly sTimeInterval time; + /// <summary>Slice of the tokens</summary> + public readonly int firstToken, countTokens; + } + + /// <summary>Token flags</summary> + [Flags] + public enum eTokenFlags: uint + { + /// <summary>The token is special</summary> + Special = 1, + } + + /// <summary>Token data</summary> + public readonly struct sToken + { + internal readonly IntPtr m_text; + /// <summary>Token text</summary> + public string? text => Marshal.PtrToStringUTF8( m_text ); + /// <summary>Start and end times of the token</summary> + public readonly sTimeInterval time; + /// <summary>Probability of the token</summary> + public readonly float probability; + /// <summary>Probability of the timestamp token</summary> + public readonly float probabilityTimestamp; + /// <summary>Sum of probabilities of all timestamp tokens</summary> + public readonly float ptsum; + /// <summary>Voice length of the token</summary> + public readonly float vlen; + /// <summary>Token id</summary> + public readonly int id; + /// <summary>Token flags</summary> + readonly eTokenFlags flags; + /// <summary>True if the token flags has the specified bit set</summary> + public bool hasFlag( eTokenFlags bit ) => flags.HasFlag( bit ); + } + + /// <summary>Output data from the model</summary> + public readonly ref struct TranscribeResult + { + /// <summary>Segments in the results</summary> + public readonly ReadOnlySpan<sSegment> segments; + /// <summary>Tokens in the results, for all segments</summary> + public readonly ReadOnlySpan<sToken> tokens; + + internal TranscribeResult( Internal.iTranscribeResult i ) + { + Internal.sTranscribeLength len = i.getSize(); + unsafe + { + // This does not copy the buffers to managed memory. + // Instead, the C# spans directly reference the native memory stored in these std::vectors + if( len.countSegments > 0 ) + segments = new ReadOnlySpan<sSegment>( (void*)i.getSegments(), len.countSegments ); + else + segments = ReadOnlySpan<sSegment>.Empty; + + if( len.countTokens > 0 ) + tokens = new ReadOnlySpan<sToken>( (void*)i.getTokens(), len.countTokens ); + else + tokens = ReadOnlySpan<sToken>.Empty; + } + } + + /// <summary>Get tokens for the specified segment</summary> + public ReadOnlySpan<sToken> getTokens( in sSegment seg ) => + tokens.Slice( seg.firstToken, seg.countTokens ); + } +}
\ No newline at end of file diff --git a/WhisperNet/Internal/sCaptureCallbacks.cs b/WhisperNet/Internal/sCaptureCallbacks.cs new file mode 100644 index 0000000..483c2f2 --- /dev/null +++ b/WhisperNet/Internal/sCaptureCallbacks.cs @@ -0,0 +1,23 @@ +using System.Runtime.InteropServices; + +namespace Whisper.Internal +{ + /// <summary>Unmanaged code calls this to check for cancellation</summary> + [UnmanagedFunctionPointer( CallingConvention.StdCall )] + public delegate int pfnShouldCancel( IntPtr pv ); + + /// <summary>Unmanaged code calls this to notify about the status</summary> + [UnmanagedFunctionPointer( CallingConvention.StdCall )] + public delegate int pfnCaptureStatus( IntPtr pv, eCaptureStatus status ); + + /// <summary>Capture callbacks for unmanaged code</summary> + public struct sCaptureCallbacks + { + /// <summary>Cancellation function pointer</summary> + public pfnShouldCancel shouldCancel; + /// <summary>Capture status function pointer</summary> + public pfnCaptureStatus captureStatus; + /// <summary>COntext pointer, only needed for C++ compatibility</summary> + public IntPtr pv; + } +}
\ No newline at end of file diff --git a/WhisperNet/Internal/sCaptureDevice.cs b/WhisperNet/Internal/sCaptureDevice.cs new file mode 100644 index 0000000..e2d524d --- /dev/null +++ b/WhisperNet/Internal/sCaptureDevice.cs @@ -0,0 +1,22 @@ +#pragma warning disable CS0649 // Field is never assigned to +using System.Runtime.InteropServices; + +namespace Whisper.Internal +{ + /// <summary>Identifiers for an audio capture device</summary> + public struct sCaptureDevice + { + readonly IntPtr m_displayName; + /// <summary>The display name is suitable for showing to the user, but might not be unique.</summary> + public string? displayName => Marshal.PtrToStringUni( m_displayName ); + + readonly IntPtr m_endpoint; + /// <summary>Endpoint ID for an audio capture device.<br/> + /// It uniquely identifies the device on the system, but is not a readable string.</summary> + public string? endpoint => Marshal.PtrToStringUni( m_endpoint ); + } + + /// <summary>Function pointer to consume a list of audio capture device IDs</summary> + [UnmanagedFunctionPointer( CallingConvention.StdCall )] + public delegate int pfnFoundCaptureDevices( int len, [In, MarshalAs( UnmanagedType.LPArray, SizeParamIndex = 0 )] sCaptureDevice[]? arr, IntPtr pv ); +}
\ No newline at end of file diff --git a/WhisperNet/Internal/sFullParams.cs b/WhisperNet/Internal/sFullParams.cs new file mode 100644 index 0000000..7347afe --- /dev/null +++ b/WhisperNet/Internal/sFullParams.cs @@ -0,0 +1,40 @@ +#pragma warning disable CS0649 // Field is never assigned to + +// Missing XML comment for publicly visible type or member +// TODO: remove this line and document them. +#pragma warning disable CS1591 + +using System.Runtime.InteropServices; + +namespace Whisper.Internals +{ + /// <summary>This callback is called on each new segment</summary> + [UnmanagedFunctionPointer( CallingConvention.Cdecl )] + delegate int pfnNewSegment( IntPtr ctx, int countNew, IntPtr userData ); + + /// <summary>The callback is called before every encoder run. If it returns S_FALSE, the processing is aborted.</summary> + [UnmanagedFunctionPointer( CallingConvention.Cdecl )] + delegate int pfnEncoderBegin( IntPtr ctx, IntPtr userData ); + + /// <summary>Transcribe parameters</summary> + public struct sFullParams + { + internal Parameters publicParams; + // The rest of these parameters are not exposed to the user-friendly public API of this DLL + + internal IntPtr prompt_tokens; + internal int prompt_n_tokens; + + /// <summary>This callback is called on each new segment</summary> + [MarshalAs( UnmanagedType.FunctionPtr )] + internal pfnNewSegment? newSegmentCallback; + /// <summary>Parameter for the above, not needed in C#</summary> + internal IntPtr newSegmentCallbackData; + + /// <summary>The callback is called before every encoder run. If it returns false, the processing is aborted</summary> + [MarshalAs( UnmanagedType.FunctionPtr )] + internal pfnEncoderBegin? encoderBeginCallback; + /// <summary>Parameter for the above, not needed in C#</summary> + internal IntPtr encoderBeginCallbackData; + } +}
\ No newline at end of file diff --git a/WhisperNet/Internal/sLoadModelCallbacks.cs b/WhisperNet/Internal/sLoadModelCallbacks.cs new file mode 100644 index 0000000..07f5199 --- /dev/null +++ b/WhisperNet/Internal/sLoadModelCallbacks.cs @@ -0,0 +1,64 @@ +using System.Runtime.InteropServices; + +namespace Whisper.Internal +{ + /// <summary>Function pointer to report model loading progress</summary> + [UnmanagedFunctionPointer( CallingConvention.StdCall )] + delegate int pfnLoadProgress( double progress, IntPtr pv ); + + /// <summary>Function pointer to implement cooperative cancellation</summary> + [UnmanagedFunctionPointer( CallingConvention.StdCall )] + delegate int pfnCancel( IntPtr pv ); + + /// <summary>Callback functions for loading models</summary> + public struct sLoadModelCallbacks + { + /// <summary>Function pointer to report model loading progress</summary> + [MarshalAs( UnmanagedType.FunctionPtr )] + pfnLoadProgress? progress; + + /// <summary>Function pointer to implement cooperative cancellation</summary> + [MarshalAs( UnmanagedType.FunctionPtr )] + pfnCancel? cancel; + + // Not needed in C#, delegates can capture things + IntPtr pv; + + /// <summary>Wrap idiomatic C# things into these low-level C callbacks</summary> + internal sLoadModelCallbacks( CancellationToken cancelToken, Action<double>? pfnProgress ) + { + if( cancelToken != CancellationToken.None ) + { + cancel = delegate ( IntPtr pv ) + { + if( cancelToken.IsCancellationRequested ) + return 1; // S_FALSE + return 0; // S_OK + }; + } + else + cancel = null; + + if( null != pfnProgress ) + { + progress = delegate ( double val, IntPtr pv ) + { + try + { + pfnProgress( val ); + return 0; // S_OK + } + catch( Exception ex ) + { + NativeLogger.captureException( ex ); + return ex.HResult; + } + }; + } + else + progress = null; + + pv = IntPtr.Zero; + } + } +}
\ No newline at end of file diff --git a/WhisperNet/Internal/sLoggerSetup.cs b/WhisperNet/Internal/sLoggerSetup.cs new file mode 100644 index 0000000..ed1baa4 --- /dev/null +++ b/WhisperNet/Internal/sLoggerSetup.cs @@ -0,0 +1,16 @@ +using System.Runtime.InteropServices; + +namespace Whisper.Internal +{ + [UnmanagedFunctionPointer( CallingConvention.StdCall )] + delegate void pfnLoggerSink( IntPtr context, eLogLevel lvl, [MarshalAs( UnmanagedType.LPUTF8Str )] string message ); + + struct sLoggerSetup + { + [MarshalAs( UnmanagedType.FunctionPtr )] + public pfnLoggerSink sink; + IntPtr context; + public eLogLevel level; + public eLoggerFlags flags; + } +}
\ No newline at end of file diff --git a/WhisperNet/Internal/sProgressSink.cs b/WhisperNet/Internal/sProgressSink.cs new file mode 100644 index 0000000..9155677 --- /dev/null +++ b/WhisperNet/Internal/sProgressSink.cs @@ -0,0 +1,20 @@ +#pragma warning disable CS0649 // Field is never assigned to +using System.Runtime.InteropServices; + +namespace Whisper.Internal +{ + /// <summary>A callback to get notified about the progress</summary> + [UnmanagedFunctionPointer( CallingConvention.StdCall )] + delegate int pfnReportProgress( double value, IntPtr context, IntPtr pv ); + + /// <summary>C structure with a progress reporting function pointer</summary> + public struct sProgressSink + { + /// <summary>A callback to get notified about the progress</summary> + [MarshalAs( UnmanagedType.FunctionPtr )] + internal pfnReportProgress? pfn; + + /// <summary>Last parameter to the callback</summary> + internal IntPtr pv; + } +}
\ No newline at end of file diff --git a/WhisperNet/Library.cs b/WhisperNet/Library.cs new file mode 100644 index 0000000..72ecb6e --- /dev/null +++ b/WhisperNet/Library.cs @@ -0,0 +1,110 @@ +using ComLight; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.X86; +using Whisper.Internal; + +namespace Whisper +{ + /// <summary>Factory methods implemented by the C++ DLL</summary> + public static class Library + { + static Library() + { + if( Environment.OSVersion.Platform != PlatformID.Win32NT ) + throw new ApplicationException( "This library requires Windows OS" ); + if( !Environment.Is64BitProcess ) + throw new ApplicationException( "This library only works in 64-bit processes" ); + if( RuntimeInformation.ProcessArchitecture != Architecture.X64 ) + throw new ApplicationException( "This library requires a processor with AMD64 instruction set" ); + if( !Sse41.IsSupported ) + throw new ApplicationException( "This library requires a CPU with SSE 4.1 support" ); + NativeLogger.startup(); + } + + const string dll = "Whisper.dll"; + + [DllImport( dll, CallingConvention = RuntimeClass.defaultCallingConvention, PreserveSig = false )] + internal static extern void setupLogger( [In] ref sLoggerSetup setup ); + + [DllImport( dll, CallingConvention = RuntimeClass.defaultCallingConvention, PreserveSig = true )] + static extern int loadModel( [MarshalAs( UnmanagedType.LPWStr )] string path, eModelImplementation impl, + [In] ref sLoadModelCallbacks callbacks, + [MarshalAs( UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof( Marshaler<iModel> ) )] out iModel model ); + + /// <summary>Load Whisper model from GGML file on disk</summary> + /// <remarks>Models are large, depending on user’s disk speed this might take a while, and this function blocks the calling thread.<br/> + /// Consider <see cref="loadModelAsync" /> instead.</remarks> + /// <seealso href="https://huggingface.co/datasets/ggerganov/whisper.cpp" /> + public static iModel loadModel( string path, eModelImplementation impl = eModelImplementation.GPU ) + { + iModel model; + sLoadModelCallbacks callbacks = default; + NativeLogger.prologue(); + int hr = loadModel( path, impl, ref callbacks, out model ); + NativeLogger.throwForHR( hr ); + return model; + } + + /// <summary>Load Whisper model on a background thread, with optional progress reporting and cancellation</summary> + public static Task<iModel> loadModelAsync( string path, CancellationToken cancelToken, Action<double>? pfnProgress = null, eModelImplementation impl = eModelImplementation.GPU ) + { + TaskCompletionSource<iModel> tcs = new TaskCompletionSource<iModel>(); + + WaitCallback wcb = delegate ( object? state ) + { + try + { + sLoadModelCallbacks callbacks = new sLoadModelCallbacks( cancelToken, pfnProgress ); + + iModel model; + NativeLogger.prologue(); + int hr = loadModel( path, impl, ref callbacks, out model ); + NativeLogger.throwForHR( hr ); + + tcs.SetResult( model ); + } + catch( Exception ex ) + { + tcs.SetException( ex ); + } + }; + + ThreadPool.QueueUserWorkItem( wcb ); + return tcs.Task; + } + + [DllImport( dll, CallingConvention = RuntimeClass.defaultCallingConvention, PreserveSig = true )] + static extern int initMediaFoundation( [MarshalAs( UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof( Marshaler<iMediaFoundation> ) )] out iMediaFoundation mf ); + + /// <summary>Initialize Media Foundation runtime</summary> + public static iMediaFoundation initMediaFoundation() + { + iMediaFoundation mf; + NativeLogger.prologue(); + int hr = initMediaFoundation( out mf ); + NativeLogger.throwForHR( hr ); + return mf; + } + + // The .NET runtime uses UTF-16 for the strings, so we only need the Unicode version of this function. + // The native DLL exports both Unicode and ASCII versions. + [DllImport( dll, CallingConvention = RuntimeClass.defaultCallingConvention, PreserveSig = true )] + static extern uint findLanguageKeyW( [MarshalAs( UnmanagedType.LPWStr )] string lang ); + + /// <summary>Try to resolve language code string like <c>"en"</c>, <c>"pl"</c> or <c>"uk"</c> into the strongly-typed enum.</summary> + /// <remarks>The function is case-sensitive, <c>"EN"</c> or <c>"UK"</c> gonna fail.</remarks> + public static eLanguage? languageFromCode( string lang ) + { + uint key = findLanguageKeyW( lang ); + if( key != uint.MaxValue ) + return (eLanguage)key; + return null; + } + + /// <summary>Set up delegate to receive log messages from the C++ library</summary> + public static void setLogSink( eLogLevel lvl, eLoggerFlags flags = eLoggerFlags.SkipFormatMessage, pfnLogMessage? pfn = null ) + { + NativeLogger.setup( lvl, flags, pfn ); + } + } +}
\ No newline at end of file diff --git a/WhisperNet/Readme.txt b/WhisperNet/Readme.txt new file mode 100644 index 0000000..50b3bb3 --- /dev/null +++ b/WhisperNet/Readme.txt @@ -0,0 +1 @@ +This project builds .NET DLL which wraps Whisper.dll into idiomatic C# API.
\ No newline at end of file diff --git a/WhisperNet/WhisperNet.csproj b/WhisperNet/WhisperNet.csproj new file mode 100644 index 0000000..105aa44 --- /dev/null +++ b/WhisperNet/WhisperNet.csproj @@ -0,0 +1,24 @@ +<Project Sdk="Microsoft.NET.Sdk"> + <PropertyGroup> + <TargetFramework>net6.0-windows</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + <CheckForOverflowUnderflow>true</CheckForOverflowUnderflow> + <AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath> + <GenerateDocumentationFile>True</GenerateDocumentationFile> + <AllowUnsafeBlocks>True</AllowUnsafeBlocks> + <RootNamespace>Whisper</RootNamespace> + <GenerateAssemblyInfo>false</GenerateAssemblyInfo> + <PlatformTarget>x64</PlatformTarget> + </PropertyGroup> + <PropertyGroup Condition="'$(Configuration)'=='Release'"> + <GeneratePackageOnBuild>True</GeneratePackageOnBuild> + <NuspecFile>WhisperNet.nuspec</NuspecFile> + </PropertyGroup> + <ItemGroup> + <Content Include="..\x64\Release\Whisper.dll" Link="Whisper.dll" /> + </ItemGroup> + <ItemGroup> + <PackageReference Include="ComLightInterop" Version="1.3.7" /> + </ItemGroup> +</Project>
\ No newline at end of file diff --git a/WhisperNet/WhisperNet.nuspec b/WhisperNet/WhisperNet.nuspec new file mode 100644 index 0000000..d0a61f7 --- /dev/null +++ b/WhisperNet/WhisperNet.nuspec @@ -0,0 +1,27 @@ +<?xml version="1.0" encoding="utf-8"?> +<package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd"> + <metadata> + <id>WhisperNet</id> + <version>1.0</version> + <authors>Konstantin, const.me</authors> + <license type="expression">MPL-2.0</license> + <projectUrl>https://github.com/Const-me/Whisper</projectUrl> + <description>High-performance GPGPU inference of OpenAI's Whisper automatic speech recognition (ASR) model</description> + <releaseNotes>Initial public version</releaseNotes> + <copyright>Copyright © const.me, 2022-2023</copyright> + <tags>whisper, gpgpu, speech recognition</tags> + <repository type="git" url="https://github.com/Const-me/Whisper.git" /> + <dependencies> + <group targetFramework="net6.0"> + <dependency id="ComLightInterop" version="1.3.7" /> + </group> + </dependencies> + </metadata> + <files> + <!-- Managed DLL with XML documentation --> + <file src="bin/Release/WhisperNet.dll" target="lib/net6.0/" /> + <file src="bin/Release/WhisperNet.xml" target="lib/net6.0/" /> + <!-- The C++ DLL --> + <file src="../x64/Release/Whisper.dll" target="runtimes/win-x64/native/" /> + </files> +</package>
\ No newline at end of file |
