From 8c4603c73675958efc960fbd4bb599a2909d106a Mon Sep 17 00:00:00 2001 From: Konstantin Date: Mon, 16 Jan 2023 14:52:43 +0100 Subject: Source codes --- WhisperNet/API/CaptureDeviceId.cs | 24 ++++ WhisperNet/API/Parameters.cs | 95 +++++++++++++ WhisperNet/API/SpecialTokens.cs | 23 ++++ WhisperNet/API/eCaptureStatus.cs | 19 +++ WhisperNet/API/eLanguage.cs | 206 +++++++++++++++++++++++++++++ WhisperNet/API/eLogLevel.cs | 34 +++++ WhisperNet/API/eModelImplementation.cs | 25 ++++ WhisperNet/API/eResultFlags.cs | 21 +++ WhisperNet/API/iAudioBuffer.cs | 27 ++++ WhisperNet/API/iAudioReader.cs | 23 ++++ WhisperNet/API/iMediaFoundation.cs | 36 +++++ WhisperNet/API/iModel.cs | 27 ++++ WhisperNet/API/sCaptureParams.cs | 37 ++++++ WhisperNet/Callbacks.cs | 44 ++++++ WhisperNet/CaptureCallbacks.cs | 49 +++++++ WhisperNet/Context.cs | 201 ++++++++++++++++++++++++++++ WhisperNet/ExtensionMethods.cs | 69 ++++++++++ WhisperNet/Internal/AssemblyInfo.cs | 8 ++ WhisperNet/Internal/NativeLogger.cs | 138 +++++++++++++++++++ WhisperNet/Internal/iContext.cs | 37 ++++++ WhisperNet/Internal/iTranscribeResult.cs | 122 +++++++++++++++++ WhisperNet/Internal/sCaptureCallbacks.cs | 23 ++++ WhisperNet/Internal/sCaptureDevice.cs | 22 +++ WhisperNet/Internal/sFullParams.cs | 40 ++++++ WhisperNet/Internal/sLoadModelCallbacks.cs | 64 +++++++++ WhisperNet/Internal/sLoggerSetup.cs | 16 +++ WhisperNet/Internal/sProgressSink.cs | 20 +++ WhisperNet/Library.cs | 110 +++++++++++++++ WhisperNet/Readme.txt | 1 + WhisperNet/WhisperNet.csproj | 24 ++++ WhisperNet/WhisperNet.nuspec | 27 ++++ 31 files changed, 1612 insertions(+) create mode 100644 WhisperNet/API/CaptureDeviceId.cs create mode 100644 WhisperNet/API/Parameters.cs create mode 100644 WhisperNet/API/SpecialTokens.cs create mode 100644 WhisperNet/API/eCaptureStatus.cs create mode 100644 WhisperNet/API/eLanguage.cs create mode 100644 WhisperNet/API/eLogLevel.cs create mode 100644 WhisperNet/API/eModelImplementation.cs create mode 100644 WhisperNet/API/eResultFlags.cs create mode 100644 WhisperNet/API/iAudioBuffer.cs create mode 100644 WhisperNet/API/iAudioReader.cs create mode 100644 WhisperNet/API/iMediaFoundation.cs create mode 100644 WhisperNet/API/iModel.cs create mode 100644 WhisperNet/API/sCaptureParams.cs create mode 100644 WhisperNet/Callbacks.cs create mode 100644 WhisperNet/CaptureCallbacks.cs create mode 100644 WhisperNet/Context.cs create mode 100644 WhisperNet/ExtensionMethods.cs create mode 100644 WhisperNet/Internal/AssemblyInfo.cs create mode 100644 WhisperNet/Internal/NativeLogger.cs create mode 100644 WhisperNet/Internal/iContext.cs create mode 100644 WhisperNet/Internal/iTranscribeResult.cs create mode 100644 WhisperNet/Internal/sCaptureCallbacks.cs create mode 100644 WhisperNet/Internal/sCaptureDevice.cs create mode 100644 WhisperNet/Internal/sFullParams.cs create mode 100644 WhisperNet/Internal/sLoadModelCallbacks.cs create mode 100644 WhisperNet/Internal/sLoggerSetup.cs create mode 100644 WhisperNet/Internal/sProgressSink.cs create mode 100644 WhisperNet/Library.cs create mode 100644 WhisperNet/Readme.txt create mode 100644 WhisperNet/WhisperNet.csproj create mode 100644 WhisperNet/WhisperNet.nuspec (limited to 'WhisperNet') diff --git a/WhisperNet/API/CaptureDeviceId.cs b/WhisperNet/API/CaptureDeviceId.cs new file mode 100644 index 0000000..9636e53 --- /dev/null +++ b/WhisperNet/API/CaptureDeviceId.cs @@ -0,0 +1,24 @@ +using Whisper.Internal; + +namespace Whisper +{ + /// Identifiers for an audio capture device + public record struct CaptureDeviceId + { + /// The display name is suitable for showing to the user, but might not be unique. + public string displayName; + + /// Endpoint ID for an audio capture device.
+ /// It uniquely identifies the device on the system, but is not a readable string. + public string endpoint; + + internal CaptureDeviceId( in sCaptureDevice rsi ) + { + displayName = rsi.displayName ?? ""; + endpoint = rsi.endpoint ?? throw new ApplicationException( "The device has no endpoint ID" ); + } + + /// Returns a String which represents the object instance + public override string ToString() => $"Capture device: \"{displayName}\""; + } +} \ No newline at end of file diff --git a/WhisperNet/API/Parameters.cs b/WhisperNet/API/Parameters.cs new file mode 100644 index 0000000..d2b53f9 --- /dev/null +++ b/WhisperNet/API/Parameters.cs @@ -0,0 +1,95 @@ +// Missing XML comment for publicly visible type or member +// TODO: remove this line and document them. +#pragma warning disable CS1591 + +namespace Whisper +{ + /// Available sampling strategies + public enum eSamplingStrategy: int + { + /// Always select the most probable token + Greedy, + /// TODO: not implemented yet! + BeamSearch, + }; + + [Flags] + public enum eFullParamsFlags: uint + { + None = 0, + Translate = 1, + NoContext = 2, + SingleSegment = 4, + PrintSpecial = 8, + PrintProgress = 0x10, + PrintRealtime = 0x20, + PrintTimestamps = 0x40, + + // Experimental + TokenTimestamps = 0x100, + SpeedupAudio = 0x200, + }; + + /// Transcribe parameters + public struct Parameters + { + /// Sampling strategy + public eSamplingStrategy strategy; + + /// Count of CPU worker threads to use + /// So far, the GPU model only uses CPU threads for MEL spectrograms + public int cpuThreads; + + public int n_max_text_ctx; + /// start offset in ms + public int offset_ms; + /// audio duration to process in ms + public int duration_ms; + public eFullParamsFlags flags; + + /// Set or clear the specified flag in the field of this structure + public void setFlag( eFullParamsFlags flag, bool set ) + { + if( flag != eFullParamsFlags.None ) + { + if( set ) + flags |= flag; + else + flags &= ~flag; + return; + } + throw new ArgumentException(); + } + + /// Language + public eLanguage language; + + // [EXPERIMENTAL] token-level timestamps + /// timestamp token probability threshold (~0.01) + public float thold_pt; + /// timestamp token sum probability threshold (~0.01) + public float thold_ptsum; + /// max segment length in characters + public int max_len; + /// max tokens per segment (0 = no limit) + public int max_tokens; + + public struct sGreedy + { + public int n_past; + } + public sGreedy greedy; + + public struct sBeamSearch + { + public int n_past; + public int beam_width; + public int n_best; + } + public sBeamSearch beamSearch; + + // [EXPERIMENTAL] speed-up techniques + /// overwrite the audio context size (0 = use default) + public int audioContextSize; + } +} \ No newline at end of file diff --git a/WhisperNet/API/SpecialTokens.cs b/WhisperNet/API/SpecialTokens.cs new file mode 100644 index 0000000..d672369 --- /dev/null +++ b/WhisperNet/API/SpecialTokens.cs @@ -0,0 +1,23 @@ +namespace Whisper +{ + /// Special tokens defined in the model + public readonly struct SpecialTokens + { + /// The end of a transcription + public readonly int TranscriptionEnd; // token_eot + /// Start of a transcription + public readonly int TranscriptionStart; // token_sot + /// Represents the previous word in the transcription. It is used to help the model predict the current word based on the context of the words that came before it. + public readonly int PreviousWord; // token_prev + /// Start of a sentence + public readonly int SentenceStart; // token_solm + /// Represents the word "not" in the transcription + public readonly int Not; // token_not + /// New transcription + public readonly int TranscriptionBegin; // token_beg + /// token_translate + public readonly int TaskTranslate; + /// token_transcribe + public readonly int TaskTranscribe; + } +} \ No newline at end of file diff --git a/WhisperNet/API/eCaptureStatus.cs b/WhisperNet/API/eCaptureStatus.cs new file mode 100644 index 0000000..41f05fb --- /dev/null +++ b/WhisperNet/API/eCaptureStatus.cs @@ -0,0 +1,19 @@ +namespace Whisper +{ + /// Status of the voice capture + [Flags] + public enum eCaptureStatus: byte + { + /// Doing nothing + None = 0, + /// Capturing the audio + Listening = 1, + /// A voice is detected in the captured audio, recording + Voice = 2, + /// Transcribing a recorded piece of the audio + Transcribing = 4, + /// The computer is unable to transcribe the audio quickly enough,
+ /// and the capture is dropping the incoming audio samples. + Stalled = 0x80, + } +} \ No newline at end of file diff --git a/WhisperNet/API/eLanguage.cs b/WhisperNet/API/eLanguage.cs new file mode 100644 index 0000000..1241077 --- /dev/null +++ b/WhisperNet/API/eLanguage.cs @@ -0,0 +1,206 @@ +// This file is generated by a tool, from the `languageCodez.tsv` file in this repository +namespace Whisper +{ + /// Supported languages + public enum eLanguage: uint + { + /// Afrikaans + Afrikaans = 0x6661, + /// Albanian + Albanian = 0x7173, + /// Amharic + Amharic = 0x6D61, + /// Arabic + Arabic = 0x7261, + /// Armenian + Armenian = 0x7968, + /// Assamese + Assamese = 0x7361, + /// Azerbaijani + Azerbaijani = 0x7A61, + /// Bashkir + Bashkir = 0x6162, + /// Basque + Basque = 0x7565, + /// Belarusian + Belarusian = 0x6562, + /// Bengali + Bengali = 0x6E62, + /// Bosnian + Bosnian = 0x7362, + /// Breton + Breton = 0x7262, + /// Bulgarian + Bulgarian = 0x6762, + /// Catalan + Catalan = 0x6163, + /// Chinese + Chinese = 0x687A, + /// Croatian + Croatian = 0x7268, + /// Czech + Czech = 0x7363, + /// Danish + Danish = 0x6164, + /// Dutch + Dutch = 0x6C6E, + /// English + English = 0x6E65, + /// Estonian + Estonian = 0x7465, + /// Faroese + Faroese = 0x6F66, + /// Finnish + Finnish = 0x6966, + /// French + French = 0x7266, + /// Galician + Galician = 0x6C67, + /// Georgian + Georgian = 0x616B, + /// German + German = 0x6564, + /// Greek + Greek = 0x6C65, + /// Gujarati + Gujarati = 0x7567, + /// Haitian Creole + HaitianCreole = 0x7468, + /// Hausa + Hausa = 0x6168, + /// Hawaiian + Hawaiian = 0x776168, + /// Hebrew + Hebrew = 0x7769, + /// Hindi + Hindi = 0x6968, + /// Hungarian + Hungarian = 0x7568, + /// Icelandic + Icelandic = 0x7369, + /// Indonesian + Indonesian = 0x6469, + /// Italian + Italian = 0x7469, + /// Japanese + Japanese = 0x616A, + /// Javanese + Javanese = 0x776A, + /// Kannada + Kannada = 0x6E6B, + /// Kazakh + Kazakh = 0x6B6B, + /// Khmer + Khmer = 0x6D6B, + /// Korean + Korean = 0x6F6B, + /// Lao + Lao = 0x6F6C, + /// Latin + Latin = 0x616C, + /// Latvian + Latvian = 0x766C, + /// Lingala + Lingala = 0x6E6C, + /// Lithuanian + Lithuanian = 0x746C, + /// Luxembourgish + Luxembourgish = 0x626C, + /// Macedonian + Macedonian = 0x6B6D, + /// Malagasy + Malagasy = 0x676D, + /// Malay + Malay = 0x736D, + /// Malayalam + Malayalam = 0x6C6D, + /// Maltese + Maltese = 0x746D, + /// Maori + Maori = 0x696D, + /// Marathi + Marathi = 0x726D, + /// Mongolian + Mongolian = 0x6E6D, + /// Myanmar + Myanmar = 0x796D, + /// Nepali + Nepali = 0x656E, + /// Norwegian + Norwegian = 0x6F6E, + /// Nynorsk + Nynorsk = 0x6E6E, + /// Occitan + Occitan = 0x636F, + /// Pashto + Pashto = 0x7370, + /// Persian + Persian = 0x6166, + /// Polish + Polish = 0x6C70, + /// Portuguese + Portuguese = 0x7470, + /// Punjabi + Punjabi = 0x6170, + /// Romanian + Romanian = 0x6F72, + /// Russian + Russian = 0x7572, + /// Sanskrit + Sanskrit = 0x6173, + /// Serbian + Serbian = 0x7273, + /// Shona + Shona = 0x6E73, + /// Sindhi + Sindhi = 0x6473, + /// Sinhala + Sinhala = 0x6973, + /// Slovak + Slovak = 0x6B73, + /// Slovenian + Slovenian = 0x6C73, + /// Somali + Somali = 0x6F73, + /// Spanish + Spanish = 0x7365, + /// Sundanese + Sundanese = 0x7573, + /// Swahili + Swahili = 0x7773, + /// Swedish + Swedish = 0x7673, + /// Tagalog + Tagalog = 0x6C74, + /// Tajik + Tajik = 0x6774, + /// Tamil + Tamil = 0x6174, + /// Tatar + Tatar = 0x7474, + /// Telugu + Telugu = 0x6574, + /// Thai + Thai = 0x6874, + /// Tibetan + Tibetan = 0x6F62, + /// Turkish + Turkish = 0x7274, + /// Turkmen + Turkmen = 0x6B74, + /// Ukrainian + Ukrainian = 0x6B75, + /// Urdu + Urdu = 0x7275, + /// Uzbek + Uzbek = 0x7A75, + /// Vietnamese + Vietnamese = 0x6976, + /// Welsh + Welsh = 0x7963, + /// Yiddish + Yiddish = 0x6979, + /// Yoruba + Yoruba = 0x6F79, + } +} \ No newline at end of file diff --git a/WhisperNet/API/eLogLevel.cs b/WhisperNet/API/eLogLevel.cs new file mode 100644 index 0000000..ae494d4 --- /dev/null +++ b/WhisperNet/API/eLogLevel.cs @@ -0,0 +1,34 @@ +namespace Whisper +{ + /// Message log level + public enum eLogLevel: byte + { + /// Error message + Error = 0, + /// Warning message + Warning = 1, + /// Informational message + Info = 2, + /// Debug message + Debug = 3 + } + + /// A delegate to receive log messages from the library + public delegate void pfnLogMessage( eLogLevel level, string message ); + + /// Log destination flags + [Flags] + public enum eLoggerFlags: byte + { + /// No special flags + None = 0, + + /// In addition to calling the delegate, print messaged to standard error + UseStandardError = 1, + + /// Don’t format error codes into messages + /// It’s recommended to use this flag in .NET.
+ /// The standard library already formats these messages automatically, as needed. + SkipFormatMessage = 2, + } +} \ No newline at end of file diff --git a/WhisperNet/API/eModelImplementation.cs b/WhisperNet/API/eModelImplementation.cs new file mode 100644 index 0000000..1b0a079 --- /dev/null +++ b/WhisperNet/API/eModelImplementation.cs @@ -0,0 +1,25 @@ +namespace Whisper +{ + /// Implementation value for the factory function + public enum eModelImplementation: uint + { + /// GPGPU implementation based on Direct3D 11.0 compute shaders + GPU = 1, + + /// A hybrid implementation which uses DirectCompute for encode, and decodes on CPU + /// + /// The build of the native DLL included into this nuget package doesn’t implement this version.
+ /// To enable, edit stdafx.h in Whisper project, change the value of BUILD_HYBRID_VERSION macro from zero to one, and build. + /// This implementation requires a CPU with AVX1, FMA3, F16C and BMI1 instruction set extensions. + /// + Hybrid = 2, + + /// A reference implementation which uses the original GGML CPU-running code. + /// + /// The build of the native DLL included into this nuget package doesn’t implement this version either.
+ /// To enable, edit stdafx.h in Whisper project, change the value of BUILD_BOTH_VERSIONS macro from zero to one, and build the project. + /// This implementation requires a CPU with AVX1, FMA3, and F16C instruction set extensions. + /// + Reference = 3, + } +} \ No newline at end of file diff --git a/WhisperNet/API/eResultFlags.cs b/WhisperNet/API/eResultFlags.cs new file mode 100644 index 0000000..1de61ab --- /dev/null +++ b/WhisperNet/API/eResultFlags.cs @@ -0,0 +1,21 @@ +namespace Whisper +{ + /// Flags for method + [Flags] + public enum eResultFlags: uint + { + /// No flags + None = 0, + + /// Return individual tokens in addition to the segments + Tokens = 1, + + /// Return timestamps + Timestamps = 2, + + /// Create a new COM object for the results. + /// Without this flag, the context returns a pointer to the COM object stored in the context.
+ /// The content of that object is replaced every time you call method. + NewObject = 0x100, + } +} \ No newline at end of file diff --git a/WhisperNet/API/iAudioBuffer.cs b/WhisperNet/API/iAudioBuffer.cs new file mode 100644 index 0000000..1b35621 --- /dev/null +++ b/WhisperNet/API/iAudioBuffer.cs @@ -0,0 +1,27 @@ +using ComLight; +using System.Runtime.InteropServices; + +namespace Whisper +{ + /// A buffer with a chunk of audio. + /// Note the interface supports both marshaling directions.
+ /// I have not tested, but you should be able to implement this interface in C#, to supply PCM audio data to the native code + [ComInterface( "013583aa-c9eb-42bc-83db-633c2c317051", eMarshalDirection.BothWays )] + public interface iAudioBuffer: IDisposable + { + /// Count of samples in the buffer + int countSamples(); + + /// Unmanaged pointer to the internal buffer containing single-channel FP32 samples. + /// If you implementing this interface in C# and your audio data is on the managed heap, use to make sure it doesn't move.
+ /// Or better yet, move the data to unmanaged buffer allocated with or method. + IntPtr getPcmMono(); + + /// Unmanaged pointer to the internal buffer containing stereo FP32 samples. + /// When the buffer doesn’t have stereo data, the method gonna return . + IntPtr getPcmStereo(); + + /// Start time of the buffer, relative to the start of the media + void getTime( out TimeSpan time ); + } +} \ No newline at end of file diff --git a/WhisperNet/API/iAudioReader.cs b/WhisperNet/API/iAudioReader.cs new file mode 100644 index 0000000..68cf916 --- /dev/null +++ b/WhisperNet/API/iAudioReader.cs @@ -0,0 +1,23 @@ +using ComLight; + +namespace Whisper +{ + /// Audio stream reader object + /// The implementation is forward-only, and these objects ain’t reusable.
+ /// To read a source file multiple time, dispose and re-create the reader. + [ComInterface( "35b988da-04a6-476a-a193-d8891d5dc390", eMarshalDirection.ToManaged )] + public interface iAudioReader: IDisposable + { + /// Get duration of the media file + [RetValIndex] + TimeSpan getDuration(); + } + + /// Audio capture reader object + /// This interface has no public methods callable from C#.
+ /// It’s only here to pass data between different functions implemented in C++. + [ComInterface( "747752c2-d9fd-40df-8847-583c781bf013", eMarshalDirection.ToManaged )] + public interface iAudioCapture: IDisposable + { + } +} \ No newline at end of file diff --git a/WhisperNet/API/iMediaFoundation.cs b/WhisperNet/API/iMediaFoundation.cs new file mode 100644 index 0000000..535f904 --- /dev/null +++ b/WhisperNet/API/iMediaFoundation.cs @@ -0,0 +1,36 @@ +using ComLight; +using System.Runtime.InteropServices; +using Whisper.Internal; + +namespace Whisper +{ + /// Exposes a small subset of MS Media Foundation framework. + /// That framework is a part of Windows OS, since Vista. + /// + [ComInterface( "fb9763a5-d77d-4b6e-aff8-f494813cebd8", eMarshalDirection.ToManaged ), CustomConventions( typeof( NativeLogger ) )] + public interface iMediaFoundation: IDisposable + { + /// Decode complete audio file into a new memory buffer. + /// + /// Under the hood, the method asks MF to resample and convert audio into the suitable type for the Whisper model.
+ /// If the path is a video file, the method will decode the first audio track. + /// + [RetValIndex( 2 )] + iAudioBuffer loadAudioFile( [MarshalAs( UnmanagedType.LPWStr )] string path, [MarshalAs( UnmanagedType.U1 )] bool stereo = false ); + + /// Create a reader to stream the audio file from disk + /// + /// Under the hood, the method asks MF to resample and convert audio into the suitable type for the Whisper model.
+ /// If the path is a video file, the method will decode the first audio track. + /// + [RetValIndex( 2 )] + iAudioReader openAudioFile( [MarshalAs( UnmanagedType.LPWStr )] string path, [MarshalAs( UnmanagedType.U1 )] bool stereo = false ); + + /// List capture devices + void listCaptureDevices( [MarshalAs( UnmanagedType.FunctionPtr )] pfnFoundCaptureDevices pfn, IntPtr pv ); + + /// Open audio capture device + [RetValIndex( 2 )] + iAudioCapture openCaptureDevice( [MarshalAs( UnmanagedType.LPWStr )] string endpoint, [In] ref sCaptureParams captureParams ); + } +} \ No newline at end of file diff --git a/WhisperNet/API/iModel.cs b/WhisperNet/API/iModel.cs new file mode 100644 index 0000000..8ec6d17 --- /dev/null +++ b/WhisperNet/API/iModel.cs @@ -0,0 +1,27 @@ +using ComLight; +using System.ComponentModel; + +namespace Whisper +{ + /// A model in VRAM, loaded from GGML file. + /// This objetc doesn't keep any mutable state, and can be safely used from multiple threads concurrently + [ComInterface( "abefb4c9-e8d8-46a3-8747-5afbadef1adb", eMarshalDirection.ToManaged ), CustomConventions( typeof( Internal.NativeLogger ) )] + public interface iModel: IDisposable + { + /// Create a context to transcribe audio with this model + /// Don't call this method, use instead. + [RetValIndex, EditorBrowsable( EditorBrowsableState.Never )] + Internal.iContext createContextInternal(); + + /// True if this model is multi-lingual + bool isMultilingual(); + + /// Retrieve integer IDs of the special tokens defined by the model + [RetValIndex] + SpecialTokens getSpecialTokens(); + + /// Try to resolve integer token ID into string. + /// Don't call this method, use instead. + IntPtr stringFromTokenInternal( int id ); + } +} \ No newline at end of file diff --git a/WhisperNet/API/sCaptureParams.cs b/WhisperNet/API/sCaptureParams.cs new file mode 100644 index 0000000..7595a69 --- /dev/null +++ b/WhisperNet/API/sCaptureParams.cs @@ -0,0 +1,37 @@ +namespace Whisper +{ + /// Flags for the audio capture + [Flags] + public enum eCaptureFlags: uint + { + /// No special flags + None = 0, + /// When the capture device supports stereo, keep stereo PCM samples in addition to mono + Stereo = 1, + } + + /// Parameters for audio capture + public struct sCaptureParams + { + /// Minimum transcribe duration in seconds + public float minDuration; + /// Maximum transcribe duration in seconds + public float maxDuration; + /// + public float dropStartSilence; + /// + public float pauseDuration; + /// Flags for the audio capture + public eCaptureFlags flags; + + /// Initialize the structure with some reasonable default values + public sCaptureParams() + { + minDuration = 7.0f; // 7 seconds + maxDuration = 11.0f; // 11 seconds + dropStartSilence = 0.25f; // 250 ms + pauseDuration = 0.333f; // 333 ms + flags = eCaptureFlags.None; + } + } +} \ No newline at end of file diff --git a/WhisperNet/Callbacks.cs b/WhisperNet/Callbacks.cs new file mode 100644 index 0000000..db38718 --- /dev/null +++ b/WhisperNet/Callbacks.cs @@ -0,0 +1,44 @@ +using Whisper.Internal; + +namespace Whisper +{ + /// Implement this abstract class to receive callbacks from the native code + public abstract class Callbacks + { + /// The callback is called before every encoder run. + /// If it returns false, the processing is aborted. + protected virtual bool onEncoderBegin( Context sender ) { return true; } + + /// This callback is called on each new segment + protected virtual void onNewSegment( Context sender, int countNew ) { } + + const int S_OK = 0; + const int S_FALSE = 1; + internal int encoderBegin( Context sender ) + { + try + { + return onEncoderBegin( sender ) ? S_OK : S_FALSE; + } + catch( Exception ex ) + { + NativeLogger.captureException( ex ); + return ex.HResult; + } + } + + internal int newSegment( Context sender, int countNew ) + { + try + { + onNewSegment( sender, countNew ); + return S_OK; + } + catch( Exception ex ) + { + NativeLogger.captureException( ex ); + return ex.HResult; + } + } + } +} \ No newline at end of file diff --git a/WhisperNet/CaptureCallbacks.cs b/WhisperNet/CaptureCallbacks.cs new file mode 100644 index 0000000..26013f9 --- /dev/null +++ b/WhisperNet/CaptureCallbacks.cs @@ -0,0 +1,49 @@ +using Whisper.Internal; + +namespace Whisper +{ + /// Implement this abstract class to provide callbacks for audio capture method + public abstract class CaptureCallbacks + { + /// Override this method to support cancellation + protected virtual bool shouldCancel( Context sender ) { return false; } + + /// Override this method to get notified about status changes + protected virtual void captureStatusChanged( Context sender, eCaptureStatus status ) { } + + internal pfnShouldCancel cancel( Context sender ) + { + const int S_OK = 0; + const int S_FALSE = 1; + return delegate ( IntPtr pv ) + { + try + { + return shouldCancel( sender ) ? S_OK : S_FALSE; + } + catch( Exception ex ) + { + NativeLogger.captureException( ex ); + return ex.HResult; + } + }; + } + + internal pfnCaptureStatus status( Context sender ) + { + return delegate ( IntPtr pv, eCaptureStatus status ) + { + try + { + captureStatusChanged( sender, status ); + return 0; + } + catch( Exception ex ) + { + NativeLogger.captureException( ex ); + return ex.HResult; + } + }; + } + } +} \ No newline at end of file diff --git a/WhisperNet/Context.cs b/WhisperNet/Context.cs new file mode 100644 index 0000000..6c6a737 --- /dev/null +++ b/WhisperNet/Context.cs @@ -0,0 +1,201 @@ +using System.Diagnostics; +using Whisper.Internal; +using Whisper.Internals; + +namespace Whisper +{ + /// Stateful context, contains methods to transcribe audio + public sealed class Context: IDisposable + { + iContext context; + // Caching the results object here saves time spent in ComLight library creating these callable proxies over and over again for the same underlying C++ object + readonly iTranscribeResult transcribeResult; + sFullParams fullParams; + sProgressSink progressSink; + bool disposed = false; + readonly Action pfnBuffer, pfnStream; + + internal Context( Internal.iContext context ) + { + this.context = context; + transcribeResult = context.getResults( eResultFlags.None ); + fullParams = context.fullDefaultParams( eSamplingStrategy.Greedy ); + pfnBuffer = processBuffer; + pfnStream = processStream; + progressSink = default; + } + + void IDisposable.Dispose() + { + if( disposed ) + return; + disposed = true; + context?.Dispose(); + GC.SuppressFinalize( this ); + } + + /// Adjustable parameters + public ref Parameters parameters => ref fullParams.publicParams; + + void processBuffer( object buffer ) + { + context.runFull( ref fullParams, (iAudioBuffer)buffer ); + } + void processStream( object reader ) + { + context.runStreamed( ref fullParams, ref progressSink, (iAudioReader)reader ); + } + + void runImpl( object source, Callbacks? callbacks, ReadOnlySpan promptTokens, Action pfn ) + { + if( null != callbacks ) + { + // TODO [very low, performance]: the following code creates 2 new GC-allocated objects on each call. + // Possible to optimize by caching these function pointers in static readonly fields, and use another [ThreadStatic] field for the callbacks object + fullParams.newSegmentCallback = delegate ( IntPtr ctx, int countNew, IntPtr userData ) + { + return callbacks.newSegment( this, countNew ); + }; + + fullParams.encoderBeginCallback = delegate ( IntPtr ctx, IntPtr userData ) + { + return callbacks.encoderBegin( this ); + }; + } + + try + { + if( promptTokens.IsEmpty ) + { + pfn( source ); + return; + } + unsafe + { + fixed( int* tokens = promptTokens ) + { + fullParams.prompt_tokens = (IntPtr)tokens; + fullParams.prompt_n_tokens = promptTokens.Length; + pfn( source ); + } + } + } + finally + { + // Reset these delegates. + // Otherwise, this class will retain the callbacks object preventing it from being garbage collected. + fullParams.newSegmentCallback = null; + fullParams.encoderBeginCallback = null; + + fullParams.prompt_tokens = IntPtr.Zero; + fullParams.prompt_n_tokens = 0; + } + } + + /// Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text + public void runFull( iAudioBuffer buffer, Callbacks? callbacks, ReadOnlySpan promptTokens ) + { + runImpl( buffer, callbacks, promptTokens, pfnBuffer ); + } + /// Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text + public void runFull( iAudioBuffer buffer, Callbacks? callbacks = null ) => + runFull( buffer, callbacks, ReadOnlySpan.Empty ); + /// Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text + public void runFull( iAudioBuffer buffer, Callbacks? callbacks, int[]? promptTokens ) => + runFull( buffer, callbacks, promptTokens ?? ReadOnlySpan.Empty ); + + /// Run the entire model, streaming audio from the provided reader object + public void runFull( iAudioReader reader, Callbacks? callbacks, Action? pfnProgress, ReadOnlySpan promptTokens ) + { + if( null != pfnProgress ) + { + progressSink.pfn = delegate ( double value, IntPtr context, IntPtr pv ) + { + try + { + pfnProgress.Invoke( value ); + return 0; + } + catch( Exception ex ) + { + return ex.HResult; + } + }; + } + try + { + runImpl( reader, callbacks, promptTokens, pfnStream ); + } + finally + { + progressSink.pfn = null; + } + } + + /// Run the entire model, streaming audio from the provided reader object + public void runFull( iAudioReader reader, Action? pfnProgress = null, Callbacks? callbacks = null ) => + runFull( reader, callbacks, pfnProgress, ReadOnlySpan.Empty ); + + /// Run the entire model, streaming audio from the provided reader object + public void runFull( iAudioReader reader, Callbacks? callbacks, Action? pfnProgress, int[]? promptTokens ) => + runFull( reader, callbacks, pfnProgress, promptTokens ?? ReadOnlySpan.Empty ); + + /// Get text results out of the context + public TranscribeResult results( eResultFlags flags = eResultFlags.None ) + { + if( flags.HasFlag( eResultFlags.NewObject ) ) + throw new ArgumentException(); + + iTranscribeResult res = context.getResults( flags ); + Debug.Assert( ReferenceEquals( res, transcribeResult ) ); + return new TranscribeResult( res ); + } + + /// Print timing data + public void timingsPrint() => context.timingsPrint(); + + /// Reset timing data + public void timingsReset() => context.timingsReset(); + + /// Continuously process audio from microphone or a similar capture device + /// It’s recommended to call this method on a background thread. + public void runCapture( iAudioCapture capture, Callbacks? callbacks, CaptureCallbacks? captureCallbacks ) + { + if( null != callbacks ) + { + // TODO [very low, performance]: the following code creates 2 new GC-allocated objects on each call. + // Possible to optimize by caching these function pointers in static readonly fields, and use another [ThreadStatic] field for the callbacks object + fullParams.newSegmentCallback = delegate ( IntPtr ctx, int countNew, IntPtr userData ) + { + return callbacks.newSegment( this, countNew ); + }; + + fullParams.encoderBeginCallback = delegate ( IntPtr ctx, IntPtr userData ) + { + return callbacks.encoderBegin( this ); + }; + } + + try + { + sCaptureCallbacks cc = default; + if( captureCallbacks != null ) + { + cc.shouldCancel = captureCallbacks.cancel( this ); + cc.captureStatus = captureCallbacks.status( this ); + } + context.runCapture( ref fullParams, ref cc, capture ); + } + finally + { + // Reset these delegates. + // Otherwise, this class will retain the callbacks object preventing it from being garbage collected. + fullParams.newSegmentCallback = null; + fullParams.encoderBeginCallback = null; + + fullParams.prompt_tokens = IntPtr.Zero; + fullParams.prompt_n_tokens = 0; + } + } + } +} \ No newline at end of file diff --git a/WhisperNet/ExtensionMethods.cs b/WhisperNet/ExtensionMethods.cs new file mode 100644 index 0000000..4380ece --- /dev/null +++ b/WhisperNet/ExtensionMethods.cs @@ -0,0 +1,69 @@ +using System.Runtime.InteropServices; +using Whisper.Internal; + +namespace Whisper +{ + /// Extension methods of these COM interfaces + public static class ExtensionMethods + { + /// Create a context to transcribe audio with this model + public static Context createContext( this iModel model ) + { + iContext ctx = model.createContextInternal(); + return new Context( ctx ); + } + + /// Convert language into a short ID string, like "en" + public static string getCode( this eLanguage lang ) + { + unsafe + { + sbyte* ptr = stackalloc sbyte[ 5 ]; + *(uint*)ptr = (uint)lang; + ptr[ 4 ] = 0; + return new string( ptr ); + } + } + + /// Resolve integer token ID into string. + /// If the token ID was not found in the model, the method returns null without raising exceptions. + public static string? stringFromToken( this iModel model, int idToken ) => + Marshal.PtrToStringUTF8( model.stringFromTokenInternal( idToken ) ); + + /// List capture devices + public static CaptureDeviceId[]? listCaptureDevices( this iMediaFoundation mf ) + { + List? list = null; + + pfnFoundCaptureDevices pfn = delegate ( int len, sCaptureDevice[]? arr, IntPtr pv ) + { + try + { + if( len == 0 || arr == null ) + return 1; + + list = new List( len ); + foreach( var i in arr ) + list.Add( new CaptureDeviceId( i ) ); + return 0; + } + catch( Exception ex ) + { + NativeLogger.captureException( ex ); + return ex.HResult; + } + }; + + mf.listCaptureDevices( pfn, IntPtr.Zero ); + + return list?.ToArray(); + } + + /// Open audio capture device + public static iAudioCapture openCaptureDevice( this iMediaFoundation mf, in CaptureDeviceId id, sCaptureParams? cp = null ) + { + sCaptureParams captureParams = cp ?? new sCaptureParams(); + return mf.openCaptureDevice( id.endpoint, ref captureParams ); + } + } +} \ No newline at end of file diff --git a/WhisperNet/Internal/AssemblyInfo.cs b/WhisperNet/Internal/AssemblyInfo.cs new file mode 100644 index 0000000..29ec638 --- /dev/null +++ b/WhisperNet/Internal/AssemblyInfo.cs @@ -0,0 +1,8 @@ +using System.Reflection; +using System.Runtime.InteropServices; +[assembly: AssemblyTitle( "WhisperNet" )] +[assembly: AssemblyCopyright( "Copyright © const.me, 2022" )] +[assembly: ComVisible( false )] +[assembly: Guid( "ced6cdb7-e040-4398-bae8-3417e5fa35f1" )] +[assembly: AssemblyVersion( "1.0.0.0" )] +[assembly: AssemblyDescription( "DirectCompute port of whisper.cpp library, C# bindings" )] \ No newline at end of file diff --git a/WhisperNet/Internal/NativeLogger.cs b/WhisperNet/Internal/NativeLogger.cs new file mode 100644 index 0000000..b4b4eb2 --- /dev/null +++ b/WhisperNet/Internal/NativeLogger.cs @@ -0,0 +1,138 @@ +using System.Runtime.CompilerServices; +using System.Runtime.ExceptionServices; +using System.Runtime.InteropServices; + +namespace Whisper.Internal +{ + /// Utility class to supply logging function pointer to the C++ library,
+ /// and provide custom calling conventions to ComLight runtime to convert error messages printed in C++ into .NET exception messages + public static class NativeLogger + { + internal static void startup() { } + + static NativeLogger() + { + sink = logSink; + sLoggerSetup setup = default; + setup.sink = sink; + setup.level = eLogLevel.Warning; + Library.setupLogger( ref setup ); + } + + internal static void setup( eLogLevel lvl, eLoggerFlags flags, pfnLogMessage? pfn ) + { + logMessage = pfn; + + sLoggerSetup setup = default; + setup.sink = sink; + setup.level = lvl; + setup.flags = flags; + Library.setupLogger( ref setup ); + } + + // This field is here to protect the function pointer from being collected by the GC + static readonly pfnLoggerSink sink; + + static void logSink( IntPtr context, eLogLevel lvl, string message ) + { + if( lvl == eLogLevel.Error ) + state.setText( message ); + logMessage?.Invoke( lvl, message ); + } + + sealed class ThreadState + { + string? errorText = null; + ExceptionDispatchInfo? dispatchInfo = null; + + public void setText( string text ) => errorText = text; + public void capture( Exception ex ) => dispatchInfo = ExceptionDispatchInfo.Capture( ex ); + + public void clear() + { + errorText = null; + dispatchInfo = null; + } + + public void Deconstruct( out string? text, out ExceptionDispatchInfo? edi ) + { + text = errorText; + edi = dispatchInfo; + errorText = null; + dispatchInfo = null; + } + } + + [ThreadStatic] + static ThreadState state = new ThreadState(); + + internal static void captureException( Exception ex ) => + state.capture( ex ); + + static pfnLogMessage? logMessage = null; + + /// Called internally by ComLight runtime + [MethodImpl( MethodImplOptions.AggressiveInlining )] + public static void prologue() + { + // https://stackoverflow.com/a/2043505/126995 + if( null != state ) + state.clear(); + else + createState(); + } + + [MethodImpl( MethodImplOptions.NoInlining )] + static void createState() + { + state = new ThreadState(); + } + + /// Epilogue implementation for unsuccessful status codes + [MethodImpl( MethodImplOptions.NoInlining )] + static void throwException( int hr ) + { + // Move state from the thread local object into local variables, and clear that object + (string? text, ExceptionDispatchInfo? edi) = state; + + if( null != edi && edi.SourceException.HResult == hr ) + { + // The error comes from a callback, and we have original context of that exception. + // Re-throw the original exception. + // This uses the original error message, and even correctly deals with the stack trace. + edi.Throw(); + } + + if( null != text ) + { + // C++ code has printed an error on the current thread, between prologue and epilogue. + // Use that text for the exception message. + Exception? ex = Marshal.GetExceptionForHR( hr ); + throw new ApplicationException( text, ex ); + } + + // We don’t have any additional info about the exception. + // Throw an exception from just the HRESULT code. + Marshal.ThrowExceptionForHR( hr ); + } + + /// Called internally by ComLight runtime + [MethodImpl( MethodImplOptions.AggressiveInlining )] + public static void throwForHR( int hr ) + { + if( hr >= 0 ) + return; // SUCCEEDED + throwException( hr ); + } + + /// Called internally by ComLight runtime + [MethodImpl( MethodImplOptions.AggressiveInlining )] + public static bool throwAndReturnBool( int hr ) + { + if( hr >= 0 ) + return 0 == hr; + throwException( hr ); + return false; + } + } +} \ No newline at end of file diff --git a/WhisperNet/Internal/iContext.cs b/WhisperNet/Internal/iContext.cs new file mode 100644 index 0000000..6adf8c5 --- /dev/null +++ b/WhisperNet/Internal/iContext.cs @@ -0,0 +1,37 @@ +using ComLight; +using System.Runtime.InteropServices; +using Whisper.Internals; + +namespace Whisper.Internal +{ + /// Stateful context, contains methods to transcribe audio + [ComInterface( "b9956374-3b18-4943-90f2-2ab18a404537", eMarshalDirection.ToManaged ), CustomConventions( typeof( NativeLogger ) )] + public interface iContext: IDisposable + { + /// Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text + void runFull( [In] ref sFullParams @params, iAudioBuffer buffer ); + + /// Run the entire model, streaming audio from the provided reader object + void runStreamed( [In] ref sFullParams @params, [In] ref sProgressSink progressSink, iAudioReader reader ); + + /// Continuously process audio from microphone or a similar capture device + void runCapture( [In] ref sFullParams @params, [In] ref sCaptureCallbacks callbacks, iAudioCapture reader ); + + /// Get text results out of the context + [RetValIndex( 1 )] + iTranscribeResult getResults( eResultFlags flags ); + + /// Get the model which was used to create this context + [RetValIndex] + iModel getModel(); + + /// Full the default parameters of the model, for the specified sampling strategy + [RetValIndex( 1 )] + sFullParams fullDefaultParams( eSamplingStrategy strategy ); + + /// Print timing data + void timingsPrint(); + /// Reset timing data + void timingsReset(); + } +} \ No newline at end of file diff --git a/WhisperNet/Internal/iTranscribeResult.cs b/WhisperNet/Internal/iTranscribeResult.cs new file mode 100644 index 0000000..cbf49dd --- /dev/null +++ b/WhisperNet/Internal/iTranscribeResult.cs @@ -0,0 +1,122 @@ +#pragma warning disable CS0649 // Field is never assigned to +using ComLight; +using System.ComponentModel; +using System.Runtime.InteropServices; + +namespace Whisper.Internal +{ + /// Size of the buffers owned by the object + public readonly struct sTranscribeLength + { + /// Count of segments + public readonly int countSegments; + /// Total count of tokens, for all segments combined + public readonly int countTokens; + } + + /// Output data from the model + [ComInterface( "2871a73f-5ce3-48f8-8779-6582ee11935e", eMarshalDirection.ToManaged ), CustomConventions( typeof( NativeLogger ) )] + public interface iTranscribeResult + { + /// Get size of the buffers + [RetValIndex, EditorBrowsable( EditorBrowsableState.Never )] + public sTranscribeLength getSize(); + + /// Pointer to segment data, a vector of structures + [EditorBrowsable( EditorBrowsableState.Never )] + public IntPtr getSegments(); + + /// Pointer to tokens data, a vector of structures + [EditorBrowsable( EditorBrowsableState.Never )] + public IntPtr getTokens(); + } +} + +namespace Whisper +{ + /// Start and end times of a segment or token + /// The times are relative to the start of the media + public readonly struct sTimeInterval + { + /// Start time + public readonly TimeSpan begin; + /// End time + public readonly TimeSpan end; + } + + /// Segment data + public readonly struct sSegment + { + internal readonly IntPtr m_text; + /// Segment text + public string? text => Marshal.PtrToStringUTF8( m_text ); + /// Start and end times of the segment + public readonly sTimeInterval time; + /// Slice of the tokens + public readonly int firstToken, countTokens; + } + + /// Token flags + [Flags] + public enum eTokenFlags: uint + { + /// The token is special + Special = 1, + } + + /// Token data + public readonly struct sToken + { + internal readonly IntPtr m_text; + /// Token text + public string? text => Marshal.PtrToStringUTF8( m_text ); + /// Start and end times of the token + public readonly sTimeInterval time; + /// Probability of the token + public readonly float probability; + /// Probability of the timestamp token + public readonly float probabilityTimestamp; + /// Sum of probabilities of all timestamp tokens + public readonly float ptsum; + /// Voice length of the token + public readonly float vlen; + /// Token id + public readonly int id; + /// Token flags + readonly eTokenFlags flags; + /// True if the token flags has the specified bit set + public bool hasFlag( eTokenFlags bit ) => flags.HasFlag( bit ); + } + + /// Output data from the model + public readonly ref struct TranscribeResult + { + /// Segments in the results + public readonly ReadOnlySpan segments; + /// Tokens in the results, for all segments + public readonly ReadOnlySpan tokens; + + internal TranscribeResult( Internal.iTranscribeResult i ) + { + Internal.sTranscribeLength len = i.getSize(); + unsafe + { + // This does not copy the buffers to managed memory. + // Instead, the C# spans directly reference the native memory stored in these std::vectors + if( len.countSegments > 0 ) + segments = new ReadOnlySpan( (void*)i.getSegments(), len.countSegments ); + else + segments = ReadOnlySpan.Empty; + + if( len.countTokens > 0 ) + tokens = new ReadOnlySpan( (void*)i.getTokens(), len.countTokens ); + else + tokens = ReadOnlySpan.Empty; + } + } + + /// Get tokens for the specified segment + public ReadOnlySpan getTokens( in sSegment seg ) => + tokens.Slice( seg.firstToken, seg.countTokens ); + } +} \ No newline at end of file diff --git a/WhisperNet/Internal/sCaptureCallbacks.cs b/WhisperNet/Internal/sCaptureCallbacks.cs new file mode 100644 index 0000000..483c2f2 --- /dev/null +++ b/WhisperNet/Internal/sCaptureCallbacks.cs @@ -0,0 +1,23 @@ +using System.Runtime.InteropServices; + +namespace Whisper.Internal +{ + /// Unmanaged code calls this to check for cancellation + [UnmanagedFunctionPointer( CallingConvention.StdCall )] + public delegate int pfnShouldCancel( IntPtr pv ); + + /// Unmanaged code calls this to notify about the status + [UnmanagedFunctionPointer( CallingConvention.StdCall )] + public delegate int pfnCaptureStatus( IntPtr pv, eCaptureStatus status ); + + /// Capture callbacks for unmanaged code + public struct sCaptureCallbacks + { + /// Cancellation function pointer + public pfnShouldCancel shouldCancel; + /// Capture status function pointer + public pfnCaptureStatus captureStatus; + /// COntext pointer, only needed for C++ compatibility + public IntPtr pv; + } +} \ No newline at end of file diff --git a/WhisperNet/Internal/sCaptureDevice.cs b/WhisperNet/Internal/sCaptureDevice.cs new file mode 100644 index 0000000..e2d524d --- /dev/null +++ b/WhisperNet/Internal/sCaptureDevice.cs @@ -0,0 +1,22 @@ +#pragma warning disable CS0649 // Field is never assigned to +using System.Runtime.InteropServices; + +namespace Whisper.Internal +{ + /// Identifiers for an audio capture device + public struct sCaptureDevice + { + readonly IntPtr m_displayName; + /// The display name is suitable for showing to the user, but might not be unique. + public string? displayName => Marshal.PtrToStringUni( m_displayName ); + + readonly IntPtr m_endpoint; + /// Endpoint ID for an audio capture device.
+ /// It uniquely identifies the device on the system, but is not a readable string. + public string? endpoint => Marshal.PtrToStringUni( m_endpoint ); + } + + /// Function pointer to consume a list of audio capture device IDs + [UnmanagedFunctionPointer( CallingConvention.StdCall )] + public delegate int pfnFoundCaptureDevices( int len, [In, MarshalAs( UnmanagedType.LPArray, SizeParamIndex = 0 )] sCaptureDevice[]? arr, IntPtr pv ); +} \ No newline at end of file diff --git a/WhisperNet/Internal/sFullParams.cs b/WhisperNet/Internal/sFullParams.cs new file mode 100644 index 0000000..7347afe --- /dev/null +++ b/WhisperNet/Internal/sFullParams.cs @@ -0,0 +1,40 @@ +#pragma warning disable CS0649 // Field is never assigned to + +// Missing XML comment for publicly visible type or member +// TODO: remove this line and document them. +#pragma warning disable CS1591 + +using System.Runtime.InteropServices; + +namespace Whisper.Internals +{ + /// This callback is called on each new segment + [UnmanagedFunctionPointer( CallingConvention.Cdecl )] + delegate int pfnNewSegment( IntPtr ctx, int countNew, IntPtr userData ); + + /// The callback is called before every encoder run. If it returns S_FALSE, the processing is aborted. + [UnmanagedFunctionPointer( CallingConvention.Cdecl )] + delegate int pfnEncoderBegin( IntPtr ctx, IntPtr userData ); + + /// Transcribe parameters + public struct sFullParams + { + internal Parameters publicParams; + // The rest of these parameters are not exposed to the user-friendly public API of this DLL + + internal IntPtr prompt_tokens; + internal int prompt_n_tokens; + + /// This callback is called on each new segment + [MarshalAs( UnmanagedType.FunctionPtr )] + internal pfnNewSegment? newSegmentCallback; + /// Parameter for the above, not needed in C# + internal IntPtr newSegmentCallbackData; + + /// The callback is called before every encoder run. If it returns false, the processing is aborted + [MarshalAs( UnmanagedType.FunctionPtr )] + internal pfnEncoderBegin? encoderBeginCallback; + /// Parameter for the above, not needed in C# + internal IntPtr encoderBeginCallbackData; + } +} \ No newline at end of file diff --git a/WhisperNet/Internal/sLoadModelCallbacks.cs b/WhisperNet/Internal/sLoadModelCallbacks.cs new file mode 100644 index 0000000..07f5199 --- /dev/null +++ b/WhisperNet/Internal/sLoadModelCallbacks.cs @@ -0,0 +1,64 @@ +using System.Runtime.InteropServices; + +namespace Whisper.Internal +{ + /// Function pointer to report model loading progress + [UnmanagedFunctionPointer( CallingConvention.StdCall )] + delegate int pfnLoadProgress( double progress, IntPtr pv ); + + /// Function pointer to implement cooperative cancellation + [UnmanagedFunctionPointer( CallingConvention.StdCall )] + delegate int pfnCancel( IntPtr pv ); + + /// Callback functions for loading models + public struct sLoadModelCallbacks + { + /// Function pointer to report model loading progress + [MarshalAs( UnmanagedType.FunctionPtr )] + pfnLoadProgress? progress; + + /// Function pointer to implement cooperative cancellation + [MarshalAs( UnmanagedType.FunctionPtr )] + pfnCancel? cancel; + + // Not needed in C#, delegates can capture things + IntPtr pv; + + /// Wrap idiomatic C# things into these low-level C callbacks + internal sLoadModelCallbacks( CancellationToken cancelToken, Action? pfnProgress ) + { + if( cancelToken != CancellationToken.None ) + { + cancel = delegate ( IntPtr pv ) + { + if( cancelToken.IsCancellationRequested ) + return 1; // S_FALSE + return 0; // S_OK + }; + } + else + cancel = null; + + if( null != pfnProgress ) + { + progress = delegate ( double val, IntPtr pv ) + { + try + { + pfnProgress( val ); + return 0; // S_OK + } + catch( Exception ex ) + { + NativeLogger.captureException( ex ); + return ex.HResult; + } + }; + } + else + progress = null; + + pv = IntPtr.Zero; + } + } +} \ No newline at end of file diff --git a/WhisperNet/Internal/sLoggerSetup.cs b/WhisperNet/Internal/sLoggerSetup.cs new file mode 100644 index 0000000..ed1baa4 --- /dev/null +++ b/WhisperNet/Internal/sLoggerSetup.cs @@ -0,0 +1,16 @@ +using System.Runtime.InteropServices; + +namespace Whisper.Internal +{ + [UnmanagedFunctionPointer( CallingConvention.StdCall )] + delegate void pfnLoggerSink( IntPtr context, eLogLevel lvl, [MarshalAs( UnmanagedType.LPUTF8Str )] string message ); + + struct sLoggerSetup + { + [MarshalAs( UnmanagedType.FunctionPtr )] + public pfnLoggerSink sink; + IntPtr context; + public eLogLevel level; + public eLoggerFlags flags; + } +} \ No newline at end of file diff --git a/WhisperNet/Internal/sProgressSink.cs b/WhisperNet/Internal/sProgressSink.cs new file mode 100644 index 0000000..9155677 --- /dev/null +++ b/WhisperNet/Internal/sProgressSink.cs @@ -0,0 +1,20 @@ +#pragma warning disable CS0649 // Field is never assigned to +using System.Runtime.InteropServices; + +namespace Whisper.Internal +{ + /// A callback to get notified about the progress + [UnmanagedFunctionPointer( CallingConvention.StdCall )] + delegate int pfnReportProgress( double value, IntPtr context, IntPtr pv ); + + /// C structure with a progress reporting function pointer + public struct sProgressSink + { + /// A callback to get notified about the progress + [MarshalAs( UnmanagedType.FunctionPtr )] + internal pfnReportProgress? pfn; + + /// Last parameter to the callback + internal IntPtr pv; + } +} \ No newline at end of file diff --git a/WhisperNet/Library.cs b/WhisperNet/Library.cs new file mode 100644 index 0000000..72ecb6e --- /dev/null +++ b/WhisperNet/Library.cs @@ -0,0 +1,110 @@ +using ComLight; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.X86; +using Whisper.Internal; + +namespace Whisper +{ + /// Factory methods implemented by the C++ DLL + public static class Library + { + static Library() + { + if( Environment.OSVersion.Platform != PlatformID.Win32NT ) + throw new ApplicationException( "This library requires Windows OS" ); + if( !Environment.Is64BitProcess ) + throw new ApplicationException( "This library only works in 64-bit processes" ); + if( RuntimeInformation.ProcessArchitecture != Architecture.X64 ) + throw new ApplicationException( "This library requires a processor with AMD64 instruction set" ); + if( !Sse41.IsSupported ) + throw new ApplicationException( "This library requires a CPU with SSE 4.1 support" ); + NativeLogger.startup(); + } + + const string dll = "Whisper.dll"; + + [DllImport( dll, CallingConvention = RuntimeClass.defaultCallingConvention, PreserveSig = false )] + internal static extern void setupLogger( [In] ref sLoggerSetup setup ); + + [DllImport( dll, CallingConvention = RuntimeClass.defaultCallingConvention, PreserveSig = true )] + static extern int loadModel( [MarshalAs( UnmanagedType.LPWStr )] string path, eModelImplementation impl, + [In] ref sLoadModelCallbacks callbacks, + [MarshalAs( UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof( Marshaler ) )] out iModel model ); + + /// Load Whisper model from GGML file on disk + /// Models are large, depending on user’s disk speed this might take a while, and this function blocks the calling thread.
+ /// Consider instead. + /// + public static iModel loadModel( string path, eModelImplementation impl = eModelImplementation.GPU ) + { + iModel model; + sLoadModelCallbacks callbacks = default; + NativeLogger.prologue(); + int hr = loadModel( path, impl, ref callbacks, out model ); + NativeLogger.throwForHR( hr ); + return model; + } + + /// Load Whisper model on a background thread, with optional progress reporting and cancellation + public static Task loadModelAsync( string path, CancellationToken cancelToken, Action? pfnProgress = null, eModelImplementation impl = eModelImplementation.GPU ) + { + TaskCompletionSource tcs = new TaskCompletionSource(); + + WaitCallback wcb = delegate ( object? state ) + { + try + { + sLoadModelCallbacks callbacks = new sLoadModelCallbacks( cancelToken, pfnProgress ); + + iModel model; + NativeLogger.prologue(); + int hr = loadModel( path, impl, ref callbacks, out model ); + NativeLogger.throwForHR( hr ); + + tcs.SetResult( model ); + } + catch( Exception ex ) + { + tcs.SetException( ex ); + } + }; + + ThreadPool.QueueUserWorkItem( wcb ); + return tcs.Task; + } + + [DllImport( dll, CallingConvention = RuntimeClass.defaultCallingConvention, PreserveSig = true )] + static extern int initMediaFoundation( [MarshalAs( UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof( Marshaler ) )] out iMediaFoundation mf ); + + /// Initialize Media Foundation runtime + public static iMediaFoundation initMediaFoundation() + { + iMediaFoundation mf; + NativeLogger.prologue(); + int hr = initMediaFoundation( out mf ); + NativeLogger.throwForHR( hr ); + return mf; + } + + // The .NET runtime uses UTF-16 for the strings, so we only need the Unicode version of this function. + // The native DLL exports both Unicode and ASCII versions. + [DllImport( dll, CallingConvention = RuntimeClass.defaultCallingConvention, PreserveSig = true )] + static extern uint findLanguageKeyW( [MarshalAs( UnmanagedType.LPWStr )] string lang ); + + /// Try to resolve language code string like "en", "pl" or "uk" into the strongly-typed enum. + /// The function is case-sensitive, "EN" or "UK" gonna fail. + public static eLanguage? languageFromCode( string lang ) + { + uint key = findLanguageKeyW( lang ); + if( key != uint.MaxValue ) + return (eLanguage)key; + return null; + } + + /// Set up delegate to receive log messages from the C++ library + public static void setLogSink( eLogLevel lvl, eLoggerFlags flags = eLoggerFlags.SkipFormatMessage, pfnLogMessage? pfn = null ) + { + NativeLogger.setup( lvl, flags, pfn ); + } + } +} \ No newline at end of file diff --git a/WhisperNet/Readme.txt b/WhisperNet/Readme.txt new file mode 100644 index 0000000..50b3bb3 --- /dev/null +++ b/WhisperNet/Readme.txt @@ -0,0 +1 @@ +This project builds .NET DLL which wraps Whisper.dll into idiomatic C# API. \ No newline at end of file diff --git a/WhisperNet/WhisperNet.csproj b/WhisperNet/WhisperNet.csproj new file mode 100644 index 0000000..105aa44 --- /dev/null +++ b/WhisperNet/WhisperNet.csproj @@ -0,0 +1,24 @@ + + + net6.0-windows + enable + enable + true + false + True + True + Whisper + false + x64 + + + True + WhisperNet.nuspec + + + + + + + + \ No newline at end of file diff --git a/WhisperNet/WhisperNet.nuspec b/WhisperNet/WhisperNet.nuspec new file mode 100644 index 0000000..d0a61f7 --- /dev/null +++ b/WhisperNet/WhisperNet.nuspec @@ -0,0 +1,27 @@ + + + + WhisperNet + 1.0 + Konstantin, const.me + MPL-2.0 + https://github.com/Const-me/Whisper + High-performance GPGPU inference of OpenAI's Whisper automatic speech recognition (ASR) model + Initial public version + Copyright © const.me, 2022-2023 + whisper, gpgpu, speech recognition + + + + + + + + + + + + + + + \ No newline at end of file -- cgit v1.2.3