Begin work on evaluation framework

Need a way to verify that beam search is actually working better than
greedy.

5 files changed, 100 insertions, 0 deletions

diff --git a/Evaluate/.gitignore b/Evaluate/.gitignore
new file mode 100644
index 0000000..c6d2776
--- /dev/null
+++ b/Evaluate/.gitignore
@@ -0,0 +1,10 @@
+# models
+*.bin
+# audio
+*.mp3
+*.wav
+# reference transcripts
+*.txt
+# binaries
+*.exe
+*.dll
diff --git a/Evaluate/.swp b/Evaluate/.swp
new file mode 100644
index 0000000..c1bc460
--- /dev/null
+++ b/Evaluate/.swp
diff --git a/Evaluate/README.md b/Evaluate/README.md
new file mode 100644
index 0000000..91038f6
--- /dev/null
+++ b/Evaluate/README.md
@@ -0,0 +1,18 @@
+## Evaluation tool
+
+This directory holds code to evaluate the accuracy of transcriptions.
+
+Example usage (in Powershell):
+```
+python3 -m pip install -r requirements.txt
+python3 evaluate.py reference_transcript.txt
+./setup.ps1
+./WhisperCLI.exe \
+  --audio_path *.mp3 \
+  --model_path *.bin
+```
+
+* setup.ps1: Downloads whisper checkpoints and audio from librivox. Copies
+  WhisperCLI.exe to CWD.
+* evaluate.py: Computes Levenshtein distance between generated transcription
+  and "correct" transcription.
diff --git a/Evaluate/evaluate.py b/Evaluate/evaluate.py
new file mode 100644
index 0000000..5e8c85d
--- /dev/null
+++ b/Evaluate/evaluate.py
@@ -0,0 +1,54 @@
+import argparse
+import editdistance
+import jiwer
+import re
+import subprocess
+import sys
+import time
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("reference_path", type=str, help="Path to reference transcript")
+    parser.add_argument("audio_path", type=str, help="Path to audio file to transcribe")
+    parser.add_argument("model_path", type=str, help="Path to Whisper model to use")
+    parser.add_argument("decode_method", type=str, help="Decoding method. Either 'greedy' or 'beam'")
+    args = parser.parse_args()
+
+    cmd = "./WhisperCLI.exe"
+    cmd_args = [
+            "--audio_path", args.audio_path,
+            "--model_path", args.model_path,
+            "--decode_method", args.decode_method,
+            ]
+
+    t0 = time.time()
+    result = subprocess.run([cmd] + cmd_args, stdout=subprocess.PIPE)
+    t1 = time.time()
+
+    if result.returncode != 0:
+        print(f"Failed to transcribe: cmd returned {result.returncode}",
+                file=sys.stderr)
+
+    test_transcript = result.stdout.decode("utf-8")
+    with open(args.reference_path, "r") as f:
+        ref_transcript = f.read()
+
+    dist = editdistance.eval(ref_transcript, test_transcript)
+    wer_transform = jiwer.Compose([
+        jiwer.ToLowerCase(),
+        jiwer.RemoveWhiteSpace(replace_by_space=True),
+        jiwer.RemoveMultipleSpaces(),
+        jiwer.RemovePunctuation(),
+        jiwer.ReduceToListOfListOfWords(word_delimiter=" "),
+        ]) 
+    wer = jiwer.wer(
+            ref_transcript,
+            test_transcript,
+            truth_transform=wer_transform,
+            hypothesis_transform=wer_transform)
+
+    print(f"Duration: {t1 - t0}")
+    print(f"Levenshtein distance: {dist}")
+    print(f"Word error rate: {wer}")
+    print(f"Transcript: {test_transcript}")
+
diff --git a/Evaluate/setup.ps1 b/Evaluate/setup.ps1
new file mode 100644
index 0000000..95e2487
--- /dev/null
+++ b/Evaluate/setup.ps1
@@ -0,0 +1,18 @@
+
+cp ../x64/Debug/WhisperCLI.exe .
+cp ../x64/Debug/Whisper.dll .
+
+$MODEL_URL = "https://huggingface.co/datasets/ggerganov/whisper.cpp/resolve/main/ggml-medium.bin"
+$MODEL_FILE = $(Split-Path -Path $MODEL_URL -Leaf)
+if (-Not (Test-Path $MODEL_FILE)) {
+  echo "Fetch model"
+  Invoke-WebRequest $MODEL_URL -OutFile $MODEL_FILE
+}
+
+#$AUDIO_URL = "https://www.archive.org/download/usconstitution_1610_librivox/constitution_01_unitedstates_64kb.mp3"
+$AUDIO_URL = "https://www.archive.org/download/usconstitution_1610_librivox/constitution_02_unitedstates_128kb.mp3"
+$AUDIO_FILE = $(Split-Path -Path $AUDIO_URL -Leaf)
+if (-Not (Test-Path $AUDIO_FILE)) {
+  echo "Fetch audio"
+  Invoke-WebRequest $AUDIO_URL -OutFile $AUDIO_FILE
+}