From a74ee78dbb79c551851dc182090e7d4292b1e80c Mon Sep 17 00:00:00 2001
From: yum <yum.food.vr@gmail.com>
Date: Fri, 3 Mar 2023 20:10:32 -0800
Subject: Use logprobs, fix beam candidate selection

Incorrect sort condition resulted in worst 5 beams being picked instead of
best 5.

Use log probabilities for joint probability calculation instead of
linear probabilities. Long beams would have probabilities converge
exponentially towards zero; now they converge linearly towards
-INFINITY.

Using both transcripts in Evaluation/setup.ps1, I see a small edit
distance regression (~5%) using beam search vs. greedy.
---
 Evaluate/evaluate.py | 22 ++++++++--------------
 1 file changed, 8 insertions(+), 14 deletions(-)

(limited to 'Evaluate/evaluate.py')

diff --git a/Evaluate/evaluate.py b/Evaluate/evaluate.py
index 5e8c85d..81b3edf 100644
--- a/Evaluate/evaluate.py
+++ b/Evaluate/evaluate.py
@@ -1,11 +1,12 @@
 import argparse
 import editdistance
-import jiwer
 import re
 import subprocess
 import sys
 import time
 
+from whisper.normalizers import EnglishTextNormalizer
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("reference_path", type=str, help="Path to reference transcript")
@@ -33,22 +34,15 @@ if __name__ == "__main__":
     with open(args.reference_path, "r") as f:
         ref_transcript = f.read()
 
+    # Normalize transcripts before computing edit distance (as described in
+    # whisper paper).
+    normalize = EnglishTextNormalizer()
+    test_transcript = normalize(test_transcript)
+    ref_transcript = normalize(ref_transcript)
+
     dist = editdistance.eval(ref_transcript, test_transcript)
-    wer_transform = jiwer.Compose([
-        jiwer.ToLowerCase(),
-        jiwer.RemoveWhiteSpace(replace_by_space=True),
-        jiwer.RemoveMultipleSpaces(),
-        jiwer.RemovePunctuation(),
-        jiwer.ReduceToListOfListOfWords(word_delimiter=" "),
-        ]) 
-    wer = jiwer.wer(
-            ref_transcript,
-            test_transcript,
-            truth_transform=wer_transform,
-            hypothesis_transform=wer_transform)
 
     print(f"Duration: {t1 - t0}")
     print(f"Levenshtein distance: {dist}")
-    print(f"Word error rate: {wer}")
     print(f"Transcript: {test_transcript}")
 
-- 
cgit v1.2.3