change multilingual wer copy logic

dashboard_data/performance_data.json

@@ -319,47 +319,3 @@
 {"model": "distil-whisper/distil-large-v3/594MB", "device": "iPhone 13 Pro", "os": "iOS 18.1", "timestamp": "2024-10-30T05:41:28Z", "speed": 4.75, "tokens_per_second": 17.58, "dataset_speed": {"librispeech-10mins": 3.17, "earnings22-10mins": 9.53}, "dataset_tokens_per_second": {"librispeech-10mins": 13.28, "earnings22-10mins": 30.58}, "average_wer": 7.66, "qoi": 0.67, "commit_hash": "a9b92c4", "commit_timestamp": "2024-10-29T190053"}
 {"model": "openai/whisper-small.en", "device": "iPhone 16 Pro", "os": "iOS 18.0", "timestamp": "2024-10-30T03:04:27Z", "speed": 17.4, "tokens_per_second": 62.94, "dataset_speed": {"librispeech-10mins": 14.83, "earnings22-10mins": 21.05}, "dataset_tokens_per_second": {"librispeech-10mins": 56.91, "earnings22-10mins": 71.54}, "average_wer": 7.87, "qoi": 0.6, "commit_hash": "a9b92c4", "commit_timestamp": "2024-10-29T190053"}
 {"model": "distil-whisper/distil-large-v3/turbo", "device": "iPad Air 11-inch (M2)", "os": "iPadOS 17.6.1", "timestamp": "2024-10-30T11:45:28Z", "speed": 1.58, "tokens_per_second": 4.83, "dataset_speed": {"earnings22-10mins": 1.58}, "dataset_tokens_per_second": {"earnings22-10mins": 4.83}, "average_wer": 8.28, "qoi": 0, "commit_hash": "a9b92c4", "commit_timestamp": "2024-10-29T190053"}
-{"model": "openai/whisper-large-v3-v20240930/turbo/632MB", "device": "iPhone 16 Plus", "os": "iOS 18.0.1", "timestamp": "2024-10-30T17:25:00Z", "speed": 12.83, "tokens_per_second": 48.07, "dataset_speed": {"earnings22-10mins": 12.83}, "dataset_tokens_per_second": {"earnings22-10mins": 48.07}, "average_wer": 7.92, "qoi": 1, "commit_hash": "a9b92c4", "commit_timestamp": "2024-10-29T190053"}
-{"model": "openai/whisper-small.en", "device": "iPhone 12 mini", "os": "iOS 17.7", "timestamp": "2024-10-30T08:02:10Z", "speed": 3.52, "tokens_per_second": 12.06, "dataset_speed": {"earnings22-10mins": 3.52}, "dataset_tokens_per_second": {"earnings22-10mins": 12.06}, "average_wer": 10.76, "qoi": 0, "commit_hash": "a9b92c4", "commit_timestamp": "2024-10-29T190053"}
-{"model": "distil-whisper/distil-large-v3/turbo", "device": "Apple M3 Max", "os": "macOS 15.2.0", "timestamp": "2024-10-29T18:27:45Z", "speed": 15.02, "tokens_per_second": 50.67, "dataset_speed": {"earnings22-10mins": 31.79, "librispeech-10mins": 9.85}, "dataset_tokens_per_second": {"earnings22-10mins": 98.06, "librispeech-10mins": 36.05}, "average_wer": 6.52, "qoi": 0.69, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-large-v3/947MB", "device": "Apple M3 Max", "os": "macOS 15.2.0", "timestamp": "2024-10-29T18:11:02Z", "speed": 2.94, "tokens_per_second": 11.75, "dataset_speed": {"librispeech-10mins": 1.96, "earnings22-10mins": 5.9}, "dataset_tokens_per_second": {"librispeech-10mins": 9.65, "earnings22-10mins": 18.1}, "average_wer": 7.84, "qoi": 0.69, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-large-v3-v20240930/turbo", "device": "Apple M3 Max", "os": "macOS 15.2.0", "timestamp": "2024-10-29T18:34:11Z", "speed": 13.6, "tokens_per_second": 44.74, "dataset_speed": {"earnings22-10mins": 28.81, "librispeech-10mins": 8.92}, "dataset_tokens_per_second": {"earnings22-10mins": 86.86, "librispeech-10mins": 31.76}, "average_wer": 6.01, "qoi": 0.74, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-small.en", "device": "Apple M3 Max", "os": "macOS 15.2.0", "timestamp": "2024-10-29T10:18:55Z", "speed": 20.41, "tokens_per_second": 73.42, "dataset_speed": {"earnings22-10mins": 35.76, "librispeech-10mins": 14.3}, "dataset_tokens_per_second": {"earnings22-10mins": 120.1, "librispeech-10mins": 54.83}, "average_wer": 7.87, "qoi": 0.6, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "distil-whisper/distil-large-v3", "device": "Apple M3 Max", "os": "macOS 15.2.0", "timestamp": "2024-10-29T18:22:58Z", "speed": 9.78, "tokens_per_second": 36.23, "dataset_speed": {"earnings22-10mins": 22.05, "librispeech-10mins": 6.29}, "dataset_tokens_per_second": {"earnings22-10mins": 71.17, "librispeech-10mins": 26.3}, "average_wer": 6.8, "qoi": 0.67, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-large-v2", "device": "Apple M3 Max", "os": "macOS 15.2.0", "timestamp": "2024-10-29T10:24:54Z", "speed": 3.16, "tokens_per_second": 11.89, "dataset_speed": {"librispeech-10mins": 2.19, "earnings22-10mins": 5.67}, "dataset_tokens_per_second": {"librispeech-10mins": 9.4, "earnings22-10mins": 18.37}, "average_wer": 6.45, "qoi": 0.64, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-base.en", "device": "iPhone 11", "os": "iOS 17.6.1", "timestamp": "2024-10-29T23:21:11Z", "speed": 4.77, "tokens_per_second": 9.44, "dataset_speed": {"librispeech-10mins": 4.9, "earnings22-10mins": 4.65}, "dataset_tokens_per_second": {"librispeech-10mins": 1.23, "earnings22-10mins": 17.27}, "average_wer": 97.03, "qoi": 0, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "distil-whisper/distil-large-v3/594MB", "device": "iPad Air 11-inch (M2)", "os": "iPadOS 17.6.1", "timestamp": "2024-10-30T04:12:28Z", "speed": 7.14, "tokens_per_second": 26.56, "dataset_speed": {"earnings22-10mins": 13.31, "librispeech-10mins": 4.89}, "dataset_tokens_per_second": {"earnings22-10mins": 42.58, "librispeech-10mins": 20.71}, "average_wer": 7.35, "qoi": 0.65, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-large-v3/turbo", "device": "Apple M3 Max", "os": "macOS 15.2.0", "timestamp": "2024-10-29T18:13:04Z", "speed": 4.4, "tokens_per_second": 14.94, "dataset_speed": {"earnings22-10mins": 6.81, "librispeech-10mins": 3.25}, "dataset_tokens_per_second": {"earnings22-10mins": 21.1, "librispeech-10mins": 12.01}, "average_wer": 5.97, "qoi": 0.73, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-base.en", "device": "Apple M3 Max", "os": "macOS 15.2.0", "timestamp": "2024-10-29T10:17:13Z", "speed": 47.56, "tokens_per_second": 163.02, "dataset_speed": {"librispeech-10mins": 31.04, "earnings22-10mins": 102.18}, "dataset_tokens_per_second": {"librispeech-10mins": 117.76, "earnings22-10mins": 312.67}, "average_wer": 9.87, "qoi": 0.51, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-tiny.en", "device": "Apple M3 Max", "os": "macOS 15.2.0", "timestamp": "2024-10-29T10:14:16Z", "speed": 57.4, "tokens_per_second": 197.25, "dataset_speed": {"librispeech-10mins": 37.59, "earnings22-10mins": 121.92}, "dataset_tokens_per_second": {"librispeech-10mins": 142.57, "earnings22-10mins": 375.32}, "average_wer": 14.74, "qoi": 0.37, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-base.en", "device": "iPad Air 11-inch (M2)", "os": "iPadOS 17.6.1", "timestamp": "2024-10-30T01:06:57Z", "speed": 33.4, "tokens_per_second": 114.46, "dataset_speed": {"librispeech-10mins": 23.02, "earnings22-10mins": 61.02}, "dataset_tokens_per_second": {"librispeech-10mins": 87.23, "earnings22-10mins": 186.94}, "average_wer": 9.91, "qoi": 0.51, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-tiny", "device": "iPad Air 11-inch (M2)", "os": "iPadOS 17.6.1", "timestamp": "2024-10-30T01:03:03Z", "speed": 40.36, "tokens_per_second": 164.49, "dataset_speed": {"earnings22-10mins": 58.06, "librispeech-10mins": 30.96}, "dataset_tokens_per_second": {"earnings22-10mins": 233.21, "librispeech-10mins": 128.01}, "average_wer": 16.74, "qoi": 0.33, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "distil-whisper/distil-large-v3/turbo/600MB", "device": "iPad Air 11-inch (M2)", "os": "iPadOS 17.6.1", "timestamp": "2024-10-30T04:34:27Z", "speed": 10.26, "tokens_per_second": 34.22, "dataset_speed": {"earnings22-10mins": 17.73, "librispeech-10mins": 7.23}, "dataset_tokens_per_second": {"earnings22-10mins": 54.65, "librispeech-10mins": 25.93}, "average_wer": 8.11, "qoi": 0.6, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "distil-whisper/distil-large-v3", "device": "iPad Air 11-inch (M2)", "os": "iPadOS 17.6.1", "timestamp": "2024-10-30T04:07:14Z", "speed": 1.12, "tokens_per_second": 4.16, "dataset_speed": {"librispeech-10mins": 0.73, "earnings22-10mins": 2.41}, "dataset_tokens_per_second": {"librispeech-10mins": 3.06, "earnings22-10mins": 7.78}, "average_wer": 6.73, "qoi": 0.68, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-tiny", "device": "Apple M3 Max", "os": "macOS 15.2.0", "timestamp": "2024-10-29T10:13:11Z", "speed": 57.13, "tokens_per_second": 221.9, "dataset_speed": {"earnings22-10mins": 102.77, "librispeech-10mins": 39.61}, "dataset_tokens_per_second": {"earnings22-10mins": 373.56, "librispeech-10mins": 163.69}, "average_wer": 16.67, "qoi": 0.32, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-tiny.en", "device": "iPhone 11", "os": "iOS 17.6.1", "timestamp": "2024-10-29T23:05:39Z", "speed": 6.24, "tokens_per_second": 22.59, "dataset_speed": {"librispeech-10mins": 4.53, "earnings22-10mins": 10.05}, "dataset_tokens_per_second": {"librispeech-10mins": 18.87, "earnings22-10mins": 30.88}, "average_wer": 14.46, "qoi": 0.39, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-base", "device": "iPad Air 11-inch (M2)", "os": "iPadOS 17.6.1", "timestamp": "2024-10-30T01:05:42Z", "speed": 31.42, "tokens_per_second": 115.63, "dataset_speed": {"librispeech-10mins": 21.39, "earnings22-10mins": 59.37}, "dataset_tokens_per_second": {"librispeech-10mins": 88.74, "earnings22-10mins": 190.59}, "average_wer": 12.56, "qoi": 0.39, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-large-v3/947MB", "device": "iPad Air 11-inch (M2)", "os": "iPadOS 17.6.1", "timestamp": "2024-10-30T03:21:55Z", "speed": 1.75, "tokens_per_second": 8.28, "dataset_speed": {"librispeech-10mins": 1.19, "earnings22-10mins": 3.34}, "dataset_tokens_per_second": {"librispeech-10mins": 7.56, "earnings22-10mins": 10.33}, "average_wer": 12.12, "qoi": 0.69, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-large-v2/949MB", "device": "iPad Air 11-inch (M2)", "os": "iPadOS 17.6.1", "timestamp": "2024-10-30T01:35:52Z", "speed": 2.02, "tokens_per_second": 8.23, "dataset_speed": {"earnings22-10mins": 2.88, "librispeech-10mins": 1.56}, "dataset_tokens_per_second": {"earnings22-10mins": 10.11, "librispeech-10mins": 7.22}, "average_wer": 7.08, "qoi": 0.62, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "distil-whisper/distil-large-v3/turbo/600MB", "device": "Apple M3 Max", "os": "macOS 15.2.0", "timestamp": "2024-10-29T18:29:33Z", "speed": 16.01, "tokens_per_second": 53.29, "dataset_speed": {"earnings22-10mins": 33.26, "librispeech-10mins": 10.55}, "dataset_tokens_per_second": {"earnings22-10mins": 102.01, "librispeech-10mins": 37.89}, "average_wer": 7.87, "qoi": 0.6, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-large-v3-v20240930/turbo", "device": "iPad Air 11-inch (M2)", "os": "iPadOS 17.6.1", "timestamp": "2024-10-30T04:50:45Z", "speed": 1.59, "tokens_per_second": 4.83, "dataset_speed": {"earnings22-10mins": 1.59}, "dataset_tokens_per_second": {"earnings22-10mins": 4.83}, "average_wer": 7.7, "qoi": 1, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-large-v3-v20240930/626MB", "device": "iPad Air 11-inch (M2)", "os": "iPadOS 17.6.1", "timestamp": "2024-10-30T04:59:59Z", "speed": 5.79, "tokens_per_second": 23.11, "dataset_speed": {"earnings22-10mins": 9.36, "librispeech-10mins": 4.19}, "dataset_tokens_per_second": {"earnings22-10mins": 35.8, "librispeech-10mins": 17.45}, "average_wer": 6.04, "qoi": 0.71, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-small", "device": "iPad Air 11-inch (M2)", "os": "iPadOS 17.6.1", "timestamp": "2024-10-30T01:09:02Z", "speed": 12.87, "tokens_per_second": 47.92, "dataset_speed": {"librispeech-10mins": 9.73, "earnings22-10mins": 19.02}, "dataset_tokens_per_second": {"librispeech-10mins": 40.89, "earnings22-10mins": 61.72}, "average_wer": 8.91, "qoi": 0.55, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-large-v2/turbo", "device": "Apple M3 Max", "os": "macOS 15.2.0", "timestamp": "2024-10-29T17:48:48Z", "speed": 4.04, "tokens_per_second": 15.21, "dataset_speed": {"earnings22-10mins": 6.67, "librispeech-10mins": 2.9}, "dataset_tokens_per_second": {"earnings22-10mins": 21.56, "librispeech-10mins": 12.46}, "average_wer": 6.42, "qoi": 0.64, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "distil-whisper/distil-large-v3/594MB", "device": "Apple M3 Max", "os": "macOS 15.2.0", "timestamp": "2024-10-29T18:26:59Z", "speed": 10.03, "tokens_per_second": 37.01, "dataset_speed": {"librispeech-10mins": 6.48, "earnings22-10mins": 22.33}, "dataset_tokens_per_second": {"librispeech-10mins": 26.93, "earnings22-10mins": 71.94}, "average_wer": 7.42, "qoi": 0.66, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-large-v3", "device": "Apple M3 Max", "os": "macOS 15.2.0", "timestamp": "2024-10-29T17:59:12Z", "speed": 3.2, "tokens_per_second": 11.72, "dataset_speed": {"earnings22-10mins": 5.79, "librispeech-10mins": 2.21}, "dataset_tokens_per_second": {"earnings22-10mins": 18.16, "librispeech-10mins": 9.27}, "average_wer": 7.63, "qoi": 0.74, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-large-v2/turbo/955MB", "device": "Apple M3 Max", "os": "macOS 15.2.0", "timestamp": "2024-10-29T17:57:24Z", "speed": 4.33, "tokens_per_second": 14.96, "dataset_speed": {"librispeech-10mins": 3.18, "earnings22-10mins": 6.8}, "dataset_tokens_per_second": {"librispeech-10mins": 12.12, "earnings22-10mins": 21.04}, "average_wer": 7.32, "qoi": 0.6, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-large-v3/turbo/954MB", "device": "iPad Air 11-inch (M2)", "os": "iPadOS 17.6.1", "timestamp": "2024-10-30T03:48:25Z", "speed": 3.05, "tokens_per_second": 10.32, "dataset_speed": {"librispeech-10mins": 2.42, "earnings22-10mins": 4.12}, "dataset_tokens_per_second": {"librispeech-10mins": 8.91, "earnings22-10mins": 12.73}, "average_wer": 6.25, "qoi": 0.68, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-large-v3-v20240930", "device": "Apple M3 Max", "os": "macOS 15.2.0", "timestamp": "2024-10-29T18:33:14Z", "speed": 8.48, "tokens_per_second": 32.37, "dataset_speed": {"librispeech-10mins": 5.57, "earnings22-10mins": 17.81}, "dataset_tokens_per_second": {"librispeech-10mins": 23.01, "earnings22-10mins": 62.41}, "average_wer": 5.97, "qoi": 0.73, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-large-v3/turbo/954MB", "device": "Apple M3 Max", "os": "macOS 15.2.0", "timestamp": "2024-10-29T18:18:02Z", "speed": 4.41, "tokens_per_second": 14.95, "dataset_speed": {"earnings22-10mins": 6.75, "librispeech-10mins": 3.28}, "dataset_tokens_per_second": {"earnings22-10mins": 20.91, "librispeech-10mins": 12.07}, "average_wer": 6.25, "qoi": 0.68, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-large-v2/turbo/955MB", "device": "iPad Air 11-inch (M2)", "os": "iPadOS 17.6.1", "timestamp": "2024-10-30T02:19:30Z", "speed": 2.98, "tokens_per_second": 10.28, "dataset_speed": {"earnings22-10mins": 4.1, "librispeech-10mins": 2.34}, "dataset_tokens_per_second": {"earnings22-10mins": 12.73, "librispeech-10mins": 8.88}, "average_wer": 7.21, "qoi": 0.61, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-small", "device": "Apple M3 Max", "os": "macOS 15.2.0", "timestamp": "2024-10-29T10:18:24Z", "speed": 19.89, "tokens_per_second": 74.1, "dataset_speed": {"librispeech-10mins": 13.23, "earnings22-10mins": 40.26}, "dataset_tokens_per_second": {"librispeech-10mins": 55.44, "earnings22-10mins": 131.12}, "average_wer": 9.01, "qoi": 0.55, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-large-v2/949MB", "device": "Apple M3 Max", "os": "macOS 15.2.0", "timestamp": "2024-10-29T17:45:23Z", "speed": 2.38, "tokens_per_second": 10.57, "dataset_speed": {"librispeech-10mins": 1.82, "earnings22-10mins": 3.45}, "dataset_tokens_per_second": {"librispeech-10mins": 8.43, "earnings22-10mins": 14.66}, "average_wer": 7.11, "qoi": 0.62, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-small.en", "device": "iPad Air 11-inch (M2)", "os": "iPadOS 17.6.1", "timestamp": "2024-10-30T01:11:27Z", "speed": 12.61, "tokens_per_second": 45.67, "dataset_speed": {"librispeech-10mins": 10.14, "earnings22-10mins": 16.67}, "dataset_tokens_per_second": {"librispeech-10mins": 38.89, "earnings22-10mins": 56.87}, "average_wer": 8.35, "qoi": 0.6, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-tiny.en", "device": "iPad Air 11-inch (M2)", "os": "iPadOS 17.6.1", "timestamp": "2024-10-30T01:04:29Z", "speed": 43.69, "tokens_per_second": 150.41, "dataset_speed": {"librispeech-10mins": 30.11, "earnings22-10mins": 79.87}, "dataset_tokens_per_second": {"librispeech-10mins": 114.73, "earnings22-10mins": 245.46}, "average_wer": 14.67, "qoi": 0.37, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-large-v3-v20240930/turbo/632MB", "device": "Apple M3 Max", "os": "macOS 15.2.0", "timestamp": "2024-10-29T18:40:13Z", "speed": 12.55, "tokens_per_second": 46.18, "dataset_speed": {"librispeech-10mins": 8.85, "earnings22-10mins": 21.63}, "dataset_tokens_per_second": {"librispeech-10mins": 32.09, "earnings22-10mins": 80.76}, "average_wer": 6.28, "qoi": 0.69, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-large-v3-v20240930/626MB", "device": "Apple M3 Max", "os": "macOS 15.2.0", "timestamp": "2024-10-29T18:38:02Z", "speed": 8.28, "tokens_per_second": 33.15, "dataset_speed": {"librispeech-10mins": 5.6, "earnings22-10mins": 16.0}, "dataset_tokens_per_second": {"librispeech-10mins": 23.34, "earnings22-10mins": 61.32}, "average_wer": 5.9, "qoi": 0.71, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-base", "device": "iPhone 11", "os": "iOS 17.6.1", "timestamp": "2024-10-29T23:12:17Z", "speed": 2.8, "tokens_per_second": 15.14, "dataset_speed": {"librispeech-10mins": 3.32, "earnings22-10mins": 2.41}, "dataset_tokens_per_second": {"librispeech-10mins": 6.72, "earnings22-10mins": 21.28}, "average_wer": 99.65, "qoi": 0, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-base", "device": "Apple M3 Max", "os": "macOS 15.2.0", "timestamp": "2024-10-29T10:15:13Z", "speed": 42.83, "tokens_per_second": 157.75, "dataset_speed": {"librispeech-10mins": 27.76, "earnings22-10mins": 94.15}, "dataset_tokens_per_second": {"librispeech-10mins": 115.24, "earnings22-10mins": 302.55}, "average_wer": 12.56, "qoi": 0.39, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "openai/whisper-large-v3-v20240930/turbo/632MB", "device": "iPad Air 11-inch (M2)", "os": "iPadOS 17.6.1", "timestamp": "2024-10-30T05:08:06Z", "speed": 7.99, "tokens_per_second": 29.43, "dataset_speed": {"earnings22-10mins": 11.93, "librispeech-10mins": 6.01}, "dataset_tokens_per_second": {"earnings22-10mins": 44.65, "librispeech-10mins": 21.79}, "average_wer": 6.28, "qoi": 0.69, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}
-{"model": "distil-whisper/distil-large-v3/turbo", "device": "iPad Air 11-inch (M2)", "os": "iPadOS 17.6.1", "timestamp": "2024-10-30T04:26:17Z", "speed": 1.69, "tokens_per_second": 5.19, "dataset_speed": {"earnings22-10mins": 1.69}, "dataset_tokens_per_second": {"earnings22-10mins": 5.19}, "average_wer": 8.28, "qoi": 0, "commit_hash": "9f493bc", "commit_timestamp": "2024-10-29T101123"}

dashboard_data/support_data.csv

@@ -7,7 +7,7 @@ openai_whisper-base,openai_whisper-base,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ ma
 openai_whisper-base.en,openai_whisper-base.en,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,✅ iOS 17.6.1,✅ iOS 17.7,✅ iOS 17.2.1,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
 openai_whisper-large-v2,openai_whisper-large-v2,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 openai_whisper-large-v2_949MB,openai_whisper-large-v2_949MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
-openai_whisper-large-v2_turbo,openai_whisper-large-v2_turbo,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,Not Supported,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
+openai_whisper-large-v2_turbo,openai_whisper-large-v2_turbo,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T073931.json>iPadOS 17.6.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 openai_whisper-large-v2_turbo_955MB,openai_whisper-large-v2_turbo_955MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,Not Supported,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
 openai_whisper-large-v3,openai_whisper-large-v3,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 openai_whisper-large-v3-v20240930,openai_whisper-large-v3-v20240930,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T073931.json>iPadOS 17.6.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported

main.py

@@ -113,7 +113,13 @@ model_to_multilingual_wer = dict(
 multilingual_models = {}
 for multilingual_model, multilingual_wer in model_to_multilingual_wer.items():
     for model in benchmark_df["model"].unique().tolist():
-        if multilingual_model in model and not model.endswith("en"):
+        if model in model_to_multilingual_wer:
+            continue
+        if model == "openai/whisper-large-v3-v20240930/turbo/632MB":
+            multilingual_models[model] = model_to_multilingual_wer["openai/whisper-large-v3-v20240930"]
+        if model.endswith("/turbo") and model.replace("/turbo", "") in model_to_multilingual_wer:
+            multilingual_models[model] = model_to_multilingual_wer[model.replace("/turbo", "")]
+        elif multilingual_model in model and not model.endswith("en"):
             multilingual_models[model] = multilingual_wer
 
 # Update the dictionary with turbo models

utils.py

@@ -407,6 +407,8 @@ def plot_metric(
         group.sort_values("commit_timestamp") 
         for _, group in grouped 
     ]
+    for i, group in enumerate(sorted_groups):
+        print(group["model"], group["device"], group["qoi"])
 
     if filter_input:
         filters = [f.strip().lower() for f in filter_input.split(";")]