Hugging Face's logo

Spaces:
natasa365
/
whisper.cpp
Running

App Files Community
ggerganov commited on
Commit
d4f72cd
·
unverified ·
1 Parent(s): 835c3e8

bench : update benches

Browse files
Files changed (1) hide show
  1. scripts/bench-all-gg.txt +100 -99
scripts/bench-all-gg.txt CHANGED
@@ -111,62 +111,61 @@ make -j && ./scripts/bench-all.sh 1 1 0
111
 
112
  | CPU | Config | Model | Th | FA | Enc. | Dec. | Bch5 | PP | Commit |
113
  | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
114
- | M2 ULTRA | METAL | tiny | 1 | 0 | 8.74 | 1.20 | 0.36 | 0.01 | ad4e350 |
115
- | M2 ULTRA | METAL | tiny-q5_0 | 1 | 0 | 10.30 | 1.15 | 0.38 | 0.01 | ad4e350 |
116
- | M2 ULTRA | METAL | tiny-q5_1 | 1 | 0 | 10.71 | 1.13 | 0.38 | 0.01 | ad4e350 |
117
- | M2 ULTRA | METAL | tiny-q8_0 | 1 | 0 | 9.97 | 1.12 | 0.37 | 0.01 | ad4e350 |
118
- | M2 ULTRA | METAL | base | 1 | 0 | 16.77 | 1.71 | 0.44 | 0.02 | ad4e350 |
119
- | M2 ULTRA | METAL | base-q5_0 | 1 | 0 | 16.92 | 1.63 | 0.44 | 0.02 | ad4e350 |
120
- | M2 ULTRA | METAL | base-q5_1 | 1 | 0 | 16.84 | 1.63 | 0.44 | 0.02 | ad4e350 |
121
- | M2 ULTRA | METAL | base-q8_0 | 1 | 0 | 16.12 | 1.63 | 0.44 | 0.02 | ad4e350 |
122
- | M2 ULTRA | METAL | small | 1 | 0 | 45.29 | 3.44 | 0.92 | 0.05 | ad4e350 |
123
- | M2 ULTRA | METAL | small-q5_0 | 1 | 0 | 50.43 | 3.34 | 0.94 | 0.06 | ad4e350 |
124
- | M2 ULTRA | METAL | small-q5_1 | 1 | 0 | 50.49 | 3.35 | 0.93 | 0.06 | ad4e350 |
125
- | M2 ULTRA | METAL | small-q8_0 | 1 | 0 | 47.37 | 3.20 | 0.91 | 0.05 | ad4e350 |
126
- | M2 ULTRA | METAL | medium | 1 | 0 | 122.81 | 7.39 | 1.99 | 0.12 | ad4e350 |
127
- | M2 ULTRA | METAL | medium-q5_0 | 1 | 0 | 140.62 | 6.73 | 2.03 | 0.14 | ad4e350 |
128
- | M2 ULTRA | METAL | medium-q5_1 | 1 | 0 | 140.44 | 6.74 | 2.04 | 0.14 | ad4e350 |
129
- | M2 ULTRA | METAL | medium-q8_0 | 1 | 0 | 131.05 | 6.54 | 1.95 | 0.13 | ad4e350 |
130
- | M2 ULTRA | METAL | medium-dis | 1 | 0 | 110.95 | 0.99 | 0.24 | 0.02 | ad4e350 |
131
- | M2 ULTRA | METAL | large-v2 | 1 | 0 | 222.19 | 10.93 | 3.01 | 0.21 | ad4e350 |
132
- | M2 ULTRA | METAL | large-v2-q5_0 | 1 | 0 | 258.47 | 9.75 | 3.01 | 0.25 | ad4e350 |
133
- | M2 ULTRA | METAL | large-v2-q5_1 | 1 | 0 | 258.40 | 9.85 | 3.01 | 0.24 | ad4e350 |
134
- | M2 ULTRA | METAL | large-v2-q8_0 | 1 | 0 | 236.68 | 9.61 | 2.85 | 0.23 | ad4e350 |
135
- | M2 ULTRA | METAL | large-v2-dis | 1 | 0 | 199.28 | 1.12 | 0.27 | 0.02 | ad4e350 |
136
- | M2 ULTRA | METAL | large-v3-turbo | 1 | 0 | 201.49 | 1.76 | 0.45 | 0.03 | ad4e350 |
137
- | M2 ULTRA | METAL | large-v3-turbo-q5_0 | 1 | 0 | 233.70 | 1.55 | 0.46 | 0.04 | ad4e350 |
138
- | M2 ULTRA | METAL | large-v3-turbo-q8_0 | 1 | 0 | 214.20 | 1.51 | 0.44 | 0.04 | ad4e350 |
139
-
140
 
141
  make -j && ./scripts/bench-all.sh 1 1 1
142
 
143
  | CPU | Config | Model | Th | FA | Enc. | Dec. | Bch5 | PP | Commit |
144
  | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
145
- | M2 ULTRA | METAL | tiny | 1 | 1 | 7.82 | 1.31 | 0.35 | 0.01 | ad4e350 |
146
- | M2 ULTRA | METAL | tiny-q5_0 | 1 | 1 | 8.32 | 1.28 | 0.37 | 0.01 | ad4e350 |
147
- | M2 ULTRA | METAL | tiny-q5_1 | 1 | 1 | 8.21 | 1.28 | 0.37 | 0.01 | ad4e350 |
148
- | M2 ULTRA | METAL | tiny-q8_0 | 1 | 1 | 7.97 | 1.23 | 0.36 | 0.01 | ad4e350 |
149
- | M2 ULTRA | METAL | base | 1 | 1 | 13.96 | 1.80 | 0.42 | 0.02 | ad4e350 |
150
- | M2 ULTRA | METAL | base-q5_0 | 1 | 1 | 15.19 | 1.75 | 0.42 | 0.02 | ad4e350 |
151
- | M2 ULTRA | METAL | base-q5_1 | 1 | 1 | 15.09 | 1.75 | 0.42 | 0.02 | ad4e350 |
152
- | M2 ULTRA | METAL | base-q8_0 | 1 | 1 | 14.45 | 1.70 | 0.41 | 0.02 | ad4e350 |
153
- | M2 ULTRA | METAL | small | 1 | 1 | 40.08 | 3.54 | 0.86 | 0.05 | ad4e350 |
154
- | M2 ULTRA | METAL | small-q5_0 | 1 | 1 | 45.07 | 3.51 | 0.88 | 0.05 | ad4e350 |
155
- | M2 ULTRA | METAL | small-q5_1 | 1 | 1 | 45.05 | 3.52 | 0.88 | 0.05 | ad4e350 |
156
- | M2 ULTRA | METAL | small-q8_0 | 1 | 1 | 42.04 | 3.34 | 0.85 | 0.05 | ad4e350 |
157
- | M2 ULTRA | METAL | medium | 1 | 1 | 107.20 | 7.28 | 1.79 | 0.11 | ad4e350 |
158
- | M2 ULTRA | METAL | medium-q5_0 | 1 | 1 | 125.02 | 6.67 | 1.83 | 0.12 | ad4e350 |
159
- | M2 ULTRA | METAL | medium-q5_1 | 1 | 1 | 124.83 | 6.70 | 1.84 | 0.12 | ad4e350 |
160
- | M2 ULTRA | METAL | medium-q8_0 | 1 | 1 | 114.56 | 6.53 | 1.79 | 0.11 | ad4e350 |
161
- | M2 ULTRA | METAL | medium-dis | 1 | 1 | 95.96 | 1.01 | 0.23 | 0.01 | ad4e350 |
162
- | M2 ULTRA | METAL | large-v2 | 1 | 1 | 194.29 | 10.57 | 2.67 | 0.20 | ad4e350 |
163
- | M2 ULTRA | METAL | large-v2-q5_0 | 1 | 1 | 230.74 | 9.57 | 2.73 | 0.23 | ad4e350 |
164
- | M2 ULTRA | METAL | large-v2-q5_1 | 1 | 1 | 229.97 | 9.69 | 2.74 | 0.23 | ad4e350 |
165
- | M2 ULTRA | METAL | large-v2-q8_0 | 1 | 1 | 208.11 | 9.37 | 2.60 | 0.21 | ad4e350 |
166
- | M2 ULTRA | METAL | large-v2-dis | 1 | 1 | 172.72 | 1.12 | 0.26 | 0.02 | ad4e350 |
167
- | M2 ULTRA | METAL | large-v3-turbo | 1 | 1 | 174.46 | 1.74 | 0.42 | 0.03 | ad4e350 |
168
- | M2 ULTRA | METAL | large-v3-turbo-q5_0 | 1 | 1 | 205.78 | 1.54 | 0.42 | 0.04 | ad4e350 |
169
- | M2 ULTRA | METAL | large-v3-turbo-q8_0 | 1 | 1 | 186.33 | 1.50 | 0.40 | 0.03 | ad4e350 |
170
 
171
 
172
  ## M4 Max
@@ -219,67 +218,69 @@ make -j && ./scripts/bench-all.sh 1 1 0
219
 
220
  | CPU | Config | Model | Th | FA | Enc. | Dec. | Bch5 | PP | Commit |
221
  | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
222
- | M4 Max | METAL | tiny | 1 | 0 | 13.12 | 0.87 | 0.29 | 0.01 | ad4e3509 |
223
- | M4 Max | METAL | tiny-q8_0 | 1 | 0 | 15.90 | 0.88 | 0.31 | 0.01 | ad4e3509 |
224
- | M4 Max | METAL | base | 1 | 0 | 23.10 | 1.42 | 0.34 | 0.02 | ad4e3509 |
225
- | M4 Max | METAL | base-q8_0 | 1 | 0 | 27.25 | 1.31 | 0.34 | 0.02 | ad4e3509 |
226
- | M4 Max | METAL | small | 1 | 0 | 71.76 | 3.02 | 0.70 | 0.06 | ad4e3509 |
227
- | M4 Max | METAL | small-q8_0 | 1 | 0 | 73.88 | 2.60 | 0.71 | 0.06 | ad4e3509 |
228
- | M4 Max | METAL | medium | 1 | 0 | 208.22 | 6.94 | 1.55 | 0.16 | ad4e3509 |
229
- | M4 Max | METAL | medium-q8_0 | 1 | 0 | 214.65 | 5.90 | 1.57 | 0.17 | ad4e3509 |
230
- | M4 Max | METAL | large-v2 | 1 | 0 | 381.72 | 11.28 | 2.51 | 0.29 | ad4e3509 |
231
- | M4 Max | METAL | large-v2-q8_0 | 1 | 0 | 394.97 | 8.90 | 2.45 | 0.30 | ad4e3509 |
 
232
 
233
 
234
  make -j && ./scripts/bench-all.sh 1 1 1
235
 
236
  | CPU | Config | Model | Th | FA | Enc. | Dec. | Bch5 | PP | Commit |
237
  | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
238
- | M4 Max | METAL | tiny | 1 | 1 | 15.22 | 0.89 | 0.26 | 0.01 | ad4e3509 |
239
- | M4 Max | METAL | tiny-q8_0 | 1 | 1 | 14.70 | 0.86 | 0.26 | 0.01 | ad4e3509 |
240
- | M4 Max | METAL | base | 1 | 1 | 25.33 | 1.36 | 0.30 | 0.02 | ad4e3509 |
241
- | M4 Max | METAL | base-q8_0 | 1 | 1 | 21.27 | 1.31 | 0.30 | 0.02 | ad4e3509 |
242
- | M4 Max | METAL | small | 1 | 1 | 58.43 | 2.78 | 0.60 | 0.05 | ad4e3509 |
243
- | M4 Max | METAL | small-q8_0 | 1 | 1 | 60.26 | 2.39 | 0.60 | 0.05 | ad4e3509 |
244
- | M4 Max | METAL | medium | 1 | 1 | 169.73 | 6.03 | 1.31 | 0.14 | ad4e3509 |
245
- | M4 Max | METAL | medium-q8_0 | 1 | 1 | 176.61 | 4.99 | 1.31 | 0.14 | ad4e3509 |
246
- | M4 Max | METAL | large-v2 | 1 | 1 | 316.18 | 9.60 | 2.08 | 0.24 | ad4e3509 |
247
- | M4 Max | METAL | large-v2-q8_0 | 1 | 1 | 329.59 | 7.55 | 2.08 | 0.25 | ad4e3509 |
 
248
 
249
 
250
  # V100
251
 
252
- WHISPER_CUDA=1 make -j && ./scripts/bench-all.sh 8 1 0
253
 
254
  | GPU | Config | Model | Th | FA | Enc. | Dec. | Bch5 | PP | Commit |
255
  | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
256
- | V100 | AVX2 CUDA | tiny | 8 | 0 | 6.15 | 1.02 | 0.30 | 0.01 | ad4e3509 |
257
- | V100 | AVX2 CUDA | tiny-q5_1 | 8 | 0 | 5.92 | 0.96 | 0.25 | 0.01 | ad4e3509 |
258
- | V100 | AVX2 CUDA | base | 8 | 0 | 10.60 | 1.43 | 0.43 | 0.02 | ad4e3509 |
259
- | V100 | AVX2 CUDA | base-q5_1 | 8 | 0 | 10.80 | 1.37 | 0.36 | 0.02 | ad4e3509 |
260
- | V100 | AVX2 CUDA | small | 8 | 0 | 31.83 | 2.82 | 0.87 | 0.04 | ad4e3509 |
261
- | V100 | AVX2 CUDA | small-q5_1 | 8 | 0 | 31.88 | 2.68 | 0.72 | 0.04 | ad4e3509 |
262
- | V100 | AVX2 CUDA | medium | 8 | 0 | 81.30 | 6.02 | 1.81 | 0.09 | ad4e3509 |
263
- | V100 | AVX2 CUDA | medium-q5_0 | 8 | 0 | 83.21 | 5.44 | 1.41 | 0.10 | ad4e3509 |
264
- | V100 | AVX2 CUDA | large-v2 | 8 | 0 | 134.81 | 8.64 | 2.69 | 0.14 | ad4e3509 |
265
- | V100 | AVX2 CUDA | large-v2-q5_0 | 8 | 0 | 138.95 | 7.57 | 2.04 | 0.15 | ad4e3509 |
266
- | V100 | AVX2 CUDA | large-v3-turbo | 8 | 0 | 124.42 | 1.37 | 0.43 | 0.02 | ad4e3509 |
267
- | V100 | AVX2 CUDA | large-v3-turbo-q5_0 | 8 | 0 | 127.81 | 1.13 | 0.32 | 0.03 | ad4e3509 |
268
-
269
-
270
- WHISPER_CUDA=1 make -j && ./scripts/bench-all.sh 8 1 1
271
 
272
  | GPU | Config | Model | Th | FA | Enc. | Dec. | Bch5 | PP | Commit |
273
  | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
274
- | V100 | AVX2 CUDA | tiny | 8 | 1 | 4.01 | 0.90 | 0.25 | 0.01 | ad4e3509 |
275
- | V100 | AVX2 CUDA | tiny-q5_1 | 8 | 1 | 4.12 | 0.88 | 0.18 | 0.01 | ad4e3509 |
276
- | V100 | AVX2 CUDA | base | 8 | 1 | 7.00 | 1.30 | 0.35 | 0.01 | ad4e3509 |
277
- | V100 | AVX2 CUDA | base-q5_1 | 8 | 1 | 7.22 | 1.21 | 0.26 | 0.02 | ad4e3509 |
278
- | V100 | AVX2 CUDA | small | 8 | 1 | 18.68 | 2.39 | 0.69 | 0.03 | ad4e3509 |
279
- | V100 | AVX2 CUDA | small-q5_1 | 8 | 1 | 19.38 | 2.32 | 0.51 | 0.03 | ad4e3509 |
280
- | V100 | AVX2 CUDA | medium | 8 | 1 | 53.17 | 5.15 | 1.45 | 0.06 | ad4e3509 |
281
- | V100 | AVX2 CUDA | medium-q5_0 | 8 | 1 | 55.09 | 4.64 | 1.05 | 0.07 | ad4e3509 |
282
- | V100 | AVX2 CUDA | large-v2 | 8 | 1 | 85.77 | 7.57 | 2.19 | 0.10 | ad4e3509 |
283
- | V100 | AVX2 CUDA | large-v2-q5_0 | 8 | 1 | 89.24 | 6.48 | 1.48 | 0.11 | ad4e3509 |
284
- | V100 | AVX2 CUDA | large-v3-turbo | 8 | 1 | 75.56 | 1.25 | 0.37 | 0.02 | ad4e3509 |
285
- | V100 | AVX2 CUDA | large-v3-turbo-q5_0 | 8 | 1 | 78.48 | 1.01 | 0.24 | 0.02 | ad4e3509 |
 
111
 
112
  | CPU | Config | Model | Th | FA | Enc. | Dec. | Bch5 | PP | Commit |
113
  | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
114
+ | M2 ULTRA | METAL | tiny | 1 | 0 | 10.15 | 1.20 | 0.36 | 0.01 | dc8dda60 |
115
+ | M2 ULTRA | METAL | tiny-q5_0 | 1 | 0 | 10.21 | 1.15 | 0.39 | 0.01 | dc8dda60 |
116
+ | M2 ULTRA | METAL | tiny-q5_1 | 1 | 0 | 9.26 | 1.15 | 0.38 | 0.01 | dc8dda60 |
117
+ | M2 ULTRA | METAL | tiny-q8_0 | 1 | 0 | 9.00 | 1.12 | 0.37 | 0.01 | dc8dda60 |
118
+ | M2 ULTRA | METAL | base | 1 | 0 | 15.77 | 1.73 | 0.45 | 0.02 | dc8dda60 |
119
+ | M2 ULTRA | METAL | base-q5_0 | 1 | 0 | 16.90 | 1.63 | 0.44 | 0.02 | dc8dda60 |
120
+ | M2 ULTRA | METAL | base-q5_1 | 1 | 0 | 16.93 | 1.64 | 0.44 | 0.02 | dc8dda60 |
121
+ | M2 ULTRA | METAL | base-q8_0 | 1 | 0 | 16.13 | 1.63 | 0.43 | 0.02 | dc8dda60 |
122
+ | M2 ULTRA | METAL | small | 1 | 0 | 45.15 | 3.45 | 0.92 | 0.05 | dc8dda60 |
123
+ | M2 ULTRA | METAL | small-q5_0 | 1 | 0 | 50.63 | 3.36 | 0.94 | 0.06 | dc8dda60 |
124
+ | M2 ULTRA | METAL | small-q5_1 | 1 | 0 | 50.56 | 3.36 | 0.94 | 0.06 | dc8dda60 |
125
+ | M2 ULTRA | METAL | small-q8_0 | 1 | 0 | 47.52 | 3.20 | 0.92 | 0.05 | dc8dda60 |
126
+ | M2 ULTRA | METAL | medium | 1 | 0 | 122.55 | 7.38 | 1.95 | 0.12 | dc8dda60 |
127
+ | M2 ULTRA | METAL | medium-q5_0 | 1 | 0 | 140.61 | 6.73 | 2.02 | 0.14 | dc8dda60 |
128
+ | M2 ULTRA | METAL | medium-q5_1 | 1 | 0 | 140.48 | 6.76 | 2.04 | 0.14 | dc8dda60 |
129
+ | M2 ULTRA | METAL | medium-q8_0 | 1 | 0 | 131.00 | 6.57 | 1.96 | 0.13 | dc8dda60 |
130
+ | M2 ULTRA | METAL | medium-dis | 1 | 0 | 110.85 | 1.00 | 0.24 | 0.02 | dc8dda60 |
131
+ | M2 ULTRA | METAL | large-v2 | 1 | 0 | 222.28 | 10.96 | 3.03 | 0.21 | dc8dda60 |
132
+ | M2 ULTRA | METAL | large-v2-q5_0 | 1 | 0 | 258.64 | 9.79 | 3.04 | 0.25 | dc8dda60 |
133
+ | M2 ULTRA | METAL | large-v2-q5_1 | 1 | 0 | 258.32 | 9.87 | 3.05 | 0.24 | dc8dda60 |
134
+ | M2 ULTRA | METAL | large-v2-q8_0 | 1 | 0 | 236.55 | 9.61 | 2.87 | 0.23 | dc8dda60 |
135
+ | M2 ULTRA | METAL | large-v2-dis | 1 | 0 | 199.84 | 1.14 | 0.27 | 0.02 | dc8dda60 |
136
+ | M2 ULTRA | METAL | large-v3-turbo | 1 | 0 | 201.52 | 1.77 | 0.45 | 0.03 | dc8dda60 |
137
+ | M2 ULTRA | METAL | large-v3-turbo-q5_0 | 1 | 0 | 233.14 | 1.56 | 0.47 | 0.04 | dc8dda60 |
138
+ | M2 ULTRA | METAL | large-v3-turbo-q8_0 | 1 | 0 | 214.23 | 1.53 | 0.44 | 0.04 | dc8dda60 |
 
139
 
140
  make -j && ./scripts/bench-all.sh 1 1 1
141
 
142
  | CPU | Config | Model | Th | FA | Enc. | Dec. | Bch5 | PP | Commit |
143
  | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
144
+ | M2 ULTRA | METAL | tiny | 1 | 1 | 7.72 | 1.05 | 0.32 | 0.01 | dc8dda60 |
145
+ | M2 ULTRA | METAL | tiny-q5_0 | 1 | 1 | 8.20 | 0.98 | 0.31 | 0.01 | dc8dda60 |
146
+ | M2 ULTRA | METAL | tiny-q5_1 | 1 | 1 | 8.13 | 0.99 | 0.31 | 0.01 | dc8dda60 |
147
+ | M2 ULTRA | METAL | tiny-q8_0 | 1 | 1 | 7.96 | 0.93 | 0.30 | 0.01 | dc8dda60 |
148
+ | M2 ULTRA | METAL | base | 1 | 1 | 13.52 | 1.39 | 0.35 | 0.02 | dc8dda60 |
149
+ | M2 ULTRA | METAL | base-q5_0 | 1 | 1 | 14.88 | 1.31 | 0.34 | 0.02 | dc8dda60 |
150
+ | M2 ULTRA | METAL | base-q5_1 | 1 | 1 | 14.76 | 1.33 | 0.34 | 0.02 | dc8dda60 |
151
+ | M2 ULTRA | METAL | base-q8_0 | 1 | 1 | 14.04 | 1.28 | 0.34 | 0.02 | dc8dda60 |
152
+ | M2 ULTRA | METAL | small | 1 | 1 | 38.78 | 2.72 | 0.67 | 0.04 | dc8dda60 |
153
+ | M2 ULTRA | METAL | small-q5_0 | 1 | 1 | 44.01 | 2.64 | 0.69 | 0.05 | dc8dda60 |
154
+ | M2 ULTRA | METAL | small-q5_1 | 1 | 1 | 44.02 | 2.66 | 0.69 | 0.05 | dc8dda60 |
155
+ | M2 ULTRA | METAL | small-q8_0 | 1 | 1 | 40.79 | 2.49 | 0.67 | 0.05 | dc8dda60 |
156
+ | M2 ULTRA | METAL | medium | 1 | 1 | 104.48 | 5.57 | 1.61 | 0.10 | dc8dda60 |
157
+ | M2 ULTRA | METAL | medium-q5_0 | 1 | 1 | 122.24 | 5.00 | 1.58 | 0.12 | dc8dda60 |
158
+ | M2 ULTRA | METAL | medium-q5_1 | 1 | 1 | 121.99 | 5.02 | 1.59 | 0.12 | dc8dda60 |
159
+ | M2 ULTRA | METAL | medium-q8_0 | 1 | 1 | 111.68 | 4.99 | 1.52 | 0.11 | dc8dda60 |
160
+ | M2 ULTRA | METAL | medium-dis | 1 | 1 | 93.23 | 0.87 | 0.21 | 0.01 | dc8dda60 |
161
+ | M2 ULTRA | METAL | large-v2 | 1 | 1 | 189.82 | 8.36 | 2.35 | 0.19 | dc8dda60 |
162
+ | M2 ULTRA | METAL | large-v2-q5_0 | 1 | 1 | 225.73 | 7.34 | 2.40 | 0.22 | dc8dda60 |
163
+ | M2 ULTRA | METAL | large-v2-q5_1 | 1 | 1 | 225.88 | 7.60 | 2.40 | 0.22 | dc8dda60 |
164
+ | M2 ULTRA | METAL | large-v2-q8_0 | 1 | 1 | 203.55 | 7.32 | 2.26 | 0.20 | dc8dda60 |
165
+ | M2 ULTRA | METAL | large-v2-dis | 1 | 1 | 168.20 | 0.98 | 0.24 | 0.02 | dc8dda60 |
166
+ | M2 ULTRA | METAL | large-v3-turbo | 1 | 1 | 170.22 | 1.46 | 0.37 | 0.03 | dc8dda60 |
167
+ | M2 ULTRA | METAL | large-v3-turbo-q5_0 | 1 | 1 | 201.88 | 1.27 | 0.38 | 0.04 | dc8dda60 |
168
+ | M2 ULTRA | METAL | large-v3-turbo-q8_0 | 1 | 1 | 182.37 | 1.24 | 0.36 | 0.03 | dc8dda60 |
169
 
170
 
171
  ## M4 Max
 
218
 
219
  | CPU | Config | Model | Th | FA | Enc. | Dec. | Bch5 | PP | Commit |
220
  | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
221
+ | M4 Max | METAL | tiny | 1 | 0 | 12.83 | 0.94 | 0.30 | 0.01 | dc8dda60 |
222
+ | M4 Max | METAL | tiny-q8_0 | 1 | 0 | 12.95 | 0.80 | 0.31 | 0.01 | dc8dda60 |
223
+ | M4 Max | METAL | base | 1 | 0 | 23.54 | 1.37 | 0.33 | 0.02 | dc8dda60 |
224
+ | M4 Max | METAL | base-q8_0 | 1 | 0 | 24.14 | 1.24 | 0.33 | 0.02 | dc8dda60 |
225
+ | M4 Max | METAL | small | 1 | 0 | 71.59 | 3.02 | 0.71 | 0.06 | dc8dda60 |
226
+ | M4 Max | METAL | small-q8_0 | 1 | 0 | 73.34 | 2.65 | 0.72 | 0.06 | dc8dda60 |
227
+ | M4 Max | METAL | medium | 1 | 0 | 208.53 | 7.02 | 1.58 | 0.16 | dc8dda60 |
228
+ | M4 Max | METAL | medium-q8_0 | 1 | 0 | 212.87 | 6.00 | 1.58 | 0.17 | dc8dda60 |
229
+ | M4 Max | METAL | large-v2 | 1 | 0 | 379.84 | 11.47 | 2.52 | 0.29 | dc8dda60 |
230
+ | M4 Max | METAL | large-v2-q8_0 | 1 | 0 | 390.45 | 9.19 | 2.48 | 0.29 | dc8dda60 |
231
+ | M4 Max | METAL | large-v3-turbo | 1 | 0 | 345.74 | 1.99 | 0.44 | 0.05 | dc8dda60 |
232
 
233
 
234
  make -j && ./scripts/bench-all.sh 1 1 1
235
 
236
  | CPU | Config | Model | Th | FA | Enc. | Dec. | Bch5 | PP | Commit |
237
  | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
238
+ | M4 Max | METAL | tiny | 1 | 1 | 11.70 | 0.74 | 0.23 | 0.01 | dc8dda60 |
239
+ | M4 Max | METAL | tiny-q8_0 | 1 | 1 | 12.36 | 0.67 | 0.23 | 0.01 | dc8dda60 |
240
+ | M4 Max | METAL | base | 1 | 1 | 21.76 | 1.12 | 0.25 | 0.02 | dc8dda60 |
241
+ | M4 Max | METAL | base-q8_0 | 1 | 1 | 22.60 | 0.94 | 0.26 | 0.02 | dc8dda60 |
242
+ | M4 Max | METAL | small | 1 | 1 | 67.26 | 2.27 | 0.50 | 0.06 | dc8dda60 |
243
+ | M4 Max | METAL | small-q8_0 | 1 | 1 | 68.67 | 1.93 | 0.53 | 0.06 | dc8dda60 |
244
+ | M4 Max | METAL | medium | 1 | 1 | 193.58 | 5.31 | 1.20 | 0.16 | dc8dda60 |
245
+ | M4 Max | METAL | medium-q8_0 | 1 | 1 | 198.60 | 4.31 | 1.21 | 0.16 | dc8dda60 |
246
+ | M4 Max | METAL | large-v2 | 1 | 1 | 357.54 | 8.73 | 1.99 | 0.27 | dc8dda60 |
247
+ | M4 Max | METAL | large-v2-q8_0 | 1 | 1 | 363.98 | 6.43 | 1.99 | 0.28 | dc8dda60 |
248
+ | M4 Max | METAL | large-v3-turbo | 1 | 1 | 322.32 | 1.66 | 0.37 | 0.05 | dc8dda60 |
249
 
250
 
251
  # V100
252
 
253
+ GGML_CUDA=1 make -j && ./scripts/bench-all.sh 8 1 0
254
 
255
  | GPU | Config | Model | Th | FA | Enc. | Dec. | Bch5 | PP | Commit |
256
  | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
257
+ | V100 | AVX2 CUDA | tiny | 8 | 0 | 5.99 | 1.01 | 0.30 | 0.01 | dc8dda60 |
258
+ | V100 | AVX2 CUDA | tiny-q5_1 | 8 | 0 | 6.07 | 1.00 | 0.26 | 0.01 | dc8dda60 |
259
+ | V100 | AVX2 CUDA | base | 8 | 0 | 10.96 | 1.44 | 0.43 | 0.02 | dc8dda60 |
260
+ | V100 | AVX2 CUDA | base-q5_1 | 8 | 0 | 11.11 | 1.41 | 0.37 | 0.02 | dc8dda60 |
261
+ | V100 | AVX2 CUDA | small | 8 | 0 | 31.04 | 2.84 | 0.86 | 0.04 | dc8dda60 |
262
+ | V100 | AVX2 CUDA | small-q5_1 | 8 | 0 | 31.69 | 2.82 | 0.71 | 0.04 | dc8dda60 |
263
+ | V100 | AVX2 CUDA | medium | 8 | 0 | 83.95 | 6.05 | 1.82 | 0.09 | dc8dda60 |
264
+ | V100 | AVX2 CUDA | medium-q5_0 | 8 | 0 | 85.86 | 5.58 | 1.45 | 0.10 | dc8dda60 |
265
+ | V100 | AVX2 CUDA | large-v2 | 8 | 0 | 138.50 | 8.70 | 2.71 | 0.15 | dc8dda60 |
266
+ | V100 | AVX2 CUDA | large-v2-q5_0 | 8 | 0 | 142.31 | 7.82 | 2.03 | 0.16 | dc8dda60 |
267
+ | V100 | AVX2 CUDA | large-v3-turbo | 8 | 0 | 128.39 | 1.42 | 0.44 | 0.02 | dc8dda60 |
268
+ | V100 | AVX2 CUDA | large-v3-turbo-q5_0 | 8 | 0 | 131.24 | 1.17 | 0.33 | 0.03 | dc8dda60 |
269
+
270
+
271
+ GGML_CUDA=1 make -j && ./scripts/bench-all.sh 8 1 1
272
 
273
  | GPU | Config | Model | Th | FA | Enc. | Dec. | Bch5 | PP | Commit |
274
  | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
275
+ | V100 | AVX2 CUDA | tiny | 8 | 1 | 4.85 | 0.97 | 0.26 | 0.01 | dc8dda60 |
276
+ | V100 | AVX2 CUDA | tiny-q5_1 | 8 | 1 | 4.97 | 0.89 | 0.19 | 0.01 | dc8dda60 |
277
+ | V100 | AVX2 CUDA | base | 8 | 1 | 7.23 | 1.28 | 0.35 | 0.02 | dc8dda60 |
278
+ | V100 | AVX2 CUDA | base-q5_1 | 8 | 1 | 7.38 | 1.24 | 0.26 | 0.02 | dc8dda60 |
279
+ | V100 | AVX2 CUDA | small | 8 | 1 | 20.87 | 2.44 | 0.71 | 0.03 | dc8dda60 |
280
+ | V100 | AVX2 CUDA | small-q5_1 | 8 | 1 | 19.80 | 2.35 | 0.51 | 0.03 | dc8dda60 |
281
+ | V100 | AVX2 CUDA | medium | 8 | 1 | 54.56 | 5.31 | 1.46 | 0.06 | dc8dda60 |
282
+ | V100 | AVX2 CUDA | medium-q5_0 | 8 | 1 | 56.09 | 4.67 | 1.05 | 0.07 | dc8dda60 |
283
+ | V100 | AVX2 CUDA | large-v2 | 8 | 1 | 87.05 | 7.65 | 2.16 | 0.10 | dc8dda60 |
284
+ | V100 | AVX2 CUDA | large-v2-q5_0 | 8 | 1 | 94.65 | 6.60 | 1.47 | 0.11 | dc8dda60 |
285
+ | V100 | AVX2 CUDA | large-v3-turbo | 8 | 1 | 76.46 | 1.29 | 0.37 | 0.02 | dc8dda60 |
286
+ | V100 | AVX2 CUDA | large-v3-turbo-q5_0 | 8 | 1 | 79.62 | 1.03 | 0.23 | 0.02 | dc8dda60 |