pytorch_awq_exllama.csv

prefill.memory,prefill.latency.unit,prefill.latency.mean,prefill.latency.stdev,prefill.latency.values,prefill.throughput.unit,prefill.throughput.value,prefill.energy,prefill.efficiency,decode.memory,decode.latency.unit,decode.latency.mean,decode.latency.stdev,decode.latency.values,decode.throughput.unit,decode.throughput.value,decode.energy,decode.efficiency,per_token.memory,per_token.latency.unit,per_token.latency.mean,per_token.latency.stdev,per_token.latency.values,per_token.throughput.unit,per_token.throughput.value,per_token.energy,per_token.efficiency,backend.name,backend.version,backend._target_,backend.model,backend.task,backend.library,backend.device,backend.device_ids,backend.seed,backend.inter_op_num_threads,backend.intra_op_num_threads,backend.hub_kwargs.revision,backend.hub_kwargs.force_download,backend.hub_kwargs.local_files_only,backend.hub_kwargs.trust_remote_code,backend.no_weights,backend.device_map,backend.torch_dtype,backend.amp_autocast,backend.amp_dtype,backend.eval_mode,backend.to_bettertransformer,backend.low_cpu_mem_usage,backend.attn_implementation,backend.cache_implementation,backend.torch_compile,backend.quantization_scheme,backend.quantization_config.version,backend.quantization_config.exllama_config.version,backend.quantization_config.exllama_config.max_input_len,backend.quantization_config.exllama_config.max_batch_size,backend.deepspeed_inference,backend.peft_type,launcher.name,launcher._target_,launcher.device_isolation,launcher.start_method,benchmark.name,benchmark._target_,benchmark.duration,benchmark.warmup_runs,benchmark.input_shapes.batch_size,benchmark.input_shapes.num_choices,benchmark.input_shapes.sequence_length,benchmark.new_tokens,benchmark.energy,benchmark.memory,benchmark.latency,benchmark.generate_kwargs.max_new_tokens,benchmark.generate_kwargs.min_new_tokens,experiment_name,task,model,device,library,environment.cpu,environment.cpu_count,environment.cpu_ram_mb,environment.system,environment.machine,environment.platform,environment.processor,environment.python_version,environment.gpu,environment.gpu_count,environment.gpu_vram_mb,environment.optimum_benchmark_version,environment.optimum_benchmark_commit,environment.transformers_version,environment.transformers_commit,environment.accelerate_version,environment.accelerate_commit,environment.diffusers_version,environment.diffusers_commit,environment.optimum_version,environment.optimum_commit,environment.timm_version,environment.timm_commit,environment.peft_version,environment.peft_commit
,s,0.06108759307861301,4.5729852488414227e-05,"[0.061031585693359, 0.06105904006958, 0.061150337219238, 0.061109409332275]",tokens/s,8381.407323433255,,,,s,3.102193259716033,0.004981575674049,"[3.093663911819458, 3.104252897262573, 3.104608381271361, 3.106247848510742]",tokens/s,40.9387776220058,,,,s,0.023945713910530003,0.00017174146877000003,"[0.023831552505493, 0.024117248535156, 0.023823360443115, 0.02353049659729, 0.023851007461547002, 0.023819263458251003, 0.024078336715698003, 0.023861248016357, 0.023973888397216002, 0.024051712036132003, 0.023943168640136, 0.023657472610473, 0.023648256301879, 0.023700479507446002, 0.023845888137817003, 0.023800832748413003, 0.023767040252685, 0.023939071655273002, 0.023824384689331003, 0.023751680374145, 0.023863296508789003, 0.023736320495605003, 0.023793664932250002, 0.023941120147705, 0.023834623336791, 0.023899135589599003, 0.023792640686035003, 0.023898111343383002, 0.024115200042724, 0.024312831878662, 0.024084480285644003, 0.024089599609375, 0.023849983215332, 0.023784448623657, 0.023871488571166003, 0.023900159835815, 0.0238960647583, 0.02391142463684, 0.023790592193603, 0.023862272262573002, 0.023808000564575, 0.023909376144409, 0.024008703231811, 0.023901184082031002, 0.023895040512084003, 0.024022016525268003, 0.023933952331542004, 0.023843839645385, 0.023814144134521, 0.024122367858886003, 0.02422374343872, 0.024133632659912002, 0.024074239730834, 0.023994367599487, 0.023772159576416003, 0.024063999176025003, 0.023820287704467, 0.023931903839111, 0.023934976577758, 0.024068096160888, 0.024182783126831003, 0.023921663284301002, 0.023763967514038004, 0.023842815399169003, 0.024083456039428002, 0.023772159576416003, 0.023569408416748, 0.023848960876464, 0.023931903839111, 0.023880704879760004, 0.023601152420043, 0.023766016006469003, 0.023783424377441003, 0.023845888137817003, 0.023617536544799003, 0.023648256301879, 0.023499776840209003, 0.023636991500854, 0.023547903060913002, 0.023656448364257003, 0.023841791152954, 0.023759872436523003, 0.023832576751708003, 0.023766016006469003, 0.023589887619018, 0.02374041557312, 0.023870464324951, 0.023773183822631, 0.023832576751708003, 0.023833599090576002, 0.023901184082031002, 0.023919616699218, 0.023799808502197, 0.023929855346679, 0.023932928085327, 0.023897087097167, 0.023941120147705, 0.023802879333496, 0.023839744567871003, 0.023957504272460003, 0.023843839645385, 0.023981056213378003, 0.023805952072143, 0.023944192886352002, 0.023922687530517003, 0.023938047409057, 0.023856128692626002, 0.024849407196044, 0.023938047409057, 0.023834623336791, 0.023912448883056003, 0.024036352157592002, 0.024071168899536, 0.023998464584350003, 0.024014848709106, 0.024135679244995, 0.024015871047973, 0.024137727737426003, 0.024083456039428002, 0.023778303146362003, 0.023748607635498003, 0.023583744049072, 0.023668735504150003, 0.023796735763549, 0.023741439819335002, 0.023749631881713, 0.023761920928955002, 0.023351295471191, 0.023758848190307002, 0.023865343093872, 0.023901184082031002, 0.023927808761596003, 0.023885824203491002, 0.023766016006469003, 0.023907327651977003, 0.023739391326904002, 0.023590911865234, 0.023863296508789003, 0.023739391326904002, 0.023564287185668, 0.023617536544799003, 0.023716863632202, 0.023947263717651, 0.023852031707763, 0.023945215225219002, 0.023919616699218, 0.024027135848999, 0.023730176925659003, 0.023735296249389002, 0.023569408416748, 0.023826431274414, 0.023977983474731002, 0.023973888397216002, 0.023989248275756003, 0.024038400650024, 0.023900159835815, 0.024095743179321002, 0.023870464324951, 0.023835647583007002, 0.023886848449707003, 0.023891967773437002, 0.023934976577758, 0.02410598373413, 0.023954431533813002, 0.024077312469482002, 0.024022016525268003, 0.023983104705810002, 0.024081407546997002, 0.023973888397216002, 0.024027135848999, 0.024047615051269, 0.023841791152954, 0.024153087615966, 0.024051712036132003, 0.024090623855590004, 0.024080383300781, 0.024138751983642, 0.024055807113647003, 0.023981056213378003, 0.024059904098510002, 0.023984128952026004, 0.024161279678344003, 0.023959552764892002, 0.024057855606079002, 0.024091648101806, 0.023985151290893, 0.024176639556884, 0.024033279418945, 0.024039424896240003, 0.024019968032836, 0.02414182472229, 0.024116224288940002, 0.024116224288940002, 0.023829504013061, 0.023871488571166003, 0.023972864151, 0.023813119888305003, 0.023822336196899004, 0.023855104446411, 0.023997440338134002, 0.023937023162841003, 0.023780351638793003, 0.024027135848999, 0.024006656646728002, 0.023817216873168002, 0.02375577545166, 0.024188928604125002, 0.024122367858886003, 0.023956480026245, 0.023923711776733, 0.023977983474731002, 0.024107007980346002, 0.024078336715698003, 0.024003583908081, 0.024287231445312002, 0.024215551376342, 0.024007680892944003, 0.0241776638031, 0.024007680892944003, 0.024088575363159, 0.024159231185913004, 0.023782400131225002, 0.024194047927856004, 0.024078336715698003, 0.024020992279052, 0.024098815917968003, 0.024111103057861002, 0.024023040771484, 0.023890943527221, 0.024084480285644003, 0.023861248016357, 0.024094720840454, 0.023927808761596003, 0.023832576751708003, 0.024978431701660003, 0.023887872695922, 0.023822336196899004, 0.023845888137817003, 0.023914495468139, 0.023909376144409, 0.023869440078735003, 0.023799808502197, 0.023968767166137002, 0.023970815658569, 0.023969791412353003, 0.023885824203491002, 0.023975936889648, 0.023934976577758, 0.023858175277709, 0.023855104446411, 0.023950336456298002, 0.024003583908081, 0.024128511428833002, 0.024123392105102, 0.023778303146362003, 0.023948287963867002, 0.023996416091918, 0.02395238494873, 0.023970815658569, 0.023957504272460003, 0.023970815658569, 0.023939071655273002, 0.024089599609375, 0.023928831100463, 0.023916543960571003, 0.023856128692626002, 0.023864320755004, 0.023909376144409, 0.023747583389282, 0.023903232574462002, 0.023779327392578, 0.023748607635498003, 0.023682048797607002, 0.023829504013061, 0.023787519454956003, 0.023828479766845, 0.023983104705810002, 0.024037376403808004, 0.023833599090576002, 0.023758848190307002, 0.023853055953979, 0.023889919281005003, 0.023806976318359003, 0.023827455520629003, 0.024093696594238, 0.023828479766845, 0.024000511169433, 0.023800832748413003, 0.023835647583007002, 0.024256511688232, 0.024129535675048, 0.023909376144409, 0.023822336196899004, 0.023908351898193, 0.023831552505493, 0.023708671569824, 0.023945215225219002, 0.024135679244995, 0.023927808761596003, 0.023835647583007002, 0.023818239212036, 0.0238960647583, 0.024007680892944003, 0.023856128692626002, 0.023900159835815, 0.023863296508789003, 0.024071168899536, 0.023939071655273002, 0.024087551116943003, 0.024024063110351, 0.024090623855590004, 0.024183807373046002, 0.024035327911376, 0.024028160095214003, 0.023954431533813002, 0.024011775970458, 0.024096767425537003, 0.023901184082031002, 0.023992319107055002, 0.024027135848999, 0.023921663284301002, 0.023872512817382, 0.023930879592895003, 0.024067071914672, 0.024028160095214003, 0.024037376403808004, 0.023950336456298002, 0.023994367599487, 0.024016895294189003, 0.024022016525268003, 0.023957504272460003, 0.023948287963867002, 0.023989248275756003, 0.024151039123535, 0.024056831359863, 0.023904256820678003, 0.023681024551391, 0.023884799957275, 0.023919616699218, 0.023844863891601002, 0.023975936889648, 0.023894016265869, 0.023859199523925, 0.023963647842407003, 0.023927808761596003, 0.023981056213378003, 0.023846912384033, 0.023927808761596003, 0.024076288223266, 0.024081407546997002, 0.024003583908081, 0.023922687530517003, 0.024025087356567002, 0.024253440856933, 0.024358911514282003, 0.024037376403808004, 0.024008703231811, 0.023859199523925, 0.023719936370849, 0.023922687530517003, 0.023836671829223003, 0.025306112289428, 0.024128511428833002, 0.024133632659912002, 0.024014848709106, 0.023999488830566, 0.023976959228515, 0.024060928344726003, 0.023860223770141003, 0.024003583908081, 0.024022016525268003, 0.023972864151, 0.023930879592895003, 0.023998464584350003, 0.023928831100463, 0.024003583908081, 0.023970815658569, 0.024138751983642, 0.024294399261474003, 0.023916543960571003, 0.024005632400512, 0.02363801574707, 0.023947263717651, 0.023986175537109, 0.023955455780029, 0.023956480026245, 0.023905279159545, 0.023890943527221, 0.023943168640136, 0.024018943786621002, 0.023958528518676, 0.023883775711059003, 0.023936000823974003, 0.023979007720947004, 0.023939071655273002, 0.023930879592895003, 0.023954431533813002, 0.024189952850341003, 0.023904256820678003, 0.023774208068847003, 0.023798784255981, 0.023877632141113003, 0.023806976318359003, 0.023794687271118, 0.023894016265869, 0.023733247756958002, 0.023837696075439, 0.023732223510742, 0.023687168121337, 0.023880704879760004, 0.023931903839111, 0.023832576751708003, 0.023919616699218, 0.023795711517333003, 0.023811071395874003, 0.02395238494873, 0.023865343093872, 0.024116224288940002, 0.023904256820678003, 0.024054784774780003, 0.023950336456298002, 0.023920640945434002, 0.023840768814086002, 0.023892992019653003, 0.023956480026245, 0.02421862411499, 0.023972864151, 0.023872512817382, 0.023975936889648, 0.023987199783325003, 0.024041471481323, 0.024083456039428002, 0.024123392105102, 0.024074239730834, 0.024036352157592002, 0.02395238494873, 0.024018943786621002, 0.024083456039428002, 0.023895040512084003, 0.024016895294189003, 0.023901184082031002, 0.024047615051269, 0.023879680633544002, 0.024061952590942, 0.024101888656616003, 0.02416230392456, 0.024027135848999, 0.02404454421997, 0.02421862411499, 0.024073215484619003, 0.02421862411499, 0.024138751983642, 0.024138751983642, 0.024046592712402, 0.024168447494506, 0.024080383300781, 0.024041471481323, 0.024107007980346002, 0.024079359054565003, 0.024102912902832, 0.023957504272460003, 0.024153087615966, 0.023954431533813002, 0.02412646484375, 0.024235008239746003, 0.024094720840454, 0.024065023422241, 0.023937023162841003, 0.024076288223266, 0.024015871047973, 0.023996416091918, 0.023905279159545, 0.024047615051269, 0.024007680892944003, 0.023986175537109, 0.023917568206787, 0.023857152938842, 0.023775232315063, 0.023838720321655002, 0.023779327392578, 0.023785472869873002, 0.023897087097167, 0.023787519454956003, 0.023777280807495003, 0.023859199523925, 0.023930879592895003, 0.024031232833862003, 0.023739391326904002, 0.024949760437011, 0.023945215225219002, 0.023883775711059003, 0.024056831359863, 0.023877632141113003, 0.023989248275756003, 0.024099840164184, 0.024063999176025003, 0.024068096160888, 0.024158208847045, 0.023838720321655002, 0.023923711776733, 0.023887872695922, 0.023965696334838003, 0.024109056472778, 0.024299520492553003, 0.024054784774780003, 0.023799808502197, 0.024101888656616003, 0.024056831359863]",tokens/s,41.76112701155336,,,pytorch,2.2.1+cu118,optimum_benchmark.backends.pytorch.backend.PyTorchBackend,TheBloke/Mistral-7B-Instruct-v0.2-AWQ,text-generation,transformers,cuda,4,42,,,main,False,False,False,True,,,False,,True,False,,,,False,awq,exllama,2,128,1,False,,process,optimum_benchmark.launchers.process.launcher.ProcessLauncher,False,spawn,inference,optimum_benchmark.benchmarks.inference.benchmark.InferenceBenchmark,10,20,1,2,512,,False,False,True,128,128,mistral-7b-instruct-awq-exllama-bs1,,,,, AMD EPYC 7742 64-Core Processor,128,540671.627264,Linux,x86_64,Linux-5.4.0-166-generic-x86_64-with-glibc2.35,x86_64,3.10.12,"['NVIDIA A100-SXM4-80GB', 'NVIDIA A100-SXM4-80GB', 'NVIDIA A100-SXM4-80GB', 'NVIDIA DGX Display', 'NVIDIA A100-SXM4-80GB']",5,347892350976,0.2.0,8b3b521bb0d0f5c537d0f47d3677667aa2767f3e,4.39.0.dev0,,0.27.2,,,,,,,,,
,s,0.171005130767822,8.164763398790689e-05,"[0.171018661499023, 0.17088777160644503, 0.17111753845214803, 0.17099655151367102]",tokens/s,11976.248845893508,,,,s,3.205203915596008,0.011015207893084002,"[3.196499370574952, 3.19660297203064, 3.204216796875, 3.22349652290344]",tokens/s,158.49225614886885,,,,s,0.023891329014395002,0.00046975251579700004,"[0.023666688919067, 0.023937023162841003, 0.023695360183715, 0.023731199264526003, 0.023649280548095, 0.023707647323608003, 0.023632896423339003, 0.023640064239501, 0.023690240859985, 0.02371481513977, 0.023720960617065003, 0.023690240859985, 0.023647232055664002, 0.023689216613769003, 0.023571456909179, 0.023710720062255, 0.023801855087280003, 0.023832576751708003, 0.023723007202148, 0.023762943267822002, 0.023995391845703002, 0.023810047149658002, 0.02362265586853, 0.023708671569824, 0.023663616180419, 0.023664640426635002, 0.023731199264526003, 0.023728128433227, 0.023602176666259002, 0.023568384170532003, 0.023418880462646, 0.023562240600585004, 0.023474176406860003, 0.023630847930908003, 0.023577600479125, 0.023641088485717002, 0.023646207809448, 0.023816192626953003, 0.023985151290893, 0.02395238494873, 0.024101888656616003, 0.024366079330444, 0.024251392364501002, 0.023785472869873002, 0.023610368728637, 0.023828479766845, 0.023766016006469003, 0.023605247497558, 0.023349248886108002, 0.023571456909179, 0.023669759750366, 0.023430143356323003, 0.023575551986694, 0.02346393585205, 0.023394304275512003, 0.023768064498901002, 0.023420927047729002, 0.023482368469238003, 0.023410688400268, 0.023912448883056003, 0.023616512298583, 0.023542783737182, 0.023632896423339003, 0.024207359313964002, 0.024182783126831003, 0.023739391326904002, 0.023410688400268, 0.023606271743774002, 0.023987199783325003, 0.024047615051269, 0.023853055953979, 0.023924736022949002, 0.023837696075439, 0.023949312210083003, 0.023809024810791002, 0.023931903839111, 0.023823360443115, 0.023878656387329, 0.023874559402465002, 0.023851007461547002, 0.023872512817382, 0.024271871566772, 0.024020992279052, 0.023974912643432003, 0.023832576751708003, 0.023883775711059003, 0.024015871047973, 0.024002559661865003, 0.024188928604125002, 0.024079359054565003, 0.024285184860229, 0.024230911254882, 0.024009727478027, 0.024133632659912002, 0.023960575103759003, 0.024004608154296003, 0.023718912124633004, 0.023970815658569, 0.023912448883056003, 0.023993343353271003, 0.023960575103759003, 0.023937023162841003, 0.023929855346679, 0.023784448623657, 0.023840768814086002, 0.024220672607421, 0.024112127304077, 0.025059328079223002, 0.023837696075439, 0.023987199783325003, 0.023908351898193, 0.024567808151245003, 0.024720384597778002, 0.024075263977050003, 0.023831552505493, 0.023739391326904002, 0.02350489616394, 0.024083456039428002, 0.024164352416992003, 0.024124416351318, 0.023788543701171002, 0.023843839645385, 0.023609344482421003, 0.023464960098266003, 0.023467008590698002, 0.02352537536621, 0.023542783737182, 0.023360511779785, 0.023715839385986003, 0.023724031448364002, 0.023618560791015, 0.023678976058959002, 0.024027135848999, 0.024062976837158002, 0.023770111083984, 0.023651327133178003, 0.023750656127929003, 0.023960575103759003, 0.023787519454956003, 0.023855104446411, 0.023729152679443002, 0.023659519195556002, 0.023749631881713, 0.023670783996582002, 0.023868415832519002, 0.023621631622314003, 0.023832576751708003, 0.02387353515625, 0.023729152679443002, 0.023735296249389002, 0.023704576492309, 0.024183807373046002, 0.024124416351318, 0.023940095901489003, 0.024081407546997002, 0.023697408676147003, 0.023798784255981, 0.023721984863281, 0.023813119888305003, 0.023583744049072, 0.023875583648681004, 0.024007680892944003, 0.02402918434143, 0.023689216613769003, 0.02395238494873, 0.024135679244995, 0.024118272781372, 0.023767040252685, 0.023742464065551003, 0.023876607894897, 0.023656448364257003, 0.023572479248046, 0.023625728607177002, 0.023554048538208004, 0.023575551986694, 0.023422975540161, 0.023580671310424003, 0.023495679855346, 0.023475200653076, 0.023481344223022, 0.023543807983398002, 0.023639039993286003, 0.023916543960571003, 0.023791616439819002, 0.023748607635498003, 0.02371481513977, 0.023518207550048003, 0.02354073524475, 0.023936000823974003, 0.0235663356781, 0.02358169555664, 0.023668735504150003, 0.023600128173828002, 0.023713792800903, 0.023672832489013, 0.02354585647583, 0.023313407897949, 0.023374847412109, 0.023506944656372, 0.023462911605834, 0.023447551727294003, 0.023826431274414, 0.023932928085327, 0.023635967254638002, 0.023385087966918003, 0.023630847930908003, 0.023733247756958002, 0.023950336456298002, 0.023974912643432003, 0.023815168380737002, 0.024161279678344003, 0.024117248535156, 0.023881727218627, 0.023936000823974003, 0.024096767425537003, 0.02402918434143, 0.024090623855590004, 0.024025087356567002, 0.023958528518676, 0.024076288223266, 0.023944192886352002, 0.023951360702514003, 0.024009727478027, 0.024068096160888, 0.02396774482727, 0.023822336196899004, 0.023966720581054, 0.023989248275756003, 0.023933952331542004, 0.023912448883056003, 0.024080383300781, 0.024139776229858002, 0.024217599868774004, 0.02404454421997, 0.02545664024353, 0.024069120407104003, 0.024103935241699, 0.023994367599487, 0.024422399520874, 0.024166400909423003, 0.02410598373413, 0.023995391845703002, 0.024073215484619003, 0.024212480545043002, 0.023977983474731002, 0.02369945526123, 0.023644159317016, 0.02369945526123, 0.023764991760253002, 0.023707647323608003, 0.02371481513977, 0.023586816787719, 0.023702527999877, 0.023790592193603, 0.023444480895996003, 0.023600128173828002, 0.023298048019409, 0.02351001548767, 0.023800832748413003, 0.023657472610473, 0.023387136459350002, 0.023526399612426002, 0.023440383911132002, 0.023331840515136003, 0.023986175537109, 0.024027135848999, 0.023496704101562002, 0.023429119110107, 0.023642112731933, 0.023376895904541003, 0.023648256301879, 0.023995391845703002, 0.02375577545166, 0.023854080200195003, 0.023989248275756003, 0.023882751464843, 0.023616512298583, 0.023627775192260003, 0.023659519195556002, 0.023707647323608003, 0.023663616180419, 0.023743488311767, 0.023611391067504, 0.023690240859985, 0.023683071136474002, 0.023742464065551003, 0.023507968902587002, 0.023708671569824, 0.023563264846801, 0.023574527740478003, 0.023720960617065003, 0.023864320755004, 0.023749631881713, 0.023762943267822002, 0.023668735504150003, 0.023797760009765002, 0.023782400131225002, 0.023909376144409, 0.023862272262573002, 0.023831552505493, 0.023939071655273002, 0.024170495986938, 0.023787519454956003, 0.023777280807495003, 0.023843839645385, 0.023779327392578, 0.023748607635498003, 0.023629823684692002, 0.023661567687988, 0.023856128692626002, 0.023644159317016, 0.023485439300537002, 0.023548927307128, 0.023680000305175003, 0.023467008590698002, 0.023659519195556002, 0.023674879074096003, 0.023842815399169003, 0.023788543701171002, 0.023633920669555, 0.023655424118041002, 0.023683071136474002, 0.023562240600585004, 0.023917568206787, 0.023839744567871003, 0.023575551986694, 0.023464960098266003, 0.023820287704467, 0.023547903060913002, 0.023503871917724004, 0.023652351379394, 0.023623680114746002, 0.023602176666259002, 0.023446527481079, 0.023564287185668, 0.023669759750366, 0.023392255783081004, 0.02395238494873, 0.023685119628906, 0.023677951812744003, 0.023928831100463, 0.023780351638793003, 0.023476224899291003, 0.024008703231811, 0.02387353515625, 0.023677951812744003, 0.023728128433227, 0.023982080459594, 0.023791616439819002, 0.023661567687988, 0.023874559402465002, 0.023889919281005003, 0.023687168121337, 0.02387353515625, 0.023698432922363, 0.023887872695922, 0.024004608154296003, 0.024001535415649002, 0.023914495468139, 0.023941120147705, 0.023836671829223003, 0.025263103485107002, 0.023973888397216002, 0.023989248275756003, 0.02371481513977, 0.023845888137817003, 0.023769088745117004, 0.023729152679443002, 0.025609216690063, 0.027024383544921003, 0.027171840667724004, 0.027449344635009, 0.026578943252563002, 0.024019968032836, 0.024281087875366, 0.024185855865478, 0.024164352416992003, 0.024171520233154, 0.02410598373413, 0.024486911773681, 0.024459264755249004, 0.023820287704467, 0.024761344909667003, 0.026677248001098, 0.026658815383911003, 0.026469375610351002, 0.026388479232788004, 0.024054784774780003, 0.024275968551635004, 0.024262655258178, 0.024340480804443003, 0.024593408584594002, 0.024668159484863002, 0.02468659210205, 0.024476671218872002, 0.024499200820922003, 0.024196096420288003, 0.024154111862182002, 0.023938047409057, 0.024030208587646002, 0.023899135589599003, 0.024165376663208, 0.024286207199096, 0.024175615310668003, 0.024087551116943003, 0.024164352416992003, 0.024255487442016003, 0.024071168899536, 0.024394752502441, 0.024189952850341003, 0.024442880630493, 0.024469503402709, 0.024194047927856004, 0.024407039642333, 0.024377344131469, 0.024572927474975, 0.024073215484619003, 0.023943168640136, 0.024323072433471003, 0.024003583908081, 0.024079359054565003, 0.023938047409057, 0.023878656387329, 0.023821311950683002, 0.023922687530517003, 0.023803903579711003, 0.023928831100463, 0.023833599090576002, 0.023886848449707003, 0.023822336196899004, 0.023750656127929003, 0.023813119888305003, 0.023828479766845, 0.023681024551391, 0.023742464065551003, 0.023811071395874003, 0.023763967514038004, 0.024109056472778, 0.02391142463684, 0.023703552246093003, 0.023783424377441003, 0.023860223770141003, 0.023946239471435003, 0.023902208328247, 0.023776256561279, 0.023645183563232003, 0.023756799697875003, 0.023736320495605003, 0.024122367858886003, 0.023825408935546, 0.023883775711059003, 0.023779327392578, 0.023704576492309, 0.023632896423339003, 0.023892992019653003, 0.023631872177124, 0.023987199783325003, 0.023674879074096003, 0.023980031967163, 0.023797760009765002, 0.024211456298828003, 0.023966720581054, 0.023942144393920003, 0.023837696075439, 0.023919616699218, 0.024056831359863, 0.023834623336791, 0.023446527481079, 0.023547903060913002, 0.023749631881713, 0.023728128433227, 0.023420927047729002, 0.0234127368927, 0.023657472610473, 0.023664640426635002, 0.023447551727294003, 0.023444480895996003, 0.023389184951782, 0.024037376403808004, 0.024142847061157, 0.0238960647583, 0.023922687530517003, 0.024089599609375, 0.02391142463684, 0.023733247756958002, 0.023645183563232003, 0.023822336196899004, 0.024117248535156, 0.025078784942626, 0.023813119888305003, 0.023653375625610002, 0.023718912124633004, 0.023800832748413003, 0.023777280807495003, 0.023602176666259002, 0.023685119628906, 0.023799808502197, 0.023834623336791, 0.023856128692626002, 0.023684095382690003, 0.023851007461547002, 0.023870464324951, 0.02396774482727, 0.023786495208740002, 0.023781375885009, 0.023988224029541, 0.023920640945434002, 0.024235008239746003]",tokens/s,167.42475889850638,,,pytorch,2.2.1+cu118,optimum_benchmark.backends.pytorch.backend.PyTorchBackend,TheBloke/Mistral-7B-Instruct-v0.2-AWQ,text-generation,transformers,cuda,4,42,,,main,False,False,False,True,,,False,,True,False,,,,False,awq,exllama,2,128,4,False,,process,optimum_benchmark.launchers.process.launcher.ProcessLauncher,False,spawn,inference,optimum_benchmark.benchmarks.inference.benchmark.InferenceBenchmark,10,20,4,2,512,,False,False,True,128,128,mistral-7b-instruct-awq-exllama-bs4,,,,, AMD EPYC 7742 64-Core Processor,128,540671.627264,Linux,x86_64,Linux-5.4.0-166-generic-x86_64-with-glibc2.35,x86_64,3.10.12,"['NVIDIA A100-SXM4-80GB', 'NVIDIA A100-SXM4-80GB', 'NVIDIA A100-SXM4-80GB', 'NVIDIA DGX Display', 'NVIDIA A100-SXM4-80GB']",5,347892350976,0.2.0,8b3b521bb0d0f5c537d0f47d3677667aa2767f3e,4.39.0.dev0,,0.27.2,,,,,,,,,
,s,2.395186279296875,0,[2.395186279296875],tokens/s,13680.77309194477,,,,s,14.812944473266603,0,[14.812944473266603],tokens/s,548.7092734782651,,,,s,0.097777623574564,0.003771736779138,"[0.09756057739257801, 0.09210060882568301, 0.09097523498535101, 0.093120513916015, 0.09122509002685501, 0.09196851348876901, 0.091255805969238, 0.093154304504394, 0.091593727111816, 0.09234022521972601, 0.091902976989746, 0.09434214019775301, 0.09193164825439401, 0.09265766143798801, 0.09232281494140601, 0.094237693786621, 0.09231053161621, 0.093003776550292, 0.09232077026367101, 0.09458585357666001, 0.0926904296875, 0.093330429077148, 0.093010940551757, 0.095310844421386, 0.093026306152343, 0.093803520202636, 0.09348300933837801, 0.09588019561767501, 0.093367294311523, 0.09410047912597601, 0.093447166442871, 0.095410179138183, 0.094289916992187, 0.09523302459716701, 0.094005249023437, 0.09621606445312501, 0.094654464721679, 0.09537126159667901, 0.094289916992187, 0.096293884277343, 0.09495654296875, 0.09551974487304601, 0.094668800354003, 0.09693286132812501, 0.095310844421386, 0.09640652465820301, 0.09536102294921801, 0.097720321655273, 0.09534464263916001, 0.09611980438232401, 0.09581977844238201, 0.09818931579589801, 0.095644668579101, 0.09640038299560501, 0.09610956573486301, 0.0984453125, 0.096267265319824, 0.09747046661376901, 0.09615974426269501, 0.098480125427246, 0.096688125610351, 0.09777049255371001, 0.09637068939208901, 0.098626556396484, 0.09717247772216701, 0.09794866943359301, 0.09727487945556601, 0.099872764587402, 0.09716838073730401, 0.09798860931396401, 0.09777049255371001, 0.100217857360839, 0.097573890686035, 0.098297859191894, 0.098111488342285, 0.10071244812011701, 0.09831014251708901, 0.09944166564941401, 0.09795891571044901, 0.10025574493408201, 0.098721794128417, 0.099843070983886, 0.099007484436035, 0.10108108520507801, 0.099125251770019, 0.10028031921386701, 0.099149826049804, 0.10148966217041001, 0.099101692199707, 0.09990962982177701, 0.099385345458984, 0.102220802307128, 0.099643394470214, 0.100800514221191, 0.09985024261474601, 0.102324226379394, 0.10034790039062501, 0.10133913421630801, 0.10004582214355401, 0.102572029113769, 0.101241859436035, 0.10196377563476501, 0.10080870056152301, 0.102988800048828, 0.10107596588134701, 0.10704179382324201, 0.10109645080566401, 0.10372608184814401, 0.101147651672363, 0.10203648376464801, 0.10188185882568301, 0.10420941162109301, 0.10202111816406201, 0.102802429199218, 0.10218701171875, 0.10417664337158201, 0.10223104095458901, 0.102870018005371, 0.10185215759277301, 0.10476850891113201, 0.10288127899169901, 0.10367897796630801, 0.10293555450439401, 0.10552524566650301, 0.10271231842041001, 0.103652351379394, 0.103020545959472]",tokens/s,654.5464868165247,,,pytorch,2.2.1+cu118,optimum_benchmark.backends.pytorch.backend.PyTorchBackend,TheBloke/Mistral-7B-Instruct-v0.2-AWQ,text-generation,transformers,cuda,4,42,,,main,False,False,False,True,,,False,,True,False,,,,False,awq,exllama,2,128,64,False,,process,optimum_benchmark.launchers.process.launcher.ProcessLauncher,False,spawn,inference,optimum_benchmark.benchmarks.inference.benchmark.InferenceBenchmark,10,20,64,2,512,,False,False,True,128,128,mistral-7b-instruct-awq-exllama-bs64,,,,, AMD EPYC 7742 64-Core Processor,128,540671.627264,Linux,x86_64,Linux-5.4.0-166-generic-x86_64-with-glibc2.35,x86_64,3.10.12,"['NVIDIA A100-SXM4-80GB', 'NVIDIA A100-SXM4-80GB', 'NVIDIA A100-SXM4-80GB', 'NVIDIA DGX Display', 'NVIDIA A100-SXM4-80GB']",5,347892350976,0.2.0,8b3b521bb0d0f5c537d0f47d3677667aa2767f3e,4.39.0.dev0,,0.27.2,,,,,,,,,
,s,0.644432800292968,0.0006251221855250001,"[0.643910827636718, 0.6453117065429681, 0.644075866699218]",tokens/s,12711.953824007398,,,,s,4.924571893056233,0.0011185622171040002,"[4.9231044273376465, 4.924794078826903, 4.925817173004151]",tokens/s,412.62469999984563,,,,s,0.033701882620183005,0.00068245497673,"[0.034259967803955, 0.033015808105468, 0.033150974273681, 0.033124351501464006, 0.033098751068115, 0.033165313720703006, 0.032962558746337, 0.03299225616455, 0.032977920532226, 0.033210369110107, 0.033200126647949, 0.033416191101074, 0.033104896545410004, 0.033382400512695004, 0.033241088867187005, 0.032994304656982006, 0.032908287048339, 0.033153022766113004, 0.033004543304443, 0.033163265228271004, 0.033131519317626, 0.033076225280761005, 0.033038337707519, 0.032892929077148005, 0.033186817169189003, 0.033498111724853, 0.033187839508056004, 0.033311744689941004, 0.033268737792968, 0.033520641326904, 0.03333222579956, 0.033263614654541, 0.033157119750976, 0.033013759613037, 0.033380352020263, 0.033174526214599, 0.033225727081298, 0.03333222579956, 0.033399806976318, 0.033496063232421006, 0.033423358917236, 0.033126399993896, 0.033224704742431, 0.033313793182373, 0.033183742523193005, 0.033188865661621006, 0.033136638641357005, 0.033172481536865, 0.03295641708374, 0.033059841156005, 0.033083393096923, 0.033424385070800006, 0.033103870391845, 0.033186817169189003, 0.03359846496582, 0.033372161865234, 0.033034240722656004, 0.033316864013671, 0.033285118103027006, 0.033460224151611005, 0.033246208190917004, 0.033279998779296, 0.033005569458007006, 0.033476608276367004, 0.033201152801513, 0.033355777740478, 0.033230846405029005, 0.033796096801757006, 0.033206272125244005, 0.033414142608642006, 0.033289215087890005, 0.033939456939697, 0.033334270477294, 0.033562625885009, 0.033361919403076, 0.0339875831604, 0.033409023284912, 0.033591297149658, 0.033556480407714, 0.034137088775634004, 0.033550334930419, 0.033754112243652004, 0.033553409576416005, 0.0341923828125, 0.033684478759765005, 0.033772544860839006, 0.033562625885009, 0.034301952362060004, 0.033718273162841005, 0.033936382293701, 0.03372134399414, 0.034366462707519, 0.033798145294189, 0.034027519226074006, 0.033788928985595004, 0.03445555114746, 0.033855487823486005, 0.034098175048828006, 0.033931262969970004, 0.034611198425292006, 0.033990657806396006, 0.034159614562988, 0.033944576263427, 0.034588672637939, 0.03402342224121, 0.034246654510498, 0.034064384460449, 0.035099647521972005, 0.034124801635742004, 0.034327552795410006, 0.034128894805908, 0.034842624664306, 0.036279296875, 0.035113983154296, 0.035092479705810004, 0.03498291015625, 0.035226623535156, 0.035149822235107, 0.035097599029541, 0.035073024749755, 0.034909183502197, 0.035087360382080005, 0.034869247436523004, 0.035133438110351, 0.034891777038574004, 0.034785278320312, 0.034831359863281, 0.034227199554443005, 0.033210369110107, 0.033150974273681, 0.033207294464111005, 0.033205249786376005, 0.033422336578369, 0.033474559783935, 0.033506305694580005, 0.033513473510742, 0.033289215087890005, 0.033249279022216, 0.033341438293457, 0.033366016387939006, 0.03331481552124, 0.033240062713623, 0.033288192749023005, 0.033205249786376005, 0.033172481536865, 0.033244159698486, 0.033108993530273, 0.033368064880371, 0.033231872558593, 0.03322777557373, 0.033511425018310004, 0.03368550491333, 0.0333762550354, 0.03320217514038, 0.033402881622314, 0.032997375488281, 0.03307417678833, 0.033052673339843006, 0.033158145904541005, 0.033081344604492004, 0.033150974273681, 0.033383422851562004, 0.033053695678710006, 0.033552383422851, 0.033279998779296, 0.033293312072753004, 0.033460224151611005, 0.03348070526123, 0.033167358398437, 0.033159168243408005, 0.033286144256591, 0.033245185852050003, 0.032954368591308, 0.033420288085937, 0.033243137359619, 0.033725440979003006, 0.033641471862792004, 0.033331199645996006, 0.033371135711669005, 0.033715198516845006, 0.03357286453247, 0.033289215087890005, 0.033311744689941004, 0.033403903961181, 0.033149951934814, 0.033268737792968, 0.033403903961181, 0.033302528381347, 0.033159168243408005, 0.033209342956542, 0.033476608276367004, 0.033244159698486, 0.033388542175292, 0.033301502227783004, 0.033785854339599006, 0.033211391448974004, 0.033460224151611005, 0.033283073425292, 0.033949695587158005, 0.033529857635498005, 0.033560577392578006, 0.03334553527832, 0.033979393005371006, 0.03341926574707, 0.033648639678955004, 0.033434623718261, 0.03411763381958, 0.033550334930419, 0.033742847442626005, 0.033520641326904, 0.0341923828125, 0.033559551239013004, 0.033795070648193, 0.033563648223876, 0.034330623626708, 0.033695743560791004, 0.033938430786132005, 0.033697792053222, 0.034369537353515, 0.03375206375122, 0.033984512329101005, 0.033742847442626005, 0.034413566589355, 0.033833984375, 0.034061313629150006, 0.033918975830078, 0.034570240020751, 0.03399270248413, 0.034189311981201, 0.033925121307373005, 0.034605056762695, 0.034028545379638, 0.03424563217163, 0.034016254425048, 0.034844673156738004, 0.034131969451904005, 0.03438899230957, 0.034141185760498, 0.034813953399658, 0.034234367370605, 0.034392063140869004, 0.034230270385742, 0.034921470642089, 0.034293758392333, 0.034516990661621005, 0.034306049346923, 0.034999294281005004, 0.0343941116333, 0.035032062530517, 0.034975742340087004, 0.035209217071533, 0.034973697662353004, 0.035217407226562004, 0.034998271942138004, 0.034310142517089005, 0.03323289489746, 0.0328130569458, 0.032820224761962, 0.03282124710083, 0.032870399475097005, 0.033033214569091, 0.032648193359375, 0.032665599822998, 0.032940032958984004, 0.033181697845458005, 0.033330177307128, 0.033784832000732005, 0.033554431915283005, 0.032989185333251, 0.033154048919677, 0.033042430877685006, 0.033092609405517004, 0.033082366943359004, 0.033177600860595, 0.033205249786376005, 0.033193984985351005, 0.033038337707519, 0.033091583251953, 0.033070079803466004, 0.033118209838867, 0.033082366943359004, 0.033106945037841, 0.033047550201416004, 0.033042430877685006, 0.033104896545410004, 0.033091583251953, 0.033164287567138004, 0.033130496978759, 0.033223678588867, 0.033139713287353004, 0.033294334411621, 0.033220607757568, 0.033007614135742006, 0.033016830444335, 0.0328642578125, 0.032921600341796003, 0.032965633392333, 0.033001472473144, 0.032983039855957005, 0.032970752716064006, 0.033028095245361, 0.033092609405517004, 0.032986110687255, 0.033267711639404, 0.033442817687988, 0.03325849533081, 0.033484798431396005, 0.033302528381347, 0.033213439941406, 0.033607681274414006, 0.033793022155761, 0.033144832611083, 0.033163265228271004, 0.033371135711669005, 0.032946174621582, 0.032941055297851005, 0.033327102661132, 0.033462272644042, 0.033278976440429, 0.033388542175292, 0.033498111724853, 0.033795070648193, 0.033436672210693005, 0.033433601379394, 0.033238014221191, 0.033975296020507, 0.03331481552124, 0.033523712158203, 0.033349632263183, 0.033950721740722, 0.03341209411621, 0.033599487304687, 0.033393665313720004, 0.034108417510986005, 0.033538047790527, 0.033787902832031, 0.033504257202148, 0.034163711547851, 0.033673217773437, 0.033829887390136, 0.033543167114257005, 0.034267135620117004, 0.03368550491333, 0.033885185241699, 0.033789951324462005, 0.03436339187622, 0.033750015258789005, 0.03395686340332, 0.036644863128662004, 0.035277824401855, 0.035347457885742, 0.035301376342773, 0.035204097747802005, 0.035524608612060006, 0.035207168579101, 0.035296257019042, 0.034775039672851005, 0.034996223449707, 0.03506687927246, 0.035022846221923004, 0.035001342773437, 0.035880958557128, 0.034149375915527004, 0.034337791442871, 0.034203647613525005, 0.0348037109375, 0.034217983245849, 0.034409473419189006, 0.034231296539306004, 0.034876415252685006, 0.034285568237304005, 0.034536449432373, 0.034263038635253006, 0.034969600677490005, 0.034403327941894005, 0.034563072204589006, 0.034357246398925005, 0.035039230346679, 0.034488319396972, 0.034676734924316, 0.034471935272216]",tokens/s,474.7509265377957,,,pytorch,2.2.1+cu118,optimum_benchmark.backends.pytorch.backend.PyTorchBackend,TheBloke/Mistral-7B-Instruct-v0.2-AWQ,text-generation,transformers,cuda,4,42,,,main,False,False,False,True,,,False,,True,False,,,,False,awq,gemm,2,128,16,False,,process,optimum_benchmark.launchers.process.launcher.ProcessLauncher,False,spawn,inference,optimum_benchmark.benchmarks.inference.benchmark.InferenceBenchmark,10,20,16,2,512,,False,False,True,128,128,mistral-7b-instruct-awq-gemm-bs16,,,,, AMD EPYC 7742 64-Core Processor,128,540671.627264,Linux,x86_64,Linux-5.4.0-166-generic-x86_64-with-glibc2.35,x86_64,3.10.12,"['NVIDIA A100-SXM4-80GB', 'NVIDIA A100-SXM4-80GB', 'NVIDIA A100-SXM4-80GB', 'NVIDIA DGX Display', 'NVIDIA A100-SXM4-80GB']",5,347892350976,0.2.0,8b3b521bb0d0f5c537d0f47d3677667aa2767f3e,4.39.0.dev0,,0.27.2,,,,,,,,,
,s,0.30794433593750004,0.00022298712051000002,"[0.307758239746093, 0.30807702636718703, 0.307701568603515, 0.30824050903320305]",tokens/s,13301.105173862717,,,,s,3.3261262240409852,0.018101565104648002,"[3.32666485977173, 3.313143274307251, 3.309279039382934, 3.355417722702026]",tokens/s,305.46044604574234,,,,s,0.02376521171735,0.001184577026906,"[0.023795711517333003, 0.022988800048828, 0.022964223861694003, 0.023105535507202003, 0.022981632232666003, 0.023273471832275003, 0.023021568298339002, 0.023037952423095, 0.022992895126342002, 0.023365631103515003, 0.023249919891357003, 0.023126016616821003, 0.023080959320068, 0.023279615402221003, 0.023339008331298, 0.023422975540161, 0.023212032318115, 0.023156736373901, 0.023379968643188, 0.023445503234863004, 0.023602176666259002, 0.023387136459350002, 0.023441408157348003, 0.02343321609497, 0.023497728347778003, 0.023573503494262002, 0.023482368469238003, 0.023703552246093003, 0.023487487792968, 0.023398399353027, 0.023490560531616002, 0.023457792282104, 0.023560192108154, 0.023966720581054, 0.023653375625610002, 0.02354073524475, 0.023852031707763, 0.023838720321655002, 0.024785919189453002, 0.025151487350463003, 0.025043968200683, 0.024445951461791002, 0.0241776638031, 0.024178688049316003, 0.023405567169189002, 0.023492607116699003, 0.023485439300537002, 0.023774208068847003, 0.023438335418701002, 0.023661567687988, 0.023501823425292, 0.023593984603881003, 0.023354368209838, 0.023417856216430003, 0.023550975799560003, 0.023660543441772004, 0.023411712646484003, 0.023355392456054003, 0.023301120758056, 0.023678976058959002, 0.023337984085083002, 0.023435264587402003, 0.023345151901245, 0.023623680114746002, 0.023420927047729002, 0.023491584777832003, 0.023413759231567, 0.023682048797607002, 0.023477247238159003, 0.023569408416748, 0.023603200912475003, 0.023900159835815, 0.023625728607177002, 0.023635967254638002, 0.023702527999877, 0.023925760269165004, 0.023733247756958002, 0.023671808242797, 0.024863744735717, 0.025112575531005002, 0.024994815826416, 0.024898559570312003, 0.024647680282592004, 0.024352767944335003, 0.023777280807495003, 0.023735296249389002, 0.023673856735229003, 0.023973888397216002, 0.023879680633544002, 0.023970815658569, 0.023827455520629003, 0.024026111602783003, 0.023778303146362003, 0.023795711517333003, 0.023774208068847003, 0.024041471481323, 0.023753728866577003, 0.023879680633544002, 0.023954431533813002, 0.024274944305419002, 0.023949312210083003, 0.023957504272460003, 0.023829504013061, 0.024151039123535, 0.023898111343383002, 0.023918592453002003, 0.023856128692626002, 0.024969215393066003, 0.023943168640136, 0.024093696594238, 0.024079359054565003, 0.024249343872070003, 0.024123392105102, 0.024098815917968003, 0.024006656646728002, 0.024289279937744, 0.023976959228515, 0.024090623855590004, 0.024001535415649002, 0.024380416870117, 0.024057855606079002, 0.024147968292236, 0.023998464584350003, 0.024382463455200003, 0.024128511428833002, 0.024209407806396, 0.024081407546997002, 0.023899135589599003, 0.023096319198608002, 0.023196672439575003, 0.023576576232910002, 0.023291904449462, 0.023373823165893003, 0.023238655090332003, 0.023539712905883, 0.023481344223022, 0.023140352249145003, 0.023174144744873, 0.023134208679199002, 0.023370752334594003, 0.023340032577514002, 0.023131135940551002, 0.023135232925415, 0.023164928436279, 0.023435264587402003, 0.023238655090332003, 0.023260160446166002, 0.023220224380493003, 0.023238655090332003, 0.023115776062011, 0.023138303756713, 0.02309836769104, 0.023169023513793, 0.023727104187011003, 0.023445503234863004, 0.023607295989990004, 0.023603200912475003, 0.023742464065551003, 0.023639039993286003, 0.023662591934204003, 0.023655424118041002, 0.023420927047729002, 0.023430143356323003, 0.023384063720703, 0.023450624465942004, 0.023404544830322, 0.023457792282104, 0.023375871658325, 0.023418880462646, 0.023464960098266003, 0.023553024291992002, 0.023539712905883, 0.023512063980102003, 0.02333081626892, 0.023655424118041002, 0.023419904708862002, 0.023544832229614003, 0.023374847412109, 0.023574527740478003, 0.023354368209838, 0.023526399612426002, 0.023343103408813, 0.023716863632202, 0.023571456909179, 0.023590911865234, 0.023388160705566003, 0.023636991500854, 0.023461887359619002, 0.023450624465942004, 0.023340032577514002, 0.023676927566528002, 0.023414783477783002, 0.023526399612426002, 0.023395328521728, 0.023759872436523003, 0.023450624465942004, 0.023531520843505002, 0.02353049659729, 0.023872512817382, 0.023499776840209003, 0.023617536544799003, 0.023528448104858, 0.023802879333496, 0.023542783737182, 0.023675903320312, 0.023595008850097, 0.024034303665161003, 0.023610368728637, 0.023693311691284, 0.023629823684692002, 0.023934976577758, 0.023609344482421003, 0.023760896682739, 0.024123392105102, 0.024128511428833002, 0.023781375885009, 0.023918592453002003, 0.023856128692626002, 0.024061952590942, 0.023784448623657, 0.023747583389282, 0.023795711517333003, 0.024037376403808004, 0.023796735763549, 0.023924736022949002, 0.024026111602783003, 0.024228864669799, 0.024030208587646002, 0.023963647842407003, 0.023882751464843, 0.024109056472778, 0.023860223770141003, 0.023940095901489003, 0.023879680633544002, 0.024806400299072003, 0.023931903839111, 0.024313856124877003, 0.0241776638031, 0.024281087875366, 0.024156160354614, 0.024095743179321002, 0.023927808761596003, 0.0242544631958, 0.024002559661865003, 0.024058879852294, 0.023991296768188, 0.02429542350769, 0.024094720840454, 0.024231935501098003, 0.024035327911376, 0.02441318321228, 0.024087551116943003, 0.024196096420288003, 0.024167423248291003, 0.023757823944091, 0.023186431884765, 0.023158784866333, 0.023292928695678002, 0.023119871139526002, 0.023320575714111003, 0.023224319458007, 0.023222272872924003, 0.023184383392333002, 0.023360511779785, 0.023302143096923, 0.023259136199951003, 0.023230464935302002, 0.023383039474487003, 0.023283712387084002, 0.023423999786376003, 0.0234895362854, 0.023364608764648003, 0.023214080810546003, 0.023529472351074002, 0.023372800827026002, 0.02316697692871, 0.023097343444824003, 0.023683071136474002, 0.023781375885009, 0.023500799179077003, 0.023221248626708, 0.02325401687622, 0.023069696426391003, 0.023271423339843, 0.023331840515136003, 0.023293952941894003, 0.023040000915527003, 0.023238655090332003, 0.023024639129638, 0.023319551467895, 0.02300108718872, 0.023616512298583, 0.023456768035888, 0.023451648712158, 0.023120895385742003, 0.023389184951782, 0.023498752593994, 0.023704576492309, 0.023377920150756, 0.023354368209838, 0.023373823165893003, 0.023636991500854, 0.023302143096923, 0.023389184951782, 0.023405567169189002, 0.023584768295288, 0.023360511779785, 0.023415807723999003, 0.023378944396972003, 0.023663616180419, 0.023339008331298, 0.023421951293945, 0.023405567169189002, 0.023563264846801, 0.023383039474487003, 0.023456768035888, 0.023387136459350002, 0.023691263198852, 0.023403520584106, 0.023473152160644002, 0.023385087966918003, 0.023737344741821, 0.023429119110107, 0.02358169555664, 0.023447551727294003, 0.023840768814086002, 0.023501823425292, 0.023601152420043, 0.02343321609497, 0.023833599090576002, 0.023586816787719, 0.023578624725341002, 0.023588863372802003, 0.023835647583007002, 0.023672832489013, 0.023661567687988, 0.023544832229614003, 0.023894016265869, 0.023642112731933, 0.02374041557312, 0.023628799438476, 0.023997440338134002, 0.023759872436523003, 0.023981056213378003, 0.023867391586303, 0.024082431793212, 0.023860223770141003, 0.023832576751708003, 0.023786495208740002, 0.024072191238403002, 0.023817216873168002, 0.023890943527221, 0.023977983474731002, 0.02422374343872, 0.023994367599487, 0.023933952331542004, 0.023800832748413003, 0.024135679244995, 0.02387353515625, 0.023921663284301002, 0.02462515258789, 0.025703424453735003, 0.023933952331542004, 0.024095743179321002, 0.023927808761596003, 0.024281087875366, 0.024080383300781, 0.024100864410400002, 0.023937023162841003, 0.024271871566772, 0.024008703231811, 0.024124416351318, 0.024051712036132003, 0.024319999694824002, 0.024094720840454, 0.024148992538452003, 0.024035327911376, 0.024327167510986, 0.024136703491210002, 0.024187904357910003, 0.024225791931152003, 0.023956480026245, 0.023207935333251002, 0.023337984085083002, 0.023437311172485, 0.02346393585205, 0.023383039474487003, 0.023488512039184003, 0.023468032836914003, 0.023373823165893003, 0.023303167343139003, 0.023258111953735002, 0.023283712387084002, 0.023265279769897, 0.023229440689086, 0.023352319717407, 0.023372800827026002, 0.023199743270874002, 0.023590911865234, 0.02342809677124, 0.023522304534912, 0.023252992630004, 0.023298048019409, 0.023513088226318, 0.023668735504150003, 0.023669759750366, 0.0235663356781, 0.023668735504150003, 0.023672832489013, 0.023376895904541003, 0.023079935073852, 0.023176191329956002, 0.023242752075195, 0.023056383132934, 0.023087104797363003, 0.02298060798645, 0.023368703842163, 0.023357440948486, 0.023216127395629, 0.023007232666015003, 0.023331840515136003, 0.02354585647583, 0.023831552505493, 0.023177215576171, 0.023301120758056, 0.023278591156005002, 0.023379968643188, 0.023354368209838, 0.023589887619018, 0.023236608505249, 0.023399423599243, 0.02351513671875, 0.02369945526123, 0.023596031188964, 0.023644159317016, 0.023542783737182, 0.023547903060913002, 0.023625728607177002, 0.023604223251342003, 0.0234127368927, 0.023647232055664002, 0.023579647064208002, 0.023564287185668, 0.023390207290649, 0.023563264846801, 0.023421951293945, 0.023481344223022, 0.023438335418701002, 0.023680000305175003, 0.023476224899291003, 0.023482368469238003, 0.023538688659667, 0.023791616439819002, 0.023569408416748, 0.023946239471435003, 0.023689216613769003, 0.023818239212036, 0.023537664413452, 0.023568384170532003, 0.023549951553344002, 0.023903232574462002, 0.023616512298583, 0.023641088485717002, 0.023654399871826003, 0.023920640945434002, 0.023685119628906, 0.0236943359375, 0.023689216613769003, 0.024025087356567002, 0.023738367080688, 0.023982080459594, 0.023835647583007002, 0.024028160095214003, 0.023800832748413003, 0.023813119888305003, 0.023733247756958002, 0.024018943786621002, 0.023762943267822002, 0.023847936630249002, 0.04108800125122, 0.036442111968994, 0.024455167770385002, 0.036813823699951, 0.024301567077636, 0.024251392364501002, 0.023946239471435003, 0.02396774482727, 0.023892992019653003, 0.024549375534057002, 0.023879680633544002, 0.024151039123535, 0.024049663543701003, 0.024286207199096, 0.024103935241699, 0.024075263977050003, 0.023943168640136, 0.024301567077636, 0.024039424896240003, 0.024118272781372, 0.024053760528564002, 0.024326143264770003, 0.024057855606079002, 0.024080383300781, 0.024006656646728002, 0.024502271652221002, 0.024154111862182002, 0.024212480545043002, 0.024204288482666002]",tokens/s,336.62649822553135,,,pytorch,2.2.1+cu118,optimum_benchmark.backends.pytorch.backend.PyTorchBackend,TheBloke/Mistral-7B-Instruct-v0.2-AWQ,text-generation,transformers,cuda,4,42,,,main,False,False,False,True,,,False,,True,False,,,,False,awq,exllama,2,128,8,False,,process,optimum_benchmark.launchers.process.launcher.ProcessLauncher,False,spawn,inference,optimum_benchmark.benchmarks.inference.benchmark.InferenceBenchmark,10,20,8,2,512,,False,False,True,128,128,mistral-7b-instruct-awq-exllama-bs8,,,,, AMD EPYC 7742 64-Core Processor,128,540671.627264,Linux,x86_64,Linux-5.4.0-166-generic-x86_64-with-glibc2.35,x86_64,3.10.12,"['NVIDIA A100-SXM4-80GB', 'NVIDIA A100-SXM4-80GB', 'NVIDIA A100-SXM4-80GB', 'NVIDIA DGX Display', 'NVIDIA A100-SXM4-80GB']",5,347892350976,0.2.0,8b3b521bb0d0f5c537d0f47d3677667aa2767f3e,4.39.0.dev0,,0.27.2,,,,,,,,,
,s,1.237770385742187,5.7983398437544416e-05,"[1.23771240234375, 1.237828369140625]",tokens/s,13236.70382546427,,,,s,7.855410322189332,0.00015091514587300001,"[7.8555612373352055, 7.855259407043458]",tokens/s,517.3504417102616,,,,s,0.052107401074386,0.0019687714746850003,"[0.053552127838134005, 0.04937216186523401, 0.048688129425048, 0.049805313110351, 0.048791553497314004, 0.04917862319946201, 0.048765953063964, 0.049907711029052004, 0.048934913635253006, 0.049334270477294, 0.048966655731201, 0.050093055725097, 0.049075199127197, 0.049540096282958, 0.049234943389892004, 0.050279422760009004, 0.049291263580322, 0.049712127685546004, 0.04935065460205, 0.050468864440917006, 0.04948992156982401, 0.049904640197753, 0.049606655120849004, 0.050658302307128006, 0.049626113891601, 0.050000896453857004, 0.049650688171386, 0.050883583068847, 0.049821697235107, 0.05027020645141601, 0.049893375396728, 0.051085311889648, 0.050008064270019005, 0.050423809051513, 0.050038784027099006, 0.051223552703857006, 0.050174976348876, 0.050587646484375005, 0.050219009399414004, 0.051492862701416, 0.050322433471679, 0.050810878753662005, 0.050380798339843005, 0.05169561767578101, 0.05054259109497, 0.050936832427978, 0.05057126235961901, 0.051846145629882004, 0.050715648651123005, 0.051165184020996006, 0.050722816467285006, 0.052099071502685, 0.050915328979492004, 0.051311614990234004, 0.050931713104248, 0.052257793426513, 0.051098625183105005, 0.051531776428222, 0.051137535095214, 0.05243904113769501, 0.051317760467529006, 0.051722240447998, 0.051299327850341005, 0.05264076614379801, 0.052025344848632006, 0.052454399108886, 0.052076545715332004, 0.053381118774414, 0.052251647949218, 0.052646911621093, 0.052229118347167, 0.053630977630615, 0.052349952697753006, 0.052815872192382006, 0.052443134307861, 0.053757953643798005, 0.052588542938232, 0.052968448638916005, 0.052580352783203006, 0.053844993591308, 0.052771839141845, 0.053200897216796, 0.052758529663085, 0.05403955078125, 0.052889598846435, 0.053410816192626005, 0.053361663818359, 0.05433343887329101, 0.053050369262695005, 0.053465087890625006, 0.053121025085449004, 0.054366207122802004, 0.053226497650146004, 0.053730304718017007, 0.053278720855712, 0.054578174591064, 0.053421054840087, 0.053874687194824, 0.053476352691650006, 0.054792190551757006, 0.05362483215332001, 0.054038528442382, 0.05361356735229401, 0.054952960968017, 0.05374873733520501, 0.054206462860107, 0.053935104370117004, 0.055613441467285, 0.054214656829833, 0.054372352600097006, 0.054052864074707004, 0.055242752075195, 0.054135807037353004, 0.054580223083496006, 0.054172672271728006, 0.055492607116699004, 0.054337535858154005, 0.05517107009887601, 0.054457344055175, 0.055705600738525006, 0.054524929046630005, 0.054933502197265005, 0.054561790466308005, 0.055815166473388, 0.054824958801269, 0.055547904968261, 0.054963199615478006, 0.053025791168212, 0.049483776092529005, 0.048628734588623004, 0.049800193786621004, 0.048773120880126, 0.049152000427246004, 0.04877107238769501, 0.049917953491210004, 0.048756736755371004, 0.04934553527832, 0.048978942871093, 0.050131969451904006, 0.049101825714111, 0.049574913024902, 0.049159168243408005, 0.050275329589843, 0.049245185852050004, 0.049741825103759005, 0.049319934844970006, 0.050501632690429, 0.049428478240966, 0.049857536315917, 0.04956774520874, 0.050654209136962, 0.049628158569335, 0.049999870300292, 0.049650688171386, 0.050860031127929, 0.04978073501586901, 0.05022310256958001, 0.049854465484619005, 0.051056640625, 0.050014209747314006, 0.050415615081787005, 0.049977344512939004, 0.05123072052001901, 0.050150398254394005, 0.05061734390258701, 0.050215934753417006, 0.051436542510986005, 0.05030502319335901, 0.050804737091064006, 0.050329601287841, 0.05166182327270501, 0.050486270904541004, 0.05095423889160101, 0.05059379196166901, 0.051814399719238007, 0.050716670989990005, 0.051187713623046006, 0.050726913452148005, 0.052108287811279005, 0.050941951751708, 0.051323902130126005, 0.050951168060302005, 0.052209663391113006, 0.05108224105834901, 0.05154304122924801, 0.051131393432617, 0.052393985748291, 0.05131980895996, 0.051700736999511006, 0.051271678924560006, 0.05265817642211901, 0.051999744415283, 0.052446208953857, 0.052059135437011, 0.053351425170898005, 0.05219839859008701, 0.052652030944824006, 0.052243457794189006, 0.05361254501342701, 0.052326400756835006, 0.052827136993408005, 0.052436992645263004, 0.053721088409423, 0.052576255798339, 0.052973567962646004, 0.052571136474609, 0.053861377716064, 0.052789249420166, 0.05318348693847601, 0.052746238708496006, 0.054060031890869005, 0.052910079956054006, 0.053491710662841, 0.053367809295654, 0.054456321716308, 0.053047294616699006, 0.053524478912353006, 0.053238784790039004, 0.054864894866943, 0.05342822265625, 0.053751808166503004, 0.053261310577392, 0.054553600311279, 0.05342822265625, 0.053883903503417006, 0.053487617492675006, 0.05478707122802701, 0.05362483215332001, 0.054076416015625, 0.05364838409423801, 0.054961151123046004, 0.053785598754882, 0.054631423950195004, 0.054069248199462, 0.055143424987792004, 0.053991424560546, 0.054371326446533, 0.054057983398437, 0.055281665802001, 0.054175743103027, 0.054620159149169005, 0.054180862426757, 0.055481342315673005, 0.054347774505615, 0.054747135162353, 0.054394878387451, 0.055744510650634, 0.054583297729492004, 0.054920192718505005, 0.055013374328613, 0.05628518295288001, 0.054815742492675006, 0.055092224121093006, 0.054744064331054006]",tokens/s,614.1162165105449,,,pytorch,2.2.1+cu118,optimum_benchmark.backends.pytorch.backend.PyTorchBackend,TheBloke/Mistral-7B-Instruct-v0.2-AWQ,text-generation,transformers,cuda,4,42,,,main,False,False,False,True,,,False,,True,False,,,,False,awq,gemm,2,128,32,False,,process,optimum_benchmark.launchers.process.launcher.ProcessLauncher,False,spawn,inference,optimum_benchmark.benchmarks.inference.benchmark.InferenceBenchmark,10,20,32,2,512,,False,False,True,128,128,mistral-7b-instruct-awq-gemm-bs32,,,,, AMD EPYC 7742 64-Core Processor,128,540671.627264,Linux,x86_64,Linux-5.4.0-166-generic-x86_64-with-glibc2.35,x86_64,3.10.12,"['NVIDIA A100-SXM4-80GB', 'NVIDIA A100-SXM4-80GB', 'NVIDIA A100-SXM4-80GB', 'NVIDIA DGX Display', 'NVIDIA A100-SXM4-80GB']",5,347892350976,0.2.0,8b3b521bb0d0f5c537d0f47d3677667aa2767f3e,4.39.0.dev0,,0.27.2,,,,,,,,,
,s,0.60403433227539,0.0019759216308590003,"[0.602058410644531, 0.60601025390625]",tokens/s,13562.143014521753,,,,s,5.207427864074708,0.000798606872558,"[5.208226470947267, 5.20662925720215]",tokens/s,390.2118383662065,,,,s,0.036247193163774005,0.0010290608320250002,"[0.036892673492431, 0.035239936828613004, 0.034665470123291, 0.035230720520019, 0.034463745117187, 0.034576385498046, 0.034354175567626, 0.034938880920410004, 0.034459648132324, 0.034729984283447, 0.034478080749511, 0.035067905426025, 0.034552833557128, 0.034776065826416, 0.034536449432373, 0.035175422668457, 0.034650112152099004, 0.034949119567871, 0.034952190399169004, 0.035563518524169, 0.035094528198242006, 0.035245056152343, 0.035035137176513, 0.035429374694824003, 0.034817024230957005, 0.035292160034179, 0.035211265563964006, 0.036046848297119, 0.035543041229248, 0.035765247344970004, 0.035337215423583, 0.035905536651611, 0.03528396987915, 0.035197952270507, 0.034995201110839, 0.035566593170166, 0.035094528198242006, 0.035292160034179, 0.035142654418945006, 0.035685375213623, 0.03521535873413, 0.035381248474121, 0.035244033813476, 0.036182014465332, 0.035629055023193, 0.035515392303466, 0.035368961334228004, 0.036231166839599004, 0.035700736999511005, 0.035880958557128, 0.035662849426269, 0.036347904205322, 0.035752960205078003, 0.035976192474365, 0.035896320343017006, 0.036744190216064, 0.035965953826904005, 0.036072448730468006, 0.035853313446044005, 0.036502529144287005, 0.035974143981933, 0.035920894622802006, 0.035630081176757006, 0.036308990478515005, 0.036033535003662, 0.036244480133056006, 0.036014080047607006, 0.036652030944824006, 0.036254718780517003, 0.036626430511474004, 0.036409343719482005, 0.036873214721679, 0.036200447082519, 0.036744190216064, 0.036503551483154005, 0.037148670196533, 0.036592639923095, 0.036804607391357005, 0.036600833892822, 0.037293056488037005, 0.036710399627685006, 0.036924415588378, 0.036745216369628006, 0.037683200836181, 0.037107711791992, 0.036973567962646, 0.036755455017089, 0.037158912658691, 0.036590591430664, 0.036776958465576, 0.036559871673583, 0.037539840698242004, 0.036971519470214, 0.036911102294921005, 0.036685825347900006, 0.037617664337158005, 0.037039104461669005, 0.037109760284423005, 0.036915199279785, 0.037725185394287, 0.037172222137451, 0.037371902465820006, 0.037097473144531004, 0.037781505584716005, 0.037222400665283005, 0.037429248809814006, 0.037211135864257006, 0.037931007385253006, 0.037432319641113, 0.037869567871093, 0.037662719726562, 0.038123519897460004, 0.03741491317749, 0.037486591339111, 0.037112831115722, 0.037720062255859006, 0.037198848724365005, 0.037754878997802006, 0.037484542846679, 0.038161407470703006, 0.037602302551269, 0.037484542846679, 0.037378047943115, 0.038264831542968, 0.037685249328613, 0.037747711181640005, 0.037677055358886005, 0.037192703247070004, 0.035088382720947005, 0.03465523147583, 0.035296257019042, 0.034435073852539, 0.034662399291992003, 0.034416641235351, 0.034983936309814, 0.034507774353027, 0.034721790313720004, 0.034522113800048, 0.03514879989624, 0.034594814300537, 0.034809856414794005, 0.034610176086425005, 0.035172351837158, 0.034675712585449006, 0.034885631561279, 0.034716670989990005, 0.035281921386718006, 0.034760704040527, 0.034969600677490005, 0.034793472290039006, 0.035421184539794, 0.034848766326904, 0.035054592132568, 0.035195903778076, 0.035784702301025005, 0.035245056152343, 0.035511295318603, 0.035300350189208005, 0.035889152526855005, 0.035350528717041, 0.035353599548339006, 0.035025920867919, 0.035681278228759, 0.035137535095214, 0.035351551055908, 0.03514060974121, 0.03604377746582, 0.035620864868164, 0.035509246826171, 0.0352102394104, 0.035907585144042, 0.035353599548339006, 0.035505153656005005, 0.035311614990234004, 0.035983360290527004, 0.035447807312011005, 0.035574783325195, 0.035412990570068005, 0.036077568054199004, 0.035581951141357, 0.03605708694458, 0.03579084777832, 0.036474880218505, 0.035746814727783, 0.036107265472412005, 0.035901439666748004, 0.036562942504882005, 0.03598745727539, 0.035888126373291, 0.035685375213623, 0.03637145614624, 0.036065280914306004, 0.036562942504882005, 0.036378623962402004, 0.037033985137939006, 0.036183040618896004, 0.036352001190185, 0.036157440185546, 0.036894718170166006, 0.036200447082519, 0.036468734741210006, 0.036250625610351, 0.036878337860107004, 0.036361217498779005, 0.036519935607910003, 0.036440063476562, 0.037334014892578, 0.036738048553466005, 0.036651008605957006, 0.036774913787841, 0.037417984008789006, 0.036799488067626, 0.037021694183349003, 0.036801536560058, 0.037513214111328005, 0.036919296264648, 0.036859905242919, 0.036728832244873, 0.037573631286621, 0.037007358551025, 0.036888576507568, 0.036670463562011, 0.03766067123413, 0.037076992034912, 0.037122047424316004, 0.036808704376220004, 0.037484542846679, 0.03687014389038, 0.037081089019775006, 0.036844543457031004, 0.037552127838134004, 0.036968448638916004, 0.037492736816406004, 0.037268478393554, 0.037971969604492006, 0.037375999450683005, 0.037316608428955005, 0.037137409210205, 0.038060031890869005, 0.037459968566894, 0.037673984527587, 0.037478401184082, 0.038134784698486, 0.037575679779052004, 0.037781505584716005, 0.037501953125, 0.037903358459472, 0.037343231201171, 0.037868545532226, 0.037646335601806004, 0.037986305236816, 0.0374026222229, 0.037615615844726, 0.037398529052734004]",tokens/s,441.413488975764,,,pytorch,2.2.1+cu118,optimum_benchmark.backends.pytorch.backend.PyTorchBackend,TheBloke/Mistral-7B-Instruct-v0.2-AWQ,text-generation,transformers,cuda,4,42,,,main,False,False,False,True,,,False,,True,False,,,,False,awq,exllama,2,128,16,False,,process,optimum_benchmark.launchers.process.launcher.ProcessLauncher,False,spawn,inference,optimum_benchmark.benchmarks.inference.benchmark.InferenceBenchmark,10,20,16,2,512,,False,False,True,128,128,mistral-7b-instruct-awq-exllama-bs16,,,,, AMD EPYC 7742 64-Core Processor,128,540671.627264,Linux,x86_64,Linux-5.4.0-166-generic-x86_64-with-glibc2.35,x86_64,3.10.12,"['NVIDIA A100-SXM4-80GB', 'NVIDIA A100-SXM4-80GB', 'NVIDIA A100-SXM4-80GB', 'NVIDIA DGX Display', 'NVIDIA A100-SXM4-80GB']",5,347892350976,0.2.0,8b3b521bb0d0f5c537d0f47d3677667aa2767f3e,4.39.0.dev0,,0.27.2,,,,,,,,,
,s,0.123411468505859,4.0685248403098025e-05,"[0.12336323547363201, 0.12346275329589801, 0.12340841674804601]",tokens/s,8297.44603477741,,,,s,4.514343948364257,0.086664515826116,"[4.618691482543945, 4.406494319915769, 4.517846042633057]",tokens/s,56.26509696764139,,,,s,0.034574271494947004,0.001050999164881,"[0.033163265228271004, 0.033212417602539006, 0.033651710510253, 0.033843200683593005, 0.033548286437988004, 0.033339393615722, 0.03345510482788, 0.033366016387939006, 0.033667072296142006, 0.033937408447265005, 0.033728511810302, 0.033591297149658, 0.033672191619873004, 0.033601535797119005, 0.034055168151855005, 0.033591297149658, 0.033916927337646, 0.033947647094726, 0.033686527252197, 0.033933311462402, 0.035528705596923005, 0.035854335784912005, 0.035885055541992006, 0.035560447692871006, 0.035477504730224, 0.035481601715087005, 0.035427326202392, 0.035892223358154, 0.035833854675292, 0.036019199371337005, 0.035924991607666004, 0.035632129669189, 0.035526657104492, 0.035827713012695, 0.035613697052001, 0.035693569183349004, 0.035776512145996, 0.035747840881347005, 0.035632129669189, 0.035399681091308005, 0.035487743377685003, 0.035119102478027006, 0.035439617156982, 0.035511295318603, 0.035531776428222, 0.035992576599121, 0.035813377380371006, 0.035507198333740005, 0.035276798248291004, 0.035622913360595004, 0.035464191436767004, 0.035639297485351, 0.035339263916015, 0.035387390136718, 0.035144702911376, 0.035405822753906004, 0.035851264953613, 0.036028415679931, 0.035745792388916, 0.035478527069091, 0.035478527069091, 0.035471359252929005, 0.035521537780761, 0.035524608612060006, 0.035708927154541, 0.035565567016601005, 0.035837951660156006, 0.035760128021240005, 0.03558604812622, 0.035585025787353, 0.035727359771728, 0.036135936737060004, 0.036007934570312004, 0.035996673583984, 0.035966976165771006, 0.035924991607666004, 0.035781631469726, 0.035633152008056, 0.035877887725830004, 0.035543041229248, 0.035702785491943, 0.035795967102050005, 0.035705856323242004, 0.035677185058593006, 0.035429374694824003, 0.035542015075683005, 0.035591167449951, 0.035531776428222, 0.035872768402099006, 0.035659774780273004, 0.035640319824218, 0.035604480743408, 0.035589118957519005, 0.035687423706054, 0.035620864868164, 0.035645439147949, 0.035577854156494004, 0.035366912841796, 0.035469310760498, 0.036022270202636, 0.036029441833496004, 0.036024318695068, 0.03611852645874, 0.036324352264404, 0.036062206268310006, 0.03585740661621, 0.035986431121826, 0.037148670196533, 0.036251647949218, 0.0360263671875, 0.035769344329833, 0.035756031036376, 0.035579902648925, 0.035757057189941, 0.035783679962158005, 0.035687423706054, 0.035644416809082, 0.035972095489501005, 0.036226047515869005, 0.036262912750244, 0.036324352264404, 0.036257793426513, 0.03559423828125, 0.035853313446044005, 0.03545087814331, 0.035344383239746, 0.035436542510986005, 0.032866302490234, 0.033251327514648, 0.033282047271728, 0.033170433044433005, 0.03336294555664, 0.033505279541015, 0.033457153320312, 0.033794048309326, 0.034198528289794, 0.033562625885009, 0.033491966247558, 0.033710079193115, 0.033542144775390005, 0.033456127166748, 0.033587200164794, 0.033758209228515, 0.033735679626464, 0.033707008361816004, 0.033360897064208, 0.033590270996093005, 0.033530879974365005, 0.03360358428955, 0.034036735534667004, 0.033718273162841005, 0.033676288604736, 0.033483776092529005, 0.033791999816894, 0.033513473510742, 0.033528831481933, 0.033373184204101, 0.033458175659179, 0.033496063232421006, 0.033794048309326, 0.033740798950195, 0.033612800598144005, 0.033380352020263, 0.033413120269775005, 0.03349401473999, 0.033850368499755, 0.033600513458251005, 0.033635326385498, 0.033409023284912, 0.033639423370361, 0.033835006713867, 0.033764350891113, 0.03359846496582, 0.033396736145019, 0.033587200164794, 0.033757183074951, 0.033713153839111006, 0.033278976440429, 0.033137664794921, 0.033305599212646, 0.033304576873779, 0.033288192749023005, 0.033391616821289, 0.033555454254150005, 0.033358848571777004, 0.033320960998535, 0.033861633300781, 0.033968128204345, 0.033989631652832, 0.033783809661865005, 0.033588222503662, 0.033760257720947005, 0.033703937530517, 0.033363967895507, 0.033532928466796, 0.034165760040283, 0.033720321655273, 0.033488895416259004, 0.033829887390136, 0.033666046142578, 0.033717247009277, 0.033508350372314005, 0.033676288604736, 0.033704959869384, 0.033636352539062005, 0.033548286437988004, 0.033435649871826005, 0.033849342346191004, 0.033565696716308005, 0.033633281707763, 0.033518592834472005, 0.033798145294189, 0.033674240112304, 0.033779712677001, 0.033576961517333005, 0.033575935363769, 0.033632255554199006, 0.033988609313964004, 0.034110462188720006, 0.033600513458251005, 0.033688575744628003, 0.033811454772949, 0.033740798950195, 0.033811454772949, 0.033767425537109, 0.033662975311279, 0.033673217773437, 0.034001918792724004, 0.033699840545654, 0.033822719573974, 0.033653759002685, 0.033392639160156, 0.033326080322265, 0.033428478240966, 0.034881534576416004, 0.033301502227783004, 0.033551361083984, 0.033805313110351, 0.033814529418945, 0.033754112243652004, 0.033470462799072, 0.033514495849609, 0.033800193786621004, 0.033579006195068005, 0.033423358917236, 0.033502208709716, 0.033694721221923005, 0.034543617248535004, 0.035607551574707005, 0.035543041229248, 0.035775489807128004, 0.035900417327880005, 0.035688449859619005, 0.035679233551025, 0.033708030700683005, 0.033820671081542006, 0.033732608795166, 0.034154495239257, 0.033750015258789005, 0.033761280059814006, 0.033762302398681006, 0.033744895935058, 0.033765377044677004, 0.034069503784179, 0.033981441497802, 0.033812480926513004, 0.03383910369873, 0.033821697235107, 0.033803264617919, 0.033822719573974, 0.033914878845214005, 0.033809406280517006, 0.033776641845703004, 0.034003967285156006, 0.033487873077392004, 0.033482753753662005, 0.033441791534423004, 0.033484798431396005, 0.033501182556152004, 0.033475582122802, 0.033552383422851, 0.033514495849609, 0.033526782989501, 0.033732608795166, 0.033545215606689, 0.033551361083984, 0.033761280059814006, 0.033576961517333005, 0.033619968414306, 0.033614849090576, 0.03401830291748, 0.034070526123046, 0.033767425537109, 0.033584129333496006, 0.03364249420166, 0.033960960388183005, 0.033826816558837, 0.033519615173339005, 0.033911808013916, 0.033777664184570004, 0.03348070526123, 0.033585151672363, 0.033729534149169, 0.033410049438476, 0.033722366333007, 0.03364249420166, 0.033813503265380004, 0.034051071166992006, 0.033562625885009, 0.033856510162353005, 0.034098175048828006, 0.033737728118896006, 0.033620990753173, 0.034046974182128, 0.033915904998779, 0.033996799468994005, 0.033627136230468, 0.034243583679199004, 0.034097152709960006, 0.033511425018310004, 0.034008064270019005, 0.034127872467041, 0.033984512329101005, 0.033882110595703, 0.033853439331054, 0.033470462799072, 0.034580478668212004, 0.035514366149902006, 0.035895294189453, 0.035682304382324004, 0.035306495666503006, 0.035529727935791004, 0.035544063568115, 0.035624961853027, 0.035786750793457, 0.036210689544677, 0.035684352874755, 0.035582977294921006, 0.03555532836914, 0.036092929840087, 0.035811328887939003, 0.035909633636474, 0.036092929840087, 0.036004863739013, 0.035428352355957, 0.035310592651367004, 0.03524607849121, 0.03514879989624, 0.035447807312011005, 0.035678207397460006, 0.035665920257568005, 0.035389438629150005, 0.035456001281738, 0.035632129669189, 0.035704830169677, 0.035988479614257, 0.036016128540039, 0.036167678833007005, 0.036297729492187, 0.035756031036376, 0.035988479614257, 0.036937728881835004, 0.035621887207031, 0.0356167678833, 0.03555532836914, 0.035735553741455005, 0.035439617156982, 0.035512321472167, 0.035400703430175005, 0.035384319305419, 0.035546112060546003, 0.035744766235351, 0.036142078399658, 0.035473407745361, 0.035237888336181, 0.035305473327636006, 0.035779582977294, 0.036070400238037, 0.036647937774658, 0.035943424224853006, 0.035729408264160004]",tokens/s,57.846482760807866,,,pytorch,2.2.1+cu118,optimum_benchmark.backends.pytorch.backend.PyTorchBackend,TheBloke/Mistral-7B-Instruct-v0.2-AWQ,text-generation,transformers,cuda,4,42,,,main,False,False,False,True,,,False,,True,False,,,,False,awq,gemm,2,128,2,False,,process,optimum_benchmark.launchers.process.launcher.ProcessLauncher,False,spawn,inference,optimum_benchmark.benchmarks.inference.benchmark.InferenceBenchmark,10,20,2,2,512,,False,False,True,128,128,mistral-7b-instruct-awq-gemm-bs2,,,,, AMD EPYC 7742 64-Core Processor,128,540671.627264,Linux,x86_64,Linux-5.4.0-166-generic-x86_64-with-glibc2.35,x86_64,3.10.12,"['NVIDIA A100-SXM4-80GB', 'NVIDIA A100-SXM4-80GB', 'NVIDIA A100-SXM4-80GB', 'NVIDIA DGX Display', 'NVIDIA A100-SXM4-80GB']",5,347892350976,0.2.0,8b3b521bb0d0f5c537d0f47d3677667aa2767f3e,4.39.0.dev0,,0.27.2,,,,,,,,,
,s,0.342703165690104,0.00045556596763000006,"[0.342171203613281, 0.343283905029296, 0.34265438842773405]",tokens/s,11952.034326125498,,,,s,4.548869178771972,0.029540604571203003,"[4.528357948303222, 4.527605941772459, 4.5906436462402365]",tokens/s,223.35221349985773,,,,s,0.033119417425841,0.0014892299790170002,"[0.032714752197265, 0.032749568939208, 0.03299225616455, 0.033094657897949, 0.032750591278076, 0.032774143218994, 0.032661502838134, 0.032476158142089005, 0.032596992492675, 0.032490497589111, 0.032373760223388004, 0.032429054260253005, 0.032985088348388, 0.032849918365478, 0.032240638732910006, 0.032557056427001005, 0.032430080413818, 0.032354305267333, 0.032924671173095, 0.032829441070556, 0.032801792144775, 0.032655361175537, 0.03349401473999, 0.033519615173339005, 0.033190910339355, 0.033101825714111, 0.033111038208007, 0.033204223632812, 0.03292569732666, 0.033196033477783, 0.033235969543457006, 0.033233921051025, 0.033210369110107, 0.03346841430664, 0.033229824066162005, 0.033474559783935, 0.03336294555664, 0.033037311553955, 0.033062911987304, 0.033525760650634, 0.033260543823242006, 0.033219585418701, 0.033526782989501, 0.033427455902099, 0.033145854949951, 0.033249279022216, 0.033170433044433005, 0.033078273773193, 0.033238014221191, 0.033180671691894, 0.03322777557373, 0.033320960998535, 0.033183742523193005, 0.033210369110107, 0.032913406372070006, 0.033004543304443, 0.033042430877685006, 0.03293081665039, 0.032947200775146006, 0.032997375488281, 0.033056766510009, 0.032891902923583, 0.033180671691894, 0.033026046752929, 0.033171455383300005, 0.032816127777099004, 0.032399360656738, 0.032503807067871, 0.032679935455322, 0.032660480499267, 0.032523265838623, 0.032825344085693, 0.032675838470458005, 0.032433151245117, 0.032487422943115005, 0.032630783081054, 0.0323583984375, 0.032660480499267, 0.032954368591308, 0.032917503356933005, 0.032497665405273005, 0.032891902923583, 0.032919551849365, 0.033057792663574004, 0.033013759613037, 0.033428478240966, 0.033384449005126, 0.033512447357177004, 0.033054721832275, 0.033082366943359004, 0.033010688781738004, 0.033117183685302004, 0.033040382385253, 0.033104896545410004, 0.033163265228271004, 0.033208320617675, 0.033239040374755, 0.03341209411621, 0.033260543823242006, 0.033298431396484, 0.032886783599853, 0.032713729858398, 0.032834560394287005, 0.03292569732666, 0.032991233825683, 0.033201152801513, 0.032744449615478004, 0.034316287994384, 0.032778240203857, 0.032927745819091005, 0.032721920013427004, 0.032515071868896, 0.033099777221679005, 0.032817150115966004, 0.032572414398193, 0.032547840118408, 0.032776191711425005, 0.032797695159912, 0.032823295593261005, 0.033380352020263, 0.032850944519042004, 0.032988159179687004, 0.032753662109375005, 0.032714752197265, 0.032709632873535004, 0.032839679718017004, 0.032814079284667, 0.032503807067871, 0.032785408020019, 0.032792575836181004, 0.03285708618164, 0.032875518798828, 0.032875518798828, 0.032915454864501, 0.0328130569458, 0.032866302490234, 0.032716800689697005, 0.032848896026611, 0.032799743652343005, 0.032805889129638, 0.032897022247314, 0.032681983947753006, 0.032837631225585, 0.032818176269531, 0.032800769805908, 0.032824321746826, 0.032837631225585, 0.032880638122558, 0.032912384033203006, 0.032691200256347, 0.033137664794921, 0.032840705871582006, 0.032688129425048, 0.032895999908447, 0.032633857727050006, 0.032586753845214006, 0.032678913116455, 0.032612350463867006, 0.033116161346435004, 0.033093631744384004, 0.032928768157958005, 0.033195007324218005, 0.032972801208496, 0.033025024414062, 0.033174526214599, 0.033136638641357005, 0.033085441589355, 0.032898048400878004, 0.032922622680664, 0.032875518798828, 0.032741374969482005, 0.03333222579956, 0.033372161865234, 0.033409023284912, 0.033356800079345, 0.032872447967529, 0.032940032958984004, 0.033311744689941004, 0.032804862976074003, 0.032913406372070006, 0.032926719665527, 0.032841728210449, 0.032918529510498, 0.032832511901855, 0.032785408020019, 0.032828414916992, 0.032884735107421, 0.032889854431152, 0.032833534240722, 0.032875518798828, 0.032927745819091005, 0.033027072906494, 0.033125377655029, 0.032983039855957005, 0.033090560913085, 0.033012737274169, 0.032876544952392, 0.032743423461914, 0.033195007324218005, 0.033367038726806006, 0.032978942871093, 0.032778240203857, 0.032605182647705004, 0.032618495941162, 0.032694271087646005, 0.032800769805908, 0.032942081451416, 0.032711681365966, 0.033369087219238, 0.033221630096435, 0.033107967376708, 0.033046527862548004, 0.033030143737792006, 0.032854015350341, 0.032822273254394005, 0.033047550201416004, 0.033318912506103006, 0.032933887481689, 0.033269760131835004, 0.033342464447021006, 0.033288192749023005, 0.033186817169189003, 0.032943103790283, 0.032716800689697005, 0.032922622680664, 0.032890880584716, 0.033073150634765, 0.033009662628173, 0.033261566162109, 0.033065982818603006, 0.032984062194824006, 0.033402881622314, 0.032771072387695, 0.032687103271484004, 0.034472961425781004, 0.033216510772705, 0.032912384033203006, 0.032880638122558, 0.03277721786499, 0.032884735107421, 0.032870399475097005, 0.032808959960937, 0.032740352630615005, 0.032881664276123004, 0.032928768157958005, 0.032830463409423, 0.033049598693847, 0.033255424499511, 0.032901119232177, 0.032781311035156004, 0.032972801208496, 0.032696319580078, 0.032870399475097005, 0.032736255645751, 0.032501758575439, 0.032727039337158, 0.032754688262939, 0.032883712768554, 0.033067008972167, 0.033224704742431, 0.033152000427246, 0.033027072906494, 0.032880638122558, 0.033370113372802004, 0.033118209838867, 0.032911361694335, 0.033034240722656004, 0.032905216217041004, 0.032775169372558005, 0.033324031829833005, 0.032912384033203006, 0.033141761779785, 0.033105918884277004, 0.032738304138183, 0.033105918884277004, 0.032960510253906006, 0.03284377670288, 0.033102848052978, 0.032805889129638, 0.032824321746826, 0.032918529510498, 0.03323289489746, 0.033643520355224, 0.033132545471191, 0.032849918365478, 0.033116161346435004, 0.033038337707519, 0.032989185333251, 0.033031169891357, 0.032975872039794005, 0.033147903442382005, 0.033038337707519, 0.033092609405517004, 0.033087486267089, 0.033091583251953, 0.033136638641357005, 0.033013759613037, 0.033160190582275005, 0.033094657897949, 0.033424385070800006, 0.033026046752929, 0.033179649353027, 0.032952320098876005, 0.032858112335205004, 0.033164287567138004, 0.032959487915039005, 0.032740352630615005, 0.032907264709472, 0.032770046234130004, 0.033053695678710006, 0.033270782470703, 0.033260543823242006, 0.034415615081787004, 0.034216960906982, 0.034427902221679005, 0.033083393096923, 0.033155071258544, 0.033165313720703006, 0.0328642578125, 0.033051647186279, 0.032893951416015005, 0.032859134674072005, 0.033461246490478005, 0.033224704742431, 0.032966655731201, 0.032804862976074003, 0.033081344604492004, 0.05723340988159101, 0.040573951721191, 0.046097408294677004, 0.033913856506347005, 0.033013759613037, 0.033004543304443, 0.033177600860595, 0.033239040374755, 0.033102848052978, 0.033031169891357, 0.032870399475097005, 0.032904193878173005, 0.033037311553955, 0.033097728729248, 0.03316633605957, 0.033156097412109, 0.033064960479736005, 0.03316633605957, 0.033199104309082, 0.033075199127197, 0.033240062713623, 0.033300479888916004, 0.033199104309082, 0.033473537445068, 0.033505279541015, 0.033639423370361, 0.033182720184326005, 0.033218559265136005, 0.033242111206054005, 0.033052673339843006, 0.032975872039794005, 0.03299225616455, 0.032835582733154005, 0.033032192230224, 0.034439167022705004, 0.033217536926269005, 0.032924671173095, 0.033157119750976, 0.032902145385742, 0.032785408020019, 0.032976894378662004, 0.033041408538818005, 0.032763904571533005, 0.032988159179687004, 0.033124351501464006, 0.033023998260498004, 0.032886783599853, 0.033007614135742006, 0.032905216217041004, 0.032996353149414, 0.032907264709472, 0.032971775054931006, 0.032534526824951004, 0.033190910339355]",tokens/s,241.55014253837615,,,pytorch,2.2.1+cu118,optimum_benchmark.backends.pytorch.backend.PyTorchBackend,TheBloke/Mistral-7B-Instruct-v0.2-AWQ,text-generation,transformers,cuda,4,42,,,main,False,False,False,True,,,False,,True,False,,,,False,awq,gemm,2,128,8,False,,process,optimum_benchmark.launchers.process.launcher.ProcessLauncher,False,spawn,inference,optimum_benchmark.benchmarks.inference.benchmark.InferenceBenchmark,10,20,8,2,512,,False,False,True,128,128,mistral-7b-instruct-awq-gemm-bs8,,,,, AMD EPYC 7742 64-Core Processor,128,540671.627264,Linux,x86_64,Linux-5.4.0-166-generic-x86_64-with-glibc2.35,x86_64,3.10.12,"['NVIDIA A100-SXM4-80GB', 'NVIDIA A100-SXM4-80GB', 'NVIDIA A100-SXM4-80GB', 'NVIDIA DGX Display', 'NVIDIA A100-SXM4-80GB']",5,347892350976,0.2.0,8b3b521bb0d0f5c537d0f47d3677667aa2767f3e,4.39.0.dev0,,0.27.2,,,,,,,,,
,s,0.19675245157877602,0.00016310696091900002,"[0.19676995849609302, 0.196544509887695, 0.196942886352539]",tokens/s,10409.01896554015,,,,s,4.43060298538208,0.009536328022351001,"[4.423868576049806, 4.423850994110108, 4.444089385986327]",tokens/s,114.65707978711882,,,,s,0.033337405777978,0.0005074512788150001,"[0.033133567810058, 0.034123775482177, 0.033967105865478, 0.03335065460205, 0.033616897583007004, 0.034097152709960006, 0.034121726989746005, 0.03410534286499, 0.034072574615478, 0.034034687042236, 0.033034240722656004, 0.033135616302490005, 0.033041408538818005, 0.033161216735839, 0.032769023895263004, 0.032728065490722005, 0.032712703704833, 0.032786430358886, 0.032854015350341, 0.03296153640747, 0.032792575836181004, 0.032836608886718, 0.032929790496826004, 0.033240062713623, 0.033184768676757, 0.033137664794921, 0.033423358917236, 0.033339393615722, 0.033189888000488, 0.033080318450927, 0.032688129425048, 0.03270348739624, 0.032723968505859, 0.032656383514404, 0.032506881713867, 0.032789505004882, 0.032626686096191, 0.032817150115966004, 0.032948223114013, 0.033100799560546006, 0.032988159179687004, 0.032946174621582, 0.033044479370117, 0.033001472473144, 0.033117183685302004, 0.033310718536376, 0.033017856597900005, 0.033090560913085, 0.032951297760009005, 0.033195007324218005, 0.033406974792480004, 0.033243137359619, 0.033181697845458005, 0.033065982818603006, 0.033012737274169, 0.033223678588867, 0.033292289733886, 0.033446910858154, 0.033409023284912, 0.033213439941406, 0.033141761779785, 0.033124351501464006, 0.033338367462158, 0.033511425018310004, 0.033590270996093005, 0.033860607147216004, 0.033068031311035, 0.033097728729248, 0.033034240722656004, 0.033173503875732, 0.033272830963134, 0.033204223632812, 0.033497089385986, 0.033940479278564, 0.033284095764160006, 0.033570816040039, 0.033845249176025, 0.034283519744873, 0.033333248138427, 0.033245185852050003, 0.033310718536376, 0.03345510482788, 0.033008640289306, 0.032578559875488, 0.032865280151367006, 0.033021953582763004, 0.032814079284667, 0.032763904571533005, 0.032766975402832, 0.032979969024658, 0.03338137435913, 0.033459201812744005, 0.0335810546875, 0.033543167114257005, 0.033129470825195004, 0.033431552886962, 0.033307647705078006, 0.033349632263183, 0.033357822418212, 0.03330662536621, 0.033456127166748, 0.033505279541015, 0.033309696197509, 0.033584129333496006, 0.034158592224121, 0.034156543731689006, 0.033274879455566, 0.034923519134521006, 0.0335810546875, 0.033816574096679, 0.033473537445068, 0.033297409057617, 0.033036289215087, 0.033098751068115, 0.033680385589599, 0.033661952972412, 0.033573886871337, 0.033060863494873, 0.032859134674072005, 0.032934913635253, 0.033126399993896, 0.033550334930419, 0.033321983337402, 0.033424385070800006, 0.033831935882568005, 0.03450675201416, 0.03338137435913, 0.033111038208007, 0.033156097412109, 0.032929790496826004, 0.033179649353027, 0.032975872039794005, 0.033115135192871, 0.033300479888916004, 0.033142784118652, 0.033099777221679005, 0.033371135711669005, 0.033000446319580004, 0.033035263061523004, 0.033124351501464006, 0.033158145904541005, 0.033143806457519, 0.033027072906494, 0.033215488433837, 0.033368064880371, 0.033145854949951, 0.033129470825195004, 0.033292289733886, 0.033061889648437, 0.033264640808105005, 0.033097728729248, 0.033062911987304, 0.032893951416015005, 0.033054721832275, 0.032947200775146006, 0.032976894378662004, 0.033021953582763004, 0.032965633392333, 0.033039360046386, 0.033003520965576, 0.033004543304443, 0.032919551849365, 0.033100799560546006, 0.032914432525634, 0.032984062194824006, 0.032832511901855, 0.033096702575683, 0.03292569732666, 0.032746494293212004, 0.032812030792236005, 0.032974849700927004, 0.032818176269531, 0.033223678588867, 0.033364990234375, 0.033157119750976, 0.033451007843017, 0.033294334411621, 0.033131519317626, 0.033264640808105005, 0.033084415435791, 0.033289215087890005, 0.033388542175292, 0.033492992401123, 0.033262592315673, 0.032995326995849006, 0.033028095245361, 0.033239040374755, 0.033033214569091, 0.033157119750976, 0.033196033477783, 0.033115135192871, 0.033112064361572005, 0.033271808624267, 0.03319705581665, 0.033564670562744, 0.033347583770751005, 0.033459201812744005, 0.033487873077392004, 0.033445888519287, 0.033515518188476, 0.033482753753662005, 0.033691646575927, 0.034598911285400005, 0.034251777648925, 0.033310718536376, 0.033359870910644004, 0.033542144775390005, 0.033390590667724006, 0.033421310424804, 0.033377281188964006, 0.033483776092529005, 0.033317886352539, 0.033483776092529005, 0.033413120269775005, 0.033635326385498, 0.0335810546875, 0.033947647094726, 0.034398208618164006, 0.033424385070800006, 0.033188865661621006, 0.033225727081298, 0.033305599212646, 0.033257472991943, 0.033171455383300005, 0.033331199645996006, 0.033100799560546006, 0.033189888000488, 0.033303550720214, 0.033903617858886, 0.033691646575927, 0.033393665313720004, 0.033471488952636005, 0.033075199127197, 0.033295360565185006, 0.034626560211181004, 0.033335296630859004, 0.033564670562744, 0.033325054168701004, 0.033161216735839, 0.03319705581665, 0.033311744689941004, 0.033005569458007006, 0.033221630096435, 0.033606655120849004, 0.033397758483886, 0.033396736145019, 0.033729534149169, 0.033640449523925, 0.03348582458496, 0.03353702545166, 0.03353702545166, 0.033250305175781, 0.033519615173339005, 0.033326080322265, 0.039286785125732006, 0.033092609405517004, 0.033396736145019, 0.033394687652587005, 0.033379329681396, 0.033754112243652004, 0.034698238372802004, 0.03465523147583, 0.034324478149414, 0.033371135711669005, 0.033187839508056004, 0.033420288085937, 0.033330177307128, 0.034597888946533005, 0.034318336486816, 0.035048446655273, 0.034929664611816, 0.034329601287841, 0.03356774520874, 0.033612800598144005, 0.033382400512695004, 0.033385471343994, 0.033250305175781, 0.033357822418212, 0.033298431396484, 0.033239040374755, 0.032941055297851005, 0.033224704742431, 0.033172481536865, 0.03322777557373, 0.033275905609130005, 0.033277950286865005, 0.033264640808105005, 0.033042430877685006, 0.03295641708374, 0.033099777221679005, 0.033042430877685006, 0.033221630096435, 0.033270782470703, 0.033073150634765, 0.033149951934814, 0.033135616302490005, 0.033187839508056004, 0.033199104309082, 0.032976894378662004, 0.033252353668212005, 0.033187839508056004, 0.033243137359619, 0.033203201293945, 0.033257472991943, 0.033268737792968, 0.033096702575683, 0.032985088348388, 0.033185791015625, 0.033134590148925, 0.033226753234863, 0.033163265228271004, 0.033351680755615, 0.033153022766113004, 0.033299457550048005, 0.033128448486328003, 0.033208320617675, 0.033180671691894, 0.033559551239013004, 0.033267711639404, 0.03386470413208, 0.03412275314331, 0.034206718444824, 0.032990207672119, 0.033259521484375006, 0.032964607238769005, 0.033245185852050003, 0.033309696197509, 0.033396736145019, 0.033273857116699, 0.033164287567138004, 0.033033214569091, 0.03331481552124, 0.033439743041992, 0.033113086700439005, 0.033082366943359004, 0.033454078674316004, 0.033210369110107, 0.033172481536865, 0.033162239074707, 0.033323009490966005, 0.033084415435791, 0.033477630615234004, 0.033348606109619004, 0.033363967895507, 0.033165313720703006, 0.033218559265136005, 0.032986110687255, 0.033100799560546006, 0.033093631744384004, 0.033190910339355, 0.033165313720703006, 0.033527809143066, 0.033241088867187005, 0.033118209838867, 0.033071102142333005, 0.033196033477783, 0.033050624847412, 0.033214462280273, 0.03422310256958, 0.034165760040283, 0.034704383850097005, 0.035491840362548, 0.033172481536865, 0.033428478240966, 0.033586177825927, 0.033359870910644004, 0.033273857116699, 0.033636352539062005, 0.033094657897949, 0.033099777221679005, 0.033117183685302004, 0.033392639160156, 0.03319705581665, 0.033286144256591, 0.033289215087890005, 0.033114112854003, 0.033231872558593, 0.03349401473999, 0.033240062713623, 0.033421310424804, 0.033703937530517]",tokens/s,119.98534099021663,,,pytorch,2.2.1+cu118,optimum_benchmark.backends.pytorch.backend.PyTorchBackend,TheBloke/Mistral-7B-Instruct-v0.2-AWQ,text-generation,transformers,cuda,4,42,,,main,False,False,False,True,,,False,,True,False,,,,False,awq,gemm,2,128,4,False,,process,optimum_benchmark.launchers.process.launcher.ProcessLauncher,False,spawn,inference,optimum_benchmark.benchmarks.inference.benchmark.InferenceBenchmark,10,20,4,2,512,,False,False,True,128,128,mistral-7b-instruct-awq-gemm-bs4,,,,, AMD EPYC 7742 64-Core Processor,128,540671.627264,Linux,x86_64,Linux-5.4.0-166-generic-x86_64-with-glibc2.35,x86_64,3.10.12,"['NVIDIA A100-SXM4-80GB', 'NVIDIA A100-SXM4-80GB', 'NVIDIA A100-SXM4-80GB', 'NVIDIA DGX Display', 'NVIDIA A100-SXM4-80GB']",5,347892350976,0.2.0,8b3b521bb0d0f5c537d0f47d3677667aa2767f3e,4.39.0.dev0,,0.27.2,,,,,,,,,
,s,1.162766662597656,0.00020220947265600002,"[1.162564453125, 1.162968872070312]",tokens/s,14090.53125362026,,,,s,9.335143241882324,0.002224113464355,"[9.332919128417968, 9.337367355346679]",tokens/s,435.34415002511963,,,,s,0.06434942188413101,0.002026856391952,"[0.06482637023925701, 0.06121574401855401, 0.060747776031494005, 0.061841407775878, 0.060647422790527004, 0.061271041870117005, 0.060862464904785, 0.062013439178466, 0.060603393554687, 0.061410305023193, 0.061077503204345004, 0.062191616058349004, 0.061224960327148005, 0.06160998535156201, 0.06141132736206, 0.062897151947021, 0.06173900985717701, 0.062237697601318004, 0.061746177673339, 0.062475265502929006, 0.061484031677246004, 0.06213529586791901, 0.062198783874511006, 0.062778369903564, 0.061652992248535, 0.06200524902343701, 0.06195916748046801, 0.063447040557861, 0.06189363098144501, 0.06276710510253901, 0.062464000701904006, 0.063710208892822, 0.062477310180664006, 0.062496768951416, 0.06211072158813401, 0.063312896728515, 0.062371841430664, 0.062705665588378, 0.062335998535156006, 0.063558654785156, 0.062443519592285, 0.062941184997558, 0.06251622390747001, 0.063789054870605, 0.06267801666259701, 0.063186943054199, 0.063190017700195, 0.06451097869873, 0.063015937805175, 0.06322790527343701, 0.062867454528808, 0.064175102233886, 0.06302719879150301, 0.06345113754272401, 0.0630302734375, 0.064310272216796, 0.06321561431884701, 0.063665153503417, 0.06385971069335901, 0.064626686096191, 0.06339891052246001, 0.06406553649902301, 0.064057342529296, 0.064822273254394, 0.064126976013183, 0.06452838134765601, 0.064169982910156, 0.065459197998046, 0.06435327911376901, 0.064716796875, 0.064334846496582, 0.065707008361816, 0.06451097869873, 0.064964607238769, 0.064592895507812, 0.066448387145996, 0.06492569732666001, 0.065728515625, 0.064773117065429, 0.066073600769042, 0.064936958312988, 0.06527999877929601, 0.06489087677001901, 0.066235389709472, 0.06503628540039, 0.065469436645507, 0.06509260559082, 0.06637670135498, 0.06518067169189401, 0.065991676330566, 0.06588313293457, 0.06715596771240201, 0.065474563598632, 0.06572338867187501, 0.06550220489501901, 0.06726246643066401, 0.065579010009765, 0.065903617858886, 0.066181121826171, 0.06739353942871, 0.065645568847656, 0.066319358825683, 0.06630604553222601, 0.067747840881347, 0.06637670135498, 0.066976768493652, 0.066492416381835, 0.06728498840332, 0.06611865234375, 0.066554878234863, 0.066168830871582, 0.067444732666015, 0.06631321716308501, 0.066736129760742, 0.06632038116455001, 0.06762700653076101, 0.066488319396972, 0.06730035400390601, 0.067115005493164, 0.06839295959472601, 0.06718361663818301, 0.067709953308105, 0.06680166625976501, 0.06798131561279201, 0.066856956481933, 0.067244033813476, 0.06689485168457, 0.064279548645019, 0.061302783966064006, 0.06078976058959901, 0.06218342590332, 0.06102732849121, 0.06125772857666, 0.060869632720947, 0.06207590484619101, 0.061029376983642004, 0.061423614501953, 0.061088768005371004, 0.062220287322998, 0.06123929595947201, 0.061627391815185006, 0.061253631591796004, 0.062379009246826, 0.061419521331787005, 0.06177484893798801, 0.06178815841674801, 0.063165439605712, 0.061769729614257, 0.061908992767333006, 0.061595649719238005, 0.06263398361206, 0.06192332839965801, 0.062629886627197, 0.062284801483154004, 0.06352896118164, 0.061976577758789, 0.062287872314453006, 0.061953025817871, 0.06317363357543901, 0.062107646942138, 0.06251622390747001, 0.062724094390869, 0.063923198699951, 0.062439422607421004, 0.06268211364746, 0.06231859207153301, 0.063581184387207, 0.062446590423583005, 0.06291046524047801, 0.062496768951416, 0.063813632965087, 0.062653438568115, 0.063010814666748, 0.06270463943481401, 0.06394163131713801, 0.062862335205078, 0.06388735961914001, 0.063483905792236, 0.06473420715332001, 0.063046657562255, 0.06345011138916, 0.063067134857177, 0.06429798126220701, 0.063197185516357, 0.063647743225097, 0.063197185516357, 0.064517120361328, 0.063430656433105, 0.063799297332763, 0.063489025115966, 0.06472499084472601, 0.064473091125488, 0.065118209838867, 0.064723968505859, 0.06607769775390601, 0.06438706970214801, 0.064673789978027, 0.06426828765869101, 0.06559846496582, 0.06448435211181601, 0.065459197998046, 0.064939010620117, 0.06576332855224601, 0.064862205505371, 0.065662979125976, 0.06530252838134701, 0.066601982116699, 0.065551361083984, 0.06592205047607401, 0.06549811553955001, 0.066233345031738, 0.065045501708984, 0.06544384002685501, 0.06504755401611301, 0.06640230560302701, 0.065191932678222, 0.065632255554199, 0.065241088867187, 0.06653132629394501, 0.065354751586914, 0.06590668487548801, 0.065589248657226, 0.06736895751953101, 0.06621183776855401, 0.066066429138183, 0.065579010009765, 0.066909187316894, 0.065804290771484, 0.066176002502441, 0.06580019378662101, 0.067776512145996, 0.065978370666503, 0.066349052429199, 0.066554878234863, 0.06769152069091701, 0.066404350280761, 0.067087356567382, 0.06671974182128901, 0.06808780670166001, 0.066921470642089, 0.067412994384765, 0.066357246398925, 0.06761164855957001, 0.066481155395507, 0.067551231384277, 0.066679809570312, 0.067875839233398, 0.066689025878906, 0.06710374450683501, 0.0666982421875, 0.06810521697998001, 0.066966529846191, 0.06786969757080001, 0.067556350708007]",tokens/s,497.2849648536051,,,pytorch,2.2.1+cu118,optimum_benchmark.backends.pytorch.backend.PyTorchBackend,TheBloke/Mistral-7B-Instruct-v0.2-AWQ,text-generation,transformers,cuda,4,42,,,main,False,False,False,True,,,False,,True,False,,,,False,awq,exllama,2,128,32,False,,process,optimum_benchmark.launchers.process.launcher.ProcessLauncher,False,spawn,inference,optimum_benchmark.benchmarks.inference.benchmark.InferenceBenchmark,10,20,32,2,512,,False,False,True,128,128,mistral-7b-instruct-awq-exllama-bs32,,,,, AMD EPYC 7742 64-Core Processor,128,540671.627264,Linux,x86_64,Linux-5.4.0-166-generic-x86_64-with-glibc2.35,x86_64,3.10.12,"['NVIDIA A100-SXM4-80GB', 'NVIDIA A100-SXM4-80GB', 'NVIDIA A100-SXM4-80GB', 'NVIDIA DGX Display', 'NVIDIA A100-SXM4-80GB']",5,347892350976,0.2.0,8b3b521bb0d0f5c537d0f47d3677667aa2767f3e,4.39.0.dev0,,0.27.2,,,,,,,,,
,s,0.096182014465332,7.246401194999651e-05,"[0.09611170959472601, 0.096303199768066, 0.096162940979003, 0.096150207519531]",tokens/s,10646.481108680582,,,,s,3.144175098419188,0.0056806976448450005,"[3.145427038192748, 3.142702163696288, 3.136369789123534, 3.152201402664183]",tokens/s,80.78430496052997,,,,s,0.023999945542943,0.00019617037279300002,"[0.023739391326904002, 0.024010751724243003, 0.023857152938842, 0.023763967514038004, 0.023888896942138003, 0.023826431274414, 0.023809024810791002, 0.024047615051269, 0.023821311950683002, 0.023840768814086002, 0.024213504791259003, 0.024007680892944003, 0.023831552505493, 0.023824384689331003, 0.023805952072143, 0.023894016265869, 0.023939071655273002, 0.023944192886352002, 0.023975936889648, 0.023949312210083003, 0.023842815399169003, 0.023718912124633004, 0.023786495208740002, 0.023928831100463, 0.023970815658569, 0.023772159576416003, 0.023689216613769003, 0.024024063110351, 0.02396774482727, 0.024325120925903002, 0.024026111602783003, 0.024168447494506, 0.024201215744018, 0.023925760269165004, 0.023562240600585004, 0.024050687789916002, 0.024180736541748002, 0.024054784774780003, 0.023936000823974003, 0.023744512557983002, 0.024093696594238, 0.024231935501098003, 0.024074239730834, 0.024015871047973, 0.023897087097167, 0.024083456039428002, 0.023779327392578, 0.023689216613769003, 0.023720960617065003, 0.024057855606079002, 0.024037376403808004, 0.023926784515380002, 0.02371481513977, 0.02410598373413, 0.024174591064453, 0.024250368118286, 0.024220672607421, 0.024169471740722002, 0.023961599349975, 0.023939071655273002, 0.023884799957275, 0.024208383560180003, 0.024236032485961, 0.024211456298828003, 0.024270847320556004, 0.024445951461791002, 0.024455167770385002, 0.024442880630493, 0.023912448883056003, 0.023912448883056003, 0.023947263717651, 0.023875583648681004, 0.024026111602783003, 0.024056831359863, 0.023825408935546, 0.024091648101806, 0.024005632400512, 0.024092672348022003, 0.02410598373413, 0.024007680892944003, 0.024007680892944003, 0.023999488830566, 0.023940095901489003, 0.024035327911376, 0.024122367858886003, 0.024089599609375, 0.024067071914672, 0.024143871307373003, 0.024169471740722002, 0.024207359313964002, 0.024226816177368, 0.024217599868774004, 0.024082431793212, 0.023945215225219002, 0.024081407546997002, 0.024073215484619003, 0.023976959228515, 0.023987199783325003, 0.023851007461547002, 0.023994367599487, 0.02401382446289, 0.023934976577758, 0.024038400650024, 0.024032255172729, 0.024072191238403002, 0.023857152938842, 0.023747583389282, 0.025249792098999004, 0.023931903839111, 0.023661567687988, 0.024061952590942, 0.024139776229858002, 0.023970815658569, 0.024027135848999, 0.023969791412353003, 0.0238960647583, 0.024135679244995, 0.023949312210083003, 0.023630847930908003, 0.023809024810791002, 0.024381439208984, 0.024233983993530002, 0.024310783386230002, 0.024004608154296003, 0.02408550453186, 0.023958528518676, 0.023964672088623, 0.023770111083984, 0.02428006362915, 0.024144895553588, 0.024171520233154, 0.024065023422241, 0.024059904098510002, 0.024020992279052, 0.024047615051269, 0.023905279159545, 0.024018943786621002, 0.023957504272460003, 0.023996416091918, 0.023883775711059003, 0.023959552764892002, 0.024027135848999, 0.024161279678344003, 0.023993343353271003, 0.024034303665161003, 0.024040447235107003, 0.024213504791259003, 0.024046592712402, 0.024028160095214003, 0.024195072174072, 0.023915519714355002, 0.024199167251586003, 0.024159231185913004, 0.024049663543701003, 0.024026111602783003, 0.023974912643432003, 0.023990272521972, 0.023998464584350003, 0.024033279418945, 0.023863296508789003, 0.02401382446289, 0.023792640686035003, 0.023820287704467, 0.023786495208740002, 0.023845888137817003, 0.023874559402465002, 0.024102912902832, 0.023894016265869, 0.023875583648681004, 0.023883775711059003, 0.024094720840454, 0.024262655258178, 0.024123392105102, 0.023955455780029, 0.024009727478027, 0.023764991760253002, 0.023557119369506004, 0.023556095123291, 0.023859199523925, 0.023799808502197, 0.023912448883056003, 0.024020992279052, 0.024079359054565003, 0.023964672088623, 0.024048639297485002, 0.023900159835815, 0.02423910331726, 0.023984128952026004, 0.023799808502197, 0.023920640945434002, 0.024036352157592002, 0.023951360702514003, 0.024025087356567002, 0.023922687530517003, 0.024031232833862003, 0.024089599609375, 0.023965696334838003, 0.023941120147705, 0.024119295120239, 0.024134656906127, 0.024172544479370003, 0.023964672088623, 0.023929855346679, 0.023891967773437002, 0.023945215225219002, 0.023882751464843, 0.023843839645385, 0.02391142463684, 0.023964672088623, 0.023897087097167, 0.023947263717651, 0.024030208587646002, 0.023754751205444003, 0.023970815658569, 0.024125440597534003, 0.023908351898193, 0.023910400390625003, 0.024143871307373003, 0.024158208847045, 0.024027135848999, 0.023898111343383002, 0.023942144393920003, 0.024076288223266, 0.023999488830566, 0.023849983215332, 0.024040447235107003, 0.023974912643432003, 0.024028160095214003, 0.024036352157592002, 0.024102912902832, 0.023960575103759003, 0.024030208587646002, 0.024024063110351, 0.024184831619262003, 0.025095167160034, 0.023996416091918, 0.024058879852294, 0.023734272003173, 0.023802879333496, 0.024119295120239, 0.024167423248291003, 0.023849983215332, 0.023825408935546, 0.023884799957275, 0.023913471221923003, 0.023870464324951, 0.023920640945434002, 0.024100864410400002, 0.024018943786621002, 0.024000511169433, 0.023971839904785002, 0.023961599349975, 0.02369945526123, 0.023686143875122002, 0.023792640686035003, 0.023912448883056003, 0.024007680892944003, 0.023815168380737002, 0.023673856735229003, 0.02363801574707, 0.023774208068847003, 0.023578624725341002, 0.023612415313720003, 0.023822336196899004, 0.023957504272460003, 0.024096767425537003, 0.024139776229858002, 0.023925760269165004, 0.024072191238403002, 0.024164352416992003, 0.024055807113647003, 0.023910400390625003, 0.023817216873168002, 0.023797760009765002, 0.023971839904785002, 0.024031232833862003, 0.023916543960571003, 0.023908351898193, 0.023951360702514003, 0.023908351898193, 0.024017919540405, 0.023913471221923003, 0.023905279159545, 0.023949312210083003, 0.023921663284301002, 0.023927808761596003, 0.023942144393920003, 0.023958528518676, 0.023909376144409, 0.023963647842407003, 0.023971839904785002, 0.023783424377441003, 0.024237056732177003, 0.023902208328247, 0.024012800216674002, 0.023976959228515, 0.023884799957275, 0.023968767166137002, 0.023903232574462002, 0.024022016525268003, 0.023924736022949002, 0.023895040512084003, 0.02408550453186, 0.023826431274414, 0.023680000305175003, 0.023718912124633004, 0.023702527999877, 0.023776256561279, 0.023719936370849, 0.023806976318359003, 0.023709695816040002, 0.023774208068847003, 0.023702527999877, 0.023795711517333003, 0.023724031448364002, 0.023820287704467, 0.023986175537109, 0.023934976577758, 0.023980031967163, 0.023892992019653003, 0.024049663543701003, 0.023963647842407003, 0.024016895294189003, 0.023992319107055002, 0.023706623077392002, 0.023701503753662003, 0.023748607635498003, 0.023584768295288, 0.023706623077392002, 0.023689216613769003, 0.023785472869873002, 0.023848960876464, 0.023606271743774002, 0.023770111083984, 0.023821311950683002, 0.024065023422241, 0.023995391845703002, 0.024306688308715, 0.02412646484375, 0.024001535415649002, 0.023592960357666, 0.023941120147705, 0.024109056472778, 0.02421862411499, 0.024065023422241, 0.024127487182617, 0.024065023422241, 0.024311807632446003, 0.023948287963867002, 0.024030208587646002, 0.023921663284301002, 0.024185855865478, 0.024139776229858002, 0.024229888916015, 0.024025087356567002, 0.024123392105102, 0.023941120147705, 0.024006656646728002, 0.023954431533813002, 0.023918592453002003, 0.023941120147705, 0.025207807540893003, 0.023976959228515, 0.024054784774780003, 0.023923711776733, 0.024098815917968003, 0.023926784515380002, 0.023979007720947004, 0.024012800216674002, 0.024183807373046002, 0.023990272521972, 0.024283136367797, 0.024025087356567002, 0.024144895553588, 0.023899135589599003, 0.023939071655273002, 0.023885824203491002, 0.023942144393920003, 0.023953407287597, 0.023932928085327, 0.023780351638793003, 0.023651327133178003, 0.023917568206787, 0.024388608932495, 0.024081407546997002, 0.023992319107055002, 0.023822336196899004, 0.024442880630493, 0.02441318321228, 0.024028160095214003, 0.023908351898193, 0.024083456039428002, 0.023937023162841003, 0.023746559143066, 0.023909376144409, 0.023951360702514003, 0.024034303665161003, 0.02404454421997, 0.023889919281005003, 0.023895040512084003, 0.024022016525268003, 0.023713792800903, 0.023607295989990004, 0.023847936630249002, 0.02391142463684, 0.02442854309082, 0.024549375534057002, 0.024333311080932, 0.024005632400512, 0.024183807373046002, 0.024036352157592002, 0.02401382446289, 0.024224767684936, 0.024284160614013003, 0.024144895553588, 0.024090623855590004, 0.024110080718994002, 0.024043519973754, 0.024228864669799, 0.02412646484375, 0.023993343353271003, 0.024035327911376, 0.024258560180664003, 0.024025087356567002, 0.024168447494506, 0.02402918434143, 0.024167423248291003, 0.024232959747314, 0.024225791931152003, 0.024087551116943003, 0.024137727737426003, 0.024179712295532, 0.024103935241699, 0.024303615570068, 0.024346624374389003, 0.02423910331726, 0.024088575363159, 0.024062976837158002, 0.024026111602783003, 0.024071168899536, 0.024047615051269, 0.024041471481323, 0.024004608154296003, 0.023918592453002003, 0.023834623336791, 0.023950336456298002, 0.023844863891601002, 0.024169471740722002, 0.024169471740722002, 0.023937023162841003, 0.024211456298828003, 0.024099840164184, 0.024047615051269, 0.02407014465332, 0.024118272781372, 0.023910400390625003, 0.023998464584350003, 0.023630847930908003, 0.024166400909423003, 0.0242544631958, 0.023847936630249002, 0.023548927307128, 0.024003583908081, 0.024189952850341003, 0.024244224548339, 0.024102912902832, 0.024209407806396, 0.023883775711059003, 0.024046592712402, 0.023929855346679, 0.024041471481323, 0.024129535675048, 0.024174591064453, 0.023932928085327, 0.024303615570068, 0.024395776748657002, 0.024040447235107003, 0.023929855346679, 0.024045568466186003, 0.024145919799804003, 0.024039424896240003, 0.023964672088623, 0.023865343093872, 0.023989248275756003, 0.023970815658569, 0.023990272521972, 0.023961599349975, 0.024005632400512, 0.025326591491699003, 0.023987199783325003, 0.023943168640136, 0.023987199783325003, 0.024088575363159, 0.024090623855590004, 0.024075263977050003, 0.023991296768188, 0.023898111343383002, 0.024140800476074004, 0.024120319366455003, 0.024022016525268003, 0.024110080718994002, 0.023981056213378003, 0.024054784774780003, 0.024019968032836, 0.024045568466186003, 0.023945215225219002, 0.024058879852294, 0.023976959228515]",tokens/s,83.33352242076309,,,pytorch,2.2.1+cu118,optimum_benchmark.backends.pytorch.backend.PyTorchBackend,TheBloke/Mistral-7B-Instruct-v0.2-AWQ,text-generation,transformers,cuda,4,42,,,main,False,False,False,True,,,False,,True,False,,,,False,awq,exllama,2,128,2,False,,process,optimum_benchmark.launchers.process.launcher.ProcessLauncher,False,spawn,inference,optimum_benchmark.benchmarks.inference.benchmark.InferenceBenchmark,10,20,2,2,512,,False,False,True,128,128,mistral-7b-instruct-awq-exllama-bs2,,,,, AMD EPYC 7742 64-Core Processor,128,540671.627264,Linux,x86_64,Linux-5.4.0-166-generic-x86_64-with-glibc2.35,x86_64,3.10.12,"['NVIDIA A100-SXM4-80GB', 'NVIDIA A100-SXM4-80GB', 'NVIDIA A100-SXM4-80GB', 'NVIDIA DGX Display', 'NVIDIA A100-SXM4-80GB']",5,347892350976,0.2.0,8b3b521bb0d0f5c537d0f47d3677667aa2767f3e,4.39.0.dev0,,0.27.2,,,,,,,,,
,s,0.140717697143554,0.000365411351117,"[0.140534042358398, 0.141227844238281, 0.140391204833984]",tokens/s,3638.4904698779833,,,,s,4.406672339121502,0.016895500232844002,"[4.384753940582274, 4.425870151519778, 4.409392925262452]",tokens/s,28.8199326445311,,,,s,0.033590194031322004,0.000598683844995,"[0.032960510253906006, 0.033561599731445, 0.033436672210693005, 0.033660926818847005, 0.033884159088134004, 0.033398784637451, 0.033402881622314, 0.033404926300048, 0.033643520355224, 0.033490943908691, 0.033512447357177004, 0.033285118103027006, 0.033068031311035, 0.033801216125488004, 0.033349632263183, 0.033507328033447005, 0.033299457550048005, 0.033290241241455, 0.033104896545410004, 0.033262592315673, 0.033351680755615, 0.033414142608642006, 0.033370113372802004, 0.033281024932861, 0.033411071777343, 0.033508350372314005, 0.033331199645996006, 0.033443840026855, 0.033547264099121, 0.033489921569824, 0.033529857635498005, 0.033366016387939006, 0.033292289733886, 0.033396736145019, 0.033380352020263, 0.033384449005126, 0.033637374877929005, 0.033514495849609, 0.033396736145019, 0.033501182556152004, 0.033583103179931004, 0.03390054321289, 0.033643520355224, 0.033367038726806006, 0.033664001464843, 0.033595390319824, 0.033212417602539006, 0.033533950805664, 0.033299457550048005, 0.033346561431884005, 0.033423358917236, 0.034151424407958, 0.033619968414306, 0.034003967285156006, 0.033588222503662, 0.033516544342041, 0.033775615692138, 0.033500160217285004, 0.033591297149658, 0.033513473510742, 0.033188865661621006, 0.033410049438476, 0.033135616302490005, 0.033377281188964006, 0.033271808624267, 0.033341438293457, 0.033361919403076, 0.033417217254638004, 0.033278976440429, 0.033451007843017, 0.033214462280273, 0.033421310424804, 0.033328128814697, 0.033388542175292, 0.033398784637451, 0.033507328033447005, 0.033477630615234004, 0.033410049438476, 0.033421310424804, 0.033459201812744005, 0.03331481552124, 0.033484798431396005, 0.033330177307128, 0.033315841674804, 0.033472511291503006, 0.033466369628906, 0.033396736145019, 0.033442817687988, 0.03325849533081, 0.033348606109619004, 0.033326080322265, 0.033475582122802, 0.033391616821289, 0.033415168762207, 0.033254398345947005, 0.03316633605957, 0.033160190582275005, 0.033204223632812, 0.033154048919677, 0.033150974273681, 0.033170433044433005, 0.033263614654541, 0.033198078155517, 0.033477630615234004, 0.033462272644042, 0.033508350372314005, 0.033360897064208, 0.034840576171875005, 0.033446910858154, 0.033421310424804, 0.033544193267822, 0.033286144256591, 0.033283073425292, 0.033264640808105005, 0.033289215087890005, 0.033008640289306, 0.033256446838378, 0.033397758483886, 0.033300479888916004, 0.033372161865234, 0.033359870910644004, 0.033239040374755, 0.033236991882324006, 0.03346841430664, 0.033393665313720004, 0.033437694549560006, 0.033436672210693005, 0.033147903442382005, 0.033245185852050003, 0.033448959350585006, 0.033358848571777004, 0.033243137359619, 0.033388542175292, 0.033447937011718006, 0.033470462799072, 0.033402881622314, 0.033380352020263, 0.033253376007080004, 0.033442817687988, 0.033276927947998004, 0.033284095764160006, 0.033245185852050003, 0.033317886352539, 0.033378303527832005, 0.033249279022216, 0.033028095245361, 0.033179649353027, 0.032979969024658, 0.033219585418701, 0.033072128295898, 0.033146881103515005, 0.03307417678833, 0.033173503875732, 0.033313793182373, 0.033304576873779, 0.033240062713623, 0.033346561431884005, 0.033261566162109, 0.033249279022216, 0.033444862365722, 0.033281024932861, 0.033104896545410004, 0.033525760650634, 0.033356800079345, 0.033326080322265, 0.033129470825195004, 0.033134590148925, 0.032951297760009005, 0.033111038208007, 0.033279998779296, 0.033344512939453, 0.033052673339843006, 0.033431552886962, 0.033349632263183, 0.033359870910644004, 0.033356800079345, 0.033393665313720004, 0.033385471343994, 0.033386497497558, 0.033352703094482, 0.0333762550354, 0.033290241241455, 0.033465343475341004, 0.033457153320312, 0.033404926300048, 0.033269760131835004, 0.033370113372802004, 0.033452030181884, 0.033462272644042, 0.033388542175292, 0.033535999298095004, 0.033671169281005005, 0.033540096282958, 0.033349632263183, 0.033426433563232, 0.033498111724853, 0.033423358917236, 0.033395713806152, 0.033294334411621, 0.033439743041992, 0.033135616302490005, 0.033069057464599004, 0.033167358398437, 0.033091583251953, 0.033031169891357, 0.033174526214599, 0.033328128814697, 0.033414142608642006, 0.033405952453613004, 0.033378303527832005, 0.03293081665039, 0.033301502227783004, 0.033206272125244005, 0.033288192749023005, 0.033478656768798, 0.033431552886962, 0.033295360565185006, 0.033639423370361, 0.034669567108154005, 0.034783233642578, 0.034794494628906006, 0.034634750366210006, 0.034623489379882, 0.034246654510498, 0.034902015686035, 0.035441665649414006, 0.034848766326904, 0.034952190399169004, 0.03479552078247, 0.034924545288085, 0.034951168060302004, 0.03494400024414, 0.034711551666259, 0.03515391921997, 0.036307968139648004, 0.035018753051757, 0.034940929412841, 0.034923519134521006, 0.034782207489013, 0.034983936309814, 0.035011585235595, 0.034802688598632, 0.034992126464843, 0.035016704559326005, 0.034991104125976, 0.034968574523925, 0.034866176605224, 0.034396160125732, 0.034561023712158, 0.034272254943847, 0.034164737701416, 0.034361343383789, 0.034483200073242, 0.034509822845458005, 0.033077247619628006, 0.033282047271728, 0.032934913635253, 0.033051647186279, 0.032873470306396, 0.033041408538818005, 0.032922622680664, 0.032982017517089006, 0.033062911987304, 0.032945152282714, 0.032884735107421, 0.03295641708374, 0.032971775054931006, 0.033229824066162005, 0.033276927947998004, 0.033242111206054005, 0.033351680755615, 0.033270782470703, 0.03340185546875, 0.033261566162109, 0.033390590667724006, 0.033441791534423004, 0.033427455902099, 0.033421310424804, 0.033295360565185006, 0.033333248138427, 0.033369087219238, 0.033317886352539, 0.033305599212646, 0.033349632263183, 0.033287166595458, 0.033452030181884, 0.033370113372802004, 0.033425407409667006, 0.033464321136474004, 0.033433601379394, 0.033548286437988004, 0.033495040893554005, 0.033437694549560006, 0.033448959350585006, 0.033150974273681, 0.033150974273681, 0.033198078155517, 0.033267711639404, 0.033154048919677, 0.033212417602539006, 0.033159168243408005, 0.033208320617675, 0.033196033477783, 0.033391616821289, 0.033416191101074, 0.033309696197509, 0.033304576873779, 0.033372161865234, 0.033117183685302004, 0.033157119750976, 0.032973823547363, 0.03295641708374, 0.033077247619628006, 0.033093631744384004, 0.032988159179687004, 0.033047550201416004, 0.033030143737792006, 0.033061889648437, 0.033085441589355, 0.033490943908691, 0.033335296630859004, 0.032980991363525, 0.033518592834472005, 0.033627136230468, 0.033430526733398004, 0.03340185546875, 0.033234943389892004, 0.033415168762207, 0.033226753234863, 0.033403903961181, 0.033252353668212005, 0.033606655120849004, 0.034728958129882005, 0.034994174957275005, 0.034898944854736005, 0.034981887817382006, 0.034893825531005, 0.034969600677490005, 0.034727935791015005, 0.034937854766845, 0.034801662445068, 0.034856960296630005, 0.034863105773925006, 0.03490816116333, 0.034834430694580004, 0.034920448303222, 0.034917377471923, 0.03500032043457, 0.034494464874267, 0.034744319915771, 0.034641918182373, 0.034809856414794005, 0.034630657196044, 0.034743297576904, 0.034684928894042004, 0.034712577819824, 0.034584575653076, 0.034767871856689, 0.034385921478271006, 0.034524158477783, 0.034513919830322, 0.034740222930908005, 0.033247230529785, 0.033111038208007, 0.032835582733154005, 0.032955390930175, 0.033144832611083, 0.033135616302490005, 0.03292569732666, 0.033487873077392004, 0.033210369110107, 0.033459201812744005, 0.033021953582763004, 0.033423358917236, 0.033474559783935, 0.033512447357177004, 0.033627136230468, 0.033798145294189, 0.033444862365722, 0.033491966247558, 0.033438720703125]",tokens/s,29.77059313999538,,,pytorch,2.2.1+cu118,optimum_benchmark.backends.pytorch.backend.PyTorchBackend,TheBloke/Mistral-7B-Instruct-v0.2-AWQ,text-generation,transformers,cuda,4,42,,,main,False,False,False,True,,,False,,True,False,,,,False,awq,gemm,2,128,1,False,,process,optimum_benchmark.launchers.process.launcher.ProcessLauncher,False,spawn,inference,optimum_benchmark.benchmarks.inference.benchmark.InferenceBenchmark,10,20,1,2,512,,False,False,True,128,128,mistral-7b-instruct-awq-gemm-bs1,,,,, AMD EPYC 7742 64-Core Processor,128,540671.627264,Linux,x86_64,Linux-5.4.0-166-generic-x86_64-with-glibc2.35,x86_64,3.10.12,"['NVIDIA A100-SXM4-80GB', 'NVIDIA A100-SXM4-80GB', 'NVIDIA A100-SXM4-80GB', 'NVIDIA DGX Display', 'NVIDIA A100-SXM4-80GB']",5,347892350976,0.2.0,8b3b521bb0d0f5c537d0f47d3677667aa2767f3e,4.39.0.dev0,,0.27.2,,,,,,,,,
,s,2.5236499023437498,0,[2.5236499023437498],tokens/s,12984.368382305282,,,,s,15.078331253051761,0,[15.078331253051761],tokens/s,539.0516936915644,,,,s,0.098855758666992,0.042394172105307,"[0.094311424255371, 0.089033729553222, 0.08798822021484301, 0.572880920410156, 0.08745574188232401, 0.08875520324707001, 0.08814387512207, 0.089899009704589, 0.08833843231201101, 0.089149436950683, 0.088834045410156, 0.09127423858642501, 0.08899686431884701, 0.092413955688476, 0.09092813110351501, 0.091599868774414, 0.089328643798828, 0.09038028717041001, 0.089821182250976, 0.091386878967285, 0.089950210571289, 0.09017651367187501, 0.08997376251220701, 0.09221631622314401, 0.09102950286865201, 0.092002304077148, 0.09062604522705, 0.09294335937499999, 0.09076121520996001, 0.09166336059570301, 0.09098751831054601, 0.092896255493164, 0.091124732971191, 0.092110847473144, 0.09111449432373, 0.09445990753173801, 0.091796478271484, 0.09253376007080001, 0.09185997009277301, 0.09378816223144501, 0.09214975738525301, 0.09283891296386701, 0.092346366882324, 0.09447219085693301, 0.092296188354492, 0.09322496032714801, 0.09286860656738201, 0.09500466918945301, 0.09264742279052701, 0.093622268676757, 0.092986366271972, 0.09531391906738201, 0.09301708984375, 0.09450905609130801, 0.09326898956298801, 0.095279106140136, 0.093356033325195, 0.09439231872558501, 0.09413529968261701, 0.096291839599609, 0.09389875030517501, 0.09468927764892501, 0.094120956420898, 0.09623654174804601, 0.094351356506347, 0.09519821166992101, 0.094445564270019, 0.09697792053222601, 0.09495654296875, 0.09618943786621001, 0.09475686645507801, 0.097001472473144, 0.0947261428833, 0.095977470397949, 0.09518284606933501, 0.09777458953857401, 0.09558016204833901, 0.09626112365722601, 0.09620377349853501, 0.09862963104248, 0.095876098632812, 0.096668670654296, 0.09589862060546801, 0.09842073822021401, 0.09632870483398401, 0.097214462280273, 0.09634099578857401, 0.09862860870361301, 0.097085441589355, 0.09819647979736301, 0.096741378784179, 0.09941401672363201, 0.09705369567871001, 0.097801216125488, 0.09713049316406201, 0.099377151489257, 0.09758310699462801, 0.09844019317626901, 0.098516990661621, 0.100961280822753, 0.09803571319580001, 0.098909187316894, 0.098078720092773, 0.10029670715332001, 0.09830604553222601, 0.09923481750488201, 0.09838899230957, 0.100980735778808, 0.09923481750488201, 0.100275199890136, 0.09875456237792901, 0.101050369262695, 0.09921740722656201, 0.100638717651367, 0.099062782287597, 0.101610496520996, 0.09934438323974601, 0.10091519927978501, 0.09988403320312501, 0.101807106018066, 0.09971814727783201, 0.10071244812011701, 0.10037964630126901, 0.10248703765869101, 0.10000793457031201, 0.10102579498291, 0.10053119659423801]",tokens/s,647.4079088866425,,,pytorch,2.2.1+cu118,optimum_benchmark.backends.pytorch.backend.PyTorchBackend,TheBloke/Mistral-7B-Instruct-v0.2-AWQ,text-generation,transformers,cuda,4,42,,,main,False,False,False,True,,,False,,True,False,,,,False,awq,gemm,2,128,64,False,,process,optimum_benchmark.launchers.process.launcher.ProcessLauncher,False,spawn,inference,optimum_benchmark.benchmarks.inference.benchmark.InferenceBenchmark,10,20,64,2,512,,False,False,True,128,128,mistral-7b-instruct-awq-gemm-bs64,,,,, AMD EPYC 7742 64-Core Processor,128,540671.627264,Linux,x86_64,Linux-5.4.0-166-generic-x86_64-with-glibc2.35,x86_64,3.10.12,"['NVIDIA A100-SXM4-80GB', 'NVIDIA A100-SXM4-80GB', 'NVIDIA A100-SXM4-80GB', 'NVIDIA DGX Display', 'NVIDIA A100-SXM4-80GB']",5,347892350976,0.2.0,8b3b521bb0d0f5c537d0f47d3677667aa2767f3e,4.39.0.dev0,,0.27.2,,,,,,,,,