{ "status": "reviewed_complete", "generated_unix": 1790046344.017151, "metrics": { "hf-A0": { "engine": "hf", "candidate": "A0", "records": 160, "failed_requests": [], "final_document_expected": 128, "final_document_present": 128, "final_document_failed": 0, "quality_complete": true, "nll": 1.93365006070158, "ppl": 6.914703322333733, "scored_tokens": 130944, "domains": { "ko": { "documents": 64, "nll": 2.1870921051688677, "ppl": 8.909268195497868 }, "en": { "documents": 32, "nll": 2.2387158178222726, "ppl": 9.38127628088293 }, "code": { "documents": 32, "nll": 1.1217002146463124, "ppl": 3.0700695456426317 } }, "delta_nll": { "delta": 0.0820362158593078, "ci95": [ 0.07676074866274643, 0.08754806751121319 ], "resampling_unit": "paired document", "replicates": 2000 }, "ppl_ratio_vs_bf16": { "ratio": 1.0854951212562869, "ci95": [ 1.0797837057905217, 1.0914947286171375 ] }, "document_delta_tail": { "mean": 0.08203621585930782, "median": 0.0820228557083264, "p95": 0.13523329350722266, "p99": 0.1744629132689386, "max": 0.1855487040410643 }, "worst20_documents": [ { "id": "final-109", "domain": "en", "delta_nll": 0.1855487040410643, "nll": 2.280070516487487 }, { "id": "final-049", "domain": "code", "delta_nll": 0.17658769676529185, "nll": 0.6997955281247374 }, { "id": "final-035", "domain": "ko", "delta_nll": 0.16871812826027968, "nll": 2.2584909539768074 }, { "id": "final-103", "domain": "en", "delta_nll": 0.15268722470787846, "nll": 2.2268263955736205 }, { "id": "final-009", "domain": "ko", "delta_nll": 0.14515656390148174, "nll": 2.3813408164567846 }, { "id": "final-075", "domain": "ko", "delta_nll": 0.1392123277352819, "nll": 2.0740082350178084 }, { "id": "final-018", "domain": "ko", "delta_nll": 0.13586595634095014, "nll": 2.4304069451922197 }, { "id": "final-003", "domain": "ko", "delta_nll": 0.13405834824458596, "nll": 1.5526132835321063 }, { "id": "final-063", "domain": "ko", "delta_nll": 0.13208126578978896, "nll": 2.06155704170145 }, { "id": "final-012", "domain": "code", "delta_nll": 0.12857145065203207, "nll": 0.4027091927775423 }, { "id": "final-001", "domain": "code", "delta_nll": 0.12701752365276378, "nll": 0.32724712274873946 }, { "id": "final-036", "domain": "ko", "delta_nll": 0.1217827195645777, "nll": 2.393973104415401 }, { "id": "final-039", "domain": "ko", "delta_nll": 0.12090000122756436, "nll": 2.3648213846126493 }, { "id": "final-032", "domain": "ko", "delta_nll": 0.11797917833076545, "nll": 1.9544753124054344 }, { "id": "final-033", "domain": "ko", "delta_nll": 0.11529606644825269, "nll": 2.4616481048270753 }, { "id": "final-113", "domain": "ko", "delta_nll": 0.11503100092226926, "nll": 2.366507243084651 }, { "id": "final-046", "domain": "code", "delta_nll": 0.11190791237272708, "nll": 1.3897814177348118 }, { "id": "final-047", "domain": "ko", "delta_nll": 0.11058930893913858, "nll": 1.6414782280796085 }, { "id": "final-019", "domain": "en", "delta_nll": 0.11052950805932546, "nll": 2.0117747268602177 }, { "id": "final-107", "domain": "ko", "delta_nll": 0.10837323178527175, "nll": 2.1305793825598873 } ], "token_delta_tail": { "mean": 0.0820362158593078, "median": 0.004975318908691406, "p95": 0.8666933059692374, "p99": 1.8084007644653302, "max": 15.288114547729492 }, "sampled_full_vocab_kl": { "mean": 0.0970104158181517, "median": 0.042478140968257146, "p95": 0.29837098349910446, "p99": 0.7911814699498793, "max": 17.646372187274768 }, "kl_positions": 4096, "sampled_top1_agreement": 0.87255859375, "tasks": {}, "speed": {}, "speed_source": "embedded workload-tail run; dedicated benchmark pending", "block_diagnostics": { "scope": "six fixed validation documents, 256 tokens, cumulative error", "layer_mean_relative_l2": { "0": 0.12903799457971124, "1": 0.1571070468040269, "2": 0.22566349600917013, "3": 0.21133631629360275, "4": 0.2285544830943772, "5": 0.23136062024308487, "6": 0.23393891724912053, "7": 0.2467944833779129, "8": 0.24475813253585876, "9": 0.24711447521186036, "10": 0.24167888034169704, "11": 0.25611256562120915, "12": 0.2604246348639825, "13": 0.2591232188818453, "14": 0.2608755204209323, "15": 0.27006129648057214, "16": 0.2737835062148852, "17": 0.2643662372116999, "18": 0.2542052663037596, "19": 0.2652235503473812, "20": 0.2567186581435374, "21": 0.2535158345391283, "22": 0.25320365483126345, "23": 0.2543835751826935, "24": 0.2530360601866923, "25": 0.25580827084285024, "26": 0.26490951443777777, "27": 0.27515065090187246, "28": 0.2654177558413169, "29": 0.271302181349774, "30": 0.2837576174019451, "31": 0.2844029517845293 } } }, "hf-B0": { "engine": "hf", "candidate": "B0", "records": 160, "failed_requests": [], "final_document_expected": 128, "final_document_present": 128, "final_document_failed": 0, "quality_complete": true, "nll": 1.851613844842272, "ppl": 6.370091571052912, "scored_tokens": 130944, "domains": { "ko": { "documents": 64, "nll": 2.0939926444639325, "ppl": 8.117259883591567 }, "en": { "documents": 32, "nll": 2.1686982321366304, "ppl": 8.746890205711475 }, "code": { "documents": 32, "nll": 1.0497718583045934, "ppl": 2.8569992430549704 } }, "delta_nll": { "delta": 0.0, "ci95": [ 0.0, 0.0 ], "resampling_unit": "paired document", "replicates": 2000 }, "ppl_ratio_vs_bf16": { "ratio": 1.0, "ci95": [ 1.0, 1.0 ] }, "document_delta_tail": { "mean": 0.0, "median": 0.0, "p95": 0.0, "p99": 0.0, "max": 0.0 }, "worst20_documents": [ { "id": "final-000", "domain": "en", "delta_nll": 0.0, "nll": 2.210419322039025 }, { "id": "final-001", "domain": "code", "delta_nll": 0.0, "nll": 0.20022959909597565 }, { "id": "final-002", "domain": "ko", "delta_nll": 0.0, "nll": 1.8950367085744908 }, { "id": "final-003", "domain": "ko", "delta_nll": 0.0, "nll": 1.4185549352875204 }, { "id": "final-004", "domain": "code", "delta_nll": 0.0, "nll": 1.1785066153413506 }, { "id": "final-005", "domain": "en", "delta_nll": 0.0, "nll": 2.2057225897631456 }, { "id": "final-006", "domain": "code", "delta_nll": 0.0, "nll": 0.7938262038916087 }, { "id": "final-007", "domain": "ko", "delta_nll": 0.0, "nll": 1.9395612649554033 }, { "id": "final-008", "domain": "en", "delta_nll": 0.0, "nll": 2.2966494592869853 }, { "id": "final-009", "domain": "ko", "delta_nll": 0.0, "nll": 2.2361842525553026 }, { "id": "final-010", "domain": "ko", "delta_nll": 0.0, "nll": 2.424708617164709 }, { "id": "final-011", "domain": "ko", "delta_nll": 0.0, "nll": 1.568956426394883 }, { "id": "final-012", "domain": "code", "delta_nll": 0.0, "nll": 0.27413774212551023 }, { "id": "final-013", "domain": "en", "delta_nll": 0.0, "nll": 2.472125922246879 }, { "id": "final-014", "domain": "ko", "delta_nll": 0.0, "nll": 2.6370440163337356 }, { "id": "final-015", "domain": "ko", "delta_nll": 0.0, "nll": 1.215205994286262 }, { "id": "final-016", "domain": "ko", "delta_nll": 0.0, "nll": 2.339994630039845 }, { "id": "final-017", "domain": "code", "delta_nll": 0.0, "nll": 1.0864892625855327 }, { "id": "final-018", "domain": "ko", "delta_nll": 0.0, "nll": 2.2945409888512693 }, { "id": "final-019", "domain": "en", "delta_nll": 0.0, "nll": 1.9012452188008924 } ], "token_delta_tail": { "mean": 0.0, "median": 0.0, "p95": 0.0, "p99": 0.0, "max": 0.0 }, "sampled_full_vocab_kl": null, "kl_positions": 0, "tasks": {}, "speed": {}, "speed_source": "embedded workload-tail run; dedicated benchmark pending", "block_diagnostics": { "scope": "six fixed validation documents, 256 tokens, cumulative error", "layer_mean_relative_l2": { "0": 0.0, "1": 0.0, "2": 0.0, "3": 0.0, "4": 0.0, "5": 0.0, "6": 0.0, "7": 0.0, "8": 0.0, "9": 0.0, "10": 0.0, "11": 0.0, "12": 0.0, "13": 0.0, "14": 0.0, "15": 0.0, "16": 0.0, "17": 0.0, "18": 0.0, "19": 0.0, "20": 0.0, "21": 0.0, "22": 0.0, "23": 0.0, "24": 0.0, "25": 0.0, "26": 0.0, "27": 0.0, "28": 0.0, "29": 0.0, "30": 0.0, "31": 0.0 } } }, "hf-G0": { "engine": "hf", "candidate": "G0", "records": 160, "failed_requests": [], "final_document_expected": 128, "final_document_present": 128, "final_document_failed": 0, "quality_complete": true, "nll": 1.8885224431476402, "ppl": 6.609595409550301, "scored_tokens": 130944, "domains": { "ko": { "documents": 64, "nll": 2.132348335019421, "ppl": 8.434650958668149 }, "en": { "documents": 32, "nll": 2.2070701874479406, "ppl": 9.08904813830095 }, "code": { "documents": 32, "nll": 1.082322915103778, "ppl": 2.9515277424747435 } }, "delta_nll": { "delta": 0.036908598305368, "ci95": [ 0.03381318624021225, 0.040242480689282815 ], "resampling_unit": "paired document", "replicates": 2000 }, "ppl_ratio_vs_bf16": { "ratio": 1.0375981782719963, "ci95": [ 1.0343913501399735, 1.0410631813070161 ] }, "document_delta_tail": { "mean": 0.036908598305368, "median": 0.03632262118983712, "p95": 0.06311846073072325, "p99": 0.10133215486595487, "max": 0.13963556429507795 }, "worst20_documents": [ { "id": "final-049", "domain": "code", "delta_nll": 0.13963556429507795, "nll": 0.6628433956545235 }, { "id": "final-103", "domain": "en", "delta_nll": 0.10965255185422776, "nll": 2.18379172271997 }, { "id": "final-036", "domain": "ko", "delta_nll": 0.07883626671247584, "nll": 2.3510266515632994 }, { "id": "final-035", "domain": "ko", "delta_nll": 0.07751580999976612, "nll": 2.167288635716294 }, { "id": "final-013", "domain": "en", "delta_nll": 0.07282334175743782, "nll": 2.5449492640043165 }, { "id": "final-046", "domain": "code", "delta_nll": 0.0649968172448122, "nll": 1.3428703226068968 }, { "id": "final-063", "domain": "ko", "delta_nll": 0.06427158568262005, "nll": 1.9937473615942813 }, { "id": "final-019", "domain": "en", "delta_nll": 0.06097694296291497, "nll": 1.9622221617638074 }, { "id": "final-047", "domain": "ko", "delta_nll": 0.058374237105294054, "nll": 1.589263156245764 }, { "id": "final-073", "domain": "en", "delta_nll": 0.05784474649736958, "nll": 2.2561646747682333 }, { "id": "final-083", "domain": "code", "delta_nll": 0.05595014708249567, "nll": 1.1111934219986812 }, { "id": "final-024", "domain": "ko", "delta_nll": 0.05479627014488302, "nll": 2.1866303319921707 }, { "id": "final-124", "domain": "ko", "delta_nll": 0.05446133282643725, "nll": 2.1020163781249046 }, { "id": "final-030", "domain": "ko", "delta_nll": 0.05397154415574358, "nll": 1.5014877142211676 }, { "id": "final-039", "domain": "ko", "delta_nll": 0.05318891314583906, "nll": 2.297110296530924 }, { "id": "final-032", "domain": "ko", "delta_nll": 0.053079588322345814, "nll": 1.8895757223970147 }, { "id": "final-003", "domain": "ko", "delta_nll": 0.052185427758001515, "nll": 1.4707403630455218 }, { "id": "final-091", "domain": "en", "delta_nll": 0.05165458168335558, "nll": 2.3481659865915367 }, { "id": "final-101", "domain": "en", "delta_nll": 0.05118013756715657, "nll": 2.287601511033283 }, { "id": "final-126", "domain": "ko", "delta_nll": 0.05110409252804395, "nll": 2.4185564548272427 } ], "token_delta_tail": { "mean": 0.036908598305368, "median": 0.001071929931640625, "p95": 0.5307848930358877, "p99": 1.1659850120544428, "max": 9.068389892578125 }, "sampled_full_vocab_kl": { "mean": 0.044962490019749626, "median": 0.016390751744274946, "p95": 0.14909470627543128, "p99": 0.5072163384931685, "max": 4.771928180072513 }, "kl_positions": 4096, "sampled_top1_agreement": 0.916748046875, "tasks": {}, "speed": {}, "speed_source": "embedded workload-tail run; dedicated benchmark pending", "block_diagnostics": { "scope": "six fixed validation documents, 256 tokens, cumulative error", "layer_mean_relative_l2": { "0": 0.061502795778974495, "1": 0.08756348352111992, "2": 0.13211320323248357, "3": 0.13194254501285904, "4": 0.1496387714938382, "5": 0.15543279143382632, "6": 0.15926372939334799, "7": 0.1705592222909812, "8": 0.17304152740954593, "9": 0.1773070721051936, "10": 0.17448204471676795, "11": 0.1832057436036161, "12": 0.18618751560964827, "13": 0.18606091406986847, "14": 0.1868976624418859, "15": 0.19186519409745897, "16": 0.19513437758205152, "17": 0.18753628823447765, "18": 0.17669351825942817, "19": 0.1808796693581447, "20": 0.17552757239029024, "21": 0.17439488314077153, "22": 0.17805623219860633, "23": 0.17567599658009236, "24": 0.17677755667138217, "25": 0.1811603875257962, "26": 0.19488573127430076, "27": 0.19370679826542161, "28": 0.18927246627180824, "29": 0.1957854662306057, "30": 0.20543273546404092, "31": 0.20172448927764497 } } }, "hf-R0": { "engine": "hf", "candidate": "R0", "records": 160, "failed_requests": [], "final_document_expected": 128, "final_document_present": 128, "final_document_failed": 0, "quality_complete": true, "nll": 1.9407359235214465, "ppl": 6.963873963990896, "scored_tokens": 130944, "domains": { "ko": { "documents": 64, "nll": 2.198646108003539, "ppl": 9.012802873688717 }, "en": { "documents": 32, "nll": 2.2330728216371694, "ppl": 9.328486859951811 }, "code": { "documents": 32, "nll": 1.1325786564415385, "ppl": 3.103649435788002 } }, "delta_nll": { "delta": 0.08912207867917427, "ci95": [ 0.08320109213785225, 0.09499921878273593 ], "resampling_unit": "paired document", "replicates": 2000 }, "ppl_ratio_vs_bf16": { "ratio": 1.0932141063146188, "ci95": [ 1.0867603255336724, 1.0996579960539563 ] }, "document_delta_tail": { "mean": 0.08912207867917427, "median": 0.08879938806140411, "p95": 0.14887570314043774, "p99": 0.16015912470812904, "max": 0.19972954560817743 }, "worst20_documents": [ { "id": "final-049", "domain": "code", "delta_nll": 0.19972954560817743, "nll": 0.7229373769676231 }, { "id": "final-063", "domain": "ko", "delta_nll": 0.16074700602571518, "nll": 2.0902227819373764 }, { "id": "final-103", "domain": "en", "delta_nll": 0.15856966781243317, "nll": 2.2327088386781755 }, { "id": "final-003", "domain": "ko", "delta_nll": 0.15642737689722086, "nll": 1.5749823121847413 }, { "id": "final-001", "domain": "code", "delta_nll": 0.15544412842244346, "nll": 0.35567372751841914 }, { "id": "final-035", "domain": "ko", "delta_nll": 0.15013684415397757, "nll": 2.239909669870505 }, { "id": "final-085", "domain": "ko", "delta_nll": 0.14948467728213014, "nll": 2.78201802385168 }, { "id": "final-012", "domain": "code", "delta_nll": 0.14774475116300909, "nll": 0.42188249328851934 }, { "id": "final-018", "domain": "ko", "delta_nll": 0.14653743863222304, "nll": 2.4410784274834927 }, { "id": "final-033", "domain": "ko", "delta_nll": 0.13925958728510612, "nll": 2.4856116256639287 }, { "id": "final-047", "domain": "ko", "delta_nll": 0.13845634553672514, "nll": 1.6693452646771951 }, { "id": "final-036", "domain": "ko", "delta_nll": 0.13749841837356283, "nll": 2.4096888032243866 }, { "id": "final-075", "domain": "ko", "delta_nll": 0.13698122205389326, "nll": 2.0717771293364198 }, { "id": "final-126", "domain": "ko", "delta_nll": 0.12869443734953015, "nll": 2.4961467996487285 }, { "id": "final-118", "domain": "ko", "delta_nll": 0.12809968017995765, "nll": 2.2436153443561153 }, { "id": "final-052", "domain": "code", "delta_nll": 0.12747464221943158, "nll": 1.2281017331433786 }, { "id": "final-009", "domain": "ko", "delta_nll": 0.12590000380052732, "nll": 2.36208425635583 }, { "id": "final-057", "domain": "ko", "delta_nll": 0.12252216395045305, "nll": 2.6040335633771043 }, { "id": "final-039", "domain": "ko", "delta_nll": 0.12172212349005108, "nll": 2.365643506875136 }, { "id": "final-046", "domain": "code", "delta_nll": 0.12160308852806352, "nll": 1.3994765938901481 } ], "token_delta_tail": { "mean": 0.08912207867917427, "median": 0.00594329833984375, "p95": 0.9231513023376459, "p99": 1.928824300765982, "max": 11.330570220947266 }, "sampled_full_vocab_kl": { "mean": 0.1059899188939109, "median": 0.046785345441875806, "p95": 0.33871690559947343, "p99": 0.9872271586607808, "max": 11.07881304550959 }, "kl_positions": 4096, "sampled_top1_agreement": 0.865234375, "tasks": {}, "speed": {}, "speed_source": "embedded workload-tail run; dedicated benchmark pending", "block_diagnostics": { "scope": "six fixed validation documents, 256 tokens, cumulative error", "layer_mean_relative_l2": { "0": 0.1295213972207483, "1": 0.1625967985442728, "2": 0.23553434934653958, "3": 0.21979621509447492, "4": 0.2376477745018263, "5": 0.24059236440198561, "6": 0.24654153530486314, "7": 0.260415532026211, "8": 0.2598454006934941, "9": 0.2624094256982629, "10": 0.25614311139116425, "11": 0.26872622260569584, "12": 0.2726684851022761, "13": 0.27115011713649545, "14": 0.27342810173514115, "15": 0.2818072557633838, "16": 0.2858158542715014, "17": 0.2768645915943102, "18": 0.2662182572484813, "19": 0.2752910905346593, "20": 0.266664676018941, "21": 0.2632310743354779, "22": 0.2639017946895043, "23": 0.26235592497209725, "24": 0.2594317109714475, "25": 0.2626580358224639, "26": 0.2774713093128148, "27": 0.27985080119603567, "28": 0.27082239671685315, "29": 0.2777480940112989, "30": 0.28839128452507495, "31": 0.2912248048843746 } } }, "native-B0": { "engine": "native", "candidate": "B0", "records": 4714, "failed_requests": [], "final_document_expected": 128, "final_document_present": 128, "final_document_failed": 0, "quality_complete": true, "nll": 1.8514976427891694, "ppl": 6.369351396339647, "scored_tokens": 130944, "domains": { "ko": { "documents": 64, "nll": 2.0938041075234968, "ppl": 8.11572962450811 }, "en": { "documents": 32, "nll": 2.168783994885714, "ppl": 8.747640395230148 }, "code": { "documents": 32, "nll": 1.0495983612239705, "ppl": 2.856503605024078 } }, "delta_nll": { "delta": 0.0, "ci95": [ 0.0, 0.0 ], "resampling_unit": "paired document", "replicates": 2000 }, "ppl_ratio_vs_bf16": { "ratio": 1.0, "ci95": [ 1.0, 1.0 ] }, "document_delta_tail": { "mean": 0.0, "median": 0.0, "p95": 0.0, "p99": 0.0, "max": 0.0 }, "worst20_documents": [ { "id": "final-000", "domain": "en", "delta_nll": 0.0, "nll": 2.210358508974271 }, { "id": "final-001", "domain": "code", "delta_nll": 0.0, "nll": 0.19993765938142033 }, { "id": "final-002", "domain": "ko", "delta_nll": 0.0, "nll": 1.8954136529580778 }, { "id": "final-003", "domain": "ko", "delta_nll": 0.0, "nll": 1.4191517381421672 }, { "id": "final-004", "domain": "code", "delta_nll": 0.0, "nll": 1.1780465211209783 }, { "id": "final-005", "domain": "en", "delta_nll": 0.0, "nll": 2.2044185917991137 }, { "id": "final-006", "domain": "code", "delta_nll": 0.0, "nll": 0.7937105457941359 }, { "id": "final-007", "domain": "ko", "delta_nll": 0.0, "nll": 1.94125673385177 }, { "id": "final-008", "domain": "en", "delta_nll": 0.0, "nll": 2.2967991607329763 }, { "id": "final-009", "domain": "ko", "delta_nll": 0.0, "nll": 2.236059863709279 }, { "id": "final-010", "domain": "ko", "delta_nll": 0.0, "nll": 2.423272220184445 }, { "id": "final-011", "domain": "ko", "delta_nll": 0.0, "nll": 1.56812581664109 }, { "id": "final-012", "domain": "code", "delta_nll": 0.0, "nll": 0.27485824072762477 }, { "id": "final-013", "domain": "en", "delta_nll": 0.0, "nll": 2.4718865563630583 }, { "id": "final-014", "domain": "ko", "delta_nll": 0.0, "nll": 2.6355058926572874 }, { "id": "final-015", "domain": "ko", "delta_nll": 0.0, "nll": 1.2157928098594526 }, { "id": "final-016", "domain": "ko", "delta_nll": 0.0, "nll": 2.339425756856219 }, { "id": "final-017", "domain": "code", "delta_nll": 0.0, "nll": 1.0859001591590602 }, { "id": "final-018", "domain": "ko", "delta_nll": 0.0, "nll": 2.2943346452131936 }, { "id": "final-019", "domain": "en", "delta_nll": 0.0, "nll": 1.9004068108754806 } ], "token_delta_tail": { "mean": 0.0, "median": 0.0, "p95": 0.0, "p99": 0.0, "max": 0.0 }, "sampled_full_vocab_kl": null, "kl_positions": 0, "tasks": { "kmmlu": { "n": 500, "accuracy": 0.482, "failures": 0, "subject_scores": { "Accounting": { "n": 12, "accuracy": 0.5833333333333334 }, "Agricultural-Sciences": { "n": 12, "accuracy": 0.5 }, "Aviation-Engineering-and-Maintenance": { "n": 12, "accuracy": 0.5 }, "Biology": { "n": 12, "accuracy": 0.3333333333333333 }, "Chemical-Engineering": { "n": 12, "accuracy": 0.4166666666666667 }, "Chemistry": { "n": 11, "accuracy": 0.7272727272727273 }, "Civil-Engineering": { "n": 11, "accuracy": 0.2727272727272727 }, "Computer-Science": { "n": 11, "accuracy": 0.8181818181818182 }, "Construction": { "n": 11, "accuracy": 0.18181818181818182 }, "Criminal-Law": { "n": 11, "accuracy": 0.2727272727272727 }, "Ecology": { "n": 11, "accuracy": 0.5454545454545454 }, "Economics": { "n": 11, "accuracy": 0.6363636363636364 }, "Education": { "n": 11, "accuracy": 0.8181818181818182 }, "Electrical-Engineering": { "n": 11, "accuracy": 0.2727272727272727 }, "Electronics-Engineering": { "n": 11, "accuracy": 0.5454545454545454 }, "Energy-Management": { "n": 11, "accuracy": 0.5454545454545454 }, "Environmental-Science": { "n": 11, "accuracy": 0.36363636363636365 }, "Fashion": { "n": 11, "accuracy": 0.6363636363636364 }, "Food-Processing": { "n": 11, "accuracy": 0.45454545454545453 }, "Gas-Technology-and-Engineering": { "n": 11, "accuracy": 0.45454545454545453 }, "Geomatics": { "n": 11, "accuracy": 0.09090909090909091 }, "Health": { "n": 11, "accuracy": 0.45454545454545453 }, "Industrial-Engineer": { "n": 11, "accuracy": 0.5454545454545454 }, "Information-Technology": { "n": 11, "accuracy": 1.0 }, "Interior-Architecture-and-Design": { "n": 11, "accuracy": 0.5454545454545454 }, "Korean-History": { "n": 11, "accuracy": 0.2727272727272727 }, "Law": { "n": 11, "accuracy": 0.09090909090909091 }, "Machine-Design-and-Manufacturing": { "n": 11, "accuracy": 0.7272727272727273 }, "Management": { "n": 11, "accuracy": 0.7272727272727273 }, "Maritime-Engineering": { "n": 11, "accuracy": 0.36363636363636365 }, "Marketing": { "n": 11, "accuracy": 0.9090909090909091 }, "Materials-Engineering": { "n": 11, "accuracy": 0.2727272727272727 }, "Math": { "n": 11, "accuracy": 0.18181818181818182 }, "Mechanical-Engineering": { "n": 11, "accuracy": 0.36363636363636365 }, "Nondestructive-Testing": { "n": 11, "accuracy": 0.6363636363636364 }, "Patent": { "n": 11, "accuracy": 0.18181818181818182 }, "Political-Science-and-Sociology": { "n": 11, "accuracy": 0.45454545454545453 }, "Psychology": { "n": 11, "accuracy": 0.36363636363636365 }, "Public-Safety": { "n": 11, "accuracy": 0.6363636363636364 }, "Railway-and-Automotive-Engineering": { "n": 11, "accuracy": 0.45454545454545453 }, "Real-Estate": { "n": 11, "accuracy": 0.36363636363636365 }, "Refrigerating-Machinery": { "n": 11, "accuracy": 0.45454545454545453 }, "Social-Welfare": { "n": 11, "accuracy": 0.7272727272727273 }, "Taxation": { "n": 11, "accuracy": 0.36363636363636365 }, "Telecommunications-and-Wireless-Technology": { "n": 11, "accuracy": 0.6363636363636364 } }, "macro_accuracy": 0.48215488215488206, "delta_accuracy": { "delta": 0.0, "ci95": [ 0.0, 0.0 ], "resampling_unit": "paired item", "replicates": 2000 }, "regressions": [], "improvements": [] }, "arc": { "n": 500, "accuracy": 0.518, "failures": 0, "accuracy_norm": 0.562, "delta_accuracy": { "delta": 0.0, "ci95": [ 0.0, 0.0 ], "resampling_unit": "paired item", "replicates": 2000 }, "regressions": [], "improvements": [], "delta_accuracy_norm": { "delta": 0.0, "ci95": [ 0.0, 0.0 ], "resampling_unit": "paired item", "replicates": 2000 } }, "ifeval": { "n": 541, "accuracy": 0.8262476894639557, "failures": 0, "instruction_strict": 0.8776978417266187, "truncated": 13, "delta_accuracy": { "delta": 0.0, "ci95": [ 0.0, 0.0 ], "resampling_unit": "paired item", "replicates": 2000 }, "regressions": [], "improvements": [] } }, "speed": { "512": { "measured": 5, "failed": 0, "ttft_seconds": { "median": 0.0382353, "min": 0.0380698, "max": 0.0384302 }, "prefill_seconds": { "median": 0.0382093, "min": 0.0380427, "max": 0.0384085 }, "decode_seconds": { "median": 1.6359083, "min": 1.5475906, "max": 1.6526967 }, "seconds": { "median": 1.6739789, "min": 1.5858268, "max": 1.6911273 }, "prefill_tokens_per_second": { "median": 13399.87908702855, "min": 13330.382597602094, "max": 13458.56103799152 }, "decode_tokens_per_second": { "median": 155.87670775923075, "min": 154.29328321403437, "max": 164.77225953685684 }, "peak_device_vram_bytes": 12694835200 }, "4096": { "measured": 5, "failed": 0, "ttft_seconds": { "median": 0.3036894, "min": 0.3015319, "max": 0.3110883 }, "prefill_seconds": { "median": 0.3036635, "min": 0.3015077, "max": 0.311065 }, "decode_seconds": { "median": 1.6379478, "min": 1.569075, "max": 1.6604678 }, "seconds": { "median": 1.9427159, "min": 1.8709703, "max": 1.9620004 }, "prefill_tokens_per_second": { "median": 13488.614864809238, "min": 13167.665921913427, "max": 13585.059353376382 }, "decode_tokens_per_second": { "median": 155.68261699182355, "min": 153.57118036254602, "max": 162.51613211605564 }, "peak_device_vram_bytes": 12694835200 } }, "speed_source": "dedicated serial benchmark", "runtime_memory_breakdown": { "source": "llama.cpp CUDA0 buffer log", "model_mib": 8023.67, "kv_mib": 256.0, "recurrent_state_mib": 50.25, "compute_mib": 495.0 }, "block_diagnostics": { "scope": "six fixed validation documents, 256 tokens, cumulative error", "layer_mean_relative_l2": { "0": 0.0, "1": 0.0, "2": 0.0, "3": 0.0, "4": 0.0, "5": 0.0, "6": 0.0, "7": 0.0, "8": 0.0, "9": 0.0, "10": 0.0, "11": 0.0, "12": 0.0, "13": 0.0, "14": 0.0, "15": 0.0, "16": 0.0, "17": 0.0, "18": 0.0, "19": 0.0, "20": 0.0, "21": 0.0, "22": 0.0, "23": 0.0, "24": 0.0, "25": 0.0, "26": 0.0, "27": 0.0, "28": 0.0, "29": 0.0, "30": 0.0, "31": 0.0 } } }, "native-I4": { "engine": "native", "candidate": "I4", "records": 4714, "failed_requests": [], "final_document_expected": 128, "final_document_present": 128, "final_document_failed": 0, "quality_complete": true, "nll": 1.8817406798660223, "ppl": 6.564922350369657, "scored_tokens": 130944, "domains": { "ko": { "documents": 64, "nll": 2.1255431679640813, "ppl": 8.377446613167 }, "en": { "documents": 32, "nll": 2.196320837668671, "ppl": 8.991870017239366 }, "code": { "documents": 32, "nll": 1.079555545867256, "ppl": 2.94337106687159 } }, "delta_nll": { "delta": 0.030243037076852775, "ci95": [ 0.027901765067122962, 0.03277231822680255 ], "resampling_unit": "paired document", "replicates": 2000 }, "ppl_ratio_vs_bf16": { "ratio": 1.0307050030466838, "ci95": [ 1.028294665002294, 1.0333152454089167 ] }, "document_delta_tail": { "mean": 0.030243037076852775, "median": 0.027785089374209863, "p95": 0.05720809367587929, "p99": 0.07025643392598449, "max": 0.07380433181319565 }, "worst20_documents": [ { "id": "final-049", "domain": "code", "delta_nll": 0.07380433181319565, "nll": 0.597948293947374 }, { "id": "final-003", "domain": "ko", "delta_nll": 0.07043613565274903, "nll": 1.4895878737949164 }, { "id": "final-012", "domain": "code", "delta_nll": 0.06977057370176921, "nll": 0.34462881442939397 }, { "id": "final-001", "domain": "code", "delta_nll": 0.063903892333006, "nll": 0.26384155171442636 }, { "id": "final-046", "domain": "code", "delta_nll": 0.06124876146265229, "nll": 1.3369401815437045 }, { "id": "final-126", "domain": "ko", "delta_nll": 0.059376084507181084, "nll": 2.4266440783292293 }, { "id": "final-075", "domain": "ko", "delta_nll": 0.05821306199108402, "nll": 1.9939955712261537 }, { "id": "final-121", "domain": "code", "delta_nll": 0.05534172394764201, "nll": 1.159221655349253 }, { "id": "final-087", "domain": "en", "delta_nll": 0.053838557034368656, "nll": 2.007992978732109 }, { "id": "final-073", "domain": "en", "delta_nll": 0.05356137828003158, "nll": 2.2537412219693067 }, { "id": "final-077", "domain": "ko", "delta_nll": 0.05005930666352927, "nll": 2.2500460776789875 }, { "id": "final-023", "domain": "code", "delta_nll": 0.048608831579902, "nll": 0.43438310067466784 }, { "id": "final-090", "domain": "code", "delta_nll": 0.047893375054855136, "nll": 0.4292391852312617 }, { "id": "final-067", "domain": "code", "delta_nll": 0.04773035563201086, "nll": 0.30584377546058883 }, { "id": "final-024", "domain": "ko", "delta_nll": 0.04722641881378296, "nll": 2.178383172753178 }, { "id": "final-007", "domain": "ko", "delta_nll": 0.045400679665202066, "nll": 1.9866574135169721 }, { "id": "final-035", "domain": "ko", "delta_nll": 0.04402014692743672, "nll": 2.133459971062297 }, { "id": "final-109", "domain": "en", "delta_nll": 0.04381307316671137, "nll": 2.139233080749062 }, { "id": "final-031", "domain": "ko", "delta_nll": 0.043743082225102155, "nll": 1.9211921424129628 }, { "id": "final-118", "domain": "ko", "delta_nll": 0.043503716774552975, "nll": 2.1575480676744987 } ], "token_delta_tail": { "mean": 0.030243037076852657, "median": 0.0004181608805549075, "p95": 0.4711779953857906, "p99": 0.9814744929429333, "max": 9.69035419171897 }, "sampled_full_vocab_kl": { "mean": 0.03342629550417496, "median": 0.013606945214072328, "p95": 0.10202216382966092, "p99": 0.32428529985428034, "max": 10.737569786955627 }, "kl_positions": 4096, "sampled_top1_agreement": 0.9345703125, "tasks": { "kmmlu": { "n": 500, "accuracy": 0.478, "failures": 0, "subject_scores": { "Accounting": { "n": 12, "accuracy": 0.5 }, "Agricultural-Sciences": { "n": 12, "accuracy": 0.4166666666666667 }, "Aviation-Engineering-and-Maintenance": { "n": 12, "accuracy": 0.5 }, "Biology": { "n": 12, "accuracy": 0.25 }, "Chemical-Engineering": { "n": 12, "accuracy": 0.25 }, "Chemistry": { "n": 11, "accuracy": 0.7272727272727273 }, "Civil-Engineering": { "n": 11, "accuracy": 0.2727272727272727 }, "Computer-Science": { "n": 11, "accuracy": 0.8181818181818182 }, "Construction": { "n": 11, "accuracy": 0.18181818181818182 }, "Criminal-Law": { "n": 11, "accuracy": 0.2727272727272727 }, "Ecology": { "n": 11, "accuracy": 0.5454545454545454 }, "Economics": { "n": 11, "accuracy": 0.7272727272727273 }, "Education": { "n": 11, "accuracy": 0.8181818181818182 }, "Electrical-Engineering": { "n": 11, "accuracy": 0.2727272727272727 }, "Electronics-Engineering": { "n": 11, "accuracy": 0.36363636363636365 }, "Energy-Management": { "n": 11, "accuracy": 0.36363636363636365 }, "Environmental-Science": { "n": 11, "accuracy": 0.2727272727272727 }, "Fashion": { "n": 11, "accuracy": 0.6363636363636364 }, "Food-Processing": { "n": 11, "accuracy": 0.5454545454545454 }, "Gas-Technology-and-Engineering": { "n": 11, "accuracy": 0.45454545454545453 }, "Geomatics": { "n": 11, "accuracy": 0.36363636363636365 }, "Health": { "n": 11, "accuracy": 0.45454545454545453 }, "Industrial-Engineer": { "n": 11, "accuracy": 0.5454545454545454 }, "Information-Technology": { "n": 11, "accuracy": 1.0 }, "Interior-Architecture-and-Design": { "n": 11, "accuracy": 0.6363636363636364 }, "Korean-History": { "n": 11, "accuracy": 0.36363636363636365 }, "Law": { "n": 11, "accuracy": 0.09090909090909091 }, "Machine-Design-and-Manufacturing": { "n": 11, "accuracy": 0.7272727272727273 }, "Management": { "n": 11, "accuracy": 0.5454545454545454 }, "Maritime-Engineering": { "n": 11, "accuracy": 0.45454545454545453 }, "Marketing": { "n": 11, "accuracy": 1.0 }, "Materials-Engineering": { "n": 11, "accuracy": 0.36363636363636365 }, "Math": { "n": 11, "accuracy": 0.09090909090909091 }, "Mechanical-Engineering": { "n": 11, "accuracy": 0.36363636363636365 }, "Nondestructive-Testing": { "n": 11, "accuracy": 0.5454545454545454 }, "Patent": { "n": 11, "accuracy": 0.18181818181818182 }, "Political-Science-and-Sociology": { "n": 11, "accuracy": 0.2727272727272727 }, "Psychology": { "n": 11, "accuracy": 0.45454545454545453 }, "Public-Safety": { "n": 11, "accuracy": 0.6363636363636364 }, "Railway-and-Automotive-Engineering": { "n": 11, "accuracy": 0.6363636363636364 }, "Real-Estate": { "n": 11, "accuracy": 0.45454545454545453 }, "Refrigerating-Machinery": { "n": 11, "accuracy": 0.36363636363636365 }, "Social-Welfare": { "n": 11, "accuracy": 0.8181818181818182 }, "Taxation": { "n": 11, "accuracy": 0.36363636363636365 }, "Telecommunications-and-Wireless-Technology": { "n": 11, "accuracy": 0.6363636363636364 } }, "macro_accuracy": 0.47895622895622886, "delta_accuracy": { "delta": -0.004, "ci95": [ -0.03, 0.022 ], "resampling_unit": "paired item", "replicates": 2000 }, "regressions": [ "kmmlu-Accounting-96", "kmmlu-Agricultural-Sciences-226", "kmmlu-Biology-379", "kmmlu-Chemical-Engineering-811", "kmmlu-Chemical-Engineering-584", "kmmlu-Chemistry-309", "kmmlu-Electrical-Engineering-481", "kmmlu-Electronics-Engineering-810", "kmmlu-Electronics-Engineering-83", "kmmlu-Energy-Management-645", "kmmlu-Energy-Management-431", "kmmlu-Environmental-Science-342", "kmmlu-Interior-Architecture-and-Design-849", "kmmlu-Management-192", "kmmlu-Management-22", "kmmlu-Math-208", "kmmlu-Nondestructive-Testing-286", "kmmlu-Patent-99", "kmmlu-Political-Science-and-Sociology-76", "kmmlu-Political-Science-and-Sociology-123", "kmmlu-Refrigerating-Machinery-247", "kmmlu-Social-Welfare-952" ], "improvements": [ "kmmlu-Chemistry-130", "kmmlu-Economics-26", "kmmlu-Electrical-Engineering-112", "kmmlu-Food-Processing-919", "kmmlu-Geomatics-951", "kmmlu-Geomatics-272", "kmmlu-Geomatics-900", "kmmlu-Interior-Architecture-and-Design-723", "kmmlu-Interior-Architecture-and-Design-561", "kmmlu-Korean-History-89", "kmmlu-Maritime-Engineering-234", "kmmlu-Marketing-80", "kmmlu-Materials-Engineering-321", "kmmlu-Patent-48", "kmmlu-Psychology-892", "kmmlu-Railway-and-Automotive-Engineering-834", "kmmlu-Railway-and-Automotive-Engineering-177", "kmmlu-Real-Estate-155", "kmmlu-Social-Welfare-606", "kmmlu-Social-Welfare-706" ] }, "arc": { "n": 500, "accuracy": 0.524, "failures": 0, "accuracy_norm": 0.554, "delta_accuracy": { "delta": 0.006, "ci95": [ -0.014, 0.026 ], "resampling_unit": "paired item", "replicates": 2000 }, "regressions": [ "arc-TIMSS_2003_8_pg87", "arc-Mercury_7268328", "arc-ACTAAP_2010_7_12", "arc-Mercury_SC_407689", "arc-Mercury_7211033", "arc-Mercury_SC_415453", "arc-Mercury_SC_401366", "arc-Mercury_7108990", "arc-MCAS_1999_4_23", "arc-NYSEDREGENTS_2008_8_27", "arc-Mercury_7212328", "arc-Mercury_7201740" ], "improvements": [ "arc-Mercury_7143308", "arc-Mercury_SC_400594", "arc-MEAP_2005_8_39", "arc-Mercury_183033", "arc-TIMSS_2007_8_pg60", "arc-Mercury_189053", "arc-NYSEDREGENTS_2008_4_15", "arc-NYSEDREGENTS_2012_4_29", "arc-Mercury_SC_409241", "arc-OHAT_2007_5_15", "arc-Mercury_7216248", "arc-Mercury_189070", "arc-NCEOGA_2013_8_28", "arc-Mercury_411732", "arc-Mercury_7210245" ], "delta_accuracy_norm": { "delta": -0.008, "ci95": [ -0.026, 0.01 ], "resampling_unit": "paired item", "replicates": 2000 } }, "ifeval": { "n": 541, "accuracy": 0.8188539741219963, "failures": 0, "instruction_strict": 0.8717026378896883, "truncated": 13, "delta_accuracy": { "delta": -0.0073937153419593345, "ci95": [ -0.03512014787430684, 0.02033271719038817 ], "resampling_unit": "paired item", "replicates": 2000 }, "regressions": [ "ifeval-1075", "ifeval-1129", "ifeval-1130", "ifeval-1379", "ifeval-1516", "ifeval-1566", "ifeval-1609", "ifeval-1670", "ifeval-1825", "ifeval-1996", "ifeval-2034", "ifeval-2035", "ifeval-2416", "ifeval-2439", "ifeval-2497", "ifeval-2605", "ifeval-2844", "ifeval-2880", "ifeval-2912", "ifeval-2969", "ifeval-301", "ifeval-3048", "ifeval-3069", "ifeval-3098", "ifeval-3243", "ifeval-3326", "ifeval-3351", "ifeval-3362", "ifeval-3428", "ifeval-3455", "ifeval-371" ], "improvements": [ "ifeval-1021", "ifeval-1092", "ifeval-1219", "ifeval-1265", "ifeval-1348", "ifeval-1372", "ifeval-1389", "ifeval-1582", "ifeval-1629", "ifeval-1645", "ifeval-2230", "ifeval-2266", "ifeval-2273", "ifeval-2299", "ifeval-2386", "ifeval-2534", "ifeval-2571", "ifeval-2591", "ifeval-2616", "ifeval-2662", "ifeval-2674", "ifeval-2736", "ifeval-2929", "ifeval-3035", "ifeval-3114", "ifeval-3329", "ifeval-3409" ] } }, "speed": { "512": { "measured": 5, "failed": 0, "ttft_seconds": { "median": 0.0321546, "min": 0.0318315, "max": 0.0434728 }, "prefill_seconds": { "median": 0.0321327, "min": 0.0318092, "max": 0.0434489 }, "decode_seconds": { "median": 0.8676278, "min": 0.8527025, "max": 0.9072525 }, "seconds": { "median": 0.8995087, "min": 0.8845344, "max": 0.9394076 }, "prefill_tokens_per_second": { "median": 15933.924008875694, "min": 11783.957706639294, "max": 16095.972234447894 }, "decode_tokens_per_second": { "median": 293.90482877565705, "min": 281.0683905527954, "max": 299.04919945702045 }, "peak_device_vram_bytes": 7447752704 }, "4096": { "measured": 5, "failed": 0, "ttft_seconds": { "median": 0.2543446, "min": 0.2522706, "max": 0.3004729 }, "prefill_seconds": { "median": 0.254317, "min": 0.2522488, "max": 0.3004501 }, "decode_seconds": { "median": 0.8634475, "min": 0.8604546, "max": 0.9443029 }, "seconds": { "median": 1.1258522, "min": 1.1138621, "max": 1.1965739 }, "prefill_tokens_per_second": { "median": 16105.88360196133, "min": 13632.879469835423, "max": 16237.936513473998 }, "decode_tokens_per_second": { "median": 295.32774140871334, "min": 270.04047112425474, "max": 296.3549732897006 }, "peak_device_vram_bytes": 7447752704 } }, "speed_source": "dedicated serial benchmark", "runtime_memory_breakdown": { "source": "llama.cpp CUDA0 buffer log", "model_mib": 3029.92, "kv_mib": 256.0, "recurrent_state_mib": 50.25, "compute_mib": 495.0 }, "block_diagnostics": { "scope": "six fixed validation documents, 256 tokens, cumulative error", "layer_mean_relative_l2": { "0": 0.05945174334560872, "1": 0.0783909388325922, "2": 0.11083954845195043, "3": 0.10881876938408984, "4": 0.12105555181836157, "5": 0.1248304704592816, "6": 0.12745386693640845, "7": 0.13842950514868071, "8": 0.14042917735344837, "9": 0.14341035886328066, "10": 0.1416100387087346, "11": 0.1514952923869246, "12": 0.1542053223190083, "13": 0.15443041750249387, "14": 0.15634749036682782, "15": 0.16304742884408438, "16": 0.16570423427182082, "17": 0.16057487752414987, "18": 0.15351608660486268, "19": 0.16190285638406396, "20": 0.15751221313613437, "21": 0.1564099730934164, "22": 0.1591623240098972, "23": 0.15705860661558005, "24": 0.1569009512306538, "25": 0.1588038424931714, "26": 0.17036119028647567, "27": 0.1690551457664339, "28": 0.16344070332987504, "29": 0.16843632358156227, "30": 0.17760223034667236, "31": 0.1767388234421695 } } }, "native-K0": { "engine": "native", "candidate": "K0", "records": 4714, "failed_requests": [], "final_document_expected": 128, "final_document_present": 128, "final_document_failed": 0, "quality_complete": true, "nll": 1.8860503484055466, "ppl": 6.593276043311702, "scored_tokens": 130944, "domains": { "ko": { "documents": 64, "nll": 2.1341204545084773, "ppl": 8.449611419962743 }, "en": { "documents": 32, "nll": 2.1928519689817065, "ppl": 8.960732438209085 }, "code": { "documents": 32, "nll": 1.0831085156235258, "ppl": 2.953847375236333 } }, "delta_nll": { "delta": 0.03455270561637727, "ci95": [ 0.03165340037672631, 0.0372973156617618 ], "resampling_unit": "paired document", "replicates": 2000 }, "ppl_ratio_vs_bf16": { "ratio": 1.0351565855042542, "ci95": [ 1.0321596971384261, 1.0380015890941678 ] }, "document_delta_tail": { "mean": 0.03455270561637727, "median": 0.03247442800900782, "p95": 0.06356939068752121, "p99": 0.07378202164310624, "max": 0.08018221477548558 }, "worst20_documents": [ { "id": "final-049", "domain": "code", "delta_nll": 0.08018221477548558, "nll": 0.604326176909664 }, { "id": "final-063", "domain": "ko", "delta_nll": 0.07451020032595729, "nll": 2.004895962536779 }, { "id": "final-012", "domain": "code", "delta_nll": 0.07181324224132374, "nll": 0.3466714829689485 }, { "id": "final-126", "domain": "ko", "delta_nll": 0.0702623463149742, "nll": 2.4375303401370223 }, { "id": "final-124", "domain": "ko", "delta_nll": 0.0671015043645476, "nll": 2.1144498761124653 }, { "id": "final-009", "domain": "ko", "delta_nll": 0.06675581638980901, "nll": 2.302815680099088 }, { "id": "final-023", "domain": "code", "delta_nll": 0.06359360603544861, "nll": 0.4493678751302145 }, { "id": "final-089", "domain": "ko", "delta_nll": 0.06352441932708461, "nll": 2.7434548353146533 }, { "id": "final-035", "domain": "ko", "delta_nll": 0.06284441766894783, "nll": 2.1522842418038084 }, { "id": "final-001", "domain": "code", "delta_nll": 0.06176999885005194, "nll": 0.26170765823147224 }, { "id": "final-034", "domain": "ko", "delta_nll": 0.06131229495963942, "nll": 2.0421069230633924 }, { "id": "final-103", "domain": "en", "delta_nll": 0.06106495530604327, "nll": 2.13662150953151 }, { "id": "final-104", "domain": "ko", "delta_nll": 0.05810418220928784, "nll": 2.661881415109549 }, { "id": "final-081", "domain": "code", "delta_nll": 0.05599665249793446, "nll": 0.4963711207885065 }, { "id": "final-051", "domain": "ko", "delta_nll": 0.05571570246230585, "nll": 2.746760436051575 }, { "id": "final-110", "domain": "ko", "delta_nll": 0.0550843839661047, "nll": 2.3278257452490507 }, { "id": "final-085", "domain": "ko", "delta_nll": 0.05464285702737248, "nll": 2.6868628290525396 }, { "id": "final-075", "domain": "ko", "delta_nll": 0.05126414635944629, "nll": 1.9870466555945159 }, { "id": "final-046", "domain": "code", "delta_nll": 0.050723821104224756, "nll": 1.326415241185277 }, { "id": "final-003", "domain": "ko", "delta_nll": 0.049201561411098535, "nll": 1.4683532995532658 } ], "token_delta_tail": { "mean": 0.03455270561637709, "median": 0.0007564102433814668, "p95": 0.5346366993519095, "p99": 1.1218961794474132, "max": 16.885432344463545 }, "sampled_full_vocab_kl": { "mean": 0.04303441856732754, "median": 0.017307583372364023, "p95": 0.13528341212096812, "p99": 0.439438270231327, "max": 3.120973678948202 }, "kl_positions": 4096, "sampled_top1_agreement": 0.9169921875, "tasks": { "kmmlu": { "n": 500, "accuracy": 0.494, "failures": 0, "subject_scores": { "Accounting": { "n": 12, "accuracy": 0.4166666666666667 }, "Agricultural-Sciences": { "n": 12, "accuracy": 0.5 }, "Aviation-Engineering-and-Maintenance": { "n": 12, "accuracy": 0.4166666666666667 }, "Biology": { "n": 12, "accuracy": 0.4166666666666667 }, "Chemical-Engineering": { "n": 12, "accuracy": 0.4166666666666667 }, "Chemistry": { "n": 11, "accuracy": 0.7272727272727273 }, "Civil-Engineering": { "n": 11, "accuracy": 0.2727272727272727 }, "Computer-Science": { "n": 11, "accuracy": 0.8181818181818182 }, "Construction": { "n": 11, "accuracy": 0.2727272727272727 }, "Criminal-Law": { "n": 11, "accuracy": 0.2727272727272727 }, "Ecology": { "n": 11, "accuracy": 0.45454545454545453 }, "Economics": { "n": 11, "accuracy": 0.6363636363636364 }, "Education": { "n": 11, "accuracy": 0.7272727272727273 }, "Electrical-Engineering": { "n": 11, "accuracy": 0.2727272727272727 }, "Electronics-Engineering": { "n": 11, "accuracy": 0.36363636363636365 }, "Energy-Management": { "n": 11, "accuracy": 0.36363636363636365 }, "Environmental-Science": { "n": 11, "accuracy": 0.2727272727272727 }, "Fashion": { "n": 11, "accuracy": 0.6363636363636364 }, "Food-Processing": { "n": 11, "accuracy": 0.5454545454545454 }, "Gas-Technology-and-Engineering": { "n": 11, "accuracy": 0.45454545454545453 }, "Geomatics": { "n": 11, "accuracy": 0.18181818181818182 }, "Health": { "n": 11, "accuracy": 0.5454545454545454 }, "Industrial-Engineer": { "n": 11, "accuracy": 0.7272727272727273 }, "Information-Technology": { "n": 11, "accuracy": 0.8181818181818182 }, "Interior-Architecture-and-Design": { "n": 11, "accuracy": 0.6363636363636364 }, "Korean-History": { "n": 11, "accuracy": 0.45454545454545453 }, "Law": { "n": 11, "accuracy": 0.2727272727272727 }, "Machine-Design-and-Manufacturing": { "n": 11, "accuracy": 0.7272727272727273 }, "Management": { "n": 11, "accuracy": 0.7272727272727273 }, "Maritime-Engineering": { "n": 11, "accuracy": 0.36363636363636365 }, "Marketing": { "n": 11, "accuracy": 1.0 }, "Materials-Engineering": { "n": 11, "accuracy": 0.2727272727272727 }, "Math": { "n": 11, "accuracy": 0.2727272727272727 }, "Mechanical-Engineering": { "n": 11, "accuracy": 0.2727272727272727 }, "Nondestructive-Testing": { "n": 11, "accuracy": 0.6363636363636364 }, "Patent": { "n": 11, "accuracy": 0.09090909090909091 }, "Political-Science-and-Sociology": { "n": 11, "accuracy": 0.36363636363636365 }, "Psychology": { "n": 11, "accuracy": 0.36363636363636365 }, "Public-Safety": { "n": 11, "accuracy": 0.7272727272727273 }, "Railway-and-Automotive-Engineering": { "n": 11, "accuracy": 0.6363636363636364 }, "Real-Estate": { "n": 11, "accuracy": 0.5454545454545454 }, "Refrigerating-Machinery": { "n": 11, "accuracy": 0.36363636363636365 }, "Social-Welfare": { "n": 11, "accuracy": 0.9090909090909091 }, "Taxation": { "n": 11, "accuracy": 0.36363636363636365 }, "Telecommunications-and-Wireless-Technology": { "n": 11, "accuracy": 0.7272727272727273 } }, "macro_accuracy": 0.49461279461279467, "delta_accuracy": { "delta": 0.012, "ci95": [ -0.02, 0.044 ], "resampling_unit": "paired item", "replicates": 2000 }, "regressions": [ "kmmlu-Accounting-52", "kmmlu-Accounting-96", "kmmlu-Aviation-Engineering-and-Maintenance-899", "kmmlu-Biology-379", "kmmlu-Chemical-Engineering-584", "kmmlu-Criminal-Law-73", "kmmlu-Ecology-998", "kmmlu-Economics-104", "kmmlu-Education-20", "kmmlu-Electrical-Engineering-481", "kmmlu-Electronics-Engineering-294", "kmmlu-Electronics-Engineering-83", "kmmlu-Energy-Management-645", "kmmlu-Energy-Management-155", "kmmlu-Environmental-Science-438", "kmmlu-Environmental-Science-342", "kmmlu-Fashion-419", "kmmlu-Food-Processing-148", "kmmlu-Information-Technology-977", "kmmlu-Information-Technology-635", "kmmlu-Interior-Architecture-and-Design-849", "kmmlu-Machine-Design-and-Manufacturing-699", "kmmlu-Mechanical-Engineering-305", "kmmlu-Nondestructive-Testing-286", "kmmlu-Patent-28", "kmmlu-Patent-99", "kmmlu-Political-Science-and-Sociology-123", "kmmlu-Refrigerating-Machinery-797", "kmmlu-Social-Welfare-952" ], "improvements": [ "kmmlu-Biology-596", "kmmlu-Biology-731", "kmmlu-Chemical-Engineering-970", "kmmlu-Construction-396", "kmmlu-Criminal-Law-172", "kmmlu-Economics-74", "kmmlu-Electrical-Engineering-962", "kmmlu-Environmental-Science-835", "kmmlu-Fashion-767", "kmmlu-Food-Processing-85", "kmmlu-Food-Processing-919", "kmmlu-Geomatics-900", "kmmlu-Health-58", "kmmlu-Industrial-Engineer-141", "kmmlu-Industrial-Engineer-927", "kmmlu-Interior-Architecture-and-Design-723", "kmmlu-Interior-Architecture-and-Design-561", "kmmlu-Korean-History-64", "kmmlu-Korean-History-53", "kmmlu-Law-11", "kmmlu-Law-694", "kmmlu-Machine-Design-and-Manufacturing-449", "kmmlu-Marketing-80", "kmmlu-Math-68", "kmmlu-Nondestructive-Testing-312", "kmmlu-Patent-48", "kmmlu-Public-Safety-152", "kmmlu-Railway-and-Automotive-Engineering-834", "kmmlu-Railway-and-Automotive-Engineering-177", "kmmlu-Real-Estate-172", "kmmlu-Real-Estate-155", "kmmlu-Social-Welfare-606", "kmmlu-Social-Welfare-706", "kmmlu-Social-Welfare-19", "kmmlu-Telecommunications-and-Wireless-Technology-713" ] }, "arc": { "n": 500, "accuracy": 0.498, "failures": 0, "accuracy_norm": 0.53, "delta_accuracy": { "delta": -0.02, "ci95": [ -0.04205, 0.002 ], "resampling_unit": "paired item", "replicates": 2000 }, "regressions": [ "arc-Mercury_7025060", "arc-Mercury_7268328", "arc-Mercury_7211680", "arc-MCAS_2004_8_14", "arc-Mercury_7228725", "arc-Mercury_7007420", "arc-Mercury_7093048", "arc-ACTAAP_2010_7_12", "arc-Mercury_7223493", "arc-Mercury_SC_402257", "arc-Mercury_SC_407689", "arc-Mercury_7042945", "arc-Mercury_SC_414156", "arc-MDSA_2007_8_42", "arc-Mercury_7212328", "arc-Mercury_7201740", "arc-Mercury_409295", "arc-Mercury_SC_408321", "arc-Mercury_402364", "arc-Mercury_416536", "arc-Mercury_7215670" ], "improvements": [ "arc-Mercury_SC_405885", "arc-TIMSS_1995_8_K18", "arc-NYSEDREGENTS_2008_4_15", "arc-Mercury_SC_409241", "arc-Mercury_SC_402164", "arc-Mercury_7283833", "arc-Mercury_404132", "arc-MEA_2016_8_15", "arc-Mercury_415303", "arc-Mercury_189070", "arc-Mercury_7210245" ], "delta_accuracy_norm": { "delta": -0.032, "ci95": [ -0.058, -0.01 ], "resampling_unit": "paired item", "replicates": 2000 } }, "ifeval": { "n": 541, "accuracy": 0.8040665434380776, "failures": 0, "instruction_strict": 0.8609112709832134, "truncated": 18, "delta_accuracy": { "delta": -0.022181146025878003, "ci95": [ -0.04990757855822551, 0.005545286506469501 ], "resampling_unit": "paired item", "replicates": 2000 }, "regressions": [ "ifeval-1203", "ifeval-1379", "ifeval-1392", "ifeval-1601", "ifeval-1644", "ifeval-1670", "ifeval-1713", "ifeval-179", "ifeval-1823", "ifeval-1922", "ifeval-1967", "ifeval-1996", "ifeval-2034", "ifeval-2035", "ifeval-2069", "ifeval-2084", "ifeval-2192", "ifeval-2275", "ifeval-2589", "ifeval-2605", "ifeval-2811", "ifeval-2844", "ifeval-2912", "ifeval-2918", "ifeval-2943", "ifeval-301", "ifeval-3048", "ifeval-3098", "ifeval-3109", "ifeval-3245", "ifeval-3287", "ifeval-3351", "ifeval-3362", "ifeval-3429", "ifeval-3479", "ifeval-3703", "ifeval-3724", "ifeval-3748" ], "improvements": [ "ifeval-1021", "ifeval-1132", "ifeval-1265", "ifeval-127", "ifeval-1348", "ifeval-1372", "ifeval-1389", "ifeval-1580", "ifeval-1582", "ifeval-1629", "ifeval-1944", "ifeval-2078", "ifeval-2266", "ifeval-2299", "ifeval-2386", "ifeval-2616", "ifeval-2674", "ifeval-2736", "ifeval-2749", "ifeval-2929", "ifeval-3035", "ifeval-3089", "ifeval-3114", "ifeval-3409", "ifeval-3617", "ifeval-3691" ] } }, "speed": { "512": { "measured": 5, "failed": 0, "ttft_seconds": { "median": 0.0329756, "min": 0.0329278, "max": 0.0331611 }, "prefill_seconds": { "median": 0.0329557, "min": 0.0329036, "max": 0.0331387 }, "decode_seconds": { "median": 0.892806, "min": 0.8877685, "max": 0.9660976 }, "seconds": { "median": 0.9257346, "min": 0.9208202, "max": 0.9990738 }, "prefill_tokens_per_second": { "median": 15536.007428153553, "min": 15450.213798368675, "max": 15560.60734995563 }, "decode_tokens_per_second": { "median": 285.6163601051068, "min": 263.9484871921843, "max": 287.2370443420779 }, "peak_device_vram_bytes": 7613911040 }, "4096": { "measured": 5, "failed": 0, "ttft_seconds": { "median": 0.2622093, "min": 0.2606742, "max": 0.2656708 }, "prefill_seconds": { "median": 0.2621853, "min": 0.2606478, "max": 0.2656398 }, "decode_seconds": { "median": 0.9130032, "min": 0.8931073, "max": 0.9816207 }, "seconds": { "median": 1.1786744, "min": 1.1552752, "max": 1.2436598 }, "prefill_tokens_per_second": { "median": 15622.538715938688, "min": 15419.376162758745, "max": 15714.69239333691 }, "decode_tokens_per_second": { "median": 279.2980353190438, "min": 259.77447297107733, "max": 285.520004147318 }, "peak_device_vram_bytes": 7613911040 } }, "speed_source": "dedicated serial benchmark", "runtime_memory_breakdown": { "source": "llama.cpp CUDA0 buffer log", "model_mib": 3136.17, "kv_mib": 256.0, "recurrent_state_mib": 50.25, "compute_mib": 495.0 }, "block_diagnostics": { "scope": "six fixed validation documents, 256 tokens, cumulative error", "layer_mean_relative_l2": { "0": 0.1080037999628915, "1": 0.1252887376234116, "2": 0.1620984789751663, "3": 0.15337773330650703, "4": 0.1616344719309899, "5": 0.1611653406666588, "6": 0.16217424258670984, "7": 0.17182376851528638, "8": 0.17156664729839063, "9": 0.17347707465485773, "10": 0.17056636553567153, "11": 0.18038459705435014, "12": 0.1833024384539849, "13": 0.182970077747693, "14": 0.1850569469848723, "15": 0.19286805654220993, "16": 0.19658882739647424, "17": 0.18980855088131157, "18": 0.18102348399361357, "19": 0.18936113912570404, "20": 0.18343657796264012, "21": 0.18077561092237135, "22": 0.18159410458906544, "23": 0.18179332116637817, "24": 0.1815043195036944, "25": 0.18360318369987208, "26": 0.19173765022077158, "27": 0.1967464658714716, "28": 0.19071340262263303, "29": 0.19463557370662488, "30": 0.20563378247306577, "31": 0.20309930844598156 } } }, "native-K1": { "engine": "native", "candidate": "K1", "records": 4714, "failed_requests": [], "final_document_expected": 128, "final_document_present": 128, "final_document_failed": 0, "quality_complete": true, "nll": 1.8744917661937208, "ppl": 6.517505861634886, "scored_tokens": 130944, "domains": { "ko": { "documents": 64, "nll": 2.120660871750281, "ppl": 8.33664512082013 }, "en": { "documents": 32, "nll": 2.185382178000655, "ppl": 8.894047013026917 }, "code": { "documents": 32, "nll": 1.0712631432736677, "ppl": 2.919064368751241 } }, "delta_nll": { "delta": 0.0229941234045515, "ci95": [ 0.020681926587697443, 0.02522469552367121 ], "resampling_unit": "paired document", "replicates": 2000 }, "ppl_ratio_vs_bf16": { "ratio": 1.0232605262413974, "ci95": [ 1.0208972797082416, 1.0255455301273968 ] }, "document_delta_tail": { "mean": 0.0229941234045515, "median": 0.02252778320885787, "p95": 0.04751316356663568, "p99": 0.054710523856905405, "max": 0.06654257498736983 }, "worst20_documents": [ { "id": "final-049", "domain": "code", "delta_nll": 0.06654257498736983, "nll": 0.5906865371215482 }, { "id": "final-012", "domain": "code", "delta_nll": 0.05634159304970733, "nll": 0.3311998337773321 }, { "id": "final-009", "domain": "ko", "delta_nll": 0.05030059603932973, "nll": 2.2863604597486087 }, { "id": "final-126", "domain": "ko", "delta_nll": 0.04956280792329472, "nll": 2.4168308017453426 }, { "id": "final-046", "domain": "code", "delta_nll": 0.048418691271459625, "nll": 1.3241101113525118 }, { "id": "final-015", "domain": "ko", "delta_nll": 0.048408062264993355, "nll": 1.264200872124446 }, { "id": "final-003", "domain": "ko", "delta_nll": 0.04811811956815295, "nll": 1.4672698577103203 }, { "id": "final-034", "domain": "ko", "delta_nll": 0.04638967384953223, "nll": 2.027184301953285 }, { "id": "final-124", "domain": "ko", "delta_nll": 0.046259074937886495, "nll": 2.0936074466858043 }, { "id": "final-022", "domain": "ko", "delta_nll": 0.046106623765750976, "nll": 2.1564485687087753 }, { "id": "final-103", "domain": "en", "delta_nll": 0.043579037724630364, "nll": 2.1191355919500974 }, { "id": "final-063", "domain": "ko", "delta_nll": 0.04177426178620086, "nll": 1.9721600239970225 }, { "id": "final-062", "domain": "ko", "delta_nll": 0.040493611630352006, "nll": 2.173768449703913 }, { "id": "final-014", "domain": "ko", "delta_nll": 0.04045411986351544, "nll": 2.6759600125208025 }, { "id": "final-081", "domain": "code", "delta_nll": 0.03840746082056484, "nll": 0.4787819291111369 }, { "id": "final-118", "domain": "ko", "delta_nll": 0.03802793493424073, "nll": 2.152072285834186 }, { "id": "final-033", "domain": "ko", "delta_nll": 0.03770490404312685, "nll": 2.3835211978340647 }, { "id": "final-024", "domain": "ko", "delta_nll": 0.03696994464438918, "nll": 2.1681266985837846 }, { "id": "final-089", "domain": "ko", "delta_nll": 0.03643101492258621, "nll": 2.7163614309101547 }, { "id": "final-090", "domain": "code", "delta_nll": 0.035225503689578906, "nll": 0.41657131386598545 } ], "token_delta_tail": { "mean": 0.022994123404551273, "median": 0.0005994020219404206, "p95": 0.4196738743059049, "p99": 0.8965297193657, "max": 6.741763657833058 }, "sampled_full_vocab_kl": { "mean": 0.028562542985934247, "median": 0.011532071488087464, "p95": 0.08724382438002827, "p99": 0.2767615327215544, "max": 2.6137904837426174 }, "kl_positions": 4096, "sampled_top1_agreement": 0.931884765625, "tasks": { "kmmlu": { "n": 500, "accuracy": 0.466, "failures": 0, "subject_scores": { "Accounting": { "n": 12, "accuracy": 0.5833333333333334 }, "Agricultural-Sciences": { "n": 12, "accuracy": 0.4166666666666667 }, "Aviation-Engineering-and-Maintenance": { "n": 12, "accuracy": 0.5833333333333334 }, "Biology": { "n": 12, "accuracy": 0.5 }, "Chemical-Engineering": { "n": 12, "accuracy": 0.3333333333333333 }, "Chemistry": { "n": 11, "accuracy": 0.7272727272727273 }, "Civil-Engineering": { "n": 11, "accuracy": 0.18181818181818182 }, "Computer-Science": { "n": 11, "accuracy": 0.8181818181818182 }, "Construction": { "n": 11, "accuracy": 0.18181818181818182 }, "Criminal-Law": { "n": 11, "accuracy": 0.36363636363636365 }, "Ecology": { "n": 11, "accuracy": 0.45454545454545453 }, "Economics": { "n": 11, "accuracy": 0.6363636363636364 }, "Education": { "n": 11, "accuracy": 0.8181818181818182 }, "Electrical-Engineering": { "n": 11, "accuracy": 0.2727272727272727 }, "Electronics-Engineering": { "n": 11, "accuracy": 0.36363636363636365 }, "Energy-Management": { "n": 11, "accuracy": 0.5454545454545454 }, "Environmental-Science": { "n": 11, "accuracy": 0.36363636363636365 }, "Fashion": { "n": 11, "accuracy": 0.6363636363636364 }, "Food-Processing": { "n": 11, "accuracy": 0.5454545454545454 }, "Gas-Technology-and-Engineering": { "n": 11, "accuracy": 0.45454545454545453 }, "Geomatics": { "n": 11, "accuracy": 0.0 }, "Health": { "n": 11, "accuracy": 0.45454545454545453 }, "Industrial-Engineer": { "n": 11, "accuracy": 0.45454545454545453 }, "Information-Technology": { "n": 11, "accuracy": 0.9090909090909091 }, "Interior-Architecture-and-Design": { "n": 11, "accuracy": 0.45454545454545453 }, "Korean-History": { "n": 11, "accuracy": 0.36363636363636365 }, "Law": { "n": 11, "accuracy": 0.09090909090909091 }, "Machine-Design-and-Manufacturing": { "n": 11, "accuracy": 0.6363636363636364 }, "Management": { "n": 11, "accuracy": 0.7272727272727273 }, "Maritime-Engineering": { "n": 11, "accuracy": 0.2727272727272727 }, "Marketing": { "n": 11, "accuracy": 0.9090909090909091 }, "Materials-Engineering": { "n": 11, "accuracy": 0.2727272727272727 }, "Math": { "n": 11, "accuracy": 0.2727272727272727 }, "Mechanical-Engineering": { "n": 11, "accuracy": 0.36363636363636365 }, "Nondestructive-Testing": { "n": 11, "accuracy": 0.5454545454545454 }, "Patent": { "n": 11, "accuracy": 0.09090909090909091 }, "Political-Science-and-Sociology": { "n": 11, "accuracy": 0.45454545454545453 }, "Psychology": { "n": 11, "accuracy": 0.45454545454545453 }, "Public-Safety": { "n": 11, "accuracy": 0.6363636363636364 }, "Railway-and-Automotive-Engineering": { "n": 11, "accuracy": 0.5454545454545454 }, "Real-Estate": { "n": 11, "accuracy": 0.36363636363636365 }, "Refrigerating-Machinery": { "n": 11, "accuracy": 0.2727272727272727 }, "Social-Welfare": { "n": 11, "accuracy": 0.6363636363636364 }, "Taxation": { "n": 11, "accuracy": 0.36363636363636365 }, "Telecommunications-and-Wireless-Technology": { "n": 11, "accuracy": 0.6363636363636364 } }, "macro_accuracy": 0.4658249158249158, "delta_accuracy": { "delta": -0.016, "ci95": [ -0.042, 0.008 ], "resampling_unit": "paired item", "replicates": 2000 }, "regressions": [ "kmmlu-Agricultural-Sciences-226", "kmmlu-Chemical-Engineering-811", "kmmlu-Chemistry-486", "kmmlu-Civil-Engineering-837", "kmmlu-Ecology-95", "kmmlu-Economics-129", "kmmlu-Electronics-Engineering-810", "kmmlu-Electronics-Engineering-83", "kmmlu-Energy-Management-645", "kmmlu-Geomatics-629", "kmmlu-Industrial-Engineer-130", "kmmlu-Industrial-Engineer-706", "kmmlu-Information-Technology-635", "kmmlu-Interior-Architecture-and-Design-849", "kmmlu-Interior-Architecture-and-Design-560", "kmmlu-Machine-Design-and-Manufacturing-345", "kmmlu-Machine-Design-and-Manufacturing-699", "kmmlu-Maritime-Engineering-413", "kmmlu-Nondestructive-Testing-286", "kmmlu-Patent-99", "kmmlu-Political-Science-and-Sociology-76", "kmmlu-Public-Safety-176", "kmmlu-Refrigerating-Machinery-797", "kmmlu-Refrigerating-Machinery-247", "kmmlu-Social-Welfare-952" ], "improvements": [ "kmmlu-Aviation-Engineering-and-Maintenance-216", "kmmlu-Biology-596", "kmmlu-Biology-731", "kmmlu-Chemistry-130", "kmmlu-Criminal-Law-172", "kmmlu-Economics-26", "kmmlu-Energy-Management-826", "kmmlu-Food-Processing-919", "kmmlu-Industrial-Engineer-927", "kmmlu-Interior-Architecture-and-Design-947", "kmmlu-Korean-History-53", "kmmlu-Machine-Design-and-Manufacturing-449", "kmmlu-Math-205", "kmmlu-Political-Science-and-Sociology-198", "kmmlu-Psychology-892", "kmmlu-Public-Safety-152", "kmmlu-Railway-and-Automotive-Engineering-834" ] }, "arc": { "n": 500, "accuracy": 0.514, "failures": 0, "accuracy_norm": 0.544, "delta_accuracy": { "delta": -0.004, "ci95": [ -0.026, 0.018 ], "resampling_unit": "paired item", "replicates": 2000 }, "regressions": [ "arc-Mercury_SC_414271", "arc-Mercury_7268328", "arc-Mercury_7211680", "arc-MCAS_2004_8_14", "arc-Mercury_7228725", "arc-Mercury_7007420", "arc-Mercury_7093048", "arc-ACTAAP_2010_7_12", "arc-Mercury_SC_407689", "arc-Mercury_7233765", "arc-MDSA_2007_8_42", "arc-Mercury_7212328", "arc-Mercury_7201740", "arc-Mercury_409295", "arc-Mercury_402364", "arc-Mercury_416536", "arc-Mercury_7215670" ], "improvements": [ "arc-ACTAAP_2011_5_1", "arc-Mercury_7143308", "arc-Mercury_SC_400594", "arc-Mercury_416230", "arc-TIMSS_1995_8_K18", "arc-NYSEDREGENTS_2008_4_15", "arc-NYSEDREGENTS_2012_4_29", "arc-Mercury_417143", "arc-Mercury_SC_402164", "arc-MDSA_2009_5_20", "arc-Mercury_404132", "arc-Mercury_189070", "arc-NCEOGA_2013_8_28", "arc-AKDE&ED_2008_8_48", "arc-Mercury_SC_400708" ], "delta_accuracy_norm": { "delta": -0.018, "ci95": [ -0.038, 0.0 ], "resampling_unit": "paired item", "replicates": 2000 } }, "ifeval": { "n": 541, "accuracy": 0.8096118299445472, "failures": 0, "instruction_strict": 0.8657074340527577, "truncated": 17, "delta_accuracy": { "delta": -0.0166358595194085, "ci95": [ -0.04621072088724584, 0.012939001848428836 ], "resampling_unit": "paired item", "replicates": 2000 }, "regressions": [ "ifeval-1075", "ifeval-1130", "ifeval-1379", "ifeval-1402", "ifeval-1436", "ifeval-1566", "ifeval-1670", "ifeval-1686", "ifeval-1713", "ifeval-179", "ifeval-1823", "ifeval-1825", "ifeval-1996", "ifeval-2084", "ifeval-2107", "ifeval-2250", "ifeval-2284", "ifeval-2311", "ifeval-279", "ifeval-2844", "ifeval-2912", "ifeval-2943", "ifeval-301", "ifeval-3048", "ifeval-3069", "ifeval-3071", "ifeval-3098", "ifeval-3109", "ifeval-3276", "ifeval-3326", "ifeval-3351", "ifeval-3362", "ifeval-3445", "ifeval-3453", "ifeval-3455", "ifeval-3479", "ifeval-3724" ], "improvements": [ "ifeval-1021", "ifeval-1092", "ifeval-1122", "ifeval-1219", "ifeval-1265", "ifeval-127", "ifeval-1348", "ifeval-1372", "ifeval-1580", "ifeval-1629", "ifeval-1645", "ifeval-2266", "ifeval-2273", "ifeval-2299", "ifeval-2571", "ifeval-2591", "ifeval-2616", "ifeval-2617", "ifeval-2674", "ifeval-2736", "ifeval-2929", "ifeval-3035", "ifeval-3089", "ifeval-3114", "ifeval-331", "ifeval-3329", "ifeval-3409", "ifeval-3691" ] } }, "speed": { "512": { "measured": 5, "failed": 0, "ttft_seconds": { "median": 0.0330675, "min": 0.0327596, "max": 0.0332567 }, "prefill_seconds": { "median": 0.0330462, "min": 0.0327358, "max": 0.033232 }, "decode_seconds": { "median": 0.8875964, "min": 0.8829836, "max": 0.9592417 }, "seconds": { "median": 0.9208536, "min": 0.9160517, "max": 0.9923819 }, "prefill_tokens_per_second": { "median": 15493.460670213217, "min": 15406.836783822822, "max": 15640.36925934298 }, "decode_tokens_per_second": { "median": 287.2927380056972, "min": 265.8349819445923, "max": 288.79358574723244 }, "peak_device_vram_bytes": 7518580736 }, "4096": { "measured": 5, "failed": 0, "ttft_seconds": { "median": 0.2629375, "min": 0.2624594, "max": 0.2783277 }, "prefill_seconds": { "median": 0.2629137, "min": 0.262434, "max": 0.2782977 }, "decode_seconds": { "median": 0.9103343, "min": 0.8922731, "max": 0.9743246 }, "seconds": { "median": 1.1761772, "min": 1.1551104, "max": 1.2405745 }, "prefill_tokens_per_second": { "median": 15579.256615383681, "min": 14718.051927845614, "max": 15607.733754010533 }, "decode_tokens_per_second": { "median": 280.11687574553656, "min": 261.7197595134106, "max": 285.7869412402996 }, "peak_device_vram_bytes": 7518580736 } }, "speed_source": "dedicated serial benchmark", "runtime_memory_breakdown": { "source": "llama.cpp CUDA0 buffer log", "model_mib": 3136.17, "kv_mib": 256.0, "recurrent_state_mib": 50.25, "compute_mib": 495.0 }, "block_diagnostics": { "scope": "six fixed validation documents, 256 tokens, cumulative error", "layer_mean_relative_l2": { "0": 0.066935607120169, "1": 0.08254003021591332, "2": 0.11221443734004773, "3": 0.1086405011262029, "4": 0.1191259817536946, "5": 0.12189637429590934, "6": 0.1239021199882645, "7": 0.13310617688155568, "8": 0.13458171267299124, "9": 0.1373012953631716, "10": 0.1355068317332434, "11": 0.1443239783354933, "12": 0.14671421864208473, "13": 0.1469534411375638, "14": 0.14865202202634462, "15": 0.15518710912462844, "16": 0.1583989896747106, "17": 0.15376421139806937, "18": 0.14641406686190905, "19": 0.15523886716075944, "20": 0.15056736005456164, "21": 0.14889112185060185, "22": 0.14890720506931343, "23": 0.1503209591401525, "24": 0.14933887836812265, "25": 0.1512918316579788, "26": 0.15762591483628324, "27": 0.16218375556570344, "28": 0.15676492841174577, "29": 0.16071543863577667, "30": 0.16840353121018434, "31": 0.16740250101368215 } } }, "native-KM": { "engine": "native", "candidate": "KM", "records": 4714, "failed_requests": [], "final_document_expected": 128, "final_document_present": 128, "final_document_failed": 0, "quality_complete": true, "nll": 1.8698531427248721, "ppl": 6.487343615720602, "scored_tokens": 130944, "domains": { "ko": { "documents": 64, "nll": 2.1160934507276865, "ppl": 8.298654977171498 }, "en": { "documents": 32, "nll": 2.179184125238734, "ppl": 8.839091724192476 }, "code": { "documents": 32, "nll": 1.068041544205382, "ppl": 2.9096754454942153 } }, "delta_nll": { "delta": 0.0183554999357027, "ci95": [ 0.016150725366032757, 0.02042449977022933 ], "resampling_unit": "paired document", "replicates": 2000 }, "ppl_ratio_vs_bf16": { "ratio": 1.0185249976078823, "ci95": [ 1.0162818533169937, 1.0206345071943617 ] }, "document_delta_tail": { "mean": 0.018355499935702697, "median": 0.01692394045877729, "p95": 0.03945486485349074, "p99": 0.048711495942498746, "max": 0.061088601142256546 }, "worst20_documents": [ { "id": "final-049", "domain": "code", "delta_nll": 0.061088601142256546, "nll": 0.5852325632764349 }, { "id": "final-012", "domain": "code", "delta_nll": 0.049407672854810375, "nll": 0.3242659135824352 }, { "id": "final-124", "domain": "ko", "delta_nll": 0.046829239846248746, "nll": 2.0941776115941666 }, { "id": "final-046", "domain": "code", "delta_nll": 0.04628819745892788, "nll": 1.32197961753998 }, { "id": "final-022", "domain": "ko", "delta_nll": 0.04345562407381191, "nll": 2.1537975690168363 }, { "id": "final-034", "domain": "ko", "delta_nll": 0.042037504139628326, "nll": 2.0228321322433813 }, { "id": "final-015", "domain": "ko", "delta_nll": 0.039921493092944904, "nll": 1.2557143029523974 }, { "id": "final-103", "domain": "en", "delta_nll": 0.038588269551647326, "nll": 2.1141448237771145 }, { "id": "final-009", "domain": "ko", "delta_nll": 0.03695002993118691, "nll": 2.2730098936404657 }, { "id": "final-126", "domain": "ko", "delta_nll": 0.03652271726803952, "nll": 2.4037907110900876 }, { "id": "final-063", "domain": "ko", "delta_nll": 0.03526436306020703, "nll": 1.9656501252710286 }, { "id": "final-033", "domain": "ko", "delta_nll": 0.034737506750652604, "nll": 2.3805538005415907 }, { "id": "final-062", "domain": "ko", "delta_nll": 0.03320915698318952, "nll": 2.1664839950567507 }, { "id": "final-010", "domain": "ko", "delta_nll": 0.03225646427612542, "nll": 2.4555286844605706 }, { "id": "final-003", "domain": "ko", "delta_nll": 0.03192312662065355, "nll": 1.4510748647628209 }, { "id": "final-089", "domain": "ko", "delta_nll": 0.03140233410873115, "nll": 2.7113327500963 }, { "id": "final-090", "domain": "code", "delta_nll": 0.031072726603585708, "nll": 0.41241853677999224 }, { "id": "final-083", "domain": "code", "delta_nll": 0.03068540944944374, "nll": 1.087048177123753 }, { "id": "final-002", "domain": "ko", "delta_nll": 0.030056604720066264, "nll": 1.9254702576781442 }, { "id": "final-081", "domain": "code", "delta_nll": 0.02982791185957918, "nll": 0.4702023801501512 } ], "token_delta_tail": { "mean": 0.018355499935702454, "median": 0.00038291399268430837, "p95": 0.3769361662507298, "p99": 0.8090497631786076, "max": 6.117191639708771 }, "sampled_full_vocab_kl": { "mean": 0.024161768090731783, "median": 0.009382512824791842, "p95": 0.07177273292391109, "p99": 0.21476818007555457, "max": 3.3884346600221567 }, "kl_positions": 4096, "sampled_top1_agreement": 0.942138671875, "tasks": { "kmmlu": { "n": 500, "accuracy": 0.488, "failures": 0, "subject_scores": { "Accounting": { "n": 12, "accuracy": 0.5833333333333334 }, "Agricultural-Sciences": { "n": 12, "accuracy": 0.4166666666666667 }, "Aviation-Engineering-and-Maintenance": { "n": 12, "accuracy": 0.5 }, "Biology": { "n": 12, "accuracy": 0.4166666666666667 }, "Chemical-Engineering": { "n": 12, "accuracy": 0.5 }, "Chemistry": { "n": 11, "accuracy": 0.7272727272727273 }, "Civil-Engineering": { "n": 11, "accuracy": 0.18181818181818182 }, "Computer-Science": { "n": 11, "accuracy": 0.8181818181818182 }, "Construction": { "n": 11, "accuracy": 0.18181818181818182 }, "Criminal-Law": { "n": 11, "accuracy": 0.36363636363636365 }, "Ecology": { "n": 11, "accuracy": 0.5454545454545454 }, "Economics": { "n": 11, "accuracy": 0.6363636363636364 }, "Education": { "n": 11, "accuracy": 0.8181818181818182 }, "Electrical-Engineering": { "n": 11, "accuracy": 0.18181818181818182 }, "Electronics-Engineering": { "n": 11, "accuracy": 0.5454545454545454 }, "Energy-Management": { "n": 11, "accuracy": 0.36363636363636365 }, "Environmental-Science": { "n": 11, "accuracy": 0.36363636363636365 }, "Fashion": { "n": 11, "accuracy": 0.6363636363636364 }, "Food-Processing": { "n": 11, "accuracy": 0.45454545454545453 }, "Gas-Technology-and-Engineering": { "n": 11, "accuracy": 0.45454545454545453 }, "Geomatics": { "n": 11, "accuracy": 0.09090909090909091 }, "Health": { "n": 11, "accuracy": 0.45454545454545453 }, "Industrial-Engineer": { "n": 11, "accuracy": 0.5454545454545454 }, "Information-Technology": { "n": 11, "accuracy": 1.0 }, "Interior-Architecture-and-Design": { "n": 11, "accuracy": 0.45454545454545453 }, "Korean-History": { "n": 11, "accuracy": 0.2727272727272727 }, "Law": { "n": 11, "accuracy": 0.09090909090909091 }, "Machine-Design-and-Manufacturing": { "n": 11, "accuracy": 0.7272727272727273 }, "Management": { "n": 11, "accuracy": 0.7272727272727273 }, "Maritime-Engineering": { "n": 11, "accuracy": 0.36363636363636365 }, "Marketing": { "n": 11, "accuracy": 0.9090909090909091 }, "Materials-Engineering": { "n": 11, "accuracy": 0.2727272727272727 }, "Math": { "n": 11, "accuracy": 0.2727272727272727 }, "Mechanical-Engineering": { "n": 11, "accuracy": 0.36363636363636365 }, "Nondestructive-Testing": { "n": 11, "accuracy": 0.5454545454545454 }, "Patent": { "n": 11, "accuracy": 0.18181818181818182 }, "Political-Science-and-Sociology": { "n": 11, "accuracy": 0.45454545454545453 }, "Psychology": { "n": 11, "accuracy": 0.5454545454545454 }, "Public-Safety": { "n": 11, "accuracy": 0.7272727272727273 }, "Railway-and-Automotive-Engineering": { "n": 11, "accuracy": 0.5454545454545454 }, "Real-Estate": { "n": 11, "accuracy": 0.45454545454545453 }, "Refrigerating-Machinery": { "n": 11, "accuracy": 0.45454545454545453 }, "Social-Welfare": { "n": 11, "accuracy": 0.8181818181818182 }, "Taxation": { "n": 11, "accuracy": 0.36363636363636365 }, "Telecommunications-and-Wireless-Technology": { "n": 11, "accuracy": 0.6363636363636364 } }, "macro_accuracy": 0.48804713804713795, "delta_accuracy": { "delta": 0.006, "ci95": [ -0.018, 0.028 ], "resampling_unit": "paired item", "replicates": 2000 }, "regressions": [ "kmmlu-Agricultural-Sciences-226", "kmmlu-Chemistry-486", "kmmlu-Civil-Engineering-837", "kmmlu-Economics-129", "kmmlu-Electrical-Engineering-481", "kmmlu-Electronics-Engineering-83", "kmmlu-Energy-Management-645", "kmmlu-Energy-Management-431", "kmmlu-Geomatics-629", "kmmlu-Industrial-Engineer-130", "kmmlu-Interior-Architecture-and-Design-849", "kmmlu-Interior-Architecture-and-Design-560", "kmmlu-Machine-Design-and-Manufacturing-699", "kmmlu-Nondestructive-Testing-286", "kmmlu-Patent-99", "kmmlu-Political-Science-and-Sociology-76", "kmmlu-Refrigerating-Machinery-797" ], "improvements": [ "kmmlu-Biology-731", "kmmlu-Chemical-Engineering-673", "kmmlu-Chemistry-130", "kmmlu-Criminal-Law-172", "kmmlu-Economics-26", "kmmlu-Electronics-Engineering-202", "kmmlu-Geomatics-15", "kmmlu-Industrial-Engineer-141", "kmmlu-Interior-Architecture-and-Design-723", "kmmlu-Machine-Design-and-Manufacturing-449", "kmmlu-Math-205", "kmmlu-Patent-48", "kmmlu-Political-Science-and-Sociology-198", "kmmlu-Psychology-892", "kmmlu-Psychology-939", "kmmlu-Public-Safety-152", "kmmlu-Railway-and-Automotive-Engineering-834", "kmmlu-Real-Estate-184", "kmmlu-Refrigerating-Machinery-302", "kmmlu-Social-Welfare-19" ] }, "arc": { "n": 500, "accuracy": 0.514, "failures": 0, "accuracy_norm": 0.538, "delta_accuracy": { "delta": -0.004, "ci95": [ -0.026, 0.018 ], "resampling_unit": "paired item", "replicates": 2000 }, "regressions": [ "arc-Mercury_7025060", "arc-Mercury_SC_414271", "arc-Mercury_7268328", "arc-Mercury_7211680", "arc-MCAS_2004_8_14", "arc-Mercury_7007420", "arc-Mercury_7093048", "arc-ACTAAP_2010_7_12", "arc-Mercury_SC_407689", "arc-Mercury_187198", "arc-NCEOGA_2013_8_42", "arc-Mercury_7201740", "arc-Mercury_409295", "arc-Mercury_402364", "arc-Mercury_416536", "arc-Mercury_7215670" ], "improvements": [ "arc-Mercury_7143308", "arc-Mercury_SC_400594", "arc-TIMSS_1995_8_K18", "arc-NYSEDREGENTS_2008_4_15", "arc-NYSEDREGENTS_2012_4_29", "arc-Mercury_SC_409241", "arc-Mercury_7136885", "arc-Mercury_417143", "arc-Mercury_SC_402164", "arc-MDSA_2009_5_20", "arc-Mercury_189070", "arc-NCEOGA_2013_8_28", "arc-Mercury_SC_400708", "arc-Mercury_7056298" ], "delta_accuracy_norm": { "delta": -0.024, "ci95": [ -0.042, -0.006 ], "resampling_unit": "paired item", "replicates": 2000 } }, "ifeval": { "n": 541, "accuracy": 0.7985212569316081, "failures": 0, "instruction_strict": 0.8585131894484412, "truncated": 21, "delta_accuracy": { "delta": -0.027726432532347505, "ci95": [ -0.05545286506469501, 0.0 ], "resampling_unit": "paired item", "replicates": 2000 }, "regressions": [ "ifeval-1087", "ifeval-1129", "ifeval-1130", "ifeval-1379", "ifeval-1566", "ifeval-1609", "ifeval-1670", "ifeval-1686", "ifeval-179", "ifeval-1823", "ifeval-1837", "ifeval-2034", "ifeval-2063", "ifeval-2139", "ifeval-2180", "ifeval-2275", "ifeval-2303", "ifeval-2311", "ifeval-2416", "ifeval-2439", "ifeval-2605", "ifeval-279", "ifeval-2844", "ifeval-2859", "ifeval-2871", "ifeval-2918", "ifeval-2943", "ifeval-3098", "ifeval-3109", "ifeval-3223", "ifeval-3276", "ifeval-3287", "ifeval-3351", "ifeval-3362", "ifeval-3429", "ifeval-3453", "ifeval-3455", "ifeval-3479", "ifeval-3724" ], "improvements": [ "ifeval-1021", "ifeval-1092", "ifeval-1122", "ifeval-1219", "ifeval-1265", "ifeval-127", "ifeval-1348", "ifeval-1372", "ifeval-1580", "ifeval-1582", "ifeval-1629", "ifeval-1645", "ifeval-1944", "ifeval-2266", "ifeval-2299", "ifeval-2337", "ifeval-2591", "ifeval-2616", "ifeval-2617", "ifeval-2736", "ifeval-2929", "ifeval-3114", "ifeval-3409", "ifeval-3456" ] } }, "speed": { "512": { "measured": 5, "failed": 0, "ttft_seconds": { "median": 0.0339146, "min": 0.03385, "max": 0.0419931 }, "prefill_seconds": { "median": 0.0338926, "min": 0.0338276, "max": 0.0419684 }, "decode_seconds": { "median": 0.9326449, "min": 0.8984739, "max": 0.9876433 }, "seconds": { "median": 0.974639, "min": 0.9323247, "max": 1.0214961 }, "prefill_tokens_per_second": { "median": 15106.542431091151, "min": 12199.654978507639, "max": 15135.569771429247 }, "decode_tokens_per_second": { "median": 273.41595927882094, "min": 258.1903810819149, "max": 283.8145882701768 }, "peak_device_vram_bytes": 7735070720 }, "4096": { "measured": 5, "failed": 0, "ttft_seconds": { "median": 0.270575, "min": 0.2692284, "max": 0.3358204 }, "prefill_seconds": { "median": 0.2705501, "min": 0.2692013, "max": 0.3357967 }, "decode_seconds": { "median": 0.9245138, "min": 0.9161905, "max": 1.0056731 }, "seconds": { "median": 1.1996949, "min": 1.1883828, "max": 1.2755291 }, "prefill_tokens_per_second": { "median": 15139.524990011092, "min": 12197.85661979406, "max": 15215.379717705673 }, "decode_tokens_per_second": { "median": 275.8206529745689, "min": 253.56152014009322, "max": 278.3263960933889 }, "peak_device_vram_bytes": 7735070720 } }, "speed_source": "dedicated serial benchmark", "runtime_memory_breakdown": { "source": "llama.cpp CUDA0 buffer log", "model_mib": 3293.43, "kv_mib": 256.0, "recurrent_state_mib": 50.25, "compute_mib": 495.0 }, "block_diagnostics": { "scope": "six fixed validation documents, 256 tokens, cumulative error", "layer_mean_relative_l2": { "0": 0.06221998385965822, "1": 0.0736281802169728, "2": 0.09711888425739844, "3": 0.09315695809951617, "4": 0.10560414343263456, "5": 0.11069646739596915, "6": 0.10908242706046767, "7": 0.1205631162176491, "8": 0.1233846479318429, "9": 0.12405128962390997, "10": 0.12408944420636558, "11": 0.1338146160702853, "12": 0.1343382014730076, "13": 0.13585275683917755, "14": 0.13853996215654016, "15": 0.14370073753881193, "16": 0.14785750032340572, "17": 0.14457445721315573, "18": 0.13532864274733278, "19": 0.14613327878598575, "20": 0.14250442456763796, "21": 0.13964477766013275, "22": 0.1437853252215492, "23": 0.14390431999820316, "24": 0.14250248377598732, "25": 0.14520986439265834, "26": 0.15554735659349533, "27": 0.1550027367062706, "28": 0.1483145894470747, "29": 0.15122062429009922, "30": 0.15901377426428184, "31": 0.15746219683377075 } } }, "vllm-A0": { "engine": "vllm", "candidate": "A0", "records": 4714, "failed_requests": [], "final_document_expected": 128, "final_document_present": 128, "final_document_failed": 0, "quality_complete": true, "nll": 1.933766899369248, "ppl": 6.915511274256328, "scored_tokens": 130944, "domains": { "ko": { "documents": 64, "nll": 2.1874108605731073, "ppl": 8.91210852554423 }, "en": { "documents": 32, "nll": 2.2382833058531633, "ppl": 9.377219643941181 }, "code": { "documents": 32, "nll": 1.1219625704776142, "ppl": 3.070875101957011 } }, "delta_nll": { "delta": 0.08193356100092503, "ci95": [ 0.07663695831162505, 0.08749693774992422 ], "resampling_unit": "paired document", "replicates": 2000 }, "ppl_ratio_vs_bf16": { "ratio": 1.0853836956276282, "ci95": [ 1.0796500472594357, 1.091438922178912 ] }, "document_delta_tail": { "mean": 0.08193356100092503, "median": 0.08127681749337565, "p95": 0.13445226782526293, "p99": 0.17382731818548527, "max": 0.18086348787961604 }, "worst20_documents": [ { "id": "final-109", "domain": "en", "delta_nll": 0.18086348787961604, "nll": 2.278217233155557 }, { "id": "final-049", "domain": "code", "delta_nll": 0.17546424062354768, "nll": 0.7017778313493083 }, { "id": "final-035", "domain": "ko", "delta_nll": 0.16940156492702016, "nll": 2.2589829856584447 }, { "id": "final-103", "domain": "en", "delta_nll": 0.1492712258299406, "nll": 2.224580187152183 }, { "id": "final-009", "domain": "ko", "delta_nll": 0.14808116910993888, "nll": 2.38311079912873 }, { "id": "final-075", "domain": "ko", "delta_nll": 0.1374624848805378, "nll": 2.072410943751995 }, { "id": "final-018", "domain": "ko", "delta_nll": 0.1348537893556687, "nll": 2.4294281420680353 }, { "id": "final-003", "domain": "ko", "delta_nll": 0.13370658498308086, "nll": 1.5522493284111323 }, { "id": "final-063", "domain": "ko", "delta_nll": 0.13283096110976852, "nll": 2.0636811395313632 }, { "id": "final-012", "domain": "code", "delta_nll": 0.12851868521360962, "nll": 0.40393711445697955 }, { "id": "final-001", "domain": "code", "delta_nll": 0.12633690652101745, "nll": 0.3265917188627491 }, { "id": "final-039", "domain": "ko", "delta_nll": 0.12100410085645907, "nll": 2.364925483530263 }, { "id": "final-036", "domain": "ko", "delta_nll": 0.11785046078056192, "nll": 2.3910732522763407 }, { "id": "final-113", "domain": "ko", "delta_nll": 0.11768524093174021, "nll": 2.367358109734549 }, { "id": "final-032", "domain": "ko", "delta_nll": 0.1170312009950744, "nll": 1.9540231333082116 }, { "id": "final-033", "domain": "ko", "delta_nll": 0.11413480868680381, "nll": 2.4609519868459198 }, { "id": "final-046", "domain": "code", "delta_nll": 0.1134328493972261, "nll": 1.3898805659945797 }, { "id": "final-047", "domain": "ko", "delta_nll": 0.11254767220010566, "nll": 1.6434353927459193 }, { "id": "final-019", "domain": "en", "delta_nll": 0.11151946451968953, "nll": 2.011857288835099 }, { "id": "final-107", "domain": "ko", "delta_nll": 0.10808832735124835, "nll": 2.130467384430413 } ], "token_delta_tail": { "mean": 0.08193356100092504, "median": 0.0050355298444628716, "p95": 0.8673044264316554, "p99": 1.8089004468917764, "max": 15.382938370108604 }, "sampled_full_vocab_kl": null, "kl_positions": 0, "tasks": { "kmmlu": { "n": 500, "accuracy": 0.438, "failures": 0, "subject_scores": { "Accounting": { "n": 12, "accuracy": 0.5 }, "Agricultural-Sciences": { "n": 12, "accuracy": 0.16666666666666666 }, "Aviation-Engineering-and-Maintenance": { "n": 12, "accuracy": 0.5833333333333334 }, "Biology": { "n": 12, "accuracy": 0.3333333333333333 }, "Chemical-Engineering": { "n": 12, "accuracy": 0.4166666666666667 }, "Chemistry": { "n": 11, "accuracy": 0.6363636363636364 }, "Civil-Engineering": { "n": 11, "accuracy": 0.45454545454545453 }, "Computer-Science": { "n": 11, "accuracy": 0.8181818181818182 }, "Construction": { "n": 11, "accuracy": 0.36363636363636365 }, "Criminal-Law": { "n": 11, "accuracy": 0.2727272727272727 }, "Ecology": { "n": 11, "accuracy": 0.36363636363636365 }, "Economics": { "n": 11, "accuracy": 0.45454545454545453 }, "Education": { "n": 11, "accuracy": 0.7272727272727273 }, "Electrical-Engineering": { "n": 11, "accuracy": 0.36363636363636365 }, "Electronics-Engineering": { "n": 11, "accuracy": 0.18181818181818182 }, "Energy-Management": { "n": 11, "accuracy": 0.45454545454545453 }, "Environmental-Science": { "n": 11, "accuracy": 0.18181818181818182 }, "Fashion": { "n": 11, "accuracy": 0.45454545454545453 }, "Food-Processing": { "n": 11, "accuracy": 0.45454545454545453 }, "Gas-Technology-and-Engineering": { "n": 11, "accuracy": 0.45454545454545453 }, "Geomatics": { "n": 11, "accuracy": 0.18181818181818182 }, "Health": { "n": 11, "accuracy": 0.45454545454545453 }, "Industrial-Engineer": { "n": 11, "accuracy": 0.6363636363636364 }, "Information-Technology": { "n": 11, "accuracy": 0.8181818181818182 }, "Interior-Architecture-and-Design": { "n": 11, "accuracy": 0.45454545454545453 }, "Korean-History": { "n": 11, "accuracy": 0.18181818181818182 }, "Law": { "n": 11, "accuracy": 0.09090909090909091 }, "Machine-Design-and-Manufacturing": { "n": 11, "accuracy": 0.45454545454545453 }, "Management": { "n": 11, "accuracy": 0.6363636363636364 }, "Maritime-Engineering": { "n": 11, "accuracy": 0.36363636363636365 }, "Marketing": { "n": 11, "accuracy": 1.0 }, "Materials-Engineering": { "n": 11, "accuracy": 0.2727272727272727 }, "Math": { "n": 11, "accuracy": 0.09090909090909091 }, "Mechanical-Engineering": { "n": 11, "accuracy": 0.36363636363636365 }, "Nondestructive-Testing": { "n": 11, "accuracy": 0.45454545454545453 }, "Patent": { "n": 11, "accuracy": 0.18181818181818182 }, "Political-Science-and-Sociology": { "n": 11, "accuracy": 0.5454545454545454 }, "Psychology": { "n": 11, "accuracy": 0.36363636363636365 }, "Public-Safety": { "n": 11, "accuracy": 0.5454545454545454 }, "Railway-and-Automotive-Engineering": { "n": 11, "accuracy": 0.45454545454545453 }, "Real-Estate": { "n": 11, "accuracy": 0.45454545454545453 }, "Refrigerating-Machinery": { "n": 11, "accuracy": 0.2727272727272727 }, "Social-Welfare": { "n": 11, "accuracy": 0.7272727272727273 }, "Taxation": { "n": 11, "accuracy": 0.5454545454545454 }, "Telecommunications-and-Wireless-Technology": { "n": 11, "accuracy": 0.5454545454545454 } }, "macro_accuracy": 0.4383838383838384, "delta_accuracy": { "delta": -0.05, "ci95": [ -0.09, -0.014 ], "resampling_unit": "paired item", "replicates": 2000 }, "regressions": [ "kmmlu-Accounting-96", "kmmlu-Agricultural-Sciences-999", "kmmlu-Agricultural-Sciences-736", "kmmlu-Agricultural-Sciences-660", "kmmlu-Agricultural-Sciences-226", "kmmlu-Aviation-Engineering-and-Maintenance-899", "kmmlu-Biology-731", "kmmlu-Chemical-Engineering-403", "kmmlu-Chemistry-367", "kmmlu-Criminal-Law-73", "kmmlu-Ecology-998", "kmmlu-Ecology-577", "kmmlu-Economics-38", "kmmlu-Economics-129", "kmmlu-Economics-106", "kmmlu-Education-20", "kmmlu-Electrical-Engineering-890", "kmmlu-Electrical-Engineering-135", "kmmlu-Electronics-Engineering-294", "kmmlu-Electronics-Engineering-810", "kmmlu-Electronics-Engineering-83", "kmmlu-Electronics-Engineering-78", "kmmlu-Energy-Management-155", "kmmlu-Environmental-Science-192", "kmmlu-Environmental-Science-342", "kmmlu-Fashion-419", "kmmlu-Fashion-249", "kmmlu-Food-Processing-225", "kmmlu-Gas-Technology-and-Engineering-361", "kmmlu-Geomatics-272", "kmmlu-Information-Technology-532", "kmmlu-Information-Technology-635", "kmmlu-Interior-Architecture-and-Design-849", "kmmlu-Interior-Architecture-and-Design-560", "kmmlu-Interior-Architecture-and-Design-926", "kmmlu-Korean-History-93", "kmmlu-Korean-History-28", "kmmlu-Law-755", "kmmlu-Machine-Design-and-Manufacturing-690", "kmmlu-Machine-Design-and-Manufacturing-815", "kmmlu-Machine-Design-and-Manufacturing-345", "kmmlu-Machine-Design-and-Manufacturing-326", "kmmlu-Management-192", "kmmlu-Maritime-Engineering-145", "kmmlu-Maritime-Engineering-115", "kmmlu-Materials-Engineering-35", "kmmlu-Math-208", "kmmlu-Math-246", "kmmlu-Nondestructive-Testing-500", "kmmlu-Nondestructive-Testing-820", "kmmlu-Patent-99", "kmmlu-Public-Safety-941", "kmmlu-Public-Safety-262", "kmmlu-Railway-and-Automotive-Engineering-685", "kmmlu-Refrigerating-Machinery-797", "kmmlu-Refrigerating-Machinery-911", "kmmlu-Social-Welfare-952", "kmmlu-Social-Welfare-383", "kmmlu-Telecommunications-and-Wireless-Technology-118", "kmmlu-Telecommunications-and-Wireless-Technology-456" ], "improvements": [ "kmmlu-Aviation-Engineering-and-Maintenance-285", "kmmlu-Aviation-Engineering-and-Maintenance-380", "kmmlu-Chemical-Engineering-837", "kmmlu-Civil-Engineering-594", "kmmlu-Civil-Engineering-924", "kmmlu-Construction-396", "kmmlu-Construction-787", "kmmlu-Criminal-Law-40", "kmmlu-Economics-74", "kmmlu-Electrical-Engineering-118", "kmmlu-Electrical-Engineering-386", "kmmlu-Environmental-Science-835", "kmmlu-Food-Processing-919", "kmmlu-Gas-Technology-and-Engineering-705", "kmmlu-Geomatics-951", "kmmlu-Industrial-Engineer-927", "kmmlu-Interior-Architecture-and-Design-723", "kmmlu-Interior-Architecture-and-Design-947", "kmmlu-Korean-History-89", "kmmlu-Law-277", "kmmlu-Machine-Design-and-Manufacturing-449", "kmmlu-Maritime-Engineering-24", "kmmlu-Maritime-Engineering-234", "kmmlu-Marketing-80", "kmmlu-Materials-Engineering-321", "kmmlu-Math-144", "kmmlu-Patent-48", "kmmlu-Political-Science-and-Sociology-213", "kmmlu-Public-Safety-152", "kmmlu-Railway-and-Automotive-Engineering-177", "kmmlu-Real-Estate-172", "kmmlu-Social-Welfare-706", "kmmlu-Taxation-71", "kmmlu-Taxation-148", "kmmlu-Telecommunications-and-Wireless-Technology-713" ] }, "arc": { "n": 500, "accuracy": 0.494, "failures": 0, "accuracy_norm": 0.532, "delta_accuracy": { "delta": -0.028, "ci95": [ -0.056, -0.002 ], "resampling_unit": "paired item", "replicates": 2000 }, "regressions": [ "arc-Mercury_7025060", "arc-Mercury_7162400", "arc-Mercury_SC_414271", "arc-Mercury_7086608", "arc-MCAS_2005_9_6", "arc-Mercury_7007420", "arc-Mercury_7093048", "arc-Mercury_SC_407689", "arc-Mercury_SC_406451", "arc-Mercury_SC_415453", "arc-TIMSS_2007_8_pg113", "arc-Mercury_417138", "arc-Mercury_SC_409241", "arc-Mercury_SC_400839", "arc-Mercury_7136885", "arc-MCAS_2004_9_2", "arc-Mercury_7221043", "arc-Mercury_7042945", "arc-Mercury_7227710", "arc-Mercury_7012740", "arc-MDSA_2007_8_42", "arc-Mercury_7270113", "arc-Mercury_SC_406720", "arc-NCEOGA_2013_8_42", "arc-MCAS_2010_8_12005", "arc-Mercury_7212328", "arc-Mercury_7201740", "arc-Mercury_409295", "arc-Mercury_7171430", "arc-MEAP_2005_5_12" ], "improvements": [ "arc-ACTAAP_2011_5_1", "arc-MEAP_2005_8_39", "arc-Mercury_SC_407613", "arc-Mercury_7206395", "arc-MCAS_2013_5_29411", "arc-Mercury_7210210", "arc-MDSA_2008_8_27", "arc-Mercury_SC_LBS10337", "arc-Mercury_7216248", "arc-Mercury_417143", "arc-Mercury_417118", "arc-Mercury_7268240", "arc-Mercury_7283833", "arc-Mercury_7185343", "arc-NCEOGA_2013_8_28", "arc-Mercury_SC_400708" ], "delta_accuracy_norm": { "delta": -0.026, "ci95": [ -0.05, -0.002 ], "resampling_unit": "paired item", "replicates": 2000 } }, "ifeval": { "n": 541, "accuracy": 0.7818853974121996, "failures": 0, "instruction_strict": 0.8465227817745803, "truncated": 49, "delta_accuracy": { "delta": -0.04990757855822551, "ci95": [ -0.07948243992606285, -0.02033271719038817 ], "resampling_unit": "paired item", "replicates": 2000 }, "regressions": [ "ifeval-1000", "ifeval-1122", "ifeval-1129", "ifeval-1216", "ifeval-1219", "ifeval-122", "ifeval-1233", "ifeval-1242", "ifeval-1332", "ifeval-1342", "ifeval-1348", "ifeval-1392", "ifeval-1481", "ifeval-1670", "ifeval-1691", "ifeval-1713", "ifeval-179", "ifeval-1825", "ifeval-1893", "ifeval-1908", "ifeval-1928", "ifeval-1967", "ifeval-2035", "ifeval-2063", "ifeval-2118", "ifeval-2250", "ifeval-2386", "ifeval-2416", "ifeval-2439", "ifeval-2563", "ifeval-2591", "ifeval-2596", "ifeval-2605", "ifeval-2716", "ifeval-279", "ifeval-2811", "ifeval-2820", "ifeval-2905", "ifeval-2908", "ifeval-2912", "ifeval-2918", "ifeval-2929", "ifeval-3069", "ifeval-3098", "ifeval-3186", "ifeval-3223", "ifeval-3276", "ifeval-3324", "ifeval-3442", "ifeval-3445", "ifeval-3455", "ifeval-3502" ], "improvements": [ "ifeval-1092", "ifeval-1203", "ifeval-1389", "ifeval-1629", "ifeval-2034", "ifeval-2078", "ifeval-2084", "ifeval-2180", "ifeval-2266", "ifeval-2299", "ifeval-2350", "ifeval-2617", "ifeval-2662", "ifeval-2736", "ifeval-2941", "ifeval-3089", "ifeval-3109", "ifeval-3256", "ifeval-3329", "ifeval-3409", "ifeval-3428", "ifeval-3429", "ifeval-3453", "ifeval-3484", "ifeval-3691" ] } }, "speed": { "512": { "measured": 5, "failed": 0, "ttft_seconds": { "median": 0.045508384704589844, "min": 0.044849395751953125, "max": 0.09570884704589844 }, "prefill_seconds": { "median": 0.04440697999999088, "min": 0.04364459399999987, "max": 0.09454871199999104 }, "decode_seconds": { "median": 5.094352267999994, "min": 4.720216910999994, "max": 7.162745676 }, "seconds": { "median": 5.1390060800000015, "min": 4.765158537000005, "max": 7.2196227419999985 }, "prefill_tokens_per_second": { "median": 11529.718976613702, "min": 5415.198040984932, "max": 11731.120697330844 }, "decode_tokens_per_second": { "median": 50.055431306110115, "min": 35.60087311970617, "max": 54.02294106564212 }, "peak_device_vram_bytes": 30523822080 }, "4096": { "measured": 5, "failed": 0, "ttft_seconds": { "median": 0.2110755443572998, "min": 0.20933222770690918, "max": 0.2458353042602539 }, "prefill_seconds": { "median": 0.20952049599999611, "min": 0.20781107700000234, "max": 0.24418523199999242 }, "decode_seconds": { "median": 5.667101024000004, "min": 4.674583480999999, "max": 6.1266377329999955 }, "seconds": { "median": 5.877772003000004, "min": 4.883624401000006, "max": 6.372081207999997 }, "prefill_tokens_per_second": { "median": 19549.40007396735, "min": 16774.151190273977, "max": 19710.210154004224 }, "decode_tokens_per_second": { "median": 44.99655095613835, "min": 41.621524090854905, "max": 54.55031470428457 }, "peak_device_vram_bytes": 31200645120 } }, "speed_source": "dedicated serial benchmark", "runtime_memory_breakdown": { "source": "vLLM startup profile, rounded GiB", "weight_gib": 3.11, "profile_peak_activation_gib": 0.42, "non_torch_gib": 0.15, "reserved_kv_cache_gib": 21.77 } }, "vllm-B0": { "engine": "vllm", "candidate": "B0", "records": 4714, "failed_requests": [], "final_document_expected": 128, "final_document_present": 128, "final_document_failed": 0, "quality_complete": true, "nll": 1.851833338368323, "ppl": 6.371489918371587, "scored_tokens": 130944, "domains": { "ko": { "documents": 64, "nll": 2.0939456000989347, "ppl": 8.116878021237127 }, "en": { "documents": 32, "nll": 2.1695132429391744, "ppl": 8.754021921535749 }, "code": { "documents": 32, "nll": 1.0499289103362484, "ppl": 2.8574479758268034 } }, "delta_nll": { "delta": 0.0, "ci95": [ 0.0, 0.0 ], "resampling_unit": "paired document", "replicates": 2000 }, "ppl_ratio_vs_bf16": { "ratio": 1.0, "ci95": [ 1.0, 1.0 ] }, "document_delta_tail": { "mean": 0.0, "median": 0.0, "p95": 0.0, "p99": 0.0, "max": 0.0 }, "worst20_documents": [ { "id": "final-000", "domain": "en", "delta_nll": 0.0, "nll": 2.2103152485675994 }, { "id": "final-001", "domain": "code", "delta_nll": 0.0, "nll": 0.20025481234173167 }, { "id": "final-002", "domain": "ko", "delta_nll": 0.0, "nll": 1.8941187276930938 }, { "id": "final-003", "domain": "ko", "delta_nll": 0.0, "nll": 1.4185427434280515 }, { "id": "final-004", "domain": "code", "delta_nll": 0.0, "nll": 1.177325957679266 }, { "id": "final-005", "domain": "en", "delta_nll": 0.0, "nll": 2.2068631005641315 }, { "id": "final-006", "domain": "code", "delta_nll": 0.0, "nll": 0.7944584573940342 }, { "id": "final-007", "domain": "ko", "delta_nll": 0.0, "nll": 1.93964734846893 }, { "id": "final-008", "domain": "en", "delta_nll": 0.0, "nll": 2.29841878664696 }, { "id": "final-009", "domain": "ko", "delta_nll": 0.0, "nll": 2.2350296300187913 }, { "id": "final-010", "domain": "ko", "delta_nll": 0.0, "nll": 2.4237570619428 }, { "id": "final-011", "domain": "ko", "delta_nll": 0.0, "nll": 1.5659503719011485 }, { "id": "final-012", "domain": "code", "delta_nll": 0.0, "nll": 0.2754184292433699 }, { "id": "final-013", "domain": "en", "delta_nll": 0.0, "nll": 2.4716811972016886 }, { "id": "final-014", "domain": "ko", "delta_nll": 0.0, "nll": 2.636997431389201 }, { "id": "final-015", "domain": "ko", "delta_nll": 0.0, "nll": 1.2153571033151187 }, { "id": "final-016", "domain": "ko", "delta_nll": 0.0, "nll": 2.3393618316037705 }, { "id": "final-017", "domain": "code", "delta_nll": 0.0, "nll": 1.0857602734252005 }, { "id": "final-018", "domain": "ko", "delta_nll": 0.0, "nll": 2.2945743527123668 }, { "id": "final-019", "domain": "en", "delta_nll": 0.0, "nll": 1.9003378243154094 } ], "token_delta_tail": { "mean": 0.0, "median": 0.0, "p95": 0.0, "p99": 0.0, "max": 0.0 }, "sampled_full_vocab_kl": null, "kl_positions": 0, "tasks": { "kmmlu": { "n": 500, "accuracy": 0.488, "failures": 0, "subject_scores": { "Accounting": { "n": 12, "accuracy": 0.5833333333333334 }, "Agricultural-Sciences": { "n": 12, "accuracy": 0.5 }, "Aviation-Engineering-and-Maintenance": { "n": 12, "accuracy": 0.5 }, "Biology": { "n": 12, "accuracy": 0.4166666666666667 }, "Chemical-Engineering": { "n": 12, "accuracy": 0.4166666666666667 }, "Chemistry": { "n": 11, "accuracy": 0.7272727272727273 }, "Civil-Engineering": { "n": 11, "accuracy": 0.2727272727272727 }, "Computer-Science": { "n": 11, "accuracy": 0.8181818181818182 }, "Construction": { "n": 11, "accuracy": 0.18181818181818182 }, "Criminal-Law": { "n": 11, "accuracy": 0.2727272727272727 }, "Ecology": { "n": 11, "accuracy": 0.5454545454545454 }, "Economics": { "n": 11, "accuracy": 0.6363636363636364 }, "Education": { "n": 11, "accuracy": 0.8181818181818182 }, "Electrical-Engineering": { "n": 11, "accuracy": 0.36363636363636365 }, "Electronics-Engineering": { "n": 11, "accuracy": 0.5454545454545454 }, "Energy-Management": { "n": 11, "accuracy": 0.5454545454545454 }, "Environmental-Science": { "n": 11, "accuracy": 0.2727272727272727 }, "Fashion": { "n": 11, "accuracy": 0.6363636363636364 }, "Food-Processing": { "n": 11, "accuracy": 0.45454545454545453 }, "Gas-Technology-and-Engineering": { "n": 11, "accuracy": 0.45454545454545453 }, "Geomatics": { "n": 11, "accuracy": 0.18181818181818182 }, "Health": { "n": 11, "accuracy": 0.45454545454545453 }, "Industrial-Engineer": { "n": 11, "accuracy": 0.5454545454545454 }, "Information-Technology": { "n": 11, "accuracy": 1.0 }, "Interior-Architecture-and-Design": { "n": 11, "accuracy": 0.5454545454545454 }, "Korean-History": { "n": 11, "accuracy": 0.2727272727272727 }, "Law": { "n": 11, "accuracy": 0.09090909090909091 }, "Machine-Design-and-Manufacturing": { "n": 11, "accuracy": 0.7272727272727273 }, "Management": { "n": 11, "accuracy": 0.7272727272727273 }, "Maritime-Engineering": { "n": 11, "accuracy": 0.36363636363636365 }, "Marketing": { "n": 11, "accuracy": 0.9090909090909091 }, "Materials-Engineering": { "n": 11, "accuracy": 0.2727272727272727 }, "Math": { "n": 11, "accuracy": 0.18181818181818182 }, "Mechanical-Engineering": { "n": 11, "accuracy": 0.36363636363636365 }, "Nondestructive-Testing": { "n": 11, "accuracy": 0.6363636363636364 }, "Patent": { "n": 11, "accuracy": 0.18181818181818182 }, "Political-Science-and-Sociology": { "n": 11, "accuracy": 0.45454545454545453 }, "Psychology": { "n": 11, "accuracy": 0.36363636363636365 }, "Public-Safety": { "n": 11, "accuracy": 0.6363636363636364 }, "Railway-and-Automotive-Engineering": { "n": 11, "accuracy": 0.45454545454545453 }, "Real-Estate": { "n": 11, "accuracy": 0.36363636363636365 }, "Refrigerating-Machinery": { "n": 11, "accuracy": 0.45454545454545453 }, "Social-Welfare": { "n": 11, "accuracy": 0.8181818181818182 }, "Taxation": { "n": 11, "accuracy": 0.36363636363636365 }, "Telecommunications-and-Wireless-Technology": { "n": 11, "accuracy": 0.6363636363636364 } }, "macro_accuracy": 0.48804713804713795, "delta_accuracy": { "delta": 0.0, "ci95": [ 0.0, 0.0 ], "resampling_unit": "paired item", "replicates": 2000 }, "regressions": [], "improvements": [] }, "arc": { "n": 500, "accuracy": 0.522, "failures": 0, "accuracy_norm": 0.558, "delta_accuracy": { "delta": 0.0, "ci95": [ 0.0, 0.0 ], "resampling_unit": "paired item", "replicates": 2000 }, "regressions": [], "improvements": [], "delta_accuracy_norm": { "delta": 0.0, "ci95": [ 0.0, 0.0 ], "resampling_unit": "paired item", "replicates": 2000 } }, "ifeval": { "n": 541, "accuracy": 0.8317929759704251, "failures": 0, "instruction_strict": 0.8836930455635491, "truncated": 14, "delta_accuracy": { "delta": 0.0, "ci95": [ 0.0, 0.0 ], "resampling_unit": "paired item", "replicates": 2000 }, "regressions": [], "improvements": [] } }, "speed": { "512": { "measured": 5, "failed": 0, "ttft_seconds": { "median": 0.044440269470214844, "min": 0.043544769287109375, "max": 0.046628475189208984 }, "prefill_seconds": { "median": 0.04305331400001933, "min": 0.04249180099998284, "max": 0.04555048100002068 }, "decode_seconds": { "median": 4.408005844999991, "min": 3.7740833859999725, "max": 5.711164961999998 }, "seconds": { "median": 4.4517488000000185, "min": 3.8205934830000388, "max": 5.75528235500002 }, "prefill_tokens_per_second": { "median": 11892.232035837476, "min": 11240.276474792166, "max": 12049.383362221026 }, "decode_tokens_per_second": { "median": 57.849288083237674, "min": 44.64938444199681, "max": 67.56607470463607 }, "peak_device_vram_bytes": 30463041536 }, "4096": { "measured": 5, "failed": 0, "ttft_seconds": { "median": 0.20170092582702637, "min": 0.1910536289215088, "max": 0.21390032768249512 }, "prefill_seconds": { "median": 0.2003497110000012, "min": 0.189843097999983, "max": 0.21237152999998443 }, "decode_seconds": { "median": 4.531537686999968, "min": 4.341469549999999, "max": 5.642180597999982 }, "seconds": { "median": 4.725557533000028, "min": 4.554985737999971, "max": 5.833118721000005 }, "prefill_tokens_per_second": { "median": 20444.25210076782, "min": 19286.954329520064, "max": 21575.711959780423 }, "decode_tokens_per_second": { "median": 56.2722893669276, "min": 45.19529206321248, "max": 58.735872050513414 }, "peak_device_vram_bytes": 31085576192 } }, "speed_source": "dedicated serial benchmark", "runtime_memory_breakdown": { "source": "vLLM startup profile, rounded GiB", "weight_gib": 7.99, "profile_peak_activation_gib": 0.41, "non_torch_gib": 0.15, "reserved_kv_cache_gib": 16.9 } }, "vllm-G0": { "engine": "vllm", "candidate": "G0", "records": 4714, "failed_requests": [], "final_document_expected": 128, "final_document_present": 128, "final_document_failed": 0, "quality_complete": true, "nll": 1.8883687908364546, "ppl": 6.6085799079587035, "scored_tokens": 130944, "domains": { "ko": { "documents": 64, "nll": 2.1323308151056413, "ppl": 8.43450318560508 }, "en": { "documents": 32, "nll": 2.2066754175675585, "ppl": 9.085460763994856 }, "code": { "documents": 32, "nll": 1.0821381155669765, "ppl": 2.950982351910591 } }, "delta_nll": { "delta": 0.0365354524681315, "ci95": [ 0.03341807698834355, 0.03985762477482175 ], "resampling_unit": "paired document", "replicates": 2000 }, "ppl_ratio_vs_bf16": { "ratio": 1.037211075058518, "ci95": [ 1.0339827332769476, 1.040662599072528 ] }, "document_delta_tail": { "mean": 0.03653545246813151, "median": 0.035811706725006574, "p95": 0.06272625737042016, "p99": 0.09966297742808866, "max": 0.13673098418300175 }, "worst20_documents": [ { "id": "final-049", "domain": "code", "delta_nll": 0.13673098418300175, "nll": 0.6630445749087623 }, { "id": "final-103", "domain": "en", "delta_nll": 0.10774578183689297, "nll": 2.1830547431591354 }, { "id": "final-035", "domain": "ko", "delta_nll": 0.07780946921169136, "nll": 2.167390889943116 }, { "id": "final-036", "domain": "ko", "delta_nll": 0.07711811391275634, "nll": 2.3503409054085354 }, { "id": "final-013", "domain": "en", "delta_nll": 0.07294375213365051, "nll": 2.544624949335339 }, { "id": "final-046", "domain": "code", "delta_nll": 0.06624053336766729, "nll": 1.3426882499650208 }, { "id": "final-063", "domain": "ko", "delta_nll": 0.06309844567647593, "nll": 1.9939486240980704 }, { "id": "final-019", "domain": "en", "delta_nll": 0.06203505051631662, "nll": 1.9623728748317262 }, { "id": "final-047", "domain": "ko", "delta_nll": 0.05830257273553661, "nll": 1.5891902932813502 }, { "id": "final-024", "domain": "ko", "delta_nll": 0.056453347677868755, "nll": 2.1876879296649783 }, { "id": "final-124", "domain": "ko", "delta_nll": 0.05469006523076297, "nll": 2.1022687137347065 }, { "id": "final-030", "domain": "ko", "delta_nll": 0.05439239831284901, "nll": 1.5023053628017553 }, { "id": "final-073", "domain": "en", "delta_nll": 0.05397568013639561, "nll": 2.2551161028804056 }, { "id": "final-083", "domain": "code", "delta_nll": 0.053870922717069, "nll": 1.1102550361923647 }, { "id": "final-003", "domain": "ko", "delta_nll": 0.05277128865201296, "nll": 1.4713140320800644 }, { "id": "final-039", "domain": "ko", "delta_nll": 0.05266503023578243, "nll": 2.296586412909586 }, { "id": "final-032", "domain": "ko", "delta_nll": 0.05213395600446524, "nll": 1.8891258883176023 }, { "id": "final-126", "domain": "ko", "delta_nll": 0.051776652951370884, "nll": 2.4192633690632284 }, { "id": "final-101", "domain": "en", "delta_nll": 0.05169492799620634, "nll": 2.288882826523074 }, { "id": "final-034", "domain": "ko", "delta_nll": 0.05027131433247346, "nll": 2.032461391931003 } ], "token_delta_tail": { "mean": 0.03653545246813151, "median": 0.0010466962703503668, "p95": 0.5288243308663364, "p99": 1.1602056407928452, "max": 9.285976648330688 }, "sampled_full_vocab_kl": null, "kl_positions": 0, "tasks": { "kmmlu": { "n": 500, "accuracy": 0.432, "failures": 0, "subject_scores": { "Accounting": { "n": 12, "accuracy": 0.4166666666666667 }, "Agricultural-Sciences": { "n": 12, "accuracy": 0.25 }, "Aviation-Engineering-and-Maintenance": { "n": 12, "accuracy": 0.4166666666666667 }, "Biology": { "n": 12, "accuracy": 0.4166666666666667 }, "Chemical-Engineering": { "n": 12, "accuracy": 0.4166666666666667 }, "Chemistry": { "n": 11, "accuracy": 0.5454545454545454 }, "Civil-Engineering": { "n": 11, "accuracy": 0.18181818181818182 }, "Computer-Science": { "n": 11, "accuracy": 0.7272727272727273 }, "Construction": { "n": 11, "accuracy": 0.09090909090909091 }, "Criminal-Law": { "n": 11, "accuracy": 0.2727272727272727 }, "Ecology": { "n": 11, "accuracy": 0.2727272727272727 }, "Economics": { "n": 11, "accuracy": 0.5454545454545454 }, "Education": { "n": 11, "accuracy": 0.7272727272727273 }, "Electrical-Engineering": { "n": 11, "accuracy": 0.36363636363636365 }, "Electronics-Engineering": { "n": 11, "accuracy": 0.18181818181818182 }, "Energy-Management": { "n": 11, "accuracy": 0.36363636363636365 }, "Environmental-Science": { "n": 11, "accuracy": 0.18181818181818182 }, "Fashion": { "n": 11, "accuracy": 0.5454545454545454 }, "Food-Processing": { "n": 11, "accuracy": 0.36363636363636365 }, "Gas-Technology-and-Engineering": { "n": 11, "accuracy": 0.36363636363636365 }, "Geomatics": { "n": 11, "accuracy": 0.18181818181818182 }, "Health": { "n": 11, "accuracy": 0.45454545454545453 }, "Industrial-Engineer": { "n": 11, "accuracy": 0.45454545454545453 }, "Information-Technology": { "n": 11, "accuracy": 0.8181818181818182 }, "Interior-Architecture-and-Design": { "n": 11, "accuracy": 0.5454545454545454 }, "Korean-History": { "n": 11, "accuracy": 0.45454545454545453 }, "Law": { "n": 11, "accuracy": 0.09090909090909091 }, "Machine-Design-and-Manufacturing": { "n": 11, "accuracy": 0.6363636363636364 }, "Management": { "n": 11, "accuracy": 0.6363636363636364 }, "Maritime-Engineering": { "n": 11, "accuracy": 0.2727272727272727 }, "Marketing": { "n": 11, "accuracy": 0.9090909090909091 }, "Materials-Engineering": { "n": 11, "accuracy": 0.36363636363636365 }, "Math": { "n": 11, "accuracy": 0.09090909090909091 }, "Mechanical-Engineering": { "n": 11, "accuracy": 0.36363636363636365 }, "Nondestructive-Testing": { "n": 11, "accuracy": 0.6363636363636364 }, "Patent": { "n": 11, "accuracy": 0.09090909090909091 }, "Political-Science-and-Sociology": { "n": 11, "accuracy": 0.6363636363636364 }, "Psychology": { "n": 11, "accuracy": 0.45454545454545453 }, "Public-Safety": { "n": 11, "accuracy": 0.6363636363636364 }, "Railway-and-Automotive-Engineering": { "n": 11, "accuracy": 0.2727272727272727 }, "Real-Estate": { "n": 11, "accuracy": 0.36363636363636365 }, "Refrigerating-Machinery": { "n": 11, "accuracy": 0.6363636363636364 }, "Social-Welfare": { "n": 11, "accuracy": 0.8181818181818182 }, "Taxation": { "n": 11, "accuracy": 0.2727272727272727 }, "Telecommunications-and-Wireless-Technology": { "n": 11, "accuracy": 0.7272727272727273 } }, "macro_accuracy": 0.43249158249158237, "delta_accuracy": { "delta": -0.056, "ci95": [ -0.09, -0.023950000000000273 ], "resampling_unit": "paired item", "replicates": 2000 }, "regressions": [ "kmmlu-Accounting-18", "kmmlu-Accounting-52", "kmmlu-Accounting-35", "kmmlu-Agricultural-Sciences-999", "kmmlu-Agricultural-Sciences-660", "kmmlu-Agricultural-Sciences-226", "kmmlu-Aviation-Engineering-and-Maintenance-899", "kmmlu-Aviation-Engineering-and-Maintenance-808", "kmmlu-Aviation-Engineering-and-Maintenance-282", "kmmlu-Chemical-Engineering-584", "kmmlu-Chemistry-309", "kmmlu-Chemistry-486", "kmmlu-Civil-Engineering-837", "kmmlu-Computer-Science-66", "kmmlu-Construction-270", "kmmlu-Criminal-Law-61", "kmmlu-Ecology-998", "kmmlu-Ecology-577", "kmmlu-Ecology-248", "kmmlu-Economics-129", "kmmlu-Education-20", "kmmlu-Electrical-Engineering-135", "kmmlu-Electronics-Engineering-294", "kmmlu-Electronics-Engineering-810", "kmmlu-Electronics-Engineering-83", "kmmlu-Electronics-Engineering-78", "kmmlu-Energy-Management-645", "kmmlu-Energy-Management-431", "kmmlu-Energy-Management-155", "kmmlu-Environmental-Science-192", "kmmlu-Environmental-Science-342", "kmmlu-Fashion-331", "kmmlu-Food-Processing-148", "kmmlu-Gas-Technology-and-Engineering-348", "kmmlu-Industrial-Engineer-706", "kmmlu-Information-Technology-258", "kmmlu-Information-Technology-635", "kmmlu-Interior-Architecture-and-Design-748", "kmmlu-Interior-Architecture-and-Design-849", "kmmlu-Interior-Architecture-and-Design-560", "kmmlu-Law-755", "kmmlu-Machine-Design-and-Manufacturing-699", "kmmlu-Management-192", "kmmlu-Maritime-Engineering-115", "kmmlu-Marketing-501", "kmmlu-Math-246", "kmmlu-Patent-28", "kmmlu-Public-Safety-262", "kmmlu-Railway-and-Automotive-Engineering-685", "kmmlu-Railway-and-Automotive-Engineering-250", "kmmlu-Social-Welfare-19", "kmmlu-Taxation-95" ], "improvements": [ "kmmlu-Accounting-24", "kmmlu-Aviation-Engineering-and-Maintenance-303", "kmmlu-Aviation-Engineering-and-Maintenance-380", "kmmlu-Chemical-Engineering-583", "kmmlu-Criminal-Law-117", "kmmlu-Electrical-Engineering-386", "kmmlu-Energy-Management-826", "kmmlu-Environmental-Science-318", "kmmlu-Interior-Architecture-and-Design-723", "kmmlu-Interior-Architecture-and-Design-561", "kmmlu-Interior-Architecture-and-Design-947", "kmmlu-Korean-History-89", "kmmlu-Korean-History-1", "kmmlu-Law-694", "kmmlu-Marketing-80", "kmmlu-Materials-Engineering-321", "kmmlu-Political-Science-and-Sociology-96", "kmmlu-Political-Science-and-Sociology-198", "kmmlu-Psychology-939", "kmmlu-Public-Safety-152", "kmmlu-Refrigerating-Machinery-473", "kmmlu-Refrigerating-Machinery-302", "kmmlu-Social-Welfare-606", "kmmlu-Telecommunications-and-Wireless-Technology-875" ] }, "arc": { "n": 500, "accuracy": 0.524, "failures": 0, "accuracy_norm": 0.548, "delta_accuracy": { "delta": 0.002, "ci95": [ -0.022, 0.024 ], "resampling_unit": "paired item", "replicates": 2000 }, "regressions": [ "arc-Mercury_7025060", "arc-Mercury_SC_414271", "arc-Mercury_7211680", "arc-MCAS_2005_9_6", "arc-Mercury_7007420", "arc-Mercury_7093048", "arc-Mercury_SC_407689", "arc-Mercury_SC_406451", "arc-TIMSS_2007_8_pg113", "arc-Mercury_417138", "arc-Mercury_SC_409241", "arc-Mercury_SC_400839", "arc-Mercury_7236740", "arc-MCAS_1999_4_23", "arc-MCAS_2010_8_12005", "arc-Mercury_7212328", "arc-Mercury_7201740" ], "improvements": [ "arc-Mercury_7143308", "arc-Mercury_SC_400594", "arc-MSA_2012_5_2", "arc-Mercury_SC_407613", "arc-Mercury_183033", "arc-Mercury_7082320", "arc-NYSEDREGENTS_2008_4_15", "arc-Mercury_7216248", "arc-Mercury_SC_405937", "arc-Mercury_SC_402164", "arc-AIMS_2009_4_5", "arc-Mercury_415303", "arc-Mercury_189070", "arc-Mercury_7185343", "arc-Mercury_7252245", "arc-Mercury_411732", "arc-Mercury_7159075", "arc-Mercury_SC_400708" ], "delta_accuracy_norm": { "delta": -0.01, "ci95": [ -0.034, 0.014 ], "resampling_unit": "paired item", "replicates": 2000 } }, "ifeval": { "n": 541, "accuracy": 0.7966728280961183, "failures": 0, "instruction_strict": 0.8573141486810552, "truncated": 19, "delta_accuracy": { "delta": -0.03512014787430684, "ci95": [ -0.06469500924214418, -0.0073937153419593345 ], "resampling_unit": "paired item", "replicates": 2000 }, "regressions": [ "ifeval-1122", "ifeval-1139", "ifeval-1219", "ifeval-1242", "ifeval-1265", "ifeval-1348", "ifeval-1392", "ifeval-1436", "ifeval-1609", "ifeval-1643", "ifeval-1670", "ifeval-1713", "ifeval-179", "ifeval-1823", "ifeval-1908", "ifeval-1967", "ifeval-1996", "ifeval-2041", "ifeval-2063", "ifeval-2069", "ifeval-2247", "ifeval-2250", "ifeval-2275", "ifeval-2589", "ifeval-2602", "ifeval-2605", "ifeval-2653", "ifeval-2716", "ifeval-2790", "ifeval-2832", "ifeval-2912", "ifeval-2929", "ifeval-296", "ifeval-3069", "ifeval-3098", "ifeval-3156", "ifeval-3276", "ifeval-3315", "ifeval-3445", "ifeval-3623" ], "improvements": [ "ifeval-1092", "ifeval-1203", "ifeval-1629", "ifeval-2084", "ifeval-2180", "ifeval-2266", "ifeval-2299", "ifeval-2571", "ifeval-2662", "ifeval-2674", "ifeval-2941", "ifeval-3035", "ifeval-3195", "ifeval-3256", "ifeval-331", "ifeval-3351", "ifeval-3409", "ifeval-3415", "ifeval-3453", "ifeval-3534", "ifeval-3691" ] } }, "speed": { "512": { "measured": 5, "failed": 0, "ttft_seconds": { "median": 0.04959464073181152, "min": 0.04455304145812988, "max": 0.10157060623168945 }, "prefill_seconds": { "median": 0.048156779999999344, "min": 0.043370059999972455, "max": 0.10035488899995926 }, "decode_seconds": { "median": 5.650424885000007, "min": 4.513579437000033, "max": 6.6403984089999994 }, "seconds": { "median": 5.695060162999994, "min": 4.557910619999973, "max": 6.689486945999988 }, "prefill_tokens_per_second": { "median": 10631.940092340206, "min": 5101.893939618705, "max": 11805.379102549667 }, "decode_tokens_per_second": { "median": 45.129349595804726, "min": 38.40131032716173, "max": 56.49618081597046 }, "peak_device_vram_bytes": 30429831168 }, "4096": { "measured": 5, "failed": 0, "ttft_seconds": { "median": 0.2177879810333252, "min": 0.21390390396118164, "max": 0.21947574615478516 }, "prefill_seconds": { "median": 0.2161841930000037, "min": 0.21227339899996878, "max": 0.21789716699998962 }, "decode_seconds": { "median": 5.081278778000012, "min": 4.358565473999988, "max": 7.047872522999967 }, "seconds": { "median": 5.300327787000015, "min": 4.576107018000016, "max": 7.261361922999981 }, "prefill_tokens_per_second": { "median": 18946.806161724922, "min": 18797.85798224809, "max": 19295.87041662532 }, "decode_tokens_per_second": { "median": 50.18421762333769, "min": 36.18113113820308, "max": 58.50548799166684 }, "peak_device_vram_bytes": 31086006272 } }, "speed_source": "dedicated serial benchmark", "runtime_memory_breakdown": { "source": "vLLM startup profile, rounded GiB", "weight_gib": 3.11, "profile_peak_activation_gib": 0.42, "non_torch_gib": 0.15, "reserved_kv_cache_gib": 21.77 } }, "vllm-R0": { "engine": "vllm", "candidate": "R0", "records": 4714, "failed_requests": [], "final_document_expected": 128, "final_document_present": 128, "final_document_failed": 0, "quality_complete": true, "nll": 1.9406551411408473, "ppl": 6.963311428395681, "scored_tokens": 130944, "domains": { "ko": { "documents": 64, "nll": 2.1986095356203292, "ppl": 9.012473260035641 }, "en": { "documents": 32, "nll": 2.2330683326369427, "ppl": 9.328444984466172 }, "code": { "documents": 32, "nll": 1.132333160685786, "ppl": 3.1028875965421494 } }, "delta_nll": { "delta": 0.08882180277252402, "ci95": [ 0.08290276643344494, 0.09480845128102602 ], "resampling_unit": "paired document", "replicates": 2000 }, "ppl_ratio_vs_bf16": { "ratio": 1.0928858897379137, "ci95": [ 1.0864361653490975, 1.09944823705355 ] }, "document_delta_tail": { "mean": 0.08882180277252402, "median": 0.087994404574261, "p95": 0.14856161287850198, "p99": 0.1587154444139622, "max": 0.19832272298927292 }, "worst20_documents": [ { "id": "final-049", "domain": "code", "delta_nll": 0.19832272298927292, "nll": 0.7246363137150335 }, { "id": "final-063", "domain": "ko", "delta_nll": 0.15986515114203179, "nll": 2.090715329563626 }, { "id": "final-003", "domain": "ko", "delta_nll": 0.1556069780751074, "nll": 1.5741497215031588 }, { "id": "final-103", "domain": "en", "delta_nll": 0.15501229391128007, "nll": 2.2303212552335223 }, { "id": "final-001", "domain": "code", "delta_nll": 0.1546165162903852, "nll": 0.35487132863211684 }, { "id": "final-085", "domain": "ko", "delta_nll": 0.14946690995447537, "nll": 2.7799039448369816 }, { "id": "final-035", "domain": "ko", "delta_nll": 0.14940944534518205, "nll": 2.2389908660766067 }, { "id": "final-018", "domain": "ko", "delta_nll": 0.14698706686895333, "nll": 2.44156141958132 }, { "id": "final-012", "domain": "code", "delta_nll": 0.14389871201080529, "nll": 0.4193171412541752 }, { "id": "final-033", "domain": "ko", "delta_nll": 0.13999413301145092, "nll": 2.486811311170567 }, { "id": "final-047", "domain": "ko", "delta_nll": 0.1387310207884297, "nll": 1.6696187413342434 }, { "id": "final-036", "domain": "ko", "delta_nll": 0.13660915660273554, "nll": 2.4098319480985144 }, { "id": "final-075", "domain": "ko", "delta_nll": 0.13436946684833534, "nll": 2.0693179257197922 }, { "id": "final-126", "domain": "ko", "delta_nll": 0.1289476009822346, "nll": 2.496434317094092 }, { "id": "final-052", "domain": "code", "delta_nll": 0.1285562151468034, "nll": 1.2281253555741551 }, { "id": "final-118", "domain": "ko", "delta_nll": 0.12769279741910605, "nll": 2.2424166882698624 }, { "id": "final-009", "domain": "ko", "delta_nll": 0.12623405137679639, "nll": 2.3612636813955876 }, { "id": "final-046", "domain": "code", "delta_nll": 0.12368547436031965, "nll": 1.400133190957673 }, { "id": "final-039", "domain": "ko", "delta_nll": 0.12295584540712193, "nll": 2.3668772280809254 }, { "id": "final-057", "domain": "ko", "delta_nll": 0.12157007951263825, "nll": 2.605094729994394 } ], "token_delta_tail": { "mean": 0.08882180277252401, "median": 0.005889688618481159, "p95": 0.9199948191642747, "p99": 1.9303989338874754, "max": 11.345199823379517 }, "sampled_full_vocab_kl": null, "kl_positions": 0, "tasks": { "kmmlu": { "n": 500, "accuracy": 0.472, "failures": 0, "subject_scores": { "Accounting": { "n": 12, "accuracy": 0.3333333333333333 }, "Agricultural-Sciences": { "n": 12, "accuracy": 0.4166666666666667 }, "Aviation-Engineering-and-Maintenance": { "n": 12, "accuracy": 0.5 }, "Biology": { "n": 12, "accuracy": 0.25 }, "Chemical-Engineering": { "n": 12, "accuracy": 0.5 }, "Chemistry": { "n": 11, "accuracy": 0.45454545454545453 }, "Civil-Engineering": { "n": 11, "accuracy": 0.5454545454545454 }, "Computer-Science": { "n": 11, "accuracy": 0.8181818181818182 }, "Construction": { "n": 11, "accuracy": 0.2727272727272727 }, "Criminal-Law": { "n": 11, "accuracy": 0.2727272727272727 }, "Ecology": { "n": 11, "accuracy": 0.5454545454545454 }, "Economics": { "n": 11, "accuracy": 0.45454545454545453 }, "Education": { "n": 11, "accuracy": 0.7272727272727273 }, "Electrical-Engineering": { "n": 11, "accuracy": 0.45454545454545453 }, "Electronics-Engineering": { "n": 11, "accuracy": 0.09090909090909091 }, "Energy-Management": { "n": 11, "accuracy": 0.45454545454545453 }, "Environmental-Science": { "n": 11, "accuracy": 0.2727272727272727 }, "Fashion": { "n": 11, "accuracy": 0.36363636363636365 }, "Food-Processing": { "n": 11, "accuracy": 0.5454545454545454 }, "Gas-Technology-and-Engineering": { "n": 11, "accuracy": 0.2727272727272727 }, "Geomatics": { "n": 11, "accuracy": 0.36363636363636365 }, "Health": { "n": 11, "accuracy": 0.45454545454545453 }, "Industrial-Engineer": { "n": 11, "accuracy": 0.6363636363636364 }, "Information-Technology": { "n": 11, "accuracy": 0.9090909090909091 }, "Interior-Architecture-and-Design": { "n": 11, "accuracy": 0.6363636363636364 }, "Korean-History": { "n": 11, "accuracy": 0.2727272727272727 }, "Law": { "n": 11, "accuracy": 0.2727272727272727 }, "Machine-Design-and-Manufacturing": { "n": 11, "accuracy": 0.45454545454545453 }, "Management": { "n": 11, "accuracy": 0.7272727272727273 }, "Maritime-Engineering": { "n": 11, "accuracy": 0.2727272727272727 }, "Marketing": { "n": 11, "accuracy": 0.9090909090909091 }, "Materials-Engineering": { "n": 11, "accuracy": 0.36363636363636365 }, "Math": { "n": 11, "accuracy": 0.18181818181818182 }, "Mechanical-Engineering": { "n": 11, "accuracy": 0.5454545454545454 }, "Nondestructive-Testing": { "n": 11, "accuracy": 0.45454545454545453 }, "Patent": { "n": 11, "accuracy": 0.2727272727272727 }, "Political-Science-and-Sociology": { "n": 11, "accuracy": 0.6363636363636364 }, "Psychology": { "n": 11, "accuracy": 0.2727272727272727 }, "Public-Safety": { "n": 11, "accuracy": 0.6363636363636364 }, "Railway-and-Automotive-Engineering": { "n": 11, "accuracy": 0.45454545454545453 }, "Real-Estate": { "n": 11, "accuracy": 0.6363636363636364 }, "Refrigerating-Machinery": { "n": 11, "accuracy": 0.45454545454545453 }, "Social-Welfare": { "n": 11, "accuracy": 0.8181818181818182 }, "Taxation": { "n": 11, "accuracy": 0.45454545454545453 }, "Telecommunications-and-Wireless-Technology": { "n": 11, "accuracy": 0.6363636363636364 } }, "macro_accuracy": 0.47272727272727266, "delta_accuracy": { "delta": -0.016, "ci95": [ -0.058, 0.024 ], "resampling_unit": "paired item", "replicates": 2000 }, "regressions": [ "kmmlu-Accounting-18", "kmmlu-Accounting-52", "kmmlu-Accounting-35", "kmmlu-Agricultural-Sciences-226", "kmmlu-Aviation-Engineering-and-Maintenance-899", "kmmlu-Biology-379", "kmmlu-Biology-731", "kmmlu-Chemistry-367", "kmmlu-Chemistry-309", "kmmlu-Chemistry-486", "kmmlu-Computer-Science-714", "kmmlu-Criminal-Law-73", "kmmlu-Ecology-577", "kmmlu-Ecology-248", "kmmlu-Economics-104", "kmmlu-Economics-129", "kmmlu-Economics-106", "kmmlu-Education-20", "kmmlu-Electrical-Engineering-135", "kmmlu-Electronics-Engineering-294", "kmmlu-Electronics-Engineering-111", "kmmlu-Electronics-Engineering-810", "kmmlu-Electronics-Engineering-83", "kmmlu-Electronics-Engineering-78", "kmmlu-Energy-Management-645", "kmmlu-Energy-Management-155", "kmmlu-Environmental-Science-192", "kmmlu-Environmental-Science-342", "kmmlu-Fashion-331", "kmmlu-Fashion-419", "kmmlu-Fashion-249", "kmmlu-Food-Processing-225", "kmmlu-Gas-Technology-and-Engineering-361", "kmmlu-Gas-Technology-and-Engineering-311", "kmmlu-Geomatics-272", "kmmlu-Health-91", "kmmlu-Health-62", "kmmlu-Information-Technology-635", "kmmlu-Interior-Architecture-and-Design-849", "kmmlu-Korean-History-93", "kmmlu-Korean-History-28", "kmmlu-Machine-Design-and-Manufacturing-690", "kmmlu-Machine-Design-and-Manufacturing-815", "kmmlu-Machine-Design-and-Manufacturing-345", "kmmlu-Management-192", "kmmlu-Management-80", "kmmlu-Maritime-Engineering-145", "kmmlu-Marketing-521", "kmmlu-Nondestructive-Testing-500", "kmmlu-Nondestructive-Testing-286", "kmmlu-Patent-99", "kmmlu-Political-Science-and-Sociology-123", "kmmlu-Psychology-458", "kmmlu-Public-Safety-941", "kmmlu-Public-Safety-262", "kmmlu-Railway-and-Automotive-Engineering-685", "kmmlu-Railway-and-Automotive-Engineering-801", "kmmlu-Refrigerating-Machinery-797", "kmmlu-Refrigerating-Machinery-247", "kmmlu-Social-Welfare-952" ], "improvements": [ "kmmlu-Aviation-Engineering-and-Maintenance-81", "kmmlu-Chemical-Engineering-837", "kmmlu-Civil-Engineering-594", "kmmlu-Civil-Engineering-924", "kmmlu-Civil-Engineering-102", "kmmlu-Computer-Science-923", "kmmlu-Construction-284", "kmmlu-Criminal-Law-40", "kmmlu-Ecology-737", "kmmlu-Ecology-254", "kmmlu-Economics-74", "kmmlu-Electrical-Engineering-474", "kmmlu-Electrical-Engineering-118", "kmmlu-Energy-Management-826", "kmmlu-Environmental-Science-318", "kmmlu-Environmental-Science-147", "kmmlu-Food-Processing-85", "kmmlu-Food-Processing-919", "kmmlu-Geomatics-15", "kmmlu-Geomatics-543", "kmmlu-Geomatics-900", "kmmlu-Health-40", "kmmlu-Health-58", "kmmlu-Industrial-Engineer-927", "kmmlu-Interior-Architecture-and-Design-618", "kmmlu-Interior-Architecture-and-Design-561", "kmmlu-Korean-History-89", "kmmlu-Korean-History-64", "kmmlu-Law-11", "kmmlu-Law-277", "kmmlu-Management-162", "kmmlu-Management-229", "kmmlu-Marketing-80", "kmmlu-Materials-Engineering-321", "kmmlu-Mechanical-Engineering-629", "kmmlu-Mechanical-Engineering-605", "kmmlu-Patent-66", "kmmlu-Patent-48", "kmmlu-Political-Science-and-Sociology-96", "kmmlu-Political-Science-and-Sociology-198", "kmmlu-Political-Science-and-Sociology-213", "kmmlu-Public-Safety-651", "kmmlu-Public-Safety-152", "kmmlu-Railway-and-Automotive-Engineering-834", "kmmlu-Railway-and-Automotive-Engineering-177", "kmmlu-Real-Estate-46", "kmmlu-Real-Estate-172", "kmmlu-Real-Estate-155", "kmmlu-Refrigerating-Machinery-473", "kmmlu-Refrigerating-Machinery-302", "kmmlu-Social-Welfare-706", "kmmlu-Taxation-185" ] }, "arc": { "n": 500, "accuracy": 0.5, "failures": 0, "accuracy_norm": 0.548, "delta_accuracy": { "delta": -0.022, "ci95": [ -0.05, 0.004 ], "resampling_unit": "paired item", "replicates": 2000 }, "regressions": [ "arc-Mercury_7025060", "arc-Mercury_7162400", "arc-Mercury_SC_414271", "arc-Mercury_7086608", "arc-MCAS_2004_8_14", "arc-Mercury_7007420", "arc-Mercury_7093048", "arc-Mercury_SC_407689", "arc-Mercury_SC_406451", "arc-Mercury_415084", "arc-Mercury_7233765", "arc-TIMSS_2007_8_pg113", "arc-Mercury_417138", "arc-Mercury_SC_400839", "arc-Mercury_7136885", "arc-MCAS_2004_9_2", "arc-Mercury_7132405", "arc-Mercury_175560", "arc-Mercury_7042945", "arc-Mercury_187198", "arc-Mercury_7227710", "arc-Mercury_184730", "arc-Mercury_7012740", "arc-MDSA_2007_8_42", "arc-Mercury_SC_407400", "arc-Mercury_403907", "arc-Mercury_402364" ], "improvements": [ "arc-ACTAAP_2011_5_1", "arc-Mercury_7143308", "arc-Mercury_SC_407613", "arc-Mercury_7206395", "arc-NYSEDREGENTS_2008_4_15", "arc-MDSA_2008_8_27", "arc-Mercury_SC_LBS10337", "arc-NCEOGA_2013_8_7", "arc-Mercury_417118", "arc-Mercury_7268240", "arc-Mercury_7283833", "arc-AIMS_2009_4_5", "arc-MEA_2016_8_15", "arc-Mercury_189070", "arc-NCEOGA_2013_8_28", "arc-Mercury_7056298" ], "delta_accuracy_norm": { "delta": -0.01, "ci95": [ -0.034, 0.014 ], "resampling_unit": "paired item", "replicates": 2000 } }, "ifeval": { "n": 541, "accuracy": 0.7652495378927912, "failures": 0, "instruction_strict": 0.8309352517985612, "truncated": 45, "delta_accuracy": { "delta": -0.066543438077634, "ci95": [ -0.09611829944547134, -0.036968576709796676 ], "resampling_unit": "paired item", "replicates": 2000 }, "regressions": [ "ifeval-1219", "ifeval-1233", "ifeval-1248", "ifeval-1265", "ifeval-1268", "ifeval-1342", "ifeval-1348", "ifeval-1367", "ifeval-1381", "ifeval-1392", "ifeval-1436", "ifeval-1466", "ifeval-1571", "ifeval-1609", "ifeval-1670", "ifeval-1813", "ifeval-1825", "ifeval-1837", "ifeval-1908", "ifeval-1967", "ifeval-2035", "ifeval-2063", "ifeval-2243", "ifeval-2246", "ifeval-2247", "ifeval-2273", "ifeval-2355", "ifeval-2386", "ifeval-24", "ifeval-2416", "ifeval-2439", "ifeval-2596", "ifeval-2605", "ifeval-2616", "ifeval-2637", "ifeval-2677", "ifeval-2716", "ifeval-279", "ifeval-2908", "ifeval-2909", "ifeval-2912", "ifeval-2969", "ifeval-30", "ifeval-3025", "ifeval-3098", "ifeval-3186", "ifeval-3223", "ifeval-3243", "ifeval-3335", "ifeval-3350", "ifeval-3445", "ifeval-3479", "ifeval-3540", "ifeval-3748" ], "improvements": [ "ifeval-1092", "ifeval-1203", "ifeval-1645", "ifeval-2078", "ifeval-2084", "ifeval-2299", "ifeval-2350", "ifeval-2674", "ifeval-2941", "ifeval-3035", "ifeval-3256", "ifeval-3294", "ifeval-331", "ifeval-3329", "ifeval-3409", "ifeval-3453", "ifeval-3557", "ifeval-3691" ] } }, "speed": { "512": { "measured": 5, "failed": 0, "ttft_seconds": { "median": 0.04797053337097168, "min": 0.04520893096923828, "max": 0.09441709518432617 }, "prefill_seconds": { "median": 0.04673513299999854, "min": 0.04410053699999139, "max": 0.09323427500001458 }, "decode_seconds": { "median": 5.031832434000023, "min": 4.672067717999994, "max": 5.736560865000001 }, "seconds": { "median": 5.076876428999981, "min": 4.71994102299999, "max": 5.784405703999994 }, "prefill_tokens_per_second": { "median": 10955.355577997734, "min": 5491.542675694319, "max": 11609.835952793499 }, "decode_tokens_per_second": { "median": 50.677363235899605, "min": 44.451720464749215, "max": 54.57968835031349 }, "peak_device_vram_bytes": 30514843648 }, "4096": { "measured": 5, "failed": 0, "ttft_seconds": { "median": 0.19234180450439453, "min": 0.19208455085754395, "max": 0.24526643753051758 }, "prefill_seconds": { "median": 0.19094724900000415, "min": 0.19074504299999262, "max": 0.24380702399997745 }, "decode_seconds": { "median": 5.615284588999998, "min": 4.693565898000003, "max": 7.128385011999995 }, "seconds": { "median": 5.812666755999999, "min": 4.886212628999999, "max": 7.3204510170000106 }, "prefill_tokens_per_second": { "median": 21450.95057117011, "min": 16800.172254267698, "max": 21473.690406728725 }, "decode_tokens_per_second": { "median": 45.41176782019731, "min": 35.77247855871006, "max": 54.32969421152886 }, "peak_device_vram_bytes": 31203332096 } }, "speed_source": "dedicated serial benchmark", "runtime_memory_breakdown": { "source": "vLLM startup profile, rounded GiB", "weight_gib": 3.11, "profile_peak_activation_gib": 0.42, "non_torch_gib": 0.15, "reserved_kv_cache_gib": 21.77 } } }, "sizes": { "native-B0": { "file_bytes": 8424393504, "scope": "text GGUF including metadata", "text_tensor_payload_bytes": 8413425664, "original_text_elements": 4205751296, "effective_payload_bpw": 16.003657985201034 }, "native-K0": { "file_bytes": 3299478304, "scope": "text GGUF including metadata", "text_tensor_payload_bytes": 3288510464, "original_text_elements": 4205751296, "effective_payload_bpw": 6.255263771070119 }, "native-K1": { "file_bytes": 3299478624, "scope": "text GGUF including metadata", "text_tensor_payload_bytes": 3288510464, "original_text_elements": 4205751296, "effective_payload_bpw": 6.255263771070119 }, "native-KM": { "file_bytes": 3464383584, "scope": "text GGUF including metadata", "text_tensor_payload_bytes": 3453415424, "original_text_elements": 4205751296, "effective_payload_bpw": 6.5689389237722535 }, "native-I4": { "file_bytes": 3188067424, "scope": "text GGUF including metadata", "text_tensor_payload_bytes": 3177099264, "original_text_elements": 4205751296, "effective_payload_bpw": 6.043342157719447 }, "hf-B0": { "safetensors_file_bytes": 9319828096, "text_tensor_payload_bytes": 8411510272, "whole_directory_bytes": 9342911475, "original_text_elements": 4205751296, "effective_payload_bpw": 16.000014608567096, "scope": "text tensor payload excludes visual and MTP; files may include vision" }, "hf-R0": { "safetensors_file_bytes": 3786643328, "text_tensor_payload_bytes": 3119473792, "whole_directory_bytes": 3806655612, "original_text_elements": 4205751296, "effective_payload_bpw": 5.933729452746033, "scope": "text tensor payload excludes visual and MTP; files may include vision" }, "hf-G0": { "safetensors_file_bytes": 3786643328, "text_tensor_payload_bytes": 3119473792, "whole_directory_bytes": 3806655682, "original_text_elements": 4205751296, "effective_payload_bpw": 5.933729452746033, "scope": "text tensor payload excludes visual and MTP; files may include vision" }, "hf-A0": { "safetensors_file_bytes": 3786643328, "text_tensor_payload_bytes": 3119473792, "whole_directory_bytes": 3806656527, "original_text_elements": 4205751296, "effective_payload_bpw": 5.933729452746033, "scope": "text tensor payload excludes visual and MTP; files may include vision" } }, "task_manifest": { "seed": 20260921, "dataset_revisions": { "HAERAE-HUB/KMMLU": "d61b3f19e552c576bf5960dd24289763edc36a88", "allenai/ai2_arc": "210d026faf9955653af8916fad021475a3f00453", "google/IFEval": "966cd89545d6b6acfd7638bc708b98261ca58e84" }, "kmmlu_configs": [ "Accounting", "Agricultural-Sciences", "Aviation-Engineering-and-Maintenance", "Biology", "Chemical-Engineering", "Chemistry", "Civil-Engineering", "Computer-Science", "Construction", "Criminal-Law", "Ecology", "Economics", "Education", "Electrical-Engineering", "Electronics-Engineering", "Energy-Management", "Environmental-Science", "Fashion", "Food-Processing", "Gas-Technology-and-Engineering", "Geomatics", "Health", "Industrial-Engineer", "Information-Technology", "Interior-Architecture-and-Design", "Korean-History", "Law", "Machine-Design-and-Manufacturing", "Management", "Maritime-Engineering", "Marketing", "Materials-Engineering", "Math", "Mechanical-Engineering", "Nondestructive-Testing", "Patent", "Political-Science-and-Sociology", "Psychology", "Public-Safety", "Railway-and-Automotive-Engineering", "Real-Estate", "Refrigerating-Machinery", "Social-Welfare", "Taxation", "Telecommunications-and-Wireless-Technology" ], "counts": { "kmmlu": 500, "arc": 500, "ifeval": 541 }, "files": { "task-samples.jsonl": "149d8a55424bfa9a153dd28bcec55ba5901cc336f55d110349feea93bca94069", "task-requests.jsonl": "39c20c909b7144c7834a104d3e16506535e147503cb8acce4d3ab52271ee35e8", "document-requests.jsonl": "066f4c6fb958b272c683a181d03896a5036a5a3052e7fba89cd3377f168c3b94", "perf-requests.jsonl": "eadfe7314158d04b1e2b32c8b47691205fe6798e11e6b9d617d7887d10deb87f" }, "llm_eval_version": "0.4.13", "mc_template": "lm-eval zero-shot raw completion templates, no chat wrapper", "ifeval": "original tokenizer chat template, enable_thinking=False, greedy, max_new_tokens=2048", "kl": "32 fixed uniformly sampled target positions per document; exact full-vocabulary KL(BF16||candidate)", "nll": "each independent <=1024 token document; score positions 1..end; no added BOS, no overlap, fresh state", "created_unix": 1790017507.9748929 }, "baseline_agreement": { "scope": "two fixed validation documents; not final evaluation", "documents": [ { "id": "validation-000", "kl_hf_bf16_to_native_bf16_mean": 0.00032668747771120435, "kl_p95": 0.0010869226979381532, "kl_max": 0.0030274173200839924, "argmax_agreement": 1.0, "positions": 32, "self_kl_max": 0.0 }, { "id": "validation-001", "kl_hf_bf16_to_native_bf16_mean": 0.00026648011406643114, "kl_p95": 0.0005722330255313285, "kl_max": 0.0006650254597962712, "argmax_agreement": 0.96875, "positions": 32, "self_kl_max": 0.0 } ], "all_finite": true, "mean_kl": 0.00029658379588881774, "gate_passed": true, "interpretation": "Engine numerical difference is measured; compare quantization deltas against each engine BF16. This is not proof of identical kernels." }, "quantization_costs": { "R0": { "candidate": "R0", "pilot": false, "quantization_seconds": 0.47693753242492676, "total_seconds": 61.78158926963806, "peak_allocated_bytes": 9346222080, "peak_reserved_bytes": 9835642880, "payload_bytes": 3786643328, "packed_linear_count": 200, "path": "/models/huggingface/Qwen3.5-4B-QuantStudy-20260922-R0", "completed_at_unix": 1790015632.2326403, "reload_verified_at_creation": false, "reload_checks_now": { "hf": true, "vllm": true } }, "G0": { "candidate": "G0", "pilot": false, "quantization_seconds": 347.3705756664276, "total_seconds": 407.3032581806183, "peak_allocated_bytes": 10689680384, "peak_reserved_bytes": 11249123328, "payload_bytes": 3786643328, "packed_linear_count": 200, "path": "/models/huggingface/Qwen3.5-4B-QuantStudy-20260922-G0", "completed_at_unix": 1790016046.3189902, "reload_verified_at_creation": false, "reload_checks_now": { "hf": true, "vllm": true } }, "A0": { "candidate": "A0", "pilot": false, "quantization_seconds": 230.36617827415466, "total_seconds": 291.3701422214508, "peak_allocated_bytes": 14561832448, "peak_reserved_bytes": 15319695360, "payload_bytes": 3786643328, "packed_linear_count": 200, "path": "/models/huggingface/Qwen3.5-4B-QuantStudy-20260922-A0", "completed_at_unix": 1790017417.474299, "reload_verified_at_creation": false, "reload_checks_now": { "hf": true, "vllm": true } }, "K0": { "process_wall_seconds": 16.343690872192383, "scope": "native process including model IO; GPU peak/CPU peak not separately sampled" }, "K1": { "process_wall_seconds": 23.947975635528564, "scope": "native process including model IO; GPU peak/CPU peak not separately sampled" }, "KM": { "process_wall_seconds": 21.803329467773438, "scope": "native process including model IO; GPU peak/CPU peak not separately sampled" }, "I4": { "process_wall_seconds": 40.300732135772705, "scope": "native process including model IO; GPU peak/CPU peak not separately sampled" }, "shared_imatrix": { "process_wall_seconds": 64.55161738395691, "used_by": [ "K1", "KM", "I4" ] } }, "reboot_recovery": { "incident": "results/crash-20260922/INCIDENT.md", "bugcheck": "0xD1", "reported_related_driver": "NETIO.SYS", "root_cause_confirmed": false, "artifact_revalidation": { "ok": true, "time": 1790043521.4152524, "files": [ { "path": "/workspace/quantization-study/run\\data\\calibration-documents.jsonl", "bytes": 1409817, "digest": "bc659614d8b2eeda7c44fcff5964ec8b9abcd5a137ffc690e82fbefc05c30795", "ok": true }, { "path": "/workspace/quantization-study/run\\data\\validation-documents.jsonl", "bytes": 358229, "digest": "e72e0a1781759688ee332f787133d2346fdd3e4d698bdc47c342974b138f2f50", "ok": true }, { "path": "/workspace/quantization-study/run\\data\\final-documents.jsonl", "bytes": 1400177, "digest": "faa9b0dfc1a46b365e372bb3e55232c888c6aca16831afb12ed9e5ba6fc94426", "ok": true }, { "path": "/workspace/quantization-study/run\\data\\calibration.txt", "bytes": 520312, "digest": "d6e3c3a988fea455200d9b5608b98e650d3bf322c97fe2e58834779425ba5028", "ok": true }, { "path": "/workspace/quantization-study/run\\data\\calibration-windows.jsonl", "bytes": 1207117, "digest": "1af35598ac67eb4ca3fa34f6764428feb88d2f9822d3bffdd9bc8a0ef44ecd58", "ok": true }, { "path": "/workspace/quantization-study/run\\data\\calibration-tokenization.json", "bytes": 293, "digest": "3d3502fdbe3c80febd9a1f4534c755af61da086d3b2ef4c9f2318bb686cbd2c1", "ok": true }, { "path": "/workspace/quantization-study/run\\data\\task-samples.jsonl", "bytes": 1385218, "digest": "149d8a55424bfa9a153dd28bcec55ba5901cc336f55d110349feea93bca94069", "ok": true }, { "path": "/workspace/quantization-study/run\\data\\task-requests.jsonl", "bytes": 2594889, "digest": "39c20c909b7144c7834a104d3e16506535e147503cb8acce4d3ab52271ee35e8", "ok": true }, { "path": "/workspace/quantization-study/run\\data\\document-requests.jsonl", "bytes": 1064042, "digest": "066f4c6fb958b272c683a181d03896a5036a5a3052e7fba89cd3377f168c3b94", "ok": true }, { "path": "/workspace/quantization-study/run\\data\\perf-requests.jsonl", "bytes": 200749, "digest": "eadfe7314158d04b1e2b32c8b47691205fe6798e11e6b9d617d7887d10deb87f", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B\\.gitattributes", "bytes": 1570, "digest": "52373fe24473b1aa44333d318f578ae6bf04b49b", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B\\LICENSE", "bytes": 11544, "digest": "f938136e3adacfd92be087f6e113b5d6d97f678f", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B\\README.md", "bytes": 77661, "digest": "7950a3aadf378cd13758097bc52f0ed849a59007", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B\\chat_template.jinja", "bytes": 7756, "digest": "a585dec894e63da457d9440ec6aa7caa16d20860", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B\\config.json", "bytes": 3161, "digest": "557d961b205319c6a7da5f757f565b69b3967b7d", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B\\merges.txt", "bytes": 3353259, "digest": "a494e019ca1502219fd0128658b979e5f05ae8e8", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B\\model.safetensors-00001-of-00002.safetensors", "bytes": 5329398688, "digest": "26a93f066e1916adb13453dae5a0c707c0fbc71299ed98779571a907b8e74c61", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B\\model.safetensors-00002-of-00002.safetensors", "bytes": 3990429408, "digest": "cb544bd9bfae93dc59b0f22b292f5933573854a7f9b97835c67060d7d910e188", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B\\model.safetensors.index.json", "bytes": 76196, "digest": "fddda6039f7c1d17260c9e923b8a72fd025d9a86", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B\\preprocessor_config.json", "bytes": 390, "digest": "2ea84a437d448ff71b08df68fdd949d5cc4ebb64", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B\\tokenizer.json", "bytes": 12807982, "digest": "5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B\\tokenizer_config.json", "bytes": 16710, "digest": "eda48d3e75a8e59a8479ee4ec8b37f76e711d9c1", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B\\video_preprocessor_config.json", "bytes": 385, "digest": "3ba673a5ad7d4d13f54155ecd38b2a94a6dac8fe", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B\\vocab.json", "bytes": 6722759, "digest": "0aa0ce0658d60ac4a5d609f4eadb0e8e43514176", "ok": true }, { "path": "/models\\gguf\\Qwen3.5-4B-QuantStudy-20260922\\Qwen3.5-4B-BF16.gguf", "bytes": 8424393504, "digest": "4f851fbf3f5efe9fe17d332aa7633e06f0cd6b8ca29cd74f0a78c55461734002", "ok": true }, { "path": "/models\\gguf\\Qwen3.5-4B-QuantStudy-20260922\\Qwen3.5-4B-I4-IQ4_XS.gguf", "bytes": 3188067424, "digest": "d2f3dd7a1e6d919c32bb993aeca9bd994226899c462122bfbbca9a88d79a0dec", "ok": true }, { "path": "/models\\gguf\\Qwen3.5-4B-QuantStudy-20260922\\Qwen3.5-4B-K0-Q4_K.gguf", "bytes": 3299478304, "digest": "e2065959661a406df842b9f6ee9f16b3054c236834a43c6e6b92c4c6100e6916", "ok": true }, { "path": "/models\\gguf\\Qwen3.5-4B-QuantStudy-20260922\\Qwen3.5-4B-K1-Q4_K.gguf", "bytes": 3299478624, "digest": "fb879775d28280a13b6f983218637d7cde5916b8d49610e88ecacd1bfdbf3b17", "ok": true }, { "path": "/models\\gguf\\Qwen3.5-4B-QuantStudy-20260922\\Qwen3.5-4B-KM-Q4_K_M.gguf", "bytes": 3464383584, "digest": "9826160dc86b8dce7e329d42323a6dfdbe8afda6d668bbcbcd0bae549772127a", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B-QuantStudy-20260922-R0\\chat_template.jinja", "bytes": 7756, "digest": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B-QuantStudy-20260922-R0\\config.json", "bytes": 13618, "digest": "4b6cd0d68cb8b673b612f8763fa0e7248674184c477904d2f3de182dbb9192b0", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B-QuantStudy-20260922-R0\\generation_config.json", "bytes": 116, "digest": "4a4505ef94dbb317b8c508ff53777cb0dfed044027864a1c05822c07217251ef", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B-QuantStudy-20260922-R0\\model.safetensors", "bytes": 3786643328, "digest": "c9caeccee9284be3a9d1d5c3dde923aea6b33ff42d1980001f344966830c6454", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B-QuantStudy-20260922-R0\\recipe.yaml", "bytes": 346, "digest": "4dc9a16dfdfdd18de75049d49d353d043f2e72d22cf1e66508466b42f0e4e323", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B-QuantStudy-20260922-R0\\tokenizer.json", "bytes": 19989325, "digest": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B-QuantStudy-20260922-R0\\tokenizer_config.json", "bytes": 1123, "digest": "bee8eba30f0eb4af73c0fe2cd06d0f89b657d7819941c438157ec42f7c80ea87", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B-QuantStudy-20260922-G0\\chat_template.jinja", "bytes": 7756, "digest": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B-QuantStudy-20260922-G0\\config.json", "bytes": 13618, "digest": "4b6cd0d68cb8b673b612f8763fa0e7248674184c477904d2f3de182dbb9192b0", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B-QuantStudy-20260922-G0\\generation_config.json", "bytes": 116, "digest": "4a4505ef94dbb317b8c508ff53777cb0dfed044027864a1c05822c07217251ef", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B-QuantStudy-20260922-G0\\model.safetensors", "bytes": 3786643328, "digest": "7b23eaf13cbe94ee041f052928d9eb7f25bb6810ad83093a5b4eb7a56829d6d4", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B-QuantStudy-20260922-G0\\recipe.yaml", "bytes": 416, "digest": "c05e0f30ae6e6bfdb12546f220587124ba7e77d9e26f6b685e1601328077f56a", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B-QuantStudy-20260922-G0\\tokenizer.json", "bytes": 19989325, "digest": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B-QuantStudy-20260922-G0\\tokenizer_config.json", "bytes": 1123, "digest": "bee8eba30f0eb4af73c0fe2cd06d0f89b657d7819941c438157ec42f7c80ea87", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B-QuantStudy-20260922-A0\\chat_template.jinja", "bytes": 7756, "digest": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B-QuantStudy-20260922-A0\\config.json", "bytes": 13618, "digest": "4b6cd0d68cb8b673b612f8763fa0e7248674184c477904d2f3de182dbb9192b0", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B-QuantStudy-20260922-A0\\generation_config.json", "bytes": 116, "digest": "4a4505ef94dbb317b8c508ff53777cb0dfed044027864a1c05822c07217251ef", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B-QuantStudy-20260922-A0\\model.safetensors", "bytes": 3786643328, "digest": "03d7ff5dba4a48ff1c689fd540ba6c03928b047a4ab9680bb5f3c3994bd82dca", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B-QuantStudy-20260922-A0\\recipe.yaml", "bytes": 1261, "digest": "88fef58471d11265dee9d59b0ab98a73335c08f17ca33a2b529bc217ab9620b3", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B-QuantStudy-20260922-A0\\tokenizer.json", "bytes": 19989325, "digest": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523", "ok": true }, { "path": "/models\\huggingface\\Qwen3.5-4B-QuantStudy-20260922-A0\\tokenizer_config.json", "bytes": 1123, "digest": "bee8eba30f0eb4af73c0fe2cd06d0f89b657d7819941c438157ec42f7c80ea87", "ok": true } ], "awq_saved_requests": 4582, "awq_pending_requests": 132 }, "validation_replay": { "ok": true, "time": 1790043578.3605447, "documents": [ { "id": "validation-000", "max_abs_token_nll_change": 0.0, "result": { "id": "validation-000", "nll_sum": 1390.3210441801912, "scored_tokens": 1023, "token_nll": [ 8.896589279174805, 8.10812759399414, 15.234284400939941, 4.063816070556641, 12.620002746582031, 4.530691623687744, 1.2930597066879272, 0.35175272822380066, 0.1414705514907837, 2.2816872596740723, 2.135037899017334, 7.231238842010498, 10.643446922302246, 4.155224323272705, 1.5417677164077759, 4.567447185516357, 0.31904101371765137, 0.6470569968223572, 1.9187898635864258, 2.2457971572875977, 0.9873025417327881, 0.780295193195343, 2.6049716472625732, 0.13760657608509064, 2.043778896331787, 0.7781569361686707, 3.7662031650543213, 2.713412284851074, 0.23097831010818481, 6.861329078674316, 2.480008363723755, 0.8723762631416321, 7.548976898193359, 0.045730166137218475, 6.295568943023682, 1.7294002771377563, 3.3896212577819824, 1.2617295980453491, 2.166970729827881, 1.2645686864852905, 1.8690170049667358, 1.142823576927185, 3.6533286571502686, 0.9696319103240967, 6.805932998657227, 0.44700637459754944, 1.6254490613937378, 2.930039167404175, 3.7419345378875732, 1.2759637832641602, 0.557578980922699, 1.0683296918869019, 0.7876671552658081, 4.050053119659424, 1.163008213043213, 1.332289218902588, 5.599356651306152, 0.10200141370296478, 0.7086512446403503, 7.168918132781982, 1.7046775817871094, 10.218987464904785, 7.706588268280029, 0.16714976727962494, 4.285050868988037, 0.006876257248222828, 6.07305908203125, 0.065034881234169, 7.29325008392334, 4.127369403839111, 2.7572555541992188, 0.8364105224609375, 4.476608753204346, 1.7050001621246338, 0.005860882345587015, 5.669244289398193, 0.1138870045542717, 0.05124836787581444, 0.33780398964881897, 5.607630729675293, 0.6737068891525269, 3.84279727935791, 4.671473026275635, 0.006709665060043335, 4.875365257263184, 3.6566967964172363, 8.060627937316895, 3.104261875152588, 3.0225369930267334, 0.2533526122570038, 0.13062703609466553, 0.05253320187330246, 8.621944427490234, 0.346645712852478, 0.5980298519134521, 0.08677282929420471, 0.00010263393050990999, 0.11037678271532059, 0.019280055537819862, 0.001278298324905336, 0.09665441513061523, 0.07735737413167953, 0.004121619276702404, 0.16229917109012604, 0.11573243886232376, 7.676764488220215, 0.019920174032449722, 0.362432599067688, 0.007550511043518782, 0.0008995062671601772, 0.08015387505292892, 0.021290449425578117, 0.00517433974891901, 0.02617933228611946, 0.4574211835861206, 0.006322027184069157, 0.20277181267738342, 0.20573385059833527, 5.205392837524414, 0.007222376763820648, 0.06693047285079956, 0.0006850994541309774, 2.3245540432981215e-05, 0.005636157002300024, 0.030755937099456787, 0.0010303908493369818, 0.022629013285040855, 0.12353207170963287, 0.005144335329532623, 1.4072444438934326, 1.9218240976333618, 5.203754425048828, 14.096646308898926, 0.1556299328804016, 1.6323039531707764, 1.7160873413085938, 5.02338981628418, 2.0788657665252686, 3.962313175201416, 1.6145744323730469, 0.19444392621517181, 0.0822046622633934, 0.0967492163181305, 5.033435821533203, 6.040292263031006, 4.726674556732178, 3.2992899417877197, 0.11521605402231216, 0.017281780019402504, 1.2206934690475464, 1.0955398082733154, 0.01815686747431755, 0.01785479672253132, 0.03478417173027992, 0.9184812903404236, 1.572862148284912, 0.7331628203392029, 0.0465637631714344, 0.6297574043273926, 0.09811332076787949, 0.01116716768592596, 1.5792090892791748, 0.38798901438713074, 0.04957647621631622, 0.01992660202085972, 0.05843404680490494, 0.004388938657939434, 0.03892272338271141, 0.010440375655889511, 0.001601009746082127, 0.004732479341328144, 0.6406623721122742, 0.5419203042984009, 0.29596859216690063, 9.05449104309082, 0.27101460099220276, 1.5104501247406006, 0.029307130724191666, 0.015437163412570953, 1.2437081336975098, 1.8386118412017822, 1.6253098249435425, 0.3509577512741089, 0.0327141098678112, 0.0615178719162941, 0.03591759875416756, 0.643612265586853, 2.098417282104492, 0.26738443970680237, 0.11389615386724472, 2.7116646766662598, 0.687693178653717, 0.5618981719017029, 0.012411263771355152, 0.00812049675732851, 0.020271621644496918, 0.00651450315490365, 0.06100871041417122, 0.15099835395812988, 0.09459132701158524, 0.2844947576522827, 0.30277326703071594, 0.008552944287657738, 0.001983700320124626, 0.006830083206295967, 0.0474667027592659, 1.740710973739624, 0.4854883551597595, 1.9961540699005127, 1.4660340547561646, 0.5130776166915894, 5.532839298248291, 7.031619071960449, 7.962206840515137, 0.7050954103469849, 0.02929636463522911, 0.00456837797537446, 0.5282288789749146, 0.3820890188217163, 0.05044836178421974, 0.002377542434260249, 1.6788654327392578, 0.47631216049194336, 3.2770724296569824, 1.4253441095352173, 3.9487555027008057, 0.9937187433242798, 1.1734027862548828, 0.8599129915237427, 0.01017797365784645, 0.0010193157941102982, 0.13257113099098206, 0.1615675538778305, 0.011217622086405754, 0.006486670114099979, 0.01831243373453617, 0.0004980515805073082, 0.006735833827406168, 9.107453346252441, 0.25694912672042847, 2.387226104736328, 0.8998934626579285, 0.00038842763751745224, 0.009435346350073814, 1.305315375328064, 7.3980865478515625, 0.005131526384502649, 0.13160039484500885, 0.028242671862244606, 0.0007030876004137099, 0.00707620196044445, 0.10835885256528854, 0.10640285909175873, 0.010141038335859776, 0.9094316363334656, 0.027385415509343147, 0.09689269959926605, 6.692057132720947, 0.2717764973640442, 6.791062355041504, 2.0940048694610596, 0.21381087601184845, 0.04802927374839783, 2.421391248703003, 9.32273006439209, 0.03999534994363785, 0.3151054084300995, 3.056155204772949, 0.09402961283922195, 0.008975280448794365, 0.0009039129945449531, 0.8397841453552246, 5.025770664215088, 4.615621566772461, 2.6083695888519287, 0.20670680701732635, 0.5682010650634766, 0.874515175819397, 2.0641939640045166, 0.7174361348152161, 0.07829786837100983, 0.022562231868505478, 5.1692280769348145, 6.993217945098877, 7.933887004852295, 0.03295024856925011, 0.34944555163383484, 5.755311012268066, 0.8602891564369202, 0.00837433710694313, 0.015615100041031837, 0.0005066303419880569, 0.7579731345176697, 0.03532584756612778, 0.011098437942564487, 0.5613908767700195, 1.117959976196289, 0.6880788207054138, 0.01624138467013836, 1.5184569358825684, 0.326661616563797, 3.231532573699951, 0.15875832736492157, 0.9366632699966431, 0.11091562360525131, 0.0025527053512632847, 0.007695906795561314, 1.7117245197296143, 0.005680963862687349, 1.4774706363677979, 0.6168534755706787, 0.0028924793004989624, 0.1830269694328308, 0.04781655594706535, 0.11460214108228683, 0.0624229796230793, 0.009019820019602776, 0.010331599973142147, 0.019788116216659546, 0.0012419852428138256, 6.875721454620361, 1.9230016469955444, 7.331838607788086, 0.4252099096775055, 0.36364924907684326, 0.00404349947348237, 0.9027749300003052, 0.016164438799023628, 5.856738567352295, 0.49911415576934814, 0.04592283070087433, 0.12887001037597656, 0.029800288379192352, 0.0853920578956604, 0.0028740549460053444, 1.558236837387085, 1.6369540691375732, 0.34120723605155945, 0.04081692174077034, 0.20421874523162842, 0.004791919607669115, 1.7696548700332642, 0.012758769094944, 0.1438761204481125, 0.01981125771999359, 0.002046397188678384, 0.0268936175853014, 0.04710603505373001, 0.16385935246944427, 0.0318477489054203, 0.0007459718617610633, 1.3133518695831299, 0.012470838613808155, 0.01126866228878498, 0.007856184616684914, 0.004528267774730921, 0.02784682996571064, 1.1127396821975708, 1.05778968334198, 0.0664849653840065, 0.002535464009270072, 0.27774834632873535, 3.008345127105713, 0.6614764332771301, 0.1744779348373413, 0.0016221948899328709, 0.022806724533438683, 0.0012060280423611403, 0.31103208661079407, 0.0012657972984015942, 0.022955160588026047, 0.04872199520468712, 0.004547373857349157, 4.093289375305176, 4.757928848266602, 0.36956414580345154, 7.174106597900391, 3.126091480255127, 6.393560886383057, 1.0177867412567139, 3.441581964492798, 0.5863550901412964, 5.9294257164001465, 1.2209279537200928, 1.7995719909667969, 0.01860067807137966, 0.0006587718962691724, 0.23936496675014496, 0.022070271894335747, 5.0108771324157715, 1.5817923545837402, 2.2744064331054688, 0.8287713527679443, 0.6823278665542603, 0.22025470435619354, 0.4040948450565338, 0.07114921510219574, 0.03844849765300751, 2.651553153991699, 0.10412641614675522, 0.14505580067634583, 0.15453027188777924, 0.00758316507562995, 3.6314706802368164, 5.673055171966553, 0.9121759533882141, 0.07465813308954239, 1.713028907775879, 0.019922861829400063, 0.07012084126472473, 0.024253802374005318, 0.7852253317832947, 0.35085225105285645, 0.04621073976159096, 1.8403822183609009, 0.0003196682082489133, 1.4305012882687151e-05, 0.12860232591629028, 0.21269303560256958, 0.006926809437572956, 0.40159493684768677, 6.092898368835449, 0.09558846056461334, 1.3061774969100952, 0.00018249277491122484, 0.0003212174342479557, 0.07394185662269592, 0.0008317348547279835, 0.027401423081755638, 0.0007528808200731874, 0.011328186839818954, 0.004341937601566315, 0.5559431314468384, 0.010279921814799309, 3.2598063945770264, 0.1835893839597702, 0.99027419090271, 0.0014718662714585662, 0.05761602520942688, 0.7520503401756287, 0.023628095164895058, 0.15128891170024872, 0.7915010452270508, 0.0005570290377363563, 0.9696894288063049, 0.23200207948684692, 0.5416104793548584, 0.032183025032281876, 0.0011039836099371314, 8.797258487902582e-05, 0.0016177912475541234, 0.004669001325964928, 0.005396442022174597, 0.00051771110156551, 0.0007556205382570624, 0.007634391542524099, 0.0009015309624373913, 2.943364381790161, 0.017758769914507866, 0.028518103063106537, 0.00030214988510124385, 0.020879119634628296, 0.0028278145473450422, 0.0009950215462595224, 0.006161503493785858, 0.2904362380504608, 0.0004533693427219987, 0.5688651204109192, 0.0022555168252438307, 1.6926774978637695, 3.541337728500366, 1.2610138654708862, 0.380769819021225, 0.976841926574707, 0.21846112608909607, 0.000979539705440402, 0.0015281677478924394, 0.01708257757127285, 0.11762820929288864, 0.21288354694843292, 0.010532978922128677, 0.15096217393875122, 0.0010644490830600262, 4.275059700012207, 0.11841512471437454, 0.6050955653190613, 6.055124282836914, 0.05341324955224991, 1.7589964866638184, 0.016042560338974, 0.0028067738749086857, 0.7729501724243164, 3.0454978942871094, 0.005466512870043516, 1.4402564764022827, 0.04472146928310394, 7.281580924987793, 2.0297634601593018, 0.0025128712877631187, 0.01568751037120819, 0.0010367024224251509, 0.13267658650875092, 0.0012677022023126483, 1.0340900421142578, 0.0023247378412634134, 0.2905362844467163, 0.002643188228830695, 2.5466411113739014, 0.10955669730901718, 2.145662784576416, 0.03284147381782532, 0.009272251278162003, 0.3050839602947235, 0.01266860868781805, 0.058449339121580124, 1.4129787683486938, 0.005802453961223364, 0.00369437737390399, 0.013313560746610165, 0.0009005781612358987, 1.070143222808838, 2.5214009284973145, 9.235360145568848, 0.494171142578125, 4.442967891693115, 0.4019629657268524, 1.9771836996078491, 0.32261091470718384, 0.007637467235326767, 0.9558711051940918, 0.22663846611976624, 7.102785587310791, 6.71352481842041, 0.13189737498760223, 0.01814129762351513, 0.8429116010665894, 1.8234776258468628, 3.492260217666626, 0.790324330329895, 0.2718084454536438, 0.23802198469638824, 0.003978790249675512, 0.16575147211551666, 11.833624839782715, 0.16591618955135345, 0.17986206710338593, 0.0055458261631429195, 0.3585289418697357, 0.0036454431246966124, 2.133877992630005, 0.31397566199302673, 1.0533976554870605, 0.704717218875885, 3.654963254928589, 1.120943307876587, 7.829639911651611, 5.781947135925293, 0.042878516018390656, 6.120713233947754, 5.188594818115234, 0.07457316666841507, 3.632713794708252, 6.4895734786987305, 0.4927110970020294, 1.2006816864013672, 2.562880039215088, 0.38893187046051025, 1.137752890586853, 0.0018385426374152303, 0.3005598783493042, 2.089118003845215, 0.2551044225692749, 0.07338226586580276, 0.047065094113349915, 0.02605692483484745, 0.018409812822937965, 0.9033674001693726, 0.2610416114330292, 0.017412647604942322, 11.486262321472168, 9.762709617614746, 1.8505489826202393, 11.85927963256836, 1.3508386611938477, 1.4082854986190796, 0.7373759746551514, 0.00895732268691063, 0.32203832268714905, 0.0003631647559814155, 0.014993708580732346, 0.27039939165115356, 0.004002774599939585, 0.11479420959949493, 0.00916334893554449, 5.015369415283203, 1.524304986000061, 1.3896381855010986, 8.086215019226074, 1.8132929801940918, 0.019564513117074966, 7.052774906158447, 1.1906909942626953, 7.081691265106201, 0.6857237219810486, 0.003565622726455331, 0.01653151400387287, 0.7350987195968628, 0.9738313555717468, 0.336582750082016, 0.3849683403968811, 5.7063703536987305, 0.40877997875213623, 0.002282991772517562, 0.0037664680276066065, 0.1660735160112381, 0.1385461837053299, 0.002397878561168909, 0.0002044230350293219, 0.06832226365804672, 0.003880472155287862, 0.32116663455963135, 0.13392199575901031, 0.0028368488419800997, 0.002273952355608344, 0.0007420408655889332, 0.00018976318824570626, 0.0092361094430089, 0.0003721021639648825, 0.0013830630341544747, 0.0005029367166571319, 0.25371885299682617, 0.004007167648524046, 15.061026573181152, 0.029593611136078835, 8.691543579101562, 1.1829710006713867, 0.012860337272286415, 0.9301536083221436, 0.0015298341168090701, 0.003188884584233165, 0.3979724645614624, 0.8979056477546692, 0.0021371636539697647, 0.17863911390304565, 0.06433705240488052, 0.0007384672062471509, 0.04878046736121178, 1.7046782886609435e-05, 0.004256949760019779, 0.00524691678583622, 0.3824421167373657, 0.0015591143164783716, 0.0004892344586551189, 0.002014156896620989, 0.640900731086731, 0.005283796228468418, 0.008050612173974514, 0.0012531769461929798, 0.00031799983116798103, 0.034635063260793686, 0.06391610950231552, 0.002556034829467535, 0.5099653601646423, 0.0003067976504098624, 0.001257344032637775, 0.00035446559195406735, 0.04389968141913414, 0.003462512046098709, 0.9078320264816284, 0.006725413724780083, 0.5767203569412231, 0.9164893627166748, 0.2063876986503601, 0.2067936807870865, 0.00032145579461939633, 0.0001102625101339072, 0.0005729940603487194, 0.0013013951247557998, 0.0028028511442244053, 0.23761138319969177, 2.027841091156006, 0.6806821227073669, 7.578509330749512, 7.880727291107178, 0.5352344512939453, 2.706234931945801, 1.3803693056106567, 0.17478016018867493, 0.011698336340487003, 0.04247711971402168, 0.048449575901031494, 0.0017714299028739333, 5.134828567504883, 0.10553178191184998, 0.7957468628883362, 0.7811497449874878, 0.25955694913864136, 0.0727454423904419, 7.434291362762451, 0.18680551648139954, 4.9988017082214355, 9.935079574584961, 0.344137966632843, 6.096242427825928, 1.5712672472000122, 0.0013869914691895247, 0.0038791659753769636, 1.675551176071167, 0.15461668372154236, 0.0014488924061879516, 0.0003045333724003285, 0.06425075232982635, 0.0031708222813904285, 0.2664250433444977, 0.003941031638532877, 0.01026446558535099, 0.00045813556062057614, 0.001166854752227664, 0.01743701472878456, 0.0002584123576525599, 0.003457641461864114, 0.020596904680132866, 0.05025169253349304, 0.000724887300748378, 0.006422829814255238, 0.12891989946365356, 0.017127107828855515, 0.011645196937024593, 0.002873579505831003, 15.025128364562988, 0.1084240972995758, 0.010265881195664406, 0.000720956246368587, 0.0008454324561171234, 0.0015432840446010232, 0.025814039632678032, 0.012821970507502556, 8.160237312316895, 0.0011503038695082068, 5.788867950439453, 0.0033094894606620073, 0.005120615474879742, 0.0006690170848742127, 0.22546011209487915, 3.218599158572033e-05, 7.092700980138034e-05, 0.003561346558853984, 0.0005243833875283599, 0.00011646069469861686, 0.03904024139046669, 0.0005947966128587723, 0.01298319362103939, 0.0010355116100981832, 0.03540213778614998, 0.0007128558354452252, 0.0030607779044657946, 8.594620157964528e-05, 4.792098479811102e-05, 0.0007744412287138402, 0.00034791138023138046, 0.1828223466873169, 0.32692229747772217, 0.054980698972940445, 0.16217632591724396, 0.0006163604557514191, 0.0006735440110787749, 0.009371339343488216, 0.0011018402874469757, 3.909505844116211, 2.4928343296051025, 0.22566017508506775, 7.757230758666992, 0.904852032661438, 0.005460940767079592, 0.0017498909728601575, 0.45829087495803833, 0.06347864866256714, 0.16344286501407623, 1.77726149559021, 0.022963082417845726, 0.148712620139122, 0.3148076832294464, 3.7633533477783203, 6.183507442474365, 0.1742970049381256, 2.237128734588623, 4.538725852966309, 0.5947341918945312, 0.1507757008075714, 1.0251219272613525, 4.231703758239746, 9.741634368896484, 8.304247856140137, 0.8171319961547852, 0.012018527835607529, 0.0008760428754612803, 0.7538920640945435, 0.03375422954559326, 0.5605875253677368, 0.009123304858803749, 0.03263138607144356, 0.7920892834663391, 0.006535702850669622, 1.9576035737991333, 0.019991805776953697, 0.0397881343960762, 4.868468284606934, 0.9831074476242065, 0.018613314256072044, 0.02297857776284218, 0.4092073142528534, 0.024444719776511192, 0.12283331900835037, 0.10178878158330917, 2.1762311458587646, 0.25880205631256104, 0.0014693664852529764, 0.0217581819742918, 0.001671466277912259, 11.013932228088379, 1.9765872955322266, 0.5634450316429138, 0.6884610056877136, 0.07229187339544296, 0.00041631137719377875, 8.758684158325195, 0.3169921636581421, 0.12192768603563309, 0.12404545396566391, 0.19380038976669312, 0.0025598397478461266, 2.3704304695129395, 0.01949787698686123, 0.10137071460485458, 0.00693225534632802, 1.346453070640564, 0.00648216949775815, 0.5920891165733337, 0.30761125683784485, 3.8414738178253174, 0.21796314418315887, 0.02804424799978733, 0.06732608377933502, 0.06329985707998276, 0.06431547552347183, 0.001490316353738308, 0.0917031466960907, 0.06218495965003967, 1.2823399305343628, 0.1261129230260849, 0.08208420127630234, 0.003927020356059074, 0.0035121680703014135, 0.1446543037891388, 0.0015817285748198628, 0.06966341286897659, 0.029600324109196663, 1.7696847915649414, 4.772102355957031, 0.2724061608314514, 0.008632130920886993, 2.9936840534210205, 0.04015099257230759, 0.07470260560512543, 0.0021634523291140795, 0.0033410938922315836, 0.28520333766937256, 0.002842554822564125, 2.9906723499298096, 3.731140613555908, 0.29857349395751953, 0.07795587182044983, 0.48902422189712524, 0.024230647832155228, 0.5852993130683899, 0.004078523721545935, 1.5437309741973877, 0.14815141260623932, 7.3670783042907715, 0.48644477128982544, 8.001540184020996, 1.1641314029693604, 0.02092861942946911, 5.693403244018555, 0.05125663802027702, 0.3546053469181061, 4.964481353759766, 5.20483922958374, 1.2226676940917969, 0.7787058353424072, 0.0030269066337496042, 2.4134511947631836, 0.039405547082424164, 6.020589828491211, 10.137667655944824, 0.17680279910564423, 0.830873966217041, 1.3913600444793701, 0.26253873109817505, 0.30726632475852966, 0.001292942208237946, 1.5529899597167969, 2.3181140422821045, 0.03447948023676872, 5.88485050201416, 1.140701174736023, 6.560232162475586, 6.520287990570068, 0.1597450226545334, 1.6155089139938354, 0.16646640002727509, 0.049461666494607925, 0.0030711174476891756, 7.8635573387146, 0.4338708519935608, 0.0013550871517509222, 0.005397627595812082, 0.0007321537705138326, 2.220344066619873, 1.4188683032989502, 2.077694892883301, 0.0030409307219088078, 0.023482901975512505, 1.6139737367630005, 0.1224723681807518, 0.034719351679086685, 0.006644062697887421, 4.338037967681885, 3.50416898727417, 8.57374095916748, 0.13536079227924347, 0.0018299752846360207, 0.010605285875499249, 0.0012903229799121618, 4.373175621032715, 0.2017698884010315, 2.081063747406006, 0.24561429023742676, 2.509530544281006, 0.22902099788188934, 4.659517288208008, 2.485172986984253, 3.357473850250244, 0.4271472096443176, 0.06526362150907516, 0.5774276852607727, 0.08758144080638885, 0.0741177648305893, 0.36058688163757324, 1.8344959020614624, 0.19233858585357666, 2.233283758163452, 0.1622292399406433, 0.5341155529022217, 0.020163560286164284, 0.13763733208179474, 0.0009479796281084418, 0.367675244808197, 0.18558627367019653, 0.1630728840827942, 0.0003262225945945829, 0.008776542730629444, 0.7169497609138489, 0.004523165058344603, 4.61142110824585, 0.5674894452095032, 0.0550866425037384, 0.8033467531204224, 0.13231906294822693, 0.0038887844420969486, 1.7124874591827393, 0.0020898187067359686, 0.9397075772285461, 0.0028247239533811808, 0.004612757824361324, 0.005728020798414946, 0.47983306646347046, 1.7003567218780518, 5.742242813110352, 2.175626516342163, 0.25671520829200745, 2.1866023540496826, 0.018553519621491432, 2.289531707763672, 6.347573280334473, 0.42354899644851685, 0.31790071725845337, 0.02005525305867195, 0.12263203412294388, 0.04115217924118042, 0.2132263034582138, 1.1361019611358643, 2.642930030822754, 6.036240577697754, 0.02378525771200657, 0.11229262501001358, 0.05106533318758011, 0.3921520411968231, 0.00033802041434682906, 0.0022695516236126423, 0.15350395441055298, 0.10364827513694763, 0.364859938621521, 1.2477688789367676, 0.004083035048097372, 8.039987564086914, 0.3342653512954712, 0.0012759171659126878, 0.010327942669391632, 0.0014399646315723658, 3.5133140087127686, 2.2340569496154785, 3.0631790161132812, 0.000942858459893614, 2.6357247829437256, 2.8013839721679688, 0.8668656349182129, 0.010071411728858948, 1.0168161392211914, 0.978556752204895, 1.230711817741394, 0.08852546662092209, 0.8543406128883362 ], "kl": [], "kl_count": 0, "ok": true, "seconds": 1.6894906880000065, "gpu_observation": { "scope": "whole_device", "sample_interval_seconds": 0.1, "samples": 16, "start_used_bytes": 30730846208, "peak_used_bytes": 33393115136, "end_used_bytes": 33393115136, "max_temperature_c": 38, "max_power_w": 77.746 } } }, { "id": "validation-001", "max_abs_token_nll_change": 0.0, "result": { "id": "validation-001", "nll_sum": 2708.6642821885343, "scored_tokens": 1023, "token_nll": [ 9.52013874053955, 12.002548217773438, 2.2489867210388184, 7.736156940460205, 6.982629776000977, 7.083405494689941, 2.1704444885253906, 5.45363712310791, 0.6513281464576721, 0.49308276176452637, 0.5399232506752014, 0.26831501722335815, 4.537866592407227, 5.843966007232666, 1.4133515357971191, 0.48656949400901794, 0.48293280601501465, 8.686925888061523, 2.7491402626037598, 9.830156326293945, 1.415894627571106, 0.6228544116020203, 0.00804316159337759, 3.3856637477874756, 0.9824801683425903, 4.771305084228516, 1.818648099899292, 3.347621202468872, 3.4381213188171387, 1.0415493249893188, 4.06350564956665, 2.4780726432800293, 7.780920505523682, 1.0956788063049316, 6.527156829833984, 4.360079765319824, 3.169369697570801, 4.763039588928223, 2.159048557281494, 4.061174392700195, 0.10069207847118378, 0.8176100850105286, 0.6291301250457764, 1.9400837421417236, 0.8991982936859131, 3.692934989929199, 0.35381919145584106, 2.080996036529541, 0.008905455470085144, 0.6580124497413635, 2.7285499572753906, 7.402239799499512, 4.475652694702148, 1.0794143676757812, 0.004874963779002428, 1.2453209161758423, 7.106649875640869, 2.986445903778076, 2.411364793777466, 3.4426541328430176, 9.921222686767578, 1.1427440643310547, 0.9658037424087524, 0.24190396070480347, 0.03176495060324669, 4.765766143798828, 0.302436888217926, 0.04982035234570503, 0.05247845500707626, 0.016262026503682137, 1.9010759592056274, 1.2622658014297485, 4.2578325271606445, 0.39999738335609436, 0.04387321323156357, 0.0128423310816288, 2.3639163970947266, 0.19565477967262268, 2.530421495437622, 1.1852247714996338, 9.045199394226074, 2.6389167308807373, 2.99233341217041, 2.3178443908691406, 4.4180908203125, 2.601801872253418, 6.5359673500061035, 1.722730040550232, 0.5832419991493225, 4.109786033630371, 1.567089319229126, 0.24157801270484924, 0.62485271692276, 2.3370957374572754, 6.058615207672119, 2.318607807159424, 0.035741399973630905, 0.01756657287478447, 1.1621527671813965, 1.1670506000518799, 2.241014003753662, 1.362016201019287, 0.6177926659584045, 0.05230933427810669, 1.2536154985427856, 2.481278657913208, 0.06438980996608734, 2.299549102783203, 1.5497442483901978, 1.7808101177215576, 3.0146775245666504, 3.772451877593994, 1.4937748908996582, 0.1082671731710434, 5.252569198608398, 0.21217048168182373, 0.0007173825288191438, 0.008874026127159595, 0.37461069226264954, 0.03351124748587608, 2.449326515197754, 0.6379566788673401, 0.29393377900123596, 0.9443280696868896, 2.4744648933410645, 1.1931442022323608, 2.1352148056030273, 1.0994985103607178, 0.006576442625373602, 2.4022374153137207, 1.7561386823654175, 0.04525288939476013, 8.73627758026123, 1.6380332708358765, 2.191119909286499, 10.151932716369629, 5.58559513092041, 2.583759307861328, 0.9925829172134399, 5.943935871124268, 6.607207298278809, 2.514878749847412, 3.8420562744140625, 2.4225268363952637, 3.752835750579834, 4.020107746124268, 1.1709790229797363, 3.625967025756836, 0.21045805513858795, 2.719209909439087, 4.507439613342285, 6.339090347290039, 0.09589870274066925, 0.0006940339808352292, 0.00010215714428341016, 0.09014923125505447, 0.019719859585165977, 1.0698810815811157, 0.20958366990089417, 1.6655155420303345, 0.256828248500824, 0.5704367160797119, 3.806450366973877, 0.5062576532363892, 0.23394525051116943, 0.04056317359209061, 0.04961981251835823, 1.046252965927124, 0.09029911458492279, 0.1302337348461151, 4.780104160308838, 0.22014681994915009, 0.6323316097259521, 0.771594762802124, 0.015376473776996136, 9.026691436767578, 5.1473565101623535, 1.3199158906936646, 1.7405927181243896, 0.9141393303871155, 4.431920528411865, 0.027568893507122993, 0.14266568422317505, 4.015029430389404, 3.83156681060791, 0.09014759957790375, 0.02191493846476078, 5.682604789733887, 0.3811469078063965, 1.115633249282837, 1.4747169017791748, 0.001755721983499825, 0.3703579604625702, 1.4685672521591187, 0.09185029566287994, 1.7258481979370117, 7.845141410827637, 2.809051036834717, 9.446828842163086, 4.171982765197754, 2.538567543029785, 6.074326515197754, 4.786511421203613, 2.44923734664917, 3.678257465362549, 2.472917079925537, 8.326075553894043, 2.1317172050476074, 4.014934062957764, 0.4702262282371521, 4.5621232986450195, 6.087001800537109, 6.460465908050537, 0.7479300498962402, 4.620270252227783, 5.405426025390625, 0.5368247628211975, 7.860622882843018, 2.6945300102233887, 2.4484665393829346, 4.52205753326416, 0.4001796543598175, 0.18742570281028748, 0.015051954425871372, 2.708491086959839, 2.2739739418029785, 6.1473188400268555, 0.8747371435165405, 0.04701732471585274, 0.0035943682305514812, 5.706228256225586, 3.885612964630127, 0.005488445982336998, 0.02589663304388523, 4.088173866271973, 4.145544528961182, 0.4073341488838196, 0.03224685788154602, 1.4364643096923828, 6.434864044189453, 8.021523475646973, 0.041275039315223694, 0.3826826214790344, 1.5614206790924072, 0.3057628571987152, 4.033501625061035, 1.9354864358901978, 2.1410155296325684, 6.892793655395508, 0.42614108324050903, 7.289139747619629, 6.081064224243164, 6.611526966094971, 3.300208330154419, 0.1674354523420334, 0.22724175453186035, 6.141839504241943, 0.30532142519950867, 2.909240245819092, 3.7135496139526367, 0.0416128896176815, 2.6620123386383057, 6.482154846191406, 2.7307634353637695, 1.4799892902374268, 0.00386265991255641, 0.520313024520874, 1.2054260969161987, 0.06755954027175903, 1.430454969406128, 9.102563858032227, 6.219606399536133, 0.9666482210159302, 6.733099460601807, 4.165942668914795, 4.390035629272461, 0.6812820434570312, 4.699453353881836, 2.7545864582061768, 0.3632604479789734, 5.953596591949463, 3.083885431289673, 6.075586795806885, 0.04576011374592781, 5.4624247550964355, 3.7884788513183594, 2.235166549682617, 1.3392868041992188, 4.152890205383301, 3.6835975646972656, 0.6967176795005798, 5.094457626342773, 8.776686668395996, 0.3436611294746399, 0.21551911532878876, 0.003585103200748563, 6.546798229217529, 3.3380300998687744, 0.07188580185174942, 2.8937230110168457, 0.7332068085670471, 2.54687762260437, 3.7017414569854736, 2.196678876876831, 0.03479718416929245, 1.755513310432434, 6.002725601196289, 2.1665854454040527, 0.013308149762451649, 0.0007167869480326772, 0.0033199451863765717, 0.15833714604377747, 0.9343689680099487, 4.008975505828857, 3.011183977127075, 0.06343680620193481, 1.399986982345581, 7.201993942260742, 3.7578301429748535, 5.920069694519043, 2.7039589881896973, 0.011523472145199776, 0.005259842611849308, 0.011257699690759182, 8.002809524536133, 0.4713149070739746, 9.39640998840332, 2.0291097164154053, 0.016703981906175613, 5.01519775390625, 0.21186411380767822, 0.03029482252895832, 0.4888567328453064, 0.003639266826212406, 12.012791633605957, 3.4073851108551025, 1.3049319982528687, 2.0471951961517334, 5.640341758728027, 5.181522369384766, 1.0265579223632812, 5.712615013122559, 0.02745083160698414, 0.00933496467769146, 0.3627079427242279, 4.041610240936279, 2.598806381225586, 3.533130407333374, 6.305349826812744, 3.697490930557251, 4.097785949707031, 4.896533489227295, 0.10606230795383453, 3.846529483795166, 6.385831832885742, 8.872353553771973, 2.8422493934631348, 5.199817657470703, 4.3152971267700195, 1.1819381713867188, 0.010937736369669437, 0.22587409615516663, 0.001970613142475486, 2.647271156311035, 0.07964608073234558, 4.988974571228027, 1.4239040613174438, 6.392162322998047, 0.885921061038971, 1.6195621490478516, 2.7298929691314697, 0.056169018149375916, 0.7875789403915405, 0.023407327011227608, 5.160245895385742, 4.225543975830078, 5.985058307647705, 5.418036937713623, 0.0793180912733078, 1.556401014328003, 3.3264482021331787, 7.322381973266602, 2.9751129150390625, 4.969808101654053, 0.23901031911373138, 1.301679253578186, 0.004251252394169569, 2.4318398573086597e-05, 0.004957051947712898, 2.4136505126953125, 1.849637746810913, 0.11040934175252914, 0.15333062410354614, 2.0718252658843994, 5.133758068084717, 0.0014173472300171852, 4.864749431610107, 3.3845767974853516, 8.729533195495605, 0.6695420742034912, 6.517980098724365, 2.640540599822998, 3.370121955871582, 0.900827944278717, 10.78148078918457, 1.23940908908844, 0.6421162486076355, 8.539202690124512, 7.126778602600098, 0.02781854197382927, 0.12017833441495895, 3.43646502494812, 0.9788106679916382, 5.2755961418151855, 1.2839224338531494, 0.33191701769828796, 1.9831781387329102, 0.10616854578256607, 1.5108072757720947, 1.3412882089614868, 1.564582109451294, 0.19084884226322174, 0.26265910267829895, 0.0015185264637693763, 5.23579740524292, 1.8870505094528198, 0.02434990555047989, 1.5293018817901611, 6.365276336669922, 3.602062702178955, 4.072422027587891, 1.8939564228057861, 2.483518362045288, 0.9312278032302856, 2.547039747238159, 2.254912853240967, 0.01946946792304516, 0.0046170298010110855, 0.797711968421936, 9.116186141967773, 1.9671567678451538, 2.3043582439422607, 4.265352725982666, 1.1981608867645264, 0.0033617669250816107, 4.075191020965576, 6.724794864654541, 5.9395856857299805, 1.0080236196517944, 0.329090416431427, 0.909895122051239, 0.014253122732043266, 1.4007568359375, 1.2435028553009033, 0.04503490775823593, 1.347987174987793, 8.34876823425293, 5.598801136016846, 0.4839744567871094, 6.667313575744629, 2.858527898788452, 6.0194854736328125, 0.5928025245666504, 2.49735164642334, 5.852606296539307, 1.4082305431365967, 1.5884265899658203, 2.2071399688720703, 0.12587876617908478, 0.05913679301738739, 3.3802742958068848, 1.94303560256958, 0.04355975240468979, 4.200966835021973, 2.070208787918091, 0.2658049166202545, 4.885797500610352, 1.5254942178726196, 0.08532758802175522, 7.575620651245117, 2.584751605987549, 4.6125078201293945, 1.2440402507781982, 4.306641578674316, 6.79020881652832, 3.3119869232177734, 1.455621361732483, 4.180821418762207, 1.880200982093811, 0.9293673038482666, 4.681321620941162, 2.633932113647461, 8.6801176071167, 3.2953503131866455, 3.534891128540039, 8.849013328552246, 3.227917194366455, 8.627224922180176, 1.8106496334075928, 3.339733362197876, 0.1411752700805664, 2.240239381790161, 9.36635971069336, 0.39735886454582214, 0.0013435394503176212, 0.003944593947380781, 1.2565886974334717, 3.667970657348633, 2.355546236038208, 0.33492982387542725, 0.006448769476264715, 2.6963937282562256, 2.8459632396698, 0.36682966351509094, 0.021082470193505287, 0.5462791323661804, 3.415285110473633, 0.0002479245886206627, 0.00032395837479270995, 1.6813945770263672, 0.0017300175968557596, 2.6758229732513428, 4.184305667877197, 10.729701042175293, 0.4366479814052582, 4.48606014251709, 2.35199236869812, 5.4959025382995605, 8.303703308105469, 2.243964195251465, 0.9387340545654297, 2.0472323894500732, 6.15701961517334, 0.8106436729431152, 0.015663335099816322, 2.071047782897949, 4.933618068695068, 2.675654411315918, 9.778838157653809, 0.20248231291770935, 3.822169780731201, 4.957231521606445, 0.3530864417552948, 0.0017099058022722602, 2.0101895332336426, 0.07595711946487427, 3.4656996726989746, 1.4002364873886108, 7.853762626647949, 0.24471254646778107, 7.311191082000732, 1.3148590326309204, 6.424290657043457, 3.621105670928955, 10.533852577209473, 11.050724029541016, 0.431475430727005, 8.332113265991211, 7.237595558166504, 3.1808056831359863, 0.7082105278968811, 0.7707714438438416, 0.007443787530064583, 5.135791778564453, 9.666401863098145, 2.606980800628662, 4.553239822387695, 0.044174257665872574, 6.629244804382324, 0.01932811364531517, 3.3037967681884766, 2.5865366458892822, 2.6291961669921875, 9.131392478942871, 5.3236823081970215, 1.7625367641448975, 1.7612687349319458, 3.423616409301758, 5.459888458251953, 0.1729235202074051, 10.438714027404785, 4.035504341125488, 0.026906035840511322, 0.003479024861007929, 9.116545677185059, 0.21822510659694672, 7.562967300415039, 0.014614555984735489, 1.701676607131958, 1.4903544187545776, 9.701549530029297, 5.234119415283203, 0.0800316333770752, 0.0036674162838608027, 0.32970210909843445, 0.16542619466781616, 1.5382198095321655, 2.888974905014038, 3.7731289863586426, 2.411900520324707, 2.350750684738159, 3.102579116821289, 4.538564205169678, 3.6543455123901367, 5.9579572677612305, 0.841665506362915, 0.30645880103111267, 1.5495028495788574, 0.8527650833129883, 0.775176465511322, 4.83044958114624, 4.15620231628418, 3.8427681922912598, 1.3022202253341675, 0.002186290919780731, 0.34646540880203247, 8.748939514160156, 4.390158653259277, 1.5454524755477905, 0.061212360858917236, 4.8776631355285645, 5.541364669799805, 2.3010575771331787, 1.9942538738250732, 1.2197606563568115, 4.999879837036133, 9.67459774017334, 4.175739765167236, 0.8834035396575928, 3.6236062049865723, 1.0331720113754272, 1.617417335510254, 0.18709634244441986, 0.051303863525390625, 0.0035947246942669153, 0.7667838931083679, 2.165177583694458, 3.2173190116882324, 1.4046707153320312, 4.654242515563965, 3.0834295749664307, 0.024394812062382698, 3.847564935684204, 1.0801135301589966, 2.9893763065338135, 0.24650366604328156, 2.019948720932007, 6.755216598510742, 2.146714210510254, 7.214040279388428, 7.8522047996521, 0.00949143711477518, 1.3611578941345215, 7.005184173583984, 6.7560224533081055, 1.2511132955551147, 6.1092681884765625, 0.5832107067108154, 0.9563677310943604, 4.556482315063477, 1.4577008485794067, 0.010033882223069668, 5.412189960479736, 1.1718896627426147, 2.8535752296447754, 0.7666105031967163, 7.922016620635986, 0.4437370300292969, 0.4853051006793976, 1.8929996490478516, 5.172942161560059, 1.217557430267334, 4.61669921875, 3.7285876274108887, 3.132673740386963, 4.729178428649902, 0.16322274506092072, 2.800417900085449, 0.3483791947364807, 0.528766393661499, 0.0032086102291941643, 10.044210433959961, 9.167045593261719, 7.302003860473633, 2.3860082626342773, 3.244396209716797, 7.9691667556762695, 3.7876617908477783, 1.7936397790908813, 0.00299232080578804, 5.772668838500977, 12.293994903564453, 0.05114711448550224, 3.735440731048584, 4.050382137298584, 3.0286755561828613, 3.6071579456329346, 0.7645921111106873, 5.605196952819824, 0.08095257729291916, 1.7436370849609375, 0.8199892044067383, 0.20012736320495605, 2.8581531047821045, 5.08929967880249, 9.155624389648438, 3.307541847229004, 5.861201286315918, 5.962027549743652, 2.1287550926208496, 0.5144307613372803, 2.4444282054901123, 0.008435807190835476, 1.048807978630066, 1.3321068286895752, 5.015321254730225, 6.9317731857299805, 8.245481491088867, 3.149745464324951, 0.006526583340018988, 4.500169277191162, 6.417437553405762, 8.66666030883789, 3.3015878200531006, 1.6830711364746094, 2.9286909103393555, 8.99172592163086, 0.4224042296409607, 0.06270964443683624, 0.5819514393806458, 3.0353498458862305, 3.7481110095977783, 0.15250737965106964, 1.6968241930007935, 0.49913525581359863, 0.6820584535598755, 0.002999095479026437, 6.936326503753662, 0.24210704863071442, 0.5040844678878784, 0.41803231835365295, 4.3634772300720215, 0.5571714639663696, 1.0179333686828613, 0.12600195407867432, 8.262211799621582, 3.8760733604431152, 2.3318140506744385, 0.7588400840759277, 0.3598584830760956, 4.252270698547363, 4.130010604858398, 7.038558006286621, 3.388709783554077, 6.3310770988464355, 8.106330871582031, 0.11122968047857285, 2.0583372116088867, 0.6294359564781189, 6.721256256103516, 4.303809642791748, 6.255277156829834, 5.205979347229004, 1.839691162109375, 0.00540023623034358, 2.1055078506469727, 1.2659471035003662, 0.47792041301727295, 0.0022185256239026785, 0.7580448985099792, 4.191143989562988, 0.02819330058991909, 1.014024257659912, 1.3445292711257935, 0.027044234797358513, 1.2161685228347778, 0.44733965396881104, 1.538764476776123, 1.8816026449203491, 0.9103418588638306, 0.12783420085906982, 0.000644237850792706, 1.054574966430664, 0.0048888432793319225, 0.6401573419570923, 0.5371537804603577, 2.061683177947998, 0.9314525127410889, 4.157558441162109, 0.2316526472568512, 4.421121597290039, 1.0113506317138672, 0.7369930148124695, 5.017963409423828, 3.6184182167053223, 0.015175716951489449, 0.5572605729103088, 0.014065656810998917, 6.649529457092285, 9.322870254516602, 0.73996502161026, 1.9200117588043213, 0.4597889184951782, 4.970315933227539, 8.493839263916016, 0.043285008519887924, 0.0024915861431509256, 0.0028178293723613024, 0.0017830915749073029, 0.6787890791893005, 0.2686477601528168, 6.296095848083496, 0.11258047819137573, 2.661877155303955, 0.5130460858345032, 1.0482263565063477, 4.230615139007568, 0.010605758056044579, 2.126100540161133, 7.92125129699707, 0.07716592401266098, 2.815237045288086, 0.013848294503986835, 2.36625337600708, 2.677311897277832, 0.6388340592384338, 7.439090251922607, 6.081787586212158, 5.438658714294434, 0.6017703413963318, 4.865327835083008, 0.33695346117019653, 5.831650257110596, 0.3667183816432953, 0.004174566827714443, 2.9191391468048096, 0.15840990841388702, 0.008652222342789173, 0.8877695202827454, 0.15600532293319702, 1.324544906616211, 2.636751651763916, 0.03006337769329548, 0.06593658775091171, 0.0004462200158741325, 0.03153650090098381, 0.00023457636416424066, 1.4341776371002197, 1.7278242111206055, 1.4976445436477661, 1.7852144241333008, 3.663688898086548, 0.5727733373641968, 10.008241653442383, 0.3660668432712555, 6.698319911956787, 5.79608678817749, 0.5687733292579651, 5.901826858520508, 1.2719409465789795, 0.47378110885620117, 3.4874439239501953, 1.9948428869247437, 4.514664649963379, 1.4556742906570435, 1.1260191202163696, 0.2614597678184509, 6.348115921020508, 1.107658863067627, 0.21057626605033875, 4.469581604003906, 0.7620528936386108, 2.079153299331665, 10.954011917114258, 0.858432412147522, 0.07731357216835022, 2.9709668159484863, 1.1911300420761108, 1.7320449352264404, 2.754031181335449, 4.077364444732666, 4.807592868804932, 0.30890265107154846, 2.7425730228424072, 0.5119137167930603, 5.150711536407471, 2.3792307376861572, 0.12903495132923126, 0.0034893599804490805, 1.4003878831863403, 0.7525083422660828, 0.0027019204571843147, 0.9180143475532532, 0.9478853940963745, 2.246816396713257, 3.6573193073272705, 4.031844139099121, 0.10876214504241943, 0.11821845173835754, 0.7130547165870667, 1.147607684135437, 0.37226980924606323, 1.686330795288086, 0.49717944860458374, 0.4750874936580658, 3.8958003520965576, 5.7793402671813965, 1.0843322277069092, 2.8691487312316895, 1.3456509113311768, 0.2756218910217285, 3.369493007659912, 5.262314319610596, 7.03236722946167, 2.123485803604126, 0.00014995403762441128, 1.9857962131500244, 0.00821745302528143, 3.265122890472412, 0.1462668627500534, 0.007099401671439409, 0.22979623079299927, 1.8143384456634521, 2.4211177825927734, 2.415961742401123, 8.10753059387207, 1.218666434288025, 2.2713751792907715, 1.544224500656128, 1.446012258529663, 0.2787134647369385, 1.5722310543060303, 0.18628330528736115, 2.936514139175415, 0.09915175288915634, 4.222903728485107, 1.8265373706817627, 4.7434773445129395, 0.5559893846511841, 0.051736149936914444, 0.6992998123168945, 0.42817407846450806, 0.005474811885505915, 0.0005536930402740836, 0.013994187116622925, 0.00807272456586361, 2.0782694816589355, 1.600785255432129, 0.8166961073875427, 1.622453212738037, 1.3907418251037598, 0.5150018334388733, 0.16946382820606232, 0.011929478496313095, 7.237950801849365, 8.364789009094238, 2.4797284603118896, 0.23961369693279266, 0.0038330908864736557, 2.5522818565368652, 5.453946113586426, 0.8815202713012695, 2.2606585025787354, 2.3193674087524414, 0.19431063532829285, 1.0410575866699219, 5.070443153381348, 0.03394357115030289, 11.661639213562012, 2.4861207008361816, 1.0653032064437866, 0.00987324584275484, 1.1326056718826294, 9.483842849731445, 1.2448152303695679, 0.5639122724533081, 2.0200963020324707, 5.458817005157471, 2.3934128284454346, 0.0070048230700194836, 2.1522297859191895, 0.09415971487760544, 0.0020548435859382153, 1.0018270015716553, 2.6017065048217773, 0.9130451679229736, 9.343354225158691, 4.285202980041504, 3.3567657470703125, 2.950617551803589, 3.1496472358703613, 0.004577515181154013, 6.355692386627197, 0.3454168140888214, 1.8655452728271484, 4.214688777923584, 2.7833943367004395, 0.033402301371097565, 0.00014876213390380144, 1.198670744895935, 1.395187258720398, 3.8425283432006836, 6.181229114532471, 1.874624490737915, 0.4393899738788605, 2.459090232849121, 2.831347942352295, 2.8265445232391357, 2.639456272125244, 0.948559045791626, 0.0834505707025528, 5.866178035736084, 0.4298786520957947, 6.117578506469727, 1.6945059299468994, 0.18276287615299225, 0.003473560092970729, 0.4372120797634125, 1.778968095779419, 0.037916433066129684, 7.28893518447876, 6.342991352081299, 2.370009422302246 ], "kl": [], "kl_count": 0, "ok": true, "seconds": 0.07583724300000227, "gpu_observation": { "scope": "whole_device", "sample_interval_seconds": 0.1, "samples": 1, "start_used_bytes": 33393115136, "peak_used_bytes": 33393115136, "end_used_bytes": 33393115136, "max_temperature_c": 38, "max_power_w": 63.586 } } } ], "repeat_max_nll_delta": 0.0, "ready": { "ready": true, "load_seconds": 40.002837723999995, "engine": "vllm", "version": "0.26.0", "enforce_eager": true, "prefix_cache": false, "max_model_len": 8192, "gpu_memory_utilization": 0.8, "dtype": "bfloat16", "kv_cache_dtype": "auto", "gpu_during_load": { "scope": "whole_device", "sample_interval_seconds": 0.1, "samples": 504, "start_used_bytes": 3134623744, "peak_used_bytes": 31034703872, "end_used_bytes": 30730846208, "max_temperature_c": 41, "max_power_w": 119.236 } }, "scope": "Same pre-existing validation examples; final outputs unchanged" }, "quality_policy": "Keep all durable final outputs; resume only unanswered request IDs", "performance_policy": "All nine configurations measured in a dedicated serial pass after reboot; original workload-tail measurements retained" }, "review": { "completed_unix": 1790046920.30222, "audit": "results/final-review-audit.json", "cases": "FAILURE-CASES.json", "reviewed_plots": [ "plots/quality-delta.png", "plots/task-delta.png", "plots/block-errors.png" ], "not_an_independent_reproduction": true, "no_final_set_retuning": true } }