[ { "precision": "FP16", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 1024, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 325.890395, "ttft_ms_q25": 311.19831550000004, "ttft_ms_q75": 326.66981050000004, "prefill_tokens_per_s": 3093.0644642042917, "prefill_tokens_per_s_q25": 3085.7021551973376, "prefill_tokens_per_s_q75": 3246.32784786788, "handoff_plus_first_decode_ms": 50.954615000000004, "handoff_plus_first_decode_ms_q25": 50.2046335, "handoff_plus_first_decode_ms_q75": 53.7457075, "steady_tpot_ms": 38.1771325, "steady_tpot_ms_q25": 37.48788975, "steady_tpot_ms_q75": 38.18753075, "steady_tpot_p95_ms": 40.4363914, "steady_tpot_p95_ms_q25": 39.8993059, "steady_tpot_p95_ms_q75": 41.703343125, "decode_tokens_per_s": 25.393661298258756, "decode_tokens_per_s_q25": 25.317242057478495, "decode_tokens_per_s_q75": 25.67713548203487, "decode_ms": 590.698593, "decode_ms_q25": 584.248521, "decode_ms_q75": 592.486995, "end_to_end_ms": 916.588988, "end_to_end_ms_q25": 895.4468365, "end_to_end_ms_q75": 919.1568055, "output_tokens_per_s_e2e": 17.456024684424857, "output_tokens_per_s_e2e_q25": 17.407394231742355, "output_tokens_per_s_e2e_q75": 17.878140595271027, "all_tokens_per_s_e2e": 1117.1855798031909, "all_tokens_per_s_e2e_q25": 1114.0732308315107, "all_tokens_per_s_e2e_q75": 1144.2009980973457, "energy_net_psys_j": 28.611885485606145, "energy_net_psys_j_q25": 28.54256725633332, "energy_net_psys_j_q75": 28.867325691640882, "prefill_energy_net_psys_j": 9.723831623047879, "prefill_energy_net_psys_j_q25": 9.373954739550998, "prefill_energy_net_psys_j_q75": 9.876530738910652, "decode_energy_net_psys_j": 19.09353604290219, "decode_energy_net_psys_j_q25": 18.921476723457403, "decode_energy_net_psys_j_q75": 19.34067183622711, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "FP16", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 1024, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 325.890395, "ttft_ms_q25": 311.19831550000004, "ttft_ms_q75": 326.66981050000004, "prefill_tokens_per_s": 3093.0644642042917, "prefill_tokens_per_s_q25": 3085.7021551973376, "prefill_tokens_per_s_q75": 3246.32784786788, "handoff_plus_first_decode_ms": 44.62128, "handoff_plus_first_decode_ms_q25": 43.1801255, "handoff_plus_first_decode_ms_q75": 75.318545, "steady_tpot_ms": 37.889624, "steady_tpot_ms_q25": 37.802927249999996, "steady_tpot_ms_q75": 37.9378635, "steady_tpot_p95_ms": 38.696261199999995, "steady_tpot_p95_ms_q25": 38.664764500000004, "steady_tpot_p95_ms_q75": 38.79098082499999, "decode_tokens_per_s": 25.956409774464063, "decode_tokens_per_s_q25": 24.74549161705678, "decode_tokens_per_s_q75": 26.05161174600982, "decode_ms": 577.89194, "decode_ms_q25": 575.787804, "decode_ms_q75": 607.6260605, "end_to_end_ms": 903.782335, "end_to_end_ms_q25": 902.4576145000001, "end_to_end_ms_q75": 918.824376, "output_tokens_per_s_e2e": 17.70337766117104, "output_tokens_per_s_e2e_q25": 17.41822453030541, "output_tokens_per_s_e2e_q75": 17.729402709710655, "all_tokens_per_s_e2e": 1133.0161703149465, "all_tokens_per_s_e2e_q25": 1114.7663699395462, "all_tokens_per_s_e2e_q75": 1134.681773421482, "energy_net_psys_j": 23.784934389537618, "energy_net_psys_j_q25": 23.55439354148898, "energy_net_psys_j_q75": 24.002253128439733, "prefill_energy_net_psys_j": 9.723831623047879, "prefill_energy_net_psys_j_q25": 9.373954739550998, "prefill_energy_net_psys_j_q75": 9.876530738910652, "decode_energy_net_psys_j": 13.75570453476419, "decode_energy_net_psys_j_q25": 13.677862802578328, "decode_energy_net_psys_j_q75": 14.47559927302596, "fastest_e2e": true, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0141700634146606, "energy_reduction_vs_gpu": 0.16870440427621713 }, { "precision": "FP16", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 1024, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 372.282625, "ttft_ms_q25": 371.818852, "ttft_ms_q75": 373.5173075, "prefill_tokens_per_s": 2707.6203193742926, "prefill_tokens_per_s_q25": 2698.6996157778267, "prefill_tokens_per_s_q75": 2711.0017762854477, "handoff_plus_first_decode_ms": 44.62128, "handoff_plus_first_decode_ms_q25": 43.1801255, "handoff_plus_first_decode_ms_q75": 75.318545, "steady_tpot_ms": 37.889624, "steady_tpot_ms_q25": 37.802927249999996, "steady_tpot_ms_q75": 37.9378635, "steady_tpot_p95_ms": 38.696261199999995, "steady_tpot_p95_ms_q25": 38.664764500000004, "steady_tpot_p95_ms_q75": 38.79098082499999, "decode_tokens_per_s": 25.956409774464063, "decode_tokens_per_s_q25": 24.74549161705678, "decode_tokens_per_s_q75": 26.05161174600982, "decode_ms": 577.89194, "decode_ms_q25": 575.787804, "decode_ms_q75": 607.6260605, "end_to_end_ms": 950.174565, "end_to_end_ms_q25": 949.3051115000001, "end_to_end_ms_q75": 979.4449125, "output_tokens_per_s_e2e": 16.839011050564164, "output_tokens_per_s_e2e_q25": 16.35038584129911, "output_tokens_per_s_e2e_q75": 16.85444777204438, "all_tokens_per_s_e2e": 1077.6967072361065, "all_tokens_per_s_e2e_q25": 1046.4246938431431, "all_tokens_per_s_e2e_q75": 1078.6846574108404, "energy_net_psys_j": 20.752796211965624, "energy_net_psys_j_q25": 20.599269091711488, "energy_net_psys_j_q75": 21.559474802474096, "prefill_energy_net_psys_j": 6.9970916772014355, "prefill_energy_net_psys_j_q25": 6.92140628913316, "prefill_energy_net_psys_j_q75": 7.083875529448138, "decode_energy_net_psys_j": 13.75570453476419, "decode_energy_net_psys_j_q25": 13.677862802578328, "decode_energy_net_psys_j_q75": 14.47559927302596, "fastest_e2e": false, "lowest_energy": true, "e2e_speedup_vs_gpu": 0.9646532561098391, "energy_reduction_vs_gpu": 0.2746791810555167 }, { "precision": "FP16", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 1024, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 372.282625, "ttft_ms_q25": 371.818852, "ttft_ms_q75": 373.5173075, "prefill_tokens_per_s": 2707.6203193742926, "prefill_tokens_per_s_q25": 2698.6996157778267, "prefill_tokens_per_s_q75": 2711.0017762854477, "handoff_plus_first_decode_ms": 50.954615000000004, "handoff_plus_first_decode_ms_q25": 50.2046335, "handoff_plus_first_decode_ms_q75": 53.7457075, "steady_tpot_ms": 38.1771325, "steady_tpot_ms_q25": 37.48788975, "steady_tpot_ms_q75": 38.18753075, "steady_tpot_p95_ms": 40.4363914, "steady_tpot_p95_ms_q25": 39.8993059, "steady_tpot_p95_ms_q75": 41.703343125, "decode_tokens_per_s": 25.393661298258756, "decode_tokens_per_s_q25": 25.317242057478495, "decode_tokens_per_s_q75": 25.67713548203487, "decode_ms": 590.698593, "decode_ms_q25": 584.248521, "decode_ms_q75": 592.486995, "end_to_end_ms": 962.9812179999999, "end_to_end_ms_q25": 956.067373, "end_to_end_ms_q75": 966.0043025, "output_tokens_per_s_e2e": 16.61506964095327, "output_tokens_per_s_e2e_q25": 16.56323544083485, "output_tokens_per_s_e2e_q75": 16.736097496587764, "all_tokens_per_s_e2e": 1063.3644570210092, "all_tokens_per_s_e2e_q25": 1060.0470682134303, "all_tokens_per_s_e2e_q75": 1071.1102397816169, "energy_net_psys_j": 26.090627720103626, "energy_net_psys_j_q25": 25.842883012590566, "energy_net_psys_j_q75": 26.424547365675245, "prefill_energy_net_psys_j": 6.9970916772014355, "prefill_energy_net_psys_j_q25": 6.92140628913316, "prefill_energy_net_psys_j_q75": 7.083875529448138, "decode_energy_net_psys_j": 19.09353604290219, "decode_energy_net_psys_j_q25": 18.921476723457403, "decode_energy_net_psys_j_q75": 19.34067183622711, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.9518243667344299, "energy_reduction_vs_gpu": 0.0881192456460409 }, { "precision": "FP16", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 2048, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 675.668286, "ttft_ms_q25": 659.165996, "ttft_ms_q75": 682.1660465, "prefill_tokens_per_s": 3007.3928910139794, "prefill_tokens_per_s_q25": 2979.017189348549, "prefill_tokens_per_s_q75": 3084.6165916339287, "handoff_plus_first_decode_ms": 52.48689, "handoff_plus_first_decode_ms_q25": 51.409315, "handoff_plus_first_decode_ms_q75": 55.5279295, "steady_tpot_ms": 39.0188285, "steady_tpot_ms_q25": 38.6505935, "steady_tpot_ms_q75": 39.2626815, "steady_tpot_p95_ms": 42.16395955, "steady_tpot_p95_ms_q25": 40.912344225, "steady_tpot_p95_ms_q75": 42.25459372500001, "decode_tokens_per_s": 24.565094490356536, "decode_tokens_per_s_q25": 24.533837957858978, "decode_tokens_per_s_q75": 25.002701786304478, "decode_ms": 610.622524, "decode_ms_q25": 600.119001, "decode_ms_q75": 611.4014599999998, "end_to_end_ms": 1278.279285, "end_to_end_ms_q25": 1266.5616934999998, "end_to_end_ms_q75": 1282.2850475, "output_tokens_per_s_e2e": 12.516826477400047, "output_tokens_per_s_e2e_q25": 12.477846618621317, "output_tokens_per_s_e2e_q75": 12.63370717939705, "all_tokens_per_s_e2e": 1602.153789107206, "all_tokens_per_s_e2e_q25": 1597.1643671835286, "all_tokens_per_s_e2e_q75": 1617.1145189628223, "energy_net_psys_j": 38.90379185876414, "energy_net_psys_j_q25": 38.74473017794784, "energy_net_psys_j_q75": 38.95703506467657, "prefill_energy_net_psys_j": 20.457327416992882, "prefill_energy_net_psys_j_q25": 20.131066499921666, "prefill_energy_net_psys_j_q75": 20.46961047374662, "decode_energy_net_psys_j": 18.42189832826378, "decode_energy_net_psys_j_q25": 18.275119704201217, "decode_energy_net_psys_j_q75": 18.813685508001164, "fastest_e2e": true, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "FP16", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 2048, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 675.668286, "ttft_ms_q25": 659.165996, "ttft_ms_q75": 682.1660465, "prefill_tokens_per_s": 3007.3928910139794, "prefill_tokens_per_s_q25": 2979.017189348549, "prefill_tokens_per_s_q75": 3084.6165916339287, "handoff_plus_first_decode_ms": 74.54955100000001, "handoff_plus_first_decode_ms_q25": 74.254223, "handoff_plus_first_decode_ms_q75": 93.40781150000001, "steady_tpot_ms": 39.836130999999995, "steady_tpot_ms_q25": 39.81991425, "steady_tpot_ms_q75": 39.872106249999995, "steady_tpot_p95_ms": 40.625068549999995, "steady_tpot_p95_ms_q25": 40.58012035, "steady_tpot_p95_ms_q75": 41.282516625, "decode_tokens_per_s": 23.642482614634385, "decode_tokens_per_s_q25": 22.952130755722422, "decode_tokens_per_s_q75": 23.663828295749074, "decode_ms": 634.451138, "decode_ms_q25": 633.8793544999999, "decode_ms_q75": 654.1258695, "end_to_end_ms": 1316.4643070000002, "end_to_end_ms_q25": 1312.720082, "end_to_end_ms_q75": 1319.789626, "output_tokens_per_s_e2e": 12.153766657343942, "output_tokens_per_s_e2e_q25": 12.123221199944373, "output_tokens_per_s_e2e_q75": 12.188531574297329, "all_tokens_per_s_e2e": 1555.6821321400246, "all_tokens_per_s_e2e_q25": 1551.7723135928798, "all_tokens_per_s_e2e_q75": 1560.1320415100581, "energy_net_psys_j": 35.82245416763587, "energy_net_psys_j_q25": 35.669914634341026, "energy_net_psys_j_q75": 35.91243279987999, "prefill_energy_net_psys_j": 20.457327416992882, "prefill_energy_net_psys_j_q25": 20.131066499921666, "prefill_energy_net_psys_j_q75": 20.46961047374662, "decode_energy_net_psys_j": 15.340560637135509, "decode_energy_net_psys_j_q25": 15.200304160594403, "decode_energy_net_psys_j_q75": 15.76908324320458, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.9709942595504033, "energy_reduction_vs_gpu": 0.07920404525899993 }, { "precision": "FP16", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 2048, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 703.072141, "ttft_ms_q25": 702.349346, "ttft_ms_q75": 703.605942, "prefill_tokens_per_s": 2890.172830784942, "prefill_tokens_per_s_q25": 2887.981820885866, "prefill_tokens_per_s_q75": 2893.150201678689, "handoff_plus_first_decode_ms": 74.54955100000001, "handoff_plus_first_decode_ms_q25": 74.254223, "handoff_plus_first_decode_ms_q75": 93.40781150000001, "steady_tpot_ms": 39.836130999999995, "steady_tpot_ms_q25": 39.81991425, "steady_tpot_ms_q75": 39.872106249999995, "steady_tpot_p95_ms": 40.625068549999995, "steady_tpot_p95_ms_q25": 40.58012035, "steady_tpot_p95_ms_q75": 41.282516625, "decode_tokens_per_s": 23.642482614634385, "decode_tokens_per_s_q25": 22.952130755722422, "decode_tokens_per_s_q75": 23.663828295749074, "decode_ms": 634.451138, "decode_ms_q25": 633.8793544999999, "decode_ms_q75": 654.1258695, "end_to_end_ms": 1337.523279, "end_to_end_ms_q25": 1336.2287004999998, "end_to_end_ms_q75": 1357.7318115, "output_tokens_per_s_e2e": 11.962408618384877, "output_tokens_per_s_e2e_q25": 11.786970890332615, "output_tokens_per_s_e2e_q75": 11.97400939834125, "all_tokens_per_s_e2e": 1531.1883031532643, "all_tokens_per_s_e2e_q25": 1508.7322739625747, "all_tokens_per_s_e2e_q75": 1532.67320298768, "energy_net_psys_j": 31.684558524914166, "energy_net_psys_j_q25": 31.212141183883652, "energy_net_psys_j_q75": 31.86194934984863, "prefill_energy_net_psys_j": 15.841734325509437, "prefill_energy_net_psys_j_q25": 15.760705242154641, "prefill_energy_net_psys_j_q75": 16.092866106644045, "decode_energy_net_psys_j": 15.340560637135509, "decode_energy_net_psys_j_q25": 15.200304160594403, "decode_energy_net_psys_j_q75": 15.76908324320458, "fastest_e2e": false, "lowest_energy": true, "e2e_speedup_vs_gpu": 0.9557061959741787, "energy_reduction_vs_gpu": 0.18556631600484064 }, { "precision": "FP16", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 2048, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 703.072141, "ttft_ms_q25": 702.349346, "ttft_ms_q75": 703.605942, "prefill_tokens_per_s": 2890.172830784942, "prefill_tokens_per_s_q25": 2887.981820885866, "prefill_tokens_per_s_q75": 2893.150201678689, "handoff_plus_first_decode_ms": 52.48689, "handoff_plus_first_decode_ms_q25": 51.409315, "handoff_plus_first_decode_ms_q75": 55.5279295, "steady_tpot_ms": 39.0188285, "steady_tpot_ms_q25": 38.6505935, "steady_tpot_ms_q75": 39.2626815, "steady_tpot_p95_ms": 42.16395955, "steady_tpot_p95_ms_q25": 40.912344225, "steady_tpot_p95_ms_q75": 42.25459372500001, "decode_tokens_per_s": 24.565094490356536, "decode_tokens_per_s_q25": 24.533837957858978, "decode_tokens_per_s_q75": 25.002701786304478, "decode_ms": 610.622524, "decode_ms_q25": 600.119001, "decode_ms_q75": 611.4014599999998, "end_to_end_ms": 1312.2490750000002, "end_to_end_ms_q25": 1302.4683470000002, "end_to_end_ms_q75": 1314.284607, "output_tokens_per_s_e2e": 12.192807222973274, "output_tokens_per_s_e2e_q25": 12.173952501741823, "output_tokens_per_s_e2e_q75": 12.285060393229974, "all_tokens_per_s_e2e": 1560.679324540579, "all_tokens_per_s_e2e_q25": 1558.2659202229534, "all_tokens_per_s_e2e_q75": 1572.4877303334367, "energy_net_psys_j": 34.765896216042435, "energy_net_psys_j_q25": 34.28695672749047, "energy_net_psys_j_q75": 34.90655161464521, "prefill_energy_net_psys_j": 15.841734325509437, "prefill_energy_net_psys_j_q25": 15.760705242154641, "prefill_energy_net_psys_j_q75": 16.092866106644045, "decode_energy_net_psys_j": 18.42189832826378, "decode_energy_net_psys_j_q25": 18.275119704201217, "decode_energy_net_psys_j_q75": 18.813685508001164, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.9741133061953196, "energy_reduction_vs_gpu": 0.10636227074584059 }, { "precision": "FP16", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 4096, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1709.174643, "ttft_ms_q25": 1674.974679, "ttft_ms_q75": 1711.6359975, "prefill_tokens_per_s": 2387.117089941522, "prefill_tokens_per_s_q25": 2383.689314159418, "prefill_tokens_per_s_q75": 2436.8736578563226, "handoff_plus_first_decode_ms": 60.345018, "handoff_plus_first_decode_ms_q25": 60.0690785, "handoff_plus_first_decode_ms_q75": 63.59227800000001, "steady_tpot_ms": 39.9686285, "steady_tpot_ms_q25": 39.96859825, "steady_tpot_ms_q75": 40.370734, "steady_tpot_p95_ms": 43.72544909999999, "steady_tpot_p95_ms_q25": 43.60656337499999, "steady_tpot_p95_ms_q75": 43.8626747, "decode_tokens_per_s": 23.812829937474554, "decode_tokens_per_s_q25": 23.522170856323314, "decode_tokens_per_s_q75": 23.832966702447706, "decode_ms": 629.9125320000005, "decode_ms_q25": 629.3807605000004, "decode_ms_q75": 637.7936290000004, "end_to_end_ms": 2339.0871750000006, "end_to_end_ms_q25": 2312.7683080000006, "end_to_end_ms_q75": 2341.0167580000007, "output_tokens_per_s_e2e": 6.840275202654642, "output_tokens_per_s_e2e_q25": 6.83464174937601, "output_tokens_per_s_e2e_q75": 6.9190122567411425, "all_tokens_per_s_e2e": 1751.1104518795883, "all_tokens_per_s_e2e_q25": 1749.6682878402585, "all_tokens_per_s_e2e_q75": 1771.2671377257325, "energy_net_psys_j": 72.45499675437996, "energy_net_psys_j_q25": 72.0342799550997, "energy_net_psys_j_q75": 72.83194131203624, "prefill_energy_net_psys_j": 51.94530337567546, "prefill_energy_net_psys_j_q25": 51.755696855872586, "prefill_energy_net_psys_j_q75": 52.496566841306276, "decode_energy_net_psys_j": 20.1610555627554, "decode_energy_net_psys_j_q25": 19.914657671449696, "decode_energy_net_psys_j_q75": 20.52498099053282, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "FP16", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 4096, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1709.174643, "ttft_ms_q25": 1674.974679, "ttft_ms_q75": 1711.6359975, "prefill_tokens_per_s": 2387.117089941522, "prefill_tokens_per_s_q25": 2383.689314159418, "prefill_tokens_per_s_q75": 2436.8736578563226, "handoff_plus_first_decode_ms": 116.282667, "handoff_plus_first_decode_ms_q25": 113.9857415, "handoff_plus_first_decode_ms_q75": 132.1389845, "steady_tpot_ms": 42.9531705, "steady_tpot_ms_q25": 42.88008375, "steady_tpot_ms_q75": 43.00171325, "steady_tpot_p95_ms": 44.112806049999996, "steady_tpot_p95_ms_q25": 43.938638499999996, "steady_tpot_p95_ms_q75": 44.12715502499999, "decode_tokens_per_s": 20.805680993969492, "decode_tokens_per_s_q25": 20.381779865615734, "decode_tokens_per_s_q75": 20.918893179896052, "decode_ms": 720.9569350000002, "decode_ms_q25": 717.0761485000002, "decode_ms_q75": 736.2699080000002, "end_to_end_ms": 2427.292714, "end_to_end_ms_q25": 2409.8251550000004, "end_to_end_ms_q75": 2428.7121460000003, "output_tokens_per_s_e2e": 6.591706021987425, "output_tokens_per_s_e2e_q25": 6.587855827805798, "output_tokens_per_s_e2e_q75": 6.639834702265922, "all_tokens_per_s_e2e": 1687.4767416287807, "all_tokens_per_s_e2e_q25": 1686.4910919182844, "all_tokens_per_s_e2e_q75": 1699.797683780076, "energy_net_psys_j": 68.45919300439584, "energy_net_psys_j_q25": 68.34436544553681, "energy_net_psys_j_q75": 69.03062676639205, "prefill_energy_net_psys_j": 51.94530337567546, "prefill_energy_net_psys_j_q25": 51.755696855872586, "prefill_energy_net_psys_j_q75": 52.496566841306276, "decode_energy_net_psys_j": 16.55423022145115, "decode_energy_net_psys_j_q25": 16.41923236622673, "decode_energy_net_psys_j_q75": 16.723666444888632, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.963660938587566, "energy_reduction_vs_gpu": 0.05514876722070339 }, { "precision": "FP16", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 4096, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1647.3376349999999, "ttft_ms_q25": 1645.6815305, "ttft_ms_q75": 1649.6660885, "prefill_tokens_per_s": 2476.723601352069, "prefill_tokens_per_s_q25": 2473.2327086212017, "prefill_tokens_per_s_q75": 2479.2185220732376, "handoff_plus_first_decode_ms": 116.282667, "handoff_plus_first_decode_ms_q25": 113.9857415, "handoff_plus_first_decode_ms_q75": 132.1389845, "steady_tpot_ms": 42.9531705, "steady_tpot_ms_q25": 42.88008375, "steady_tpot_ms_q75": 43.00171325, "steady_tpot_p95_ms": 44.112806049999996, "steady_tpot_p95_ms_q25": 43.938638499999996, "steady_tpot_p95_ms_q75": 44.12715502499999, "decode_tokens_per_s": 20.805680993969492, "decode_tokens_per_s_q25": 20.381779865615734, "decode_tokens_per_s_q75": 20.918893179896052, "decode_ms": 720.9569350000002, "decode_ms_q25": 717.0761485000002, "decode_ms_q75": 736.2699080000002, "end_to_end_ms": 2372.951477, "end_to_end_ms_q25": 2365.0861325, "end_to_end_ms_q75": 2385.9359965000003, "output_tokens_per_s_e2e": 6.742657890429338, "output_tokens_per_s_e2e_q25": 6.7061622365399405, "output_tokens_per_s_e2e_q75": 6.765156133879357, "all_tokens_per_s_e2e": 1726.1204199499105, "all_tokens_per_s_e2e_q25": 1716.7775325542248, "all_tokens_per_s_e2e_q75": 1731.8799702731153, "energy_net_psys_j": 56.32058538076437, "energy_net_psys_j_q25": 56.17634495377227, "energy_net_psys_j_q75": 56.45572909818361, "prefill_energy_net_psys_j": 39.747870015777856, "prefill_energy_net_psys_j_q25": 39.7228200815273, "prefill_energy_net_psys_j_q75": 39.757112587545535, "decode_energy_net_psys_j": 16.55423022145115, "decode_energy_net_psys_j_q25": 16.41923236622673, "decode_energy_net_psys_j_q75": 16.723666444888632, "fastest_e2e": false, "lowest_energy": true, "e2e_speedup_vs_gpu": 0.9857290373072388, "energy_reduction_vs_gpu": 0.2226818314313188 }, { "precision": "FP16", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 4096, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1647.3376349999999, "ttft_ms_q25": 1645.6815305, "ttft_ms_q75": 1649.6660885, "prefill_tokens_per_s": 2476.723601352069, "prefill_tokens_per_s_q25": 2473.2327086212017, "prefill_tokens_per_s_q75": 2479.2185220732376, "handoff_plus_first_decode_ms": 60.345018, "handoff_plus_first_decode_ms_q25": 60.0690785, "handoff_plus_first_decode_ms_q75": 63.59227800000001, "steady_tpot_ms": 39.9686285, "steady_tpot_ms_q25": 39.96859825, "steady_tpot_ms_q75": 40.370734, "steady_tpot_p95_ms": 43.72544909999999, "steady_tpot_p95_ms_q25": 43.60656337499999, "steady_tpot_p95_ms_q75": 43.8626747, "decode_tokens_per_s": 23.812829937474554, "decode_tokens_per_s_q25": 23.522170856323314, "decode_tokens_per_s_q75": 23.832966702447706, "decode_ms": 629.9125320000005, "decode_ms_q25": 629.3807605000004, "decode_ms_q75": 637.7936290000004, "end_to_end_ms": 2281.9070740000006, "end_to_end_ms_q25": 2277.3907445000004, "end_to_end_ms_q75": 2287.4597175000004, "output_tokens_per_s_e2e": 7.011679039126374, "output_tokens_per_s_e2e_q25": 6.994699909531228, "output_tokens_per_s_e2e_q75": 7.025611640364678, "all_tokens_per_s_e2e": 1794.9898340163518, "all_tokens_per_s_e2e_q25": 1790.6431768399943, "all_tokens_per_s_e2e_q75": 1798.5565799333576, "energy_net_psys_j": 59.92741072206862, "energy_net_psys_j_q25": 59.67177025899524, "energy_net_psys_j_q75": 60.2570436438278, "prefill_energy_net_psys_j": 39.747870015777856, "prefill_energy_net_psys_j_q25": 39.7228200815273, "prefill_energy_net_psys_j_q75": 39.757112587545535, "decode_energy_net_psys_j": 20.1610555627554, "decode_energy_net_psys_j_q25": 19.914657671449696, "decode_energy_net_psys_j_q75": 20.52498099053282, "fastest_e2e": true, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0250580322273017, "energy_reduction_vs_gpu": 0.1729016160856296 }, { "precision": "FP16", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 8192, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 4475.55968, "ttft_ms_q25": 4385.711507, "ttft_ms_q75": 4483.825789, "prefill_tokens_per_s": 1826.810630307582, "prefill_tokens_per_s_q25": 1823.4490303814532, "prefill_tokens_per_s_q75": 1865.0184560200641, "handoff_plus_first_decode_ms": 75.730897, "handoff_plus_first_decode_ms_q25": 74.952999, "handoff_plus_first_decode_ms_q75": 76.4329355, "steady_tpot_ms": 44.182173000000006, "steady_tpot_ms_q25": 44.0600525, "steady_tpot_ms_q75": 44.4365195, "steady_tpot_p95_ms": 47.9535477, "steady_tpot_p95_ms_q25": 47.95155835, "steady_tpot_p95_ms_q75": 48.401415575, "decode_tokens_per_s": 21.236507844865102, "decode_tokens_per_s_q25": 21.16574016738894, "decode_tokens_per_s_q75": 21.35928958761327, "decode_ms": 706.3308199999992, "decode_ms_q25": 702.2937544999995, "decode_ms_q75": 708.7003605, "end_to_end_ms": 5181.8904999999995, "end_to_end_ms_q25": 5094.4118675, "end_to_end_ms_q75": 5186.119543499999, "output_tokens_per_s_e2e": 3.087676206203122, "output_tokens_per_s_e2e_q25": 3.085160398878995, "output_tokens_per_s_e2e_q75": 3.141622540105343, "all_tokens_per_s_e2e": 1580.8902175759986, "all_tokens_per_s_e2e_q25": 1579.6021242260454, "all_tokens_per_s_e2e_q75": 1608.5107405339356, "energy_net_psys_j": 156.69767674481108, "energy_net_psys_j_q25": 154.8833029194545, "energy_net_psys_j_q75": 158.35906682980513, "prefill_energy_net_psys_j": 134.60066570578104, "prefill_energy_net_psys_j_q25": 132.68328291383347, "prefill_energy_net_psys_j_q75": 135.82319718374322, "decode_energy_net_psys_j": 22.303028972212015, "decode_energy_net_psys_j_q25": 22.20002000562102, "decode_energy_net_psys_j_q75": 22.638878612652892, "fastest_e2e": true, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "FP16", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 8192, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 4475.55968, "ttft_ms_q25": 4385.711507, "ttft_ms_q75": 4483.825789, "prefill_tokens_per_s": 1826.810630307582, "prefill_tokens_per_s_q25": 1823.4490303814532, "prefill_tokens_per_s_q75": 1865.0184560200641, "handoff_plus_first_decode_ms": 152.325229, "handoff_plus_first_decode_ms_q25": 150.367657, "handoff_plus_first_decode_ms_q75": 161.8536205, "steady_tpot_ms": 47.188070499999995, "steady_tpot_ms_q25": 47.12804575, "steady_tpot_ms_q75": 47.229930249999995, "steady_tpot_p95_ms": 50.53907135000001, "steady_tpot_p95_ms_q25": 49.229488200000006, "steady_tpot_p95_ms_q75": 50.717617125000004, "decode_tokens_per_s": 18.334684491609448, "decode_tokens_per_s_q25": 18.093337313414104, "decode_tokens_per_s_q75": 18.369327745210455, "decode_ms": 818.1215230000007, "decode_ms_q25": 816.5815080000002, "decode_ms_q75": 829.1819885000004, "end_to_end_ms": 5293.681203000001, "end_to_end_ms_q25": 5214.8934955, "end_to_end_ms_q75": 5300.407297, "output_tokens_per_s_e2e": 3.022471393051131, "output_tokens_per_s_e2e_q25": 3.0186408078360607, "output_tokens_per_s_e2e_q75": 3.0688360128374303, "all_tokens_per_s_e2e": 1547.5053532421791, "all_tokens_per_s_e2e_q25": 1545.544093612063, "all_tokens_per_s_e2e_q75": 1571.2440385727643, "energy_net_psys_j": 152.65689247338162, "energy_net_psys_j_q25": 151.03686912498998, "energy_net_psys_j_q75": 153.24283607282825, "prefill_energy_net_psys_j": 134.60066570578104, "prefill_energy_net_psys_j_q25": 132.68328291383347, "prefill_energy_net_psys_j_q75": 135.82319718374322, "decode_energy_net_psys_j": 18.056226767600588, "decode_energy_net_psys_j_q25": 17.41963888908503, "decode_energy_net_psys_j_q75": 18.353586211156504, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.9788822373858388, "energy_reduction_vs_gpu": 0.025787135810635187 }, { "precision": "FP16", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 8192, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 5123.838341, "ttft_ms_q25": 5118.5012145, "ttft_ms_q75": 5138.648222, "prefill_tokens_per_s": 1595.6787579688396, "prefill_tokens_per_s_q25": 1591.0931356454546, "prefill_tokens_per_s_q75": 1597.3443292742138, "handoff_plus_first_decode_ms": 152.325229, "handoff_plus_first_decode_ms_q25": 150.367657, "handoff_plus_first_decode_ms_q75": 161.8536205, "steady_tpot_ms": 47.188070499999995, "steady_tpot_ms_q25": 47.12804575, "steady_tpot_ms_q75": 47.229930249999995, "steady_tpot_p95_ms": 50.53907135000001, "steady_tpot_p95_ms_q25": 49.229488200000006, "steady_tpot_p95_ms_q75": 50.717617125000004, "decode_tokens_per_s": 18.334684491609448, "decode_tokens_per_s_q25": 18.093337313414104, "decode_tokens_per_s_q75": 18.369327745210455, "decode_ms": 818.1215230000007, "decode_ms_q25": 816.5815080000002, "decode_ms_q75": 829.1819885000004, "end_to_end_ms": 5953.406542000001, "end_to_end_ms_q25": 5946.143188, "end_to_end_ms_q75": 5962.493084000001, "output_tokens_per_s_e2e": 2.687536939922286, "output_tokens_per_s_e2e_q25": 2.6834474998058333, "output_tokens_per_s_e2e_q75": 2.6908238446464696, "all_tokens_per_s_e2e": 1376.0189132402104, "all_tokens_per_s_e2e_q25": 1373.9251199005867, "all_tokens_per_s_e2e_q75": 1377.7018084589924, "energy_net_psys_j": 124.69915985880303, "energy_net_psys_j_q25": 121.73285955947998, "energy_net_psys_j_q75": 125.55332830227859, "prefill_energy_net_psys_j": 106.04821420409061, "prefill_energy_net_psys_j_q25": 103.37927334832348, "prefill_energy_net_psys_j_q75": 107.83632996963765, "decode_energy_net_psys_j": 18.056226767600588, "decode_energy_net_psys_j_q25": 17.41963888908503, "decode_energy_net_psys_j_q75": 18.353586211156504, "fastest_e2e": false, "lowest_energy": true, "e2e_speedup_vs_gpu": 0.8704076335863977, "energy_reduction_vs_gpu": 0.20420543272073532 }, { "precision": "FP16", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 8192, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 5123.838341, "ttft_ms_q25": 5118.5012145, "ttft_ms_q75": 5138.648222, "prefill_tokens_per_s": 1595.6787579688396, "prefill_tokens_per_s_q25": 1591.0931356454546, "prefill_tokens_per_s_q75": 1597.3443292742138, "handoff_plus_first_decode_ms": 75.730897, "handoff_plus_first_decode_ms_q25": 74.952999, "handoff_plus_first_decode_ms_q75": 76.4329355, "steady_tpot_ms": 44.182173000000006, "steady_tpot_ms_q25": 44.0600525, "steady_tpot_ms_q75": 44.4365195, "steady_tpot_p95_ms": 47.9535477, "steady_tpot_p95_ms_q25": 47.95155835, "steady_tpot_p95_ms_q75": 48.401415575, "decode_tokens_per_s": 21.236507844865102, "decode_tokens_per_s_q25": 21.16574016738894, "decode_tokens_per_s_q75": 21.35928958761327, "decode_ms": 706.3308199999992, "decode_ms_q25": 702.2937544999995, "decode_ms_q75": 708.7003605, "end_to_end_ms": 5824.233989000001, "end_to_end_ms_q25": 5823.1645095, "end_to_end_ms_q75": 5842.011456, "output_tokens_per_s_e2e": 2.747142376185188, "output_tokens_per_s_e2e_q25": 2.738808077189325, "output_tokens_per_s_e2e_q75": 2.747647007669177, "all_tokens_per_s_e2e": 1406.5368966068163, "all_tokens_per_s_e2e_q25": 1402.2697355209343, "all_tokens_per_s_e2e_q75": 1406.7952679266186, "energy_net_psys_j": 128.35124317630263, "energy_net_psys_j_q25": 125.57929335394451, "energy_net_psys_j_q75": 130.47520858229052, "prefill_energy_net_psys_j": 106.04821420409061, "prefill_energy_net_psys_j_q25": 103.37927334832348, "prefill_energy_net_psys_j_q75": 107.83632996963765, "decode_energy_net_psys_j": 22.303028972212015, "decode_energy_net_psys_j_q25": 22.20002000562102, "decode_energy_net_psys_j_q75": 22.638878612652892, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.8897119363313407, "energy_reduction_vs_gpu": 0.1808988758312724 }, { "precision": "FP16", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 1024, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 608.015883, "ttft_ms_q25": 592.866033, "ttft_ms_q75": 612.9221520000001, "prefill_tokens_per_s": 1657.851428200273, "prefill_tokens_per_s_q25": 1644.6861790917592, "prefill_tokens_per_s_q75": 1701.3264098583854, "handoff_plus_first_decode_ms": 96.843889, "handoff_plus_first_decode_ms_q25": 95.145674, "handoff_plus_first_decode_ms_q75": 101.84947550000001, "steady_tpot_ms": 86.96682750000001, "steady_tpot_ms_q25": 86.93742850000001, "steady_tpot_ms_q75": 86.9841755, "steady_tpot_p95_ms": 89.6742428, "steady_tpot_p95_ms_q25": 89.63911555, "steady_tpot_p95_ms_q75": 89.813657, "decode_tokens_per_s": 11.351113110507946, "decode_tokens_per_s_q25": 11.304144194500878, "decode_tokens_per_s_q75": 11.367143440480916, "decode_ms": 1321.456306, "decode_ms_q25": 1319.595368, "decode_ms_q75": 1326.969889, "end_to_end_ms": 1925.750313, "end_to_end_ms_q25": 1917.974984, "end_to_end_ms_q75": 1932.51752, "output_tokens_per_s_e2e": 8.308449902352423, "output_tokens_per_s_e2e_q25": 8.279457253900368, "output_tokens_per_s_e2e_q75": 8.342268844420449, "all_tokens_per_s_e2e": 531.7407937505551, "all_tokens_per_s_e2e_q25": 529.8852642496236, "all_tokens_per_s_e2e_q75": 533.9052060429087, "energy_net_psys_j": 65.31244432195557, "energy_net_psys_j_q25": 64.86409664222063, "energy_net_psys_j_q75": 65.45618557989684, "prefill_energy_net_psys_j": 22.669455582541488, "prefill_energy_net_psys_j_q25": 22.383034636950725, "prefill_energy_net_psys_j_q75": 22.80752426359583, "decode_energy_net_psys_j": 42.654333893187946, "decode_energy_net_psys_j_q25": 42.20031363656608, "decode_energy_net_psys_j_q75": 42.93508226189177, "fastest_e2e": true, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "FP16", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 1024, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 608.015883, "ttft_ms_q25": 592.866033, "ttft_ms_q75": 612.9221520000001, "prefill_tokens_per_s": 1657.851428200273, "prefill_tokens_per_s_q25": 1644.6861790917592, "prefill_tokens_per_s_q75": 1701.3264098583854, "handoff_plus_first_decode_ms": 109.747179, "handoff_plus_first_decode_ms_q25": 108.5420605, "handoff_plus_first_decode_ms_q75": 160.8111525, "steady_tpot_ms": 96.990364, "steady_tpot_ms_q25": 96.97914374999999, "steady_tpot_ms_q75": 97.06703775, "steady_tpot_p95_ms": 97.5769649, "steady_tpot_p95_ms_q25": 97.54373365000001, "steady_tpot_p95_ms_q75": 97.886416525, "decode_tokens_per_s": 10.202238934793973, "decode_tokens_per_s_q25": 9.875892479033057, "decode_tokens_per_s_q75": 10.207971083856268, "decode_ms": 1470.265507, "decode_ms_q25": 1469.4403625, "decode_ms_q75": 1520.510402, "end_to_end_ms": 2088.0939280000002, "end_to_end_ms_q25": 2082.3625145, "end_to_end_ms_q75": 2118.282704, "output_tokens_per_s_e2e": 7.662490554400002, "output_tokens_per_s_e2e_q25": 7.554822748193472, "output_tokens_per_s_e2e_q75": 7.683638702371958, "all_tokens_per_s_e2e": 490.39939548160015, "all_tokens_per_s_e2e_q25": 483.5086558843822, "all_tokens_per_s_e2e_q75": 491.7528769518053, "energy_net_psys_j": 53.469603405206925, "energy_net_psys_j_q25": 53.27340872764955, "energy_net_psys_j_q75": 53.7773644469636, "prefill_energy_net_psys_j": 22.669455582541488, "prefill_energy_net_psys_j_q25": 22.383034636950725, "prefill_energy_net_psys_j_q75": 22.80752426359583, "decode_energy_net_psys_j": 31.139532544070104, "decode_energy_net_psys_j_q25": 30.773645505810393, "decode_energy_net_psys_j_q75": 31.256261128958528, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.9222527239684593, "energy_reduction_vs_gpu": 0.1813259485186275 }, { "precision": "FP16", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 1024, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 764.6387970000001, "ttft_ms_q25": 762.237527, "ttft_ms_q75": 766.6386135, "prefill_tokens_per_s": 1318.2694939817445, "prefill_tokens_per_s_q25": 1314.8396666629214, "prefill_tokens_per_s_q75": 1322.4355507586504, "handoff_plus_first_decode_ms": 109.747179, "handoff_plus_first_decode_ms_q25": 108.5420605, "handoff_plus_first_decode_ms_q75": 160.8111525, "steady_tpot_ms": 96.990364, "steady_tpot_ms_q25": 96.97914374999999, "steady_tpot_ms_q75": 97.06703775, "steady_tpot_p95_ms": 97.5769649, "steady_tpot_p95_ms_q25": 97.54373365000001, "steady_tpot_p95_ms_q75": 97.886416525, "decode_tokens_per_s": 10.202238934793973, "decode_tokens_per_s_q25": 9.875892479033057, "decode_tokens_per_s_q75": 10.207971083856268, "decode_ms": 1470.265507, "decode_ms_q25": 1469.4403625, "decode_ms_q75": 1520.510402, "end_to_end_ms": 2234.904304, "end_to_end_ms_q25": 2231.6778894999998, "end_to_end_ms_q75": 2287.1490155, "output_tokens_per_s_e2e": 7.159143222089387, "output_tokens_per_s_e2e_q25": 6.999261040689813, "output_tokens_per_s_e2e_q75": 7.1695084302882, "all_tokens_per_s_e2e": 458.1851662137208, "all_tokens_per_s_e2e_q25": 447.95270660414803, "all_tokens_per_s_e2e_q75": 458.8485395384448, "energy_net_psys_j": 52.55018231241183, "energy_net_psys_j_q25": 52.39392568172434, "energy_net_psys_j_q75": 52.64115238927822, "prefill_energy_net_psys_j": 21.592589922074502, "prefill_energy_net_psys_j_q25": 21.384891260319687, "prefill_energy_net_psys_j_q75": 21.71125025278034, "decode_energy_net_psys_j": 31.139532544070104, "decode_energy_net_psys_j_q25": 30.773645505810393, "decode_energy_net_psys_j_q75": 31.256261128958528, "fastest_e2e": false, "lowest_energy": true, "e2e_speedup_vs_gpu": 0.8616701437969041, "energy_reduction_vs_gpu": 0.19540322127024656 }, { "precision": "FP16", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 1024, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 764.6387970000001, "ttft_ms_q25": 762.237527, "ttft_ms_q75": 766.6386135, "prefill_tokens_per_s": 1318.2694939817445, "prefill_tokens_per_s_q25": 1314.8396666629214, "prefill_tokens_per_s_q75": 1322.4355507586504, "handoff_plus_first_decode_ms": 96.843889, "handoff_plus_first_decode_ms_q25": 95.145674, "handoff_plus_first_decode_ms_q75": 101.84947550000001, "steady_tpot_ms": 86.96682750000001, "steady_tpot_ms_q25": 86.93742850000001, "steady_tpot_ms_q75": 86.9841755, "steady_tpot_p95_ms": 89.6742428, "steady_tpot_p95_ms_q25": 89.63911555, "steady_tpot_p95_ms_q75": 89.813657, "decode_tokens_per_s": 11.351113110507946, "decode_tokens_per_s_q25": 11.304144194500878, "decode_tokens_per_s_q75": 11.367143440480916, "decode_ms": 1321.456306, "decode_ms_q25": 1319.595368, "decode_ms_q75": 1326.969889, "end_to_end_ms": 2086.095103, "end_to_end_ms_q25": 2081.832895, "end_to_end_ms_q75": 2093.6085025, "output_tokens_per_s_e2e": 7.669832490853606, "output_tokens_per_s_e2e_q25": 7.642405945279544, "output_tokens_per_s_e2e_q75": 7.685567417037216, "all_tokens_per_s_e2e": 490.8692794146308, "all_tokens_per_s_e2e_q25": 489.1139804978908, "all_tokens_per_s_e2e_q75": 491.87631469038183, "energy_net_psys_j": 64.24692381526245, "energy_net_psys_j_q25": 63.911563889346425, "energy_net_psys_j_q75": 64.31997352221146, "prefill_energy_net_psys_j": 21.592589922074502, "prefill_energy_net_psys_j_q25": 21.384891260319687, "prefill_energy_net_psys_j_q75": 21.71125025278034, "decode_energy_net_psys_j": 42.654333893187946, "decode_energy_net_psys_j_q25": 42.20031363656608, "decode_energy_net_psys_j_q75": 42.93508226189177, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.9231363949949313, "energy_reduction_vs_gpu": 0.016314203483805745 }, { "precision": "FP16", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 2048, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1333.976724, "ttft_ms_q25": 1306.1093925, "ttft_ms_q75": 1335.895748, "prefill_tokens_per_s": 1523.2649591568136, "prefill_tokens_per_s_q25": 1521.0799167457399, "prefill_tokens_per_s_q75": 1556.4741069968932, "handoff_plus_first_decode_ms": 100.18574600000001, "handoff_plus_first_decode_ms_q25": 99.9964105, "handoff_plus_first_decode_ms_q75": 102.64697749999999, "steady_tpot_ms": 87.75281500000001, "steady_tpot_ms_q25": 87.71656675, "steady_tpot_ms_q75": 87.79231925, "steady_tpot_p95_ms": 90.78791505, "steady_tpot_p95_ms_q25": 90.260761025, "steady_tpot_p95_ms_q75": 90.8908207, "decode_tokens_per_s": 11.225396339072269, "decode_tokens_per_s_q25": 11.211182400383763, "decode_tokens_per_s_q75": 11.227255763351272, "decode_ms": 1336.2557139999997, "decode_ms_q25": 1336.034444, "decode_ms_q75": 1337.9520174999996, "end_to_end_ms": 2669.789898, "end_to_end_ms_q25": 2643.84014, "end_to_end_ms_q75": 2671.930192, "output_tokens_per_s_e2e": 5.99298095029349, "output_tokens_per_s_e2e_q25": 5.988184240095618, "output_tokens_per_s_e2e_q75": 6.052386189881833, "all_tokens_per_s_e2e": 767.1015616375668, "all_tokens_per_s_e2e_q25": 766.487582732239, "all_tokens_per_s_e2e_q75": 774.7054323048746, "energy_net_psys_j": 92.22111851553885, "energy_net_psys_j_q25": 91.60348249114398, "energy_net_psys_j_q75": 92.42676630440494, "prefill_energy_net_psys_j": 49.39871092110475, "prefill_energy_net_psys_j_q25": 49.38592137439346, "prefill_energy_net_psys_j_q75": 49.69224627588552, "decode_energy_net_psys_j": 42.23533688487256, "decode_energy_net_psys_j_q25": 41.911236215258455, "decode_energy_net_psys_j_q75": 42.7473095752307, "fastest_e2e": true, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "FP16", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 2048, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1333.976724, "ttft_ms_q25": 1306.1093925, "ttft_ms_q75": 1335.895748, "prefill_tokens_per_s": 1523.2649591568136, "prefill_tokens_per_s_q25": 1521.0799167457399, "prefill_tokens_per_s_q75": 1556.4741069968932, "handoff_plus_first_decode_ms": 178.34850899999998, "handoff_plus_first_decode_ms_q25": 177.11055349999998, "handoff_plus_first_decode_ms_q75": 194.3620465, "steady_tpot_ms": 100.5010975, "steady_tpot_ms_q25": 100.4861975, "steady_tpot_ms_q75": 100.5453005, "steady_tpot_p95_ms": 101.3804163, "steady_tpot_p95_ms_q25": 101.075597725, "steady_tpot_p95_ms_q75": 101.993017025, "decode_tokens_per_s": 9.428149969795225, "decode_tokens_per_s_q25": 9.345168960724997, "decode_tokens_per_s_q75": 9.452399572268277, "decode_ms": 1590.980208, "decode_ms_q25": 1586.9090815, "decode_ms_q75": 1605.233983, "end_to_end_ms": 2920.652727, "end_to_end_ms_q25": 2909.1912730000004, "end_to_end_ms_q75": 2922.8048295, "output_tokens_per_s_e2e": 5.478227470211661, "output_tokens_per_s_e2e_q25": 5.474196741791277, "output_tokens_per_s_e2e_q75": 5.499895622894383, "all_tokens_per_s_e2e": 701.2131161870926, "all_tokens_per_s_e2e_q25": 700.6971829492835, "all_tokens_per_s_e2e_q75": 703.986639730481, "energy_net_psys_j": 81.06615447658498, "energy_net_psys_j_q25": 80.8737375662419, "energy_net_psys_j_q75": 81.11999194614235, "prefill_energy_net_psys_j": 49.39871092110475, "prefill_energy_net_psys_j_q25": 49.38592137439346, "prefill_energy_net_psys_j_q75": 49.69224627588552, "decode_energy_net_psys_j": 31.308188828216643, "decode_energy_net_psys_j_q25": 31.194280837067666, "decode_energy_net_psys_j_q75": 31.5416536614058, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.9141072724323243, "energy_reduction_vs_gpu": 0.1209588890105936 }, { "precision": "FP16", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 2048, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1576.823208, "ttft_ms_q25": 1575.7440855, "ttft_ms_q75": 1580.0888515000001, "prefill_tokens_per_s": 1288.6669790821597, "prefill_tokens_per_s_q25": 1286.0091239036803, "prefill_tokens_per_s_q75": 1289.550106328152, "handoff_plus_first_decode_ms": 178.34850899999998, "handoff_plus_first_decode_ms_q25": 177.11055349999998, "handoff_plus_first_decode_ms_q75": 194.3620465, "steady_tpot_ms": 100.5010975, "steady_tpot_ms_q25": 100.4861975, "steady_tpot_ms_q75": 100.5453005, "steady_tpot_p95_ms": 101.3804163, "steady_tpot_p95_ms_q25": 101.075597725, "steady_tpot_p95_ms_q75": 101.993017025, "decode_tokens_per_s": 9.428149969795225, "decode_tokens_per_s_q25": 9.345168960724997, "decode_tokens_per_s_q75": 9.452399572268277, "decode_ms": 1590.980208, "decode_ms_q25": 1586.9090815, "decode_ms_q75": 1605.233983, "end_to_end_ms": 3174.334703, "end_to_end_ms_q25": 3166.997933, "end_to_end_ms_q75": 3184.243712, "output_tokens_per_s_e2e": 5.040426261565525, "output_tokens_per_s_e2e_q25": 5.024789680114262, "output_tokens_per_s_e2e_q75": 5.052130190017129, "all_tokens_per_s_e2e": 645.1745614803872, "all_tokens_per_s_e2e_q25": 643.1730790546255, "all_tokens_per_s_e2e_q75": 646.6726643221925, "energy_net_psys_j": 76.77005017567933, "energy_net_psys_j_q25": 76.40717616521292, "energy_net_psys_j_q75": 77.37635485110076, "prefill_energy_net_psys_j": 45.46186134746268, "prefill_energy_net_psys_j_q25": 45.21289532814525, "prefill_energy_net_psys_j_q75": 45.834701189694954, "decode_energy_net_psys_j": 31.308188828216643, "decode_energy_net_psys_j_q25": 31.194280837067666, "decode_energy_net_psys_j_q75": 31.5416536614058, "fastest_e2e": false, "lowest_energy": true, "e2e_speedup_vs_gpu": 0.8410549446713464, "energy_reduction_vs_gpu": 0.16754371003704627 }, { "precision": "FP16", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 2048, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1576.823208, "ttft_ms_q25": 1575.7440855, "ttft_ms_q75": 1580.0888515000001, "prefill_tokens_per_s": 1288.6669790821597, "prefill_tokens_per_s_q25": 1286.0091239036803, "prefill_tokens_per_s_q75": 1289.550106328152, "handoff_plus_first_decode_ms": 100.18574600000001, "handoff_plus_first_decode_ms_q25": 99.9964105, "handoff_plus_first_decode_ms_q75": 102.64697749999999, "steady_tpot_ms": 87.75281500000001, "steady_tpot_ms_q25": 87.71656675, "steady_tpot_ms_q75": 87.79231925, "steady_tpot_p95_ms": 90.78791505, "steady_tpot_p95_ms_q25": 90.260761025, "steady_tpot_p95_ms_q75": 90.8908207, "decode_tokens_per_s": 11.225396339072269, "decode_tokens_per_s_q25": 11.211182400383763, "decode_tokens_per_s_q75": 11.227255763351272, "decode_ms": 1336.2557139999997, "decode_ms_q25": 1336.034444, "decode_ms_q75": 1337.9520174999996, "end_to_end_ms": 2914.3132839999994, "end_to_end_ms_q25": 2913.6961029999993, "end_to_end_ms_q75": 2916.7404764999997, "output_tokens_per_s_e2e": 5.490144140591305, "output_tokens_per_s_e2e_q25": 5.48557926519087, "output_tokens_per_s_e2e_q75": 5.4913073128710685, "all_tokens_per_s_e2e": 702.738449995687, "all_tokens_per_s_e2e_q25": 702.1541459444313, "all_tokens_per_s_e2e_q75": 702.8873360474968, "energy_net_psys_j": 87.7946765775716, "energy_net_psys_j_q25": 87.49697138563599, "energy_net_psys_j_q75": 88.25791009531156, "prefill_energy_net_psys_j": 45.46186134746268, "prefill_energy_net_psys_j_q25": 45.21289532814525, "prefill_energy_net_psys_j_q75": 45.834701189694954, "decode_energy_net_psys_j": 42.23533688487256, "decode_energy_net_psys_j_q25": 41.911236215258455, "decode_energy_net_psys_j_q75": 42.7473095752307, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.9160957103196599, "energy_reduction_vs_gpu": 0.04799813761987082 }, { "precision": "FP16", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 4096, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 3401.502968, "ttft_ms_q25": 3349.8570529999997, "ttft_ms_q75": 3403.8467545, "prefill_tokens_per_s": 1199.4697750914913, "prefill_tokens_per_s_q25": 1198.6444245915184, "prefill_tokens_per_s_q75": 1218.2519911042018, "handoff_plus_first_decode_ms": 109.114073, "handoff_plus_first_decode_ms_q25": 108.62187650000001, "handoff_plus_first_decode_ms_q75": 111.657108, "steady_tpot_ms": 92.99677700000001, "steady_tpot_ms_q25": 92.99468750000001, "steady_tpot_ms_q75": 93.00464825, "steady_tpot_p95_ms": 94.43623765000001, "steady_tpot_p95_ms_q25": 94.29474630000001, "steady_tpot_p95_ms_q75": 94.71633805, "decode_tokens_per_s": 10.594658249592095, "decode_tokens_per_s_q25": 10.580235900469772, "decode_tokens_per_s_q75": 10.604553138881375, "decode_ms": 1415.807820000001, "decode_ms_q25": 1414.4879905000003, "decode_ms_q75": 1417.7403995000006, "end_to_end_ms": 4817.310788000001, "end_to_end_ms_q25": 4767.5974525, "end_to_end_ms_q75": 4818.334745, "output_tokens_per_s_e2e": 3.3213551510640045, "output_tokens_per_s_e2e_q25": 3.3206494711039287, "output_tokens_per_s_e2e_q75": 3.356352967637018, "all_tokens_per_s_e2e": 850.2669186723851, "all_tokens_per_s_e2e_q25": 850.0862646026058, "all_tokens_per_s_e2e_q75": 859.2263597150766, "energy_net_psys_j": 165.47752763595605, "energy_net_psys_j_q25": 162.6926125645806, "energy_net_psys_j_q75": 166.65769336677786, "prefill_energy_net_psys_j": 120.24855199467956, "prefill_energy_net_psys_j_q25": 118.71971973437182, "prefill_energy_net_psys_j_q75": 121.24681045981217, "decode_energy_net_psys_j": 45.22897564127649, "decode_energy_net_psys_j_q25": 43.97289283020879, "decode_energy_net_psys_j_q75": 45.41088290696568, "fastest_e2e": true, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "FP16", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 4096, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 3401.502968, "ttft_ms_q25": 3349.8570529999997, "ttft_ms_q75": 3403.8467545, "prefill_tokens_per_s": 1199.4697750914913, "prefill_tokens_per_s_q25": 1198.6444245915184, "prefill_tokens_per_s_q75": 1218.2519911042018, "handoff_plus_first_decode_ms": 266.97515, "handoff_plus_first_decode_ms_q25": 263.0479305, "handoff_plus_first_decode_ms_q75": 275.74849299999994, "steady_tpot_ms": 107.17812599999999, "steady_tpot_ms_q25": 107.17021025, "steady_tpot_ms_q75": 107.49997124999999, "steady_tpot_p95_ms": 109.60956010000001, "steady_tpot_p95_ms_q25": 109.233686425, "steady_tpot_p95_ms_q75": 109.8088157, "decode_tokens_per_s": 8.466271671821122, "decode_tokens_per_s_q25": 8.401582366719186, "decode_tokens_per_s_q75": 8.480237867743881, "decode_ms": 1771.7361999999998, "decode_ms_q25": 1768.823106, "decode_ms_q75": 1785.4838134999998, "end_to_end_ms": 5167.41298, "end_to_end_ms_q25": 5132.427772499999, "end_to_end_ms_q75": 5172.6698605, "output_tokens_per_s_e2e": 3.0963269361141714, "output_tokens_per_s_e2e_q25": 3.093183395947178, "output_tokens_per_s_e2e_q75": 3.1175779142206395, "all_tokens_per_s_e2e": 792.6596956452279, "all_tokens_per_s_e2e_q25": 791.8549493624776, "all_tokens_per_s_e2e_q75": 798.0999460404837, "energy_net_psys_j": 153.62109845076316, "energy_net_psys_j_q25": 152.04089340171447, "energy_net_psys_j_q75": 155.1668899905201, "prefill_energy_net_psys_j": 120.24855199467956, "prefill_energy_net_psys_j_q25": 118.71971973437182, "prefill_energy_net_psys_j_q75": 121.24681045981217, "decode_energy_net_psys_j": 33.3725464560836, "decode_energy_net_psys_j_q25": 33.321173667342634, "decode_energy_net_psys_j_q75": 33.92007953070794, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.9322480720323617, "energy_reduction_vs_gpu": 0.07164978444249281 }, { "precision": "FP16", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 4096, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 3819.786142, "ttft_ms_q25": 3813.6422555, "ttft_ms_q75": 3821.775001, "prefill_tokens_per_s": 1068.1226247560955, "prefill_tokens_per_s_q25": 1067.5670608214152, "prefill_tokens_per_s_q75": 1069.8461774772245, "handoff_plus_first_decode_ms": 266.97515, "handoff_plus_first_decode_ms_q25": 263.0479305, "handoff_plus_first_decode_ms_q75": 275.74849299999994, "steady_tpot_ms": 107.17812599999999, "steady_tpot_ms_q25": 107.17021025, "steady_tpot_ms_q75": 107.49997124999999, "steady_tpot_p95_ms": 109.60956010000001, "steady_tpot_p95_ms_q25": 109.233686425, "steady_tpot_p95_ms_q75": 109.8088157, "decode_tokens_per_s": 8.466271671821122, "decode_tokens_per_s_q25": 8.401582366719186, "decode_tokens_per_s_q75": 8.480237867743881, "decode_ms": 1771.7361999999998, "decode_ms_q25": 1768.823106, "decode_ms_q75": 1785.4838134999998, "end_to_end_ms": 5589.673872, "end_to_end_ms_q25": 5584.4542205, "end_to_end_ms_q75": 5604.3457204999995, "output_tokens_per_s_e2e": 2.862421022476425, "output_tokens_per_s_e2e_q25": 2.8549469387830966, "output_tokens_per_s_e2e_q75": 2.8650989597452066, "all_tokens_per_s_e2e": 732.7797817539648, "all_tokens_per_s_e2e_q25": 730.8664163284727, "all_tokens_per_s_e2e_q75": 733.4653336947729, "energy_net_psys_j": 138.65458698340248, "energy_net_psys_j_q25": 138.23103475952297, "energy_net_psys_j_q75": 140.5430702314295, "prefill_energy_net_psys_j": 105.28204052731888, "prefill_energy_net_psys_j_q25": 104.90986109218036, "prefill_energy_net_psys_j_q75": 106.62299070072156, "decode_energy_net_psys_j": 33.3725464560836, "decode_energy_net_psys_j_q25": 33.321173667342634, "decode_energy_net_psys_j_q75": 33.92007953070794, "fastest_e2e": false, "lowest_energy": true, "e2e_speedup_vs_gpu": 0.8618232294608548, "energy_reduction_vs_gpu": 0.16209415886103262 }, { "precision": "FP16", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 4096, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 3819.786142, "ttft_ms_q25": 3813.6422555, "ttft_ms_q75": 3821.775001, "prefill_tokens_per_s": 1068.1226247560955, "prefill_tokens_per_s_q25": 1067.5670608214152, "prefill_tokens_per_s_q75": 1069.8461774772245, "handoff_plus_first_decode_ms": 109.114073, "handoff_plus_first_decode_ms_q25": 108.62187650000001, "handoff_plus_first_decode_ms_q75": 111.657108, "steady_tpot_ms": 92.99677700000001, "steady_tpot_ms_q25": 92.99468750000001, "steady_tpot_ms_q75": 93.00464825, "steady_tpot_p95_ms": 94.43623765000001, "steady_tpot_p95_ms_q25": 94.29474630000001, "steady_tpot_p95_ms_q75": 94.71633805, "decode_tokens_per_s": 10.594658249592095, "decode_tokens_per_s_q25": 10.580235900469772, "decode_tokens_per_s_q75": 10.604553138881375, "decode_ms": 1415.807820000001, "decode_ms_q25": 1414.4879905000003, "decode_ms_q75": 1417.7403995000006, "end_to_end_ms": 5239.459121, "end_to_end_ms_q25": 5230.0628255, "end_to_end_ms_q75": 5239.5154005, "output_tokens_per_s_e2e": 3.053750326225706, "output_tokens_per_s_e2e_q25": 3.053717525158752, "output_tokens_per_s_e2e_q75": 3.059246547883142, "all_tokens_per_s_e2e": 781.7600835137807, "all_tokens_per_s_e2e_q25": 781.7516864406405, "all_tokens_per_s_e2e_q75": 783.1671162580843, "energy_net_psys_j": 150.51101616859538, "energy_net_psys_j_q25": 148.88275392238916, "energy_net_psys_j_q75": 152.03387360768724, "prefill_energy_net_psys_j": 105.28204052731888, "prefill_energy_net_psys_j_q25": 104.90986109218036, "prefill_energy_net_psys_j_q75": 106.62299070072156, "decode_energy_net_psys_j": 45.22897564127649, "decode_energy_net_psys_j_q25": 43.97289283020879, "decode_energy_net_psys_j_q75": 45.41088290696568, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.9194290243991009, "energy_reduction_vs_gpu": 0.09044437441853981 }, { "precision": "FP16", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 8192, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 8393.731064000001, "ttft_ms_q25": 8304.008112, "ttft_ms_q75": 8398.885063000002, "prefill_tokens_per_s": 974.0602763729432, "prefill_tokens_per_s_q25": 973.4629081725661, "prefill_tokens_per_s_q75": 984.699736985742, "handoff_plus_first_decode_ms": 124.933821, "handoff_plus_first_decode_ms_q25": 124.2429285, "handoff_plus_first_decode_ms_q75": 126.7821945, "steady_tpot_ms": 99.4496485, "steady_tpot_ms_q25": 99.42917675, "steady_tpot_ms_q75": 99.571631, "steady_tpot_p95_ms": 101.72606005, "steady_tpot_p95_ms_q25": 101.66135237499999, "steady_tpot_p95_ms_q75": 101.957472725, "decode_tokens_per_s": 9.825767383491202, "decode_tokens_per_s_q25": 9.823172126265263, "decode_tokens_per_s_q75": 9.843393928516315, "decode_ms": 1526.5983220000016, "decode_ms_q25": 1523.8695320000006, "decode_ms_q75": 1527.0017520000001, "end_to_end_ms": 9914.871806000001, "end_to_end_ms_q25": 9828.281073999999, "end_to_end_ms_q75": 9922.754595000002, "output_tokens_per_s_e2e": 1.6137374555178388, "output_tokens_per_s_e2e_q25": 1.6124564952560392, "output_tokens_per_s_e2e_q75": 1.6280814453823762, "all_tokens_per_s_e2e": 826.2335772251334, "all_tokens_per_s_e2e_q25": 825.5777255710921, "all_tokens_per_s_e2e_q75": 833.5777000357766, "energy_net_psys_j": 343.93872694202236, "energy_net_psys_j_q25": 342.82446973193373, "energy_net_psys_j_q75": 345.7816045764536, "prefill_energy_net_psys_j": 295.5721234930767, "prefill_energy_net_psys_j_q25": 294.5815669048591, "prefill_energy_net_psys_j_q75": 297.14477334743196, "decode_energy_net_psys_j": 48.36660344894568, "decode_energy_net_psys_j_q25": 48.242902827074644, "decode_energy_net_psys_j_q75": 48.63683122902165, "fastest_e2e": true, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "FP16", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 8192, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 8393.731064000001, "ttft_ms_q25": 8304.008112, "ttft_ms_q75": 8398.885063000002, "prefill_tokens_per_s": 974.0602763729432, "prefill_tokens_per_s_q25": 973.4629081725661, "prefill_tokens_per_s_q75": 984.699736985742, "handoff_plus_first_decode_ms": 396.897302, "handoff_plus_first_decode_ms_q25": 396.0727155, "handoff_plus_first_decode_ms_q75": 411.564028, "steady_tpot_ms": 123.84031549999999, "steady_tpot_ms_q25": 123.51343499999999, "steady_tpot_ms_q75": 123.859394, "steady_tpot_p95_ms": 125.85470939999999, "steady_tpot_p95_ms_q25": 125.586965825, "steady_tpot_p95_ms_q75": 126.02089537500001, "decode_tokens_per_s": 7.025409807563141, "decode_tokens_per_s_q25": 6.989961835045682, "decode_tokens_per_s_q75": 7.025609047548404, "decode_ms": 2135.106764, "decode_ms_q25": 2135.0462160000006, "decode_ms_q75": 2145.9896535000007, "end_to_end_ms": 10528.716732000003, "end_to_end_ms_q25": 10449.937217500003, "end_to_end_ms_q75": 10533.931279, "output_tokens_per_s_e2e": 1.519653382958921, "output_tokens_per_s_e2e_q25": 1.5189014905266522, "output_tokens_per_s_e2e_q75": 1.5311966994474127, "all_tokens_per_s_e2e": 778.0625320749675, "all_tokens_per_s_e2e_q25": 777.677563149646, "all_tokens_per_s_e2e_q75": 783.9727101170753, "energy_net_psys_j": 333.11812980832195, "energy_net_psys_j_q25": 332.83437756564086, "energy_net_psys_j_q75": 334.9759381692131, "prefill_energy_net_psys_j": 295.5721234930767, "prefill_energy_net_psys_j_q25": 294.5815669048591, "prefill_energy_net_psys_j_q75": 297.14477334743196, "decode_energy_net_psys_j": 38.116323328317065, "decode_energy_net_psys_j_q25": 37.83116482178118, "decode_energy_net_psys_j_q75": 38.53796916731765, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.9416980300994956, "energy_reduction_vs_gpu": 0.031460828008252895 }, { "precision": "FP16", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 8192, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 9824.381988000001, "ttft_ms_q25": 9823.482581, "ttft_ms_q75": 9832.498856, "prefill_tokens_per_s": 832.2151978604437, "prefill_tokens_per_s_q25": 831.5287590137207, "prefill_tokens_per_s_q75": 832.2913998290301, "handoff_plus_first_decode_ms": 396.897302, "handoff_plus_first_decode_ms_q25": 396.0727155, "handoff_plus_first_decode_ms_q75": 411.564028, "steady_tpot_ms": 123.84031549999999, "steady_tpot_ms_q25": 123.51343499999999, "steady_tpot_ms_q75": 123.859394, "steady_tpot_p95_ms": 125.85470939999999, "steady_tpot_p95_ms_q25": 125.586965825, "steady_tpot_p95_ms_q75": 126.02089537500001, "decode_tokens_per_s": 7.025409807563141, "decode_tokens_per_s_q25": 6.989961835045682, "decode_tokens_per_s_q75": 7.025609047548404, "decode_ms": 2135.106764, "decode_ms_q25": 2135.0462160000006, "decode_ms_q75": 2145.9896535000007, "end_to_end_ms": 11975.722488, "end_to_end_ms_q25": 11967.545072, "end_to_end_ms_q75": 11977.5891025, "output_tokens_per_s_e2e": 1.3360363031151095, "output_tokens_per_s_e2e_q25": 1.3358281246473362, "output_tokens_per_s_e2e_q75": 1.3369498400976598, "all_tokens_per_s_e2e": 684.050587194936, "all_tokens_per_s_e2e_q25": 683.9439998194362, "all_tokens_per_s_e2e_q75": 684.5183181300018, "energy_net_psys_j": 296.3685678572622, "energy_net_psys_j_q25": 294.1043334975039, "energy_net_psys_j_q75": 296.77729115229215, "prefill_energy_net_psys_j": 258.22639944100393, "prefill_energy_net_psys_j_q25": 256.2602461317521, "prefill_energy_net_psys_j_q75": 258.23932198497454, "decode_energy_net_psys_j": 38.116323328317065, "decode_energy_net_psys_j_q25": 37.83116482178118, "decode_energy_net_psys_j_q75": 38.53796916731765, "fastest_e2e": false, "lowest_energy": true, "e2e_speedup_vs_gpu": 0.8279142920967794, "energy_reduction_vs_gpu": 0.13830998186133037 }, { "precision": "FP16", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 8192, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 9824.381988000001, "ttft_ms_q25": 9823.482581, "ttft_ms_q75": 9832.498856, "prefill_tokens_per_s": 832.2151978604437, "prefill_tokens_per_s_q25": 831.5287590137207, "prefill_tokens_per_s_q75": 832.2913998290301, "handoff_plus_first_decode_ms": 124.933821, "handoff_plus_first_decode_ms_q25": 124.2429285, "handoff_plus_first_decode_ms_q75": 126.7821945, "steady_tpot_ms": 99.4496485, "steady_tpot_ms_q25": 99.42917675, "steady_tpot_ms_q75": 99.571631, "steady_tpot_p95_ms": 101.72606005, "steady_tpot_p95_ms_q25": 101.66135237499999, "steady_tpot_p95_ms_q75": 101.957472725, "decode_tokens_per_s": 9.825767383491202, "decode_tokens_per_s_q25": 9.823172126265263, "decode_tokens_per_s_q75": 9.843393928516315, "decode_ms": 1526.5983220000016, "decode_ms_q25": 1523.8695320000006, "decode_ms_q75": 1527.0017520000001, "end_to_end_ms": 11349.988355999998, "end_to_end_ms_q25": 11347.755543, "end_to_end_ms_q75": 11358.601201, "output_tokens_per_s_e2e": 1.4096930761644213, "output_tokens_per_s_e2e_q25": 1.4086249632641588, "output_tokens_per_s_e2e_q75": 1.4099705055170721, "all_tokens_per_s_e2e": 721.7628549961837, "all_tokens_per_s_e2e_q25": 721.2159811912493, "all_tokens_per_s_e2e_q75": 721.9048988247409, "energy_net_psys_j": 306.34560164620757, "energy_net_psys_j_q25": 304.5031489588268, "energy_net_psys_j_q75": 306.75245259212517, "prefill_energy_net_psys_j": 258.22639944100393, "prefill_energy_net_psys_j_q25": 256.2602461317521, "prefill_energy_net_psys_j_q75": 258.23932198497454, "decode_energy_net_psys_j": 48.36660344894568, "decode_energy_net_psys_j_q25": 48.242902827074644, "decode_energy_net_psys_j_q75": 48.63683122902165, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.873557883498502, "energy_reduction_vs_gpu": 0.10930180974401249 }, { "precision": "FP16", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 1024, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 207.69891900000002, "ttft_ms_q25": 194.50003650000002, "ttft_ms_q75": 207.97992950000003, "prefill_tokens_per_s": 4853.1788458658275, "prefill_tokens_per_s_q25": 4846.6303581174, "prefill_tokens_per_s_q75": 5206.494508368829, "handoff_plus_first_decode_ms": 24.224101, "handoff_plus_first_decode_ms_q25": 23.6317355, "handoff_plus_first_decode_ms_q75": 30.771002000000003, "steady_tpot_ms": 15.2328625, "steady_tpot_ms_q25": 15.22344175, "steady_tpot_ms_q75": 15.295879750000001, "steady_tpot_p95_ms": 16.608904, "steady_tpot_p95_ms_q25": 16.084898199999998, "steady_tpot_p95_ms_q75": 17.133307324999997, "decode_tokens_per_s": 61.171514515309156, "decode_tokens_per_s_q25": 60.47460600914057, "decode_tokens_per_s_q75": 61.57064257445532, "decode_ms": 245.21217299999998, "decode_ms_q25": 243.6328375, "decode_ms_q75": 248.070939, "end_to_end_ms": 450.31444200000004, "end_to_end_ms_q25": 441.27265050000005, "end_to_end_ms_q75": 451.612767, "output_tokens_per_s_e2e": 35.53072810398561, "output_tokens_per_s_e2e_q25": 35.428874929310865, "output_tokens_per_s_e2e_q75": 36.27399163704915, "all_tokens_per_s_e2e": 2273.966598655079, "all_tokens_per_s_e2e_q25": 2267.4479954758954, "all_tokens_per_s_e2e_q75": 2321.5354647711456, "energy_net_psys_j": 14.367050427195586, "energy_net_psys_j_q25": 14.34006304929965, "energy_net_psys_j_q75": 14.397856528065999, "prefill_energy_net_psys_j": 5.535363733573653, "prefill_energy_net_psys_j_q25": 5.0970728583691605, "prefill_energy_net_psys_j_q75": 5.543861215840693, "decode_energy_net_psys_j": 8.876303930828678, "decode_energy_net_psys_j_q25": 8.853995312225305, "decode_energy_net_psys_j_q75": 9.265298809533862, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "FP16", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 1024, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 207.69891900000002, "ttft_ms_q25": 194.50003650000002, "ttft_ms_q75": 207.97992950000003, "prefill_tokens_per_s": 4853.1788458658275, "prefill_tokens_per_s_q25": 4846.6303581174, "prefill_tokens_per_s_q75": 5206.494508368829, "handoff_plus_first_decode_ms": 21.64219, "handoff_plus_first_decode_ms_q25": 20.996133, "handoff_plus_first_decode_ms_q75": 56.0067405, "steady_tpot_ms": 16.6017975, "steady_tpot_ms_q25": 16.534877, "steady_tpot_ms_q75": 16.622454750000003, "steady_tpot_p95_ms": 17.67764695, "steady_tpot_p95_ms_q25": 17.65987035, "steady_tpot_p95_ms_q75": 17.85497235, "decode_tokens_per_s": 58.259557701827305, "decode_tokens_per_s_q25": 52.105704847208315, "decode_tokens_per_s_q75": 58.40052970577568, "decode_ms": 257.468484, "decode_ms_q25": 256.848482, "decode_ms_q75": 291.94855600000005, "end_to_end_ms": 465.16740300000004, "end_to_end_ms_q25": 464.8284115, "end_to_end_ms_q75": 486.4485925, "output_tokens_per_s_e2e": 34.39621928968226, "output_tokens_per_s_e2e_q25": 32.95452237384665, "output_tokens_per_s_e2e_q75": 34.421322177862876, "all_tokens_per_s_e2e": 2201.3580345396645, "all_tokens_per_s_e2e_q25": 2109.0894319261856, "all_tokens_per_s_e2e_q75": 2202.964619383224, "energy_net_psys_j": 12.934269199598155, "energy_net_psys_j_q25": 12.890027974147833, "energy_net_psys_j_q75": 13.328539196585298, "prefill_energy_net_psys_j": 5.535363733573653, "prefill_energy_net_psys_j_q25": 5.0970728583691605, "prefill_energy_net_psys_j_q75": 5.543861215840693, "decode_energy_net_psys_j": 7.381910501490423, "decode_energy_net_psys_j_q25": 7.34616675830714, "decode_energy_net_psys_j_q75": 8.222968855949098, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.9680696435214313, "energy_reduction_vs_gpu": 0.09972688791328388 }, { "precision": "FP16", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 1024, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 203.698701, "ttft_ms_q25": 202.494179, "ttft_ms_q75": 203.72010649999999, "prefill_tokens_per_s": 4948.485164861213, "prefill_tokens_per_s_q25": 4947.9652668877625, "prefill_tokens_per_s_q75": 4978.0970154479965, "handoff_plus_first_decode_ms": 21.64219, "handoff_plus_first_decode_ms_q25": 20.996133, "handoff_plus_first_decode_ms_q75": 56.0067405, "steady_tpot_ms": 16.6017975, "steady_tpot_ms_q25": 16.534877, "steady_tpot_ms_q75": 16.622454750000003, "steady_tpot_p95_ms": 17.67764695, "steady_tpot_p95_ms_q25": 17.65987035, "steady_tpot_p95_ms_q75": 17.85497235, "decode_tokens_per_s": 58.259557701827305, "decode_tokens_per_s_q25": 52.105704847208315, "decode_tokens_per_s_q75": 58.40052970577568, "decode_ms": 257.468484, "decode_ms_q25": 256.848482, "decode_ms_q75": 291.94855600000005, "end_to_end_ms": 461.167185, "end_to_end_ms_q25": 459.342661, "end_to_end_ms_q75": 495.66866250000004, "output_tokens_per_s_e2e": 34.69457611126429, "output_tokens_per_s_e2e_q25": 32.43678366547543, "output_tokens_per_s_e2e_q75": 34.83293365748071, "all_tokens_per_s_e2e": 2220.4528711209146, "all_tokens_per_s_e2e_q25": 2075.9541545904276, "all_tokens_per_s_e2e_q75": 2229.3077540787654, "energy_net_psys_j": 10.16254561750419, "energy_net_psys_j_q25": 10.086285060208787, "energy_net_psys_j_q75": 10.78111545520396, "prefill_energy_net_psys_j": 2.6281140014229605, "prefill_energy_net_psys_j_q25": 2.4818860419594584, "prefill_energy_net_psys_j_q75": 2.7401183019016466, "decode_energy_net_psys_j": 7.381910501490423, "decode_energy_net_psys_j_q25": 7.34616675830714, "decode_energy_net_psys_j_q75": 8.222968855949098, "fastest_e2e": false, "lowest_energy": true, "e2e_speedup_vs_gpu": 0.9764667926231568, "energy_reduction_vs_gpu": 0.2926491301048566 }, { "precision": "FP16", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 1024, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 203.698701, "ttft_ms_q25": 202.494179, "ttft_ms_q75": 203.72010649999999, "prefill_tokens_per_s": 4948.485164861213, "prefill_tokens_per_s_q25": 4947.9652668877625, "prefill_tokens_per_s_q75": 4978.0970154479965, "handoff_plus_first_decode_ms": 24.224101, "handoff_plus_first_decode_ms_q25": 23.6317355, "handoff_plus_first_decode_ms_q75": 30.771002000000003, "steady_tpot_ms": 15.2328625, "steady_tpot_ms_q25": 15.22344175, "steady_tpot_ms_q75": 15.295879750000001, "steady_tpot_p95_ms": 16.608904, "steady_tpot_p95_ms_q25": 16.084898199999998, "steady_tpot_p95_ms_q75": 17.133307324999997, "decode_tokens_per_s": 61.171514515309156, "decode_tokens_per_s_q25": 60.47460600914057, "decode_tokens_per_s_q75": 61.57064257445532, "decode_ms": 245.21217299999998, "decode_ms_q25": 243.6328375, "decode_ms_q75": 248.070939, "end_to_end_ms": 448.910874, "end_to_end_ms_q25": 446.12701649999997, "end_to_end_ms_q75": 451.7910455, "output_tokens_per_s_e2e": 35.6418187366074, "output_tokens_per_s_e2e_q25": 35.41604119297779, "output_tokens_per_s_e2e_q75": 35.86562222243843, "all_tokens_per_s_e2e": 2281.0763991428735, "all_tokens_per_s_e2e_q25": 2266.6266363505783, "all_tokens_per_s_e2e_q75": 2295.3998222360597, "energy_net_psys_j": 11.683809296002266, "energy_net_psys_j_q25": 11.594113614126952, "energy_net_psys_j_q75": 11.836880533368634, "prefill_energy_net_psys_j": 2.6281140014229605, "prefill_energy_net_psys_j_q25": 2.4818860419594584, "prefill_energy_net_psys_j_q75": 2.7401183019016466, "decode_energy_net_psys_j": 8.876303930828678, "decode_energy_net_psys_j_q25": 8.853995312225305, "decode_energy_net_psys_j_q75": 9.265298809533862, "fastest_e2e": true, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0031266072650316, "energy_reduction_vs_gpu": 0.18676353540975787 }, { "precision": "FP16", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 2048, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 441.579209, "ttft_ms_q25": 415.621217, "ttft_ms_q75": 442.29111650000004, "prefill_tokens_per_s": 4601.665926712596, "prefill_tokens_per_s_q25": 4594.271032528484, "prefill_tokens_per_s_q75": 4908.212707749677, "handoff_plus_first_decode_ms": 37.542289, "handoff_plus_first_decode_ms_q25": 33.3053835, "handoff_plus_first_decode_ms_q75": 42.152677, "steady_tpot_ms": 16.547641000000002, "steady_tpot_ms_q25": 16.492974500000003, "steady_tpot_ms_q75": 16.61902925, "steady_tpot_p95_ms": 18.34303435, "steady_tpot_p95_ms_q25": 18.236382024999997, "steady_tpot_p95_ms_q75": 22.249032649999997, "decode_tokens_per_s": 54.51941527970728, "decode_tokens_per_s_q25": 52.15687718422036, "decode_tokens_per_s_q75": 55.63636080975168, "decode_ms": 275.13134400000007, "decode_ms_q25": 269.71656299999995, "decode_ms_q75": 288.185203, "end_to_end_ms": 705.8809909999999, "end_to_end_ms_q25": 698.3916389999999, "end_to_end_ms_q75": 712.0076795, "output_tokens_per_s_e2e": 22.666710400195495, "output_tokens_per_s_e2e_q25": 22.473331723265623, "output_tokens_per_s_e2e_q75": 22.91241659637302, "all_tokens_per_s_e2e": 2901.3389312250233, "all_tokens_per_s_e2e_q25": 2876.586460578, "all_tokens_per_s_e2e_q75": 2932.7893243357466, "energy_net_psys_j": 21.176776688412616, "energy_net_psys_j_q25": 20.70068914507565, "energy_net_psys_j_q75": 21.378772005881416, "prefill_energy_net_psys_j": 11.988368053034002, "prefill_energy_net_psys_j_q25": 11.115381583578568, "prefill_energy_net_psys_j_q75": 12.034233480063682, "decode_energy_net_psys_j": 9.592399270316214, "decode_energy_net_psys_j_q25": 9.344538525817736, "decode_energy_net_psys_j_q75": 9.787302878965884, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "FP16", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 2048, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 441.579209, "ttft_ms_q25": 415.621217, "ttft_ms_q75": 442.29111650000004, "prefill_tokens_per_s": 4601.665926712596, "prefill_tokens_per_s_q25": 4594.271032528484, "prefill_tokens_per_s_q75": 4908.212707749677, "handoff_plus_first_decode_ms": 31.766454, "handoff_plus_first_decode_ms_q25": 31.277967500000003, "handoff_plus_first_decode_ms_q75": 67.5615095, "steady_tpot_ms": 16.9854185, "steady_tpot_ms_q25": 16.967267, "steady_tpot_ms_q75": 17.151094750000002, "steady_tpot_p95_ms": 17.955736849999997, "steady_tpot_p95_ms_q25": 17.881051425, "steady_tpot_p95_ms_q75": 19.4568709, "decode_tokens_per_s": 54.76064976081041, "decode_tokens_per_s_q25": 48.776739515156834, "decode_tokens_per_s_q75": 54.95241436007043, "decode_ms": 273.919321, "decode_ms_q25": 272.966763, "decode_ms_q75": 312.222669, "end_to_end_ms": 715.4985300000001, "end_to_end_ms_q25": 715.2578795000001, "end_to_end_ms_q75": 727.843886, "output_tokens_per_s_e2e": 22.362030569091452, "output_tokens_per_s_e2e_q25": 21.98906212724761, "output_tokens_per_s_e2e_q75": 22.369556868625512, "all_tokens_per_s_e2e": 2862.339912843706, "all_tokens_per_s_e2e_q25": 2814.599952287694, "all_tokens_per_s_e2e_q75": 2863.3032791840656, "energy_net_psys_j": 20.17528765300994, "energy_net_psys_j_q25": 20.033770512601087, "energy_net_psys_j_q75": 20.21443138737155, "prefill_energy_net_psys_j": 11.988368053034002, "prefill_energy_net_psys_j_q25": 11.115381583578568, "prefill_energy_net_psys_j_q75": 12.034233480063682, "decode_energy_net_psys_j": 8.173476214639802, "decode_energy_net_psys_j_q25": 8.038680766899018, "decode_energy_net_psys_j_q75": 9.053184376763301, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.9865582686801604, "energy_reduction_vs_gpu": 0.047291854191892524 }, { "precision": "FP16", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 2048, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 378.987701, "ttft_ms_q25": 376.7324235, "ttft_ms_q75": 381.44781850000004, "prefill_tokens_per_s": 5361.651564518712, "prefill_tokens_per_s_q25": 5327.293606253492, "prefill_tokens_per_s_q75": 5393.941951970818, "handoff_plus_first_decode_ms": 31.766454, "handoff_plus_first_decode_ms_q25": 31.277967500000003, "handoff_plus_first_decode_ms_q75": 67.5615095, "steady_tpot_ms": 16.9854185, "steady_tpot_ms_q25": 16.967267, "steady_tpot_ms_q75": 17.151094750000002, "steady_tpot_p95_ms": 17.955736849999997, "steady_tpot_p95_ms_q25": 17.881051425, "steady_tpot_p95_ms_q75": 19.4568709, "decode_tokens_per_s": 54.76064976081041, "decode_tokens_per_s_q25": 48.776739515156834, "decode_tokens_per_s_q75": 54.95241436007043, "decode_ms": 273.919321, "decode_ms_q25": 272.966763, "decode_ms_q75": 312.222669, "end_to_end_ms": 657.827257, "end_to_end_ms_q25": 652.159304, "end_to_end_ms_q75": 693.6704875, "output_tokens_per_s_e2e": 24.322494742719364, "output_tokens_per_s_e2e_q25": 23.127456508498895, "output_tokens_per_s_e2e_q75": 24.535736198821198, "all_tokens_per_s_e2e": 3113.2793270680786, "all_tokens_per_s_e2e_q25": 2960.3144330878586, "all_tokens_per_s_e2e_q75": 3140.5742334491133, "energy_net_psys_j": 14.641903385774786, "energy_net_psys_j_q25": 14.556627562261449, "energy_net_psys_j_q75": 15.729274250951343, "prefill_energy_net_psys_j": 6.567466419589879, "prefill_energy_net_psys_j_q25": 6.517946795362431, "prefill_energy_net_psys_j_q75": 6.725609498415488, "decode_energy_net_psys_j": 8.173476214639802, "decode_energy_net_psys_j_q25": 8.038680766899018, "decode_energy_net_psys_j_q75": 9.053184376763301, "fastest_e2e": false, "lowest_energy": true, "e2e_speedup_vs_gpu": 1.0730491682864394, "energy_reduction_vs_gpu": 0.30858677875247864 }, { "precision": "FP16", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 2048, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 378.987701, "ttft_ms_q25": 376.7324235, "ttft_ms_q75": 381.44781850000004, "prefill_tokens_per_s": 5361.651564518712, "prefill_tokens_per_s_q25": 5327.293606253492, "prefill_tokens_per_s_q75": 5393.941951970818, "handoff_plus_first_decode_ms": 37.542289, "handoff_plus_first_decode_ms_q25": 33.3053835, "handoff_plus_first_decode_ms_q75": 42.152677, "steady_tpot_ms": 16.547641000000002, "steady_tpot_ms_q25": 16.492974500000003, "steady_tpot_ms_q75": 16.61902925, "steady_tpot_p95_ms": 18.34303435, "steady_tpot_p95_ms_q25": 18.236382024999997, "steady_tpot_p95_ms_q75": 22.249032649999997, "decode_tokens_per_s": 54.51941527970728, "decode_tokens_per_s_q25": 52.15687718422036, "decode_tokens_per_s_q75": 55.63636080975168, "decode_ms": 275.13134400000007, "decode_ms_q25": 269.71656299999995, "decode_ms_q75": 288.185203, "end_to_end_ms": 649.6084900000001, "end_to_end_ms_q25": 648.909104, "end_to_end_ms_q75": 664.9176265000001, "output_tokens_per_s_e2e": 24.63021996525938, "output_tokens_per_s_e2e_q25": 24.07589393755326, "output_tokens_per_s_e2e_q75": 24.65679474583159, "all_tokens_per_s_e2e": 3152.6681555532004, "all_tokens_per_s_e2e_q25": 3081.7144240068174, "all_tokens_per_s_e2e_q75": 3156.0697274664435, "energy_net_psys_j": 16.15986568990609, "energy_net_psys_j_q25": 15.862485321180166, "energy_net_psys_j_q75": 16.51291237738137, "prefill_energy_net_psys_j": 6.567466419589879, "prefill_energy_net_psys_j_q25": 6.517946795362431, "prefill_energy_net_psys_j_q75": 6.725609498415488, "decode_energy_net_psys_j": 9.592399270316214, "decode_energy_net_psys_j_q25": 9.344538525817736, "decode_energy_net_psys_j_q75": 9.787302878965884, "fastest_e2e": true, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0866252548515796, "energy_reduction_vs_gpu": 0.23690626162439765 }, { "precision": "FP16", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 4096, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1102.167363, "ttft_ms_q25": 1054.916456, "ttft_ms_q75": 1115.386621, "prefill_tokens_per_s": 3701.797147118028, "prefill_tokens_per_s_q25": 3658.438333858278, "prefill_tokens_per_s_q75": 3875.379813415315, "handoff_plus_first_decode_ms": 46.685089999999995, "handoff_plus_first_decode_ms_q25": 43.7248115, "handoff_plus_first_decode_ms_q75": 48.964158, "steady_tpot_ms": 19.858069999999998, "steady_tpot_ms_q25": 19.68306975, "steady_tpot_ms_q75": 19.95097275, "steady_tpot_p95_ms": 23.73208415, "steady_tpot_p95_ms_q25": 23.58919285, "steady_tpot_p95_ms_q75": 23.794901949999996, "decode_tokens_per_s": 44.477808863252505, "decode_tokens_per_s_q25": 44.17141662359542, "decode_tokens_per_s_q75": 45.31323026243381, "decode_ms": 337.24682899999993, "decode_ms_q25": 331.14170550000006, "decode_ms_q75": 339.6024599999999, "end_to_end_ms": 1439.414192, "end_to_end_ms_q25": 1394.518916, "end_to_end_ms_q75": 1446.5283265, "output_tokens_per_s_e2e": 11.115633074152711, "output_tokens_per_s_e2e_q25": 11.061233102451435, "output_tokens_per_s_e2e_q75": 11.485394984057322, "all_tokens_per_s_e2e": 2845.602066983094, "all_tokens_per_s_e2e_q25": 2831.6756742275675, "all_tokens_per_s_e2e_q75": 2940.2611159186745, "energy_net_psys_j": 40.289082094538706, "energy_net_psys_j_q25": 39.071873587210305, "energy_net_psys_j_q75": 40.4494325020034, "prefill_energy_net_psys_j": 29.509258382133034, "prefill_energy_net_psys_j_q25": 28.141964137578384, "prefill_energy_net_psys_j_q75": 29.925564630126253, "decode_energy_net_psys_j": 10.77982371240567, "decode_energy_net_psys_j_q25": 10.52386787187714, "decode_energy_net_psys_j_q75": 10.929909449631921, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "FP16", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 4096, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1102.167363, "ttft_ms_q25": 1054.916456, "ttft_ms_q75": 1115.386621, "prefill_tokens_per_s": 3701.797147118028, "prefill_tokens_per_s_q25": 3658.438333858278, "prefill_tokens_per_s_q75": 3875.379813415315, "handoff_plus_first_decode_ms": 54.623934, "handoff_plus_first_decode_ms_q25": 53.079732500000006, "handoff_plus_first_decode_ms_q75": 88.20110700000001, "steady_tpot_ms": 19.554413999999998, "steady_tpot_ms_q25": 19.4815235, "steady_tpot_ms_q75": 19.59206975, "steady_tpot_p95_ms": 20.2453457, "steady_tpot_p95_ms_q25": 20.175806625, "steady_tpot_p95_ms_q75": 23.897542674999997, "decode_tokens_per_s": 45.60808645967518, "decode_tokens_per_s_q25": 40.88247535057374, "decode_tokens_per_s_q75": 45.700504780446224, "decode_ms": 328.889045, "decode_ms_q25": 328.22528800000003, "decode_ms_q75": 371.874015, "end_to_end_ms": 1431.056408, "end_to_end_ms_q25": 1426.7904709999998, "end_to_end_ms_q75": 1443.611909, "output_tokens_per_s_e2e": 11.180551591506518, "output_tokens_per_s_e2e_q25": 11.084149607968298, "output_tokens_per_s_e2e_q75": 11.214080383154492, "all_tokens_per_s_e2e": 2862.2212074256686, "all_tokens_per_s_e2e_q25": 2837.5422996398843, "all_tokens_per_s_e2e_q75": 2870.80457808755, "energy_net_psys_j": 38.96763849047304, "energy_net_psys_j_q25": 38.37706180776674, "energy_net_psys_j_q75": 39.5647979757121, "prefill_energy_net_psys_j": 29.509258382133034, "prefill_energy_net_psys_j_q25": 28.141964137578384, "prefill_energy_net_psys_j_q75": 29.925564630126253, "decode_energy_net_psys_j": 9.820086582831687, "decode_energy_net_psys_j_q25": 9.639233345585847, "decode_energy_net_psys_j_q75": 10.415950907434198, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0058402897001668, "energy_reduction_vs_gpu": 0.03279904965233216 }, { "precision": "FP16", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 4096, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 889.630587, "ttft_ms_q25": 889.4896515, "ttft_ms_q75": 893.698973, "prefill_tokens_per_s": 4586.173249459104, "prefill_tokens_per_s_q25": 4565.3902227043145, "prefill_tokens_per_s_q75": 4586.900022442942, "handoff_plus_first_decode_ms": 54.623934, "handoff_plus_first_decode_ms_q25": 53.079732500000006, "handoff_plus_first_decode_ms_q75": 88.20110700000001, "steady_tpot_ms": 19.554413999999998, "steady_tpot_ms_q25": 19.4815235, "steady_tpot_ms_q75": 19.59206975, "steady_tpot_p95_ms": 20.2453457, "steady_tpot_p95_ms_q25": 20.175806625, "steady_tpot_p95_ms_q75": 23.897542674999997, "decode_tokens_per_s": 45.60808645967518, "decode_tokens_per_s_q25": 40.88247535057374, "decode_tokens_per_s_q75": 45.700504780446224, "decode_ms": 328.889045, "decode_ms_q25": 328.22528800000003, "decode_ms_q75": 371.874015, "end_to_end_ms": 1226.656404, "end_to_end_ms_q25": 1221.7833255, "end_to_end_ms_q75": 1265.572988, "output_tokens_per_s_e2e": 13.043587387491435, "output_tokens_per_s_e2e_q25": 12.654460578721014, "output_tokens_per_s_e2e_q75": 13.095820019616568, "all_tokens_per_s_e2e": 3339.1583711978074, "all_tokens_per_s_e2e_q25": 3239.5419081525797, "all_tokens_per_s_e2e_q75": 3352.5299250218413, "energy_net_psys_j": 27.786917847685658, "energy_net_psys_j_q25": 27.612264668247487, "energy_net_psys_j_q75": 28.929106769659477, "prefill_energy_net_psys_j": 17.979231380469308, "prefill_energy_net_psys_j_q25": 17.97303132266164, "prefill_energy_net_psys_j_q75": 18.51935592003295, "decode_energy_net_psys_j": 9.820086582831687, "decode_energy_net_psys_j_q25": 9.639233345585847, "decode_energy_net_psys_j_q75": 10.415950907434198, "fastest_e2e": true, "lowest_energy": true, "e2e_speedup_vs_gpu": 1.1734453000092109, "energy_reduction_vs_gpu": 0.31031146893633876 }, { "precision": "FP16", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 4096, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 889.630587, "ttft_ms_q25": 889.4896515, "ttft_ms_q75": 893.698973, "prefill_tokens_per_s": 4586.173249459104, "prefill_tokens_per_s_q25": 4565.3902227043145, "prefill_tokens_per_s_q75": 4586.900022442942, "handoff_plus_first_decode_ms": 46.685089999999995, "handoff_plus_first_decode_ms_q25": 43.7248115, "handoff_plus_first_decode_ms_q75": 48.964158, "steady_tpot_ms": 19.858069999999998, "steady_tpot_ms_q25": 19.68306975, "steady_tpot_ms_q75": 19.95097275, "steady_tpot_p95_ms": 23.73208415, "steady_tpot_p95_ms_q25": 23.58919285, "steady_tpot_p95_ms_q75": 23.794901949999996, "decode_tokens_per_s": 44.477808863252505, "decode_tokens_per_s_q25": 44.17141662359542, "decode_tokens_per_s_q75": 45.31323026243381, "decode_ms": 337.24682899999993, "decode_ms_q25": 331.14170550000006, "decode_ms_q75": 339.6024599999999, "end_to_end_ms": 1231.5886779999998, "end_to_end_ms_q25": 1222.986988, "end_to_end_ms_q75": 1233.301433, "output_tokens_per_s_e2e": 12.991350347571158, "output_tokens_per_s_e2e_q25": 12.973333550289995, "output_tokens_per_s_e2e_q75": 13.083370210258263, "all_tokens_per_s_e2e": 3325.7856889782165, "all_tokens_per_s_e2e_q25": 3321.1733888742388, "all_tokens_per_s_e2e_q75": 3349.3427738261153, "energy_net_psys_j": 28.75905509287498, "energy_net_psys_j_q25": 28.496899194538784, "energy_net_psys_j_q75": 29.44926536966487, "prefill_energy_net_psys_j": 17.979231380469308, "prefill_energy_net_psys_j_q25": 17.97303132266164, "prefill_energy_net_psys_j_q75": 18.51935592003295, "decode_energy_net_psys_j": 10.77982371240567, "decode_energy_net_psys_j_q25": 10.52386787187714, "decode_energy_net_psys_j_q75": 10.929909449631921, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.1687458789711285, "energy_reduction_vs_gpu": 0.2861824197088534 }, { "precision": "FP16", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 8192, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 2856.822478, "ttft_ms_q25": 2751.7923769999998, "ttft_ms_q75": 2864.265589, "prefill_tokens_per_s": 2861.920914919376, "prefill_tokens_per_s_q25": 2854.5031726382854, "prefill_tokens_per_s_q75": 2975.4890451113583, "handoff_plus_first_decode_ms": 54.975293, "handoff_plus_first_decode_ms_q25": 54.7470285, "handoff_plus_first_decode_ms_q75": 62.376990000000006, "steady_tpot_ms": 22.787651, "steady_tpot_ms_q25": 22.78760075, "steady_tpot_ms_q75": 22.823230000000002, "steady_tpot_p95_ms": 26.0898805, "steady_tpot_p95_ms_q25": 25.599848625, "steady_tpot_p95_ms_q75": 26.176459174999998, "decode_tokens_per_s": 39.09940484893042, "decode_tokens_per_s_q25": 38.331621450663874, "decode_tokens_per_s_q75": 39.24716908652915, "decode_ms": 383.63755299999957, "decode_ms_q25": 382.1985884999999, "decode_ms_q75": 391.47888549999993, "end_to_end_ms": 3237.5821020000003, "end_to_end_ms_q25": 3141.8322980000003, "end_to_end_ms_q75": 3246.4641775, "output_tokens_per_s_e2e": 4.941959615515566, "output_tokens_per_s_e2e_q25": 4.928475686307573, "output_tokens_per_s_e2e_q75": 5.0973039522147845, "all_tokens_per_s_e2e": 2530.28332314397, "all_tokens_per_s_e2e_q25": 2523.3795513894775, "all_tokens_per_s_e2e_q75": 2609.8196235339697, "energy_net_psys_j": 89.21420497592791, "energy_net_psys_j_q25": 87.10804362204142, "energy_net_psys_j_q75": 90.54855250851622, "prefill_energy_net_psys_j": 78.6067166446301, "prefill_energy_net_psys_j_q25": 76.10534310695107, "prefill_energy_net_psys_j_q75": 79.3695969585709, "decode_energy_net_psys_j": 11.397912698882905, "decode_energy_net_psys_j_q25": 11.002700515090362, "decode_energy_net_psys_j_q75": 11.57416773373787, "fastest_e2e": true, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "FP16", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 8192, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 2856.822478, "ttft_ms_q25": 2751.7923769999998, "ttft_ms_q75": 2864.265589, "prefill_tokens_per_s": 2861.920914919376, "prefill_tokens_per_s_q25": 2854.5031726382854, "prefill_tokens_per_s_q75": 2975.4890451113583, "handoff_plus_first_decode_ms": 87.489434, "handoff_plus_first_decode_ms_q25": 81.8039895, "handoff_plus_first_decode_ms_q75": 116.94190849999998, "steady_tpot_ms": 22.611179500000002, "steady_tpot_ms_q25": 22.605743000000004, "steady_tpot_ms_q75": 22.923969, "steady_tpot_p95_ms": 24.377819849999995, "steady_tpot_p95_ms_q25": 24.140172874999998, "steady_tpot_p95_ms_q75": 27.619299924999993, "decode_tokens_per_s": 36.81603411132208, "decode_tokens_per_s_q25": 33.677503954482, "decode_tokens_per_s_q75": 37.2919916797083, "decode_ms": 407.4311740000003, "decode_ms_q25": 402.2966630000003, "decode_ms_q75": 449.3034069999999, "end_to_end_ms": 3264.2536520000003, "end_to_end_ms_q25": 3201.095784, "end_to_end_ms_q75": 3266.5622520000006, "output_tokens_per_s_e2e": 4.901579872690604, "output_tokens_per_s_e2e_q25": 4.898118191943207, "output_tokens_per_s_e2e_q75": 5.000234894835042, "all_tokens_per_s_e2e": 2509.6088948175893, "all_tokens_per_s_e2e_q25": 2507.836514274922, "all_tokens_per_s_e2e_q75": 2560.1202661555417, "energy_net_psys_j": 88.81601267346514, "energy_net_psys_j_q25": 87.01605052325792, "energy_net_psys_j_q75": 89.57725471644027, "prefill_energy_net_psys_j": 78.6067166446301, "prefill_energy_net_psys_j_q25": 76.10534310695107, "prefill_energy_net_psys_j_q75": 79.3695969585709, "decode_energy_net_psys_j": 10.209296028835045, "decode_energy_net_psys_j_q25": 10.207657757869368, "decode_energy_net_psys_j_q75": 10.910707416306865, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.9918292042091587, "energy_reduction_vs_gpu": 0.0044633284864245 }, { "precision": "FP16", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 8192, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 2978.993062, "ttft_ms_q25": 2976.052053, "ttft_ms_q75": 2985.345813, "prefill_tokens_per_s": 2744.551541355688, "prefill_tokens_per_s_q25": 2738.723597070557, "prefill_tokens_per_s_q75": 2747.2664587148565, "handoff_plus_first_decode_ms": 87.489434, "handoff_plus_first_decode_ms_q25": 81.8039895, "handoff_plus_first_decode_ms_q75": 116.94190849999998, "steady_tpot_ms": 22.611179500000002, "steady_tpot_ms_q25": 22.605743000000004, "steady_tpot_ms_q75": 22.923969, "steady_tpot_p95_ms": 24.377819849999995, "steady_tpot_p95_ms_q25": 24.140172874999998, "steady_tpot_p95_ms_q75": 27.619299924999993, "decode_tokens_per_s": 36.81603411132208, "decode_tokens_per_s_q25": 33.677503954482, "decode_tokens_per_s_q75": 37.2919916797083, "decode_ms": 407.4311740000003, "decode_ms_q25": 402.2966630000003, "decode_ms_q75": 449.3034069999999, "end_to_end_ms": 3388.860716, "end_to_end_ms_q25": 3384.701467, "end_to_end_ms_q75": 3429.514709, "output_tokens_per_s_e2e": 4.7213507254690015, "output_tokens_per_s_e2e_q25": 4.666038786532679, "output_tokens_per_s_e2e_q75": 4.727159637772772, "all_tokens_per_s_e2e": 2417.3315714401288, "all_tokens_per_s_e2e_q25": 2389.011858704732, "all_tokens_per_s_e2e_q75": 2420.305734539659, "energy_net_psys_j": 64.2541216746021, "energy_net_psys_j_q25": 63.08175203277463, "energy_net_psys_j_q75": 66.60504801560094, "prefill_energy_net_psys_j": 54.04482564576706, "prefill_energy_net_psys_j_q25": 52.87409427490526, "prefill_energy_net_psys_j_q75": 55.69434059929409, "decode_energy_net_psys_j": 10.209296028835045, "decode_energy_net_psys_j_q25": 10.207657757869368, "decode_energy_net_psys_j_q75": 10.910707416306865, "fastest_e2e": false, "lowest_energy": true, "e2e_speedup_vs_gpu": 0.9553600378776972, "energy_reduction_vs_gpu": 0.27977700757475366 }, { "precision": "FP16", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 8192, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 2978.993062, "ttft_ms_q25": 2976.052053, "ttft_ms_q75": 2985.345813, "prefill_tokens_per_s": 2744.551541355688, "prefill_tokens_per_s_q25": 2738.723597070557, "prefill_tokens_per_s_q75": 2747.2664587148565, "handoff_plus_first_decode_ms": 54.975293, "handoff_plus_first_decode_ms_q25": 54.7470285, "handoff_plus_first_decode_ms_q75": 62.376990000000006, "steady_tpot_ms": 22.787651, "steady_tpot_ms_q25": 22.78760075, "steady_tpot_ms_q75": 22.823230000000002, "steady_tpot_p95_ms": 26.0898805, "steady_tpot_p95_ms_q25": 25.599848625, "steady_tpot_p95_ms_q75": 26.176459174999998, "decode_tokens_per_s": 39.09940484893042, "decode_tokens_per_s_q25": 38.331621450663874, "decode_tokens_per_s_q75": 39.24716908652915, "decode_ms": 383.63755299999957, "decode_ms_q25": 382.1985884999999, "decode_ms_q75": 391.47888549999993, "end_to_end_ms": 3375.3361169999994, "end_to_end_ms_q25": 3364.6033924999997, "end_to_end_ms_q75": 3376.8246984999996, "output_tokens_per_s_e2e": 4.740268656331865, "output_tokens_per_s_e2e_q25": 4.738179958321345, "output_tokens_per_s_e2e_q75": 4.755437994264247, "all_tokens_per_s_e2e": 2427.0175520419148, "all_tokens_per_s_e2e_q25": 2425.9481386605285, "all_tokens_per_s_e2e_q75": 2434.7842530632943, "energy_net_psys_j": 64.65231397706488, "energy_net_psys_j_q25": 64.05304982485059, "energy_net_psys_j_q75": 66.69704111438445, "prefill_energy_net_psys_j": 54.04482564576706, "prefill_energy_net_psys_j_q25": 52.87409427490526, "prefill_energy_net_psys_j_q75": 55.69434059929409, "decode_energy_net_psys_j": 11.397912698882905, "decode_energy_net_psys_j_q25": 11.002700515090362, "decode_energy_net_psys_j_q75": 11.57416773373787, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.9591880600257272, "energy_reduction_vs_gpu": 0.27531367908832916 }, { "precision": "FP16", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 1024, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 360.493108, "ttft_ms_q25": 347.1690065, "ttft_ms_q75": 369.713306, "prefill_tokens_per_s": 2796.169961729199, "prefill_tokens_per_s_q25": 2728.133642911648, "prefill_tokens_per_s_q75": 2907.768047616809, "handoff_plus_first_decode_ms": 55.536018999999996, "handoff_plus_first_decode_ms_q25": 54.2231775, "handoff_plus_first_decode_ms_q75": 57.620129999999996, "steady_tpot_ms": 43.521603, "steady_tpot_ms_q25": 43.449562, "steady_tpot_ms_q75": 43.5519585, "steady_tpot_p95_ms": 45.06358405, "steady_tpot_p95_ms_q25": 44.741105399999995, "steady_tpot_p95_ms_q75": 45.35408905, "decode_tokens_per_s": 22.477790538086463, "decode_tokens_per_s_q25": 22.3496970718926, "decode_tokens_per_s_q75": 22.51589054630978, "decode_ms": 667.3253749999998, "decode_ms_q25": 666.1980755, "decode_ms_q75": 671.1720829999999, "end_to_end_ms": 1027.8184829999998, "end_to_end_ms_q25": 1018.3410895, "end_to_end_ms_q75": 1035.9113815, "output_tokens_per_s_e2e": 15.566951037209556, "output_tokens_per_s_e2e_q25": 15.446279353087142, "output_tokens_per_s_e2e_q75": 15.713188949387204, "all_tokens_per_s_e2e": 996.2848663814116, "all_tokens_per_s_e2e_q25": 988.5618785975771, "all_tokens_per_s_e2e_q75": 1005.6440927607811, "energy_net_psys_j": 34.01359510119007, "energy_net_psys_j_q25": 33.44256860175318, "energy_net_psys_j_q75": 34.20331383758705, "prefill_energy_net_psys_j": 11.490362435506256, "prefill_energy_net_psys_j_q25": 11.080950246895839, "prefill_energy_net_psys_j_q75": 11.726210388944478, "decode_energy_net_psys_j": 22.43097423160133, "decode_energy_net_psys_j_q25": 21.906076949205676, "decode_energy_net_psys_j_q75": 22.88651563725299, "fastest_e2e": true, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "FP16", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 1024, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 360.493108, "ttft_ms_q25": 347.1690065, "ttft_ms_q75": 369.713306, "prefill_tokens_per_s": 2796.169961729199, "prefill_tokens_per_s_q25": 2728.133642911648, "prefill_tokens_per_s_q75": 2907.768047616809, "handoff_plus_first_decode_ms": 52.85397, "handoff_plus_first_decode_ms_q25": 52.370791499999996, "handoff_plus_first_decode_ms_q75": 89.395224, "steady_tpot_ms": 47.742450999999996, "steady_tpot_ms_q25": 47.442558749999996, "steady_tpot_ms_q75": 48.24036325, "steady_tpot_p95_ms": 49.104597749999996, "steady_tpot_p95_ms_q25": 48.51265959999999, "steady_tpot_p95_ms_q75": 49.31260585, "decode_tokens_per_s": 20.557439560377347, "decode_tokens_per_s_q25": 19.68496873840741, "decode_tokens_per_s_q75": 20.773227416650087, "decode_ms": 729.6628529999998, "decode_ms_q25": 722.1611964999998, "decode_ms_q75": 763.5025669999999, "end_to_end_ms": 1093.5930439999997, "end_to_end_ms_q25": 1091.8745024999998, "end_to_end_ms_q75": 1112.390115, "output_tokens_per_s_e2e": 14.630670968313149, "output_tokens_per_s_e2e_q25": 14.387551381764876, "output_tokens_per_s_e2e_q75": 14.653735021651741, "all_tokens_per_s_e2e": 936.3629419720415, "all_tokens_per_s_e2e_q25": 920.8032884329521, "all_tokens_per_s_e2e_q75": 937.8390413857114, "energy_net_psys_j": 27.41282792159977, "energy_net_psys_j_q25": 26.87561944603535, "energy_net_psys_j_q75": 27.501389661654663, "prefill_energy_net_psys_j": 11.490362435506256, "prefill_energy_net_psys_j_q25": 11.080950246895839, "prefill_energy_net_psys_j_q75": 11.726210388944478, "decode_energy_net_psys_j": 15.627893059326851, "decode_energy_net_psys_j_q25": 15.237970797145763, "decode_energy_net_psys_j_q75": 16.1845914613206, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.9398546274952349, "energy_reduction_vs_gpu": 0.19406261407984338 }, { "precision": "FP16", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 1024, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 470.919173, "ttft_ms_q25": 470.917144, "ttft_ms_q75": 473.960953, "prefill_tokens_per_s": 2140.494712879316, "prefill_tokens_per_s_q25": 2126.845076184798, "prefill_tokens_per_s_q75": 2140.5039354836326, "handoff_plus_first_decode_ms": 52.85397, "handoff_plus_first_decode_ms_q25": 52.370791499999996, "handoff_plus_first_decode_ms_q75": 89.395224, "steady_tpot_ms": 47.742450999999996, "steady_tpot_ms_q25": 47.442558749999996, "steady_tpot_ms_q75": 48.24036325, "steady_tpot_p95_ms": 49.104597749999996, "steady_tpot_p95_ms_q25": 48.51265959999999, "steady_tpot_p95_ms_q75": 49.31260585, "decode_tokens_per_s": 20.557439560377347, "decode_tokens_per_s_q25": 19.68496873840741, "decode_tokens_per_s_q75": 20.773227416650087, "decode_ms": 729.6628529999998, "decode_ms_q25": 722.1611964999998, "decode_ms_q75": 763.5025669999999, "end_to_end_ms": 1206.6655859999998, "end_to_end_ms_q25": 1196.1221495, "end_to_end_ms_q75": 1237.461491, "output_tokens_per_s_e2e": 13.259680383393317, "output_tokens_per_s_e2e_q25": 12.937707999313213, "output_tokens_per_s_e2e_q75": 13.377599671754059, "all_tokens_per_s_e2e": 848.6195445371723, "all_tokens_per_s_e2e_q25": 828.0133119560456, "all_tokens_per_s_e2e_q75": 856.1663789922598, "energy_net_psys_j": 24.786369496670208, "energy_net_psys_j_q25": 23.823591376537124, "energy_net_psys_j_q75": 25.291509651087466, "prefill_energy_net_psys_j": 9.055359942190375, "prefill_energy_net_psys_j_q25": 8.534062331814866, "prefill_energy_net_psys_j_q75": 9.106918189766866, "decode_energy_net_psys_j": 15.627893059326851, "decode_energy_net_psys_j_q25": 15.237970797145763, "decode_energy_net_psys_j_q75": 16.1845914613206, "fastest_e2e": false, "lowest_energy": true, "e2e_speedup_vs_gpu": 0.851784036045261, "energy_reduction_vs_gpu": 0.2712805152489458 }, { "precision": "FP16", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 1024, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 470.919173, "ttft_ms_q25": 470.917144, "ttft_ms_q75": 473.960953, "prefill_tokens_per_s": 2140.494712879316, "prefill_tokens_per_s_q25": 2126.845076184798, "prefill_tokens_per_s_q75": 2140.5039354836326, "handoff_plus_first_decode_ms": 55.536018999999996, "handoff_plus_first_decode_ms_q25": 54.2231775, "handoff_plus_first_decode_ms_q75": 57.620129999999996, "steady_tpot_ms": 43.521603, "steady_tpot_ms_q25": 43.449562, "steady_tpot_ms_q75": 43.5519585, "steady_tpot_p95_ms": 45.06358405, "steady_tpot_p95_ms_q25": 44.741105399999995, "steady_tpot_p95_ms_q75": 45.35408905, "decode_tokens_per_s": 22.477790538086463, "decode_tokens_per_s_q25": 22.3496970718926, "decode_tokens_per_s_q75": 22.51589054630978, "decode_ms": 667.3253749999998, "decode_ms_q25": 666.1980755, "decode_ms_q75": 671.1720829999999, "end_to_end_ms": 1144.3281079999997, "end_to_end_ms_q25": 1140.1590285, "end_to_end_ms_q75": 1145.131007, "output_tokens_per_s_e2e": 13.982003839758873, "output_tokens_per_s_e2e_q25": 13.972207343781083, "output_tokens_per_s_e2e_q75": 14.033317749383311, "all_tokens_per_s_e2e": 894.8482457445679, "all_tokens_per_s_e2e_q25": 894.2212700019893, "all_tokens_per_s_e2e_q75": 898.1323359605319, "energy_net_psys_j": 31.589450668944686, "energy_net_psys_j_q25": 30.491697528597037, "energy_net_psys_j_q75": 31.993433827019853, "prefill_energy_net_psys_j": 9.055359942190375, "prefill_energy_net_psys_j_q25": 8.534062331814866, "prefill_energy_net_psys_j_q75": 9.106918189766866, "decode_energy_net_psys_j": 22.43097423160133, "decode_energy_net_psys_j_q25": 21.906076949205676, "decode_energy_net_psys_j_q75": 22.88651563725299, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.898185123492571, "energy_reduction_vs_gpu": 0.07126986797583679 }, { "precision": "FP16", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 2048, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 770.339746, "ttft_ms_q25": 747.4911930000001, "ttft_ms_q75": 774.9201585000001, "prefill_tokens_per_s": 2637.797167485111, "prefill_tokens_per_s_q25": 2622.297244113774, "prefill_tokens_per_s_q75": 2720.9689890943573, "handoff_plus_first_decode_ms": 65.452983, "handoff_plus_first_decode_ms_q25": 63.2263055, "handoff_plus_first_decode_ms_q75": 67.77745300000001, "steady_tpot_ms": 46.5253935, "steady_tpot_ms_q25": 46.489628499999995, "steady_tpot_ms_q75": 46.530795499999996, "steady_tpot_p95_ms": 53.8247413, "steady_tpot_p95_ms_q25": 51.810425925, "steady_tpot_p95_ms_q75": 54.267149575000005, "decode_tokens_per_s": 20.416328324438073, "decode_tokens_per_s_q25": 20.34733433084017, "decode_tokens_per_s_q75": 20.630648167284473, "decode_ms": 734.706053, "decode_ms_q25": 727.1520915000001, "decode_ms_q75": 737.2057794999998, "end_to_end_ms": 1499.0987010000001, "end_to_end_ms_q25": 1481.7234235, "end_to_end_ms_q75": 1502.0722500000002, "output_tokens_per_s_e2e": 10.67307975740818, "output_tokens_per_s_e2e_q25": 10.651992740879749, "output_tokens_per_s_e2e_q75": 10.799721583036305, "all_tokens_per_s_e2e": 1366.154208948247, "all_tokens_per_s_e2e_q25": 1363.4550708326078, "all_tokens_per_s_e2e_q75": 1382.364362628647, "energy_net_psys_j": 48.04695553438617, "energy_net_psys_j_q25": 47.793525539450656, "energy_net_psys_j_q75": 48.689665940820746, "prefill_energy_net_psys_j": 25.69547178335022, "prefill_energy_net_psys_j_q25": 24.8980657021562, "prefill_energy_net_psys_j_q75": 25.883410347253655, "decode_energy_net_psys_j": 23.261027436098225, "decode_energy_net_psys_j_q25": 22.806255593567087, "decode_energy_net_psys_j_q75": 23.35023167982559, "fastest_e2e": true, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "FP16", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 2048, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 770.339746, "ttft_ms_q25": 747.4911930000001, "ttft_ms_q75": 774.9201585000001, "prefill_tokens_per_s": 2637.797167485111, "prefill_tokens_per_s_q25": 2622.297244113774, "prefill_tokens_per_s_q75": 2720.9689890943573, "handoff_plus_first_decode_ms": 90.44789899999999, "handoff_plus_first_decode_ms_q25": 83.576206, "handoff_plus_first_decode_ms_q75": 114.51614899999998, "steady_tpot_ms": 49.289752500000006, "steady_tpot_ms_q25": 49.0961465, "steady_tpot_ms_q75": 49.381530500000004, "steady_tpot_p95_ms": 50.79916185, "steady_tpot_p95_ms_q25": 50.251029375, "steady_tpot_p95_ms_q75": 51.060063325, "decode_tokens_per_s": 19.106795571571624, "decode_tokens_per_s_q25": 18.54293616734449, "decode_tokens_per_s_q75": 19.38125119236078, "decode_ms": 785.0609979999999, "decode_ms_q25": 774.099072, "decode_ms_q75": 809.6820634999999, "end_to_end_ms": 1555.400744, "end_to_end_ms_q25": 1549.0192305, "end_to_end_ms_q75": 1557.1732565, "output_tokens_per_s_e2e": 10.286738039518387, "output_tokens_per_s_e2e_q25": 10.27504207669412, "output_tokens_per_s_e2e_q75": 10.32929174927586, "all_tokens_per_s_e2e": 1316.7024690583535, "all_tokens_per_s_e2e_q25": 1315.2053858168474, "all_tokens_per_s_e2e_q75": 1322.1493439073101, "energy_net_psys_j": 43.01036802874478, "energy_net_psys_j_q25": 42.28801534424129, "energy_net_psys_j_q75": 43.67669184408312, "prefill_energy_net_psys_j": 25.69547178335022, "prefill_energy_net_psys_j_q25": 24.8980657021562, "prefill_energy_net_psys_j_q75": 25.883410347253655, "decode_energy_net_psys_j": 17.46500303877562, "decode_energy_net_psys_j_q25": 17.389949642085092, "decode_energy_net_psys_j_q75": 17.86833489352, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.9638022270355813, "energy_reduction_vs_gpu": 0.10482636099672982 }, { "precision": "FP16", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 2048, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 953.900271, "ttft_ms_q25": 950.9068265, "ttft_ms_q75": 958.8231645000001, "prefill_tokens_per_s": 2130.201722104344, "prefill_tokens_per_s_q25": 2119.3204778965087, "prefill_tokens_per_s_q75": 2136.9287509131045, "handoff_plus_first_decode_ms": 90.44789899999999, "handoff_plus_first_decode_ms_q25": 83.576206, "handoff_plus_first_decode_ms_q75": 114.51614899999998, "steady_tpot_ms": 49.289752500000006, "steady_tpot_ms_q25": 49.0961465, "steady_tpot_ms_q75": 49.381530500000004, "steady_tpot_p95_ms": 50.79916185, "steady_tpot_p95_ms_q25": 50.251029375, "steady_tpot_p95_ms_q75": 51.060063325, "decode_tokens_per_s": 19.106795571571624, "decode_tokens_per_s_q25": 18.54293616734449, "decode_tokens_per_s_q75": 19.38125119236078, "decode_ms": 785.0609979999999, "decode_ms_q25": 774.099072, "decode_ms_q75": 809.6820634999999, "end_to_end_ms": 1732.9743799999999, "end_to_end_ms_q25": 1725.0058985, "end_to_end_ms_q75": 1765.5117834999999, "output_tokens_per_s_e2e": 9.232681212517406, "output_tokens_per_s_e2e_q25": 9.06560709898897, "output_tokens_per_s_e2e_q75": 9.275528531223998, "all_tokens_per_s_e2e": 1181.783195202228, "all_tokens_per_s_e2e_q25": 1160.3977086705881, "all_tokens_per_s_e2e_q75": 1187.2676519966717, "energy_net_psys_j": 39.48190518484798, "energy_net_psys_j_q25": 39.055324441821085, "energy_net_psys_j_q75": 39.8651912459966, "prefill_energy_net_psys_j": 21.2102384365836, "prefill_energy_net_psys_j_q25": 21.186989548301078, "prefill_energy_net_psys_j_q75": 22.071909749167126, "decode_energy_net_psys_j": 17.46500303877562, "decode_energy_net_psys_j_q25": 17.389949642085092, "decode_energy_net_psys_j_q75": 17.86833489352, "fastest_e2e": false, "lowest_energy": true, "e2e_speedup_vs_gpu": 0.8650437757769969, "energy_reduction_vs_gpu": 0.1782641637597281 }, { "precision": "FP16", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 2048, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 953.900271, "ttft_ms_q25": 950.9068265, "ttft_ms_q75": 958.8231645000001, "prefill_tokens_per_s": 2130.201722104344, "prefill_tokens_per_s_q25": 2119.3204778965087, "prefill_tokens_per_s_q75": 2136.9287509131045, "handoff_plus_first_decode_ms": 65.452983, "handoff_plus_first_decode_ms_q25": 63.2263055, "handoff_plus_first_decode_ms_q75": 67.77745300000001, "steady_tpot_ms": 46.5253935, "steady_tpot_ms_q25": 46.489628499999995, "steady_tpot_ms_q75": 46.530795499999996, "steady_tpot_p95_ms": 53.8247413, "steady_tpot_p95_ms_q25": 51.810425925, "steady_tpot_p95_ms_q75": 54.267149575000005, "decode_tokens_per_s": 20.416328324438073, "decode_tokens_per_s_q25": 20.34733433084017, "decode_tokens_per_s_q75": 20.630648167284473, "decode_ms": 734.706053, "decode_ms_q25": 727.1520915000001, "decode_ms_q75": 737.2057794999998, "end_to_end_ms": 1682.619435, "end_to_end_ms_q25": 1678.0589180000002, "end_to_end_ms_q75": 1693.0354995, "output_tokens_per_s_e2e": 9.508983236010227, "output_tokens_per_s_e2e_q25": 9.450838828027694, "output_tokens_per_s_e2e_q75": 9.534896544128493, "all_tokens_per_s_e2e": 1217.149854209309, "all_tokens_per_s_e2e_q25": 1209.7073699875448, "all_tokens_per_s_e2e_q75": 1220.4667576484471, "energy_net_psys_j": 44.60317658357152, "energy_net_psys_j_q25": 44.53722122812667, "energy_net_psys_j_q75": 44.94412069817906, "prefill_energy_net_psys_j": 21.2102384365836, "prefill_energy_net_psys_j_q25": 21.186989548301078, "prefill_energy_net_psys_j_q75": 22.071909749167126, "decode_energy_net_psys_j": 23.261027436098225, "decode_energy_net_psys_j_q25": 22.806255593567087, "decode_energy_net_psys_j_q75": 23.35023167982559, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.8909315260583568, "energy_reduction_vs_gpu": 0.07167527916206917 }, { "precision": "FP16", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 4096, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1809.143317, "ttft_ms_q25": 1753.3364430000001, "ttft_ms_q75": 1820.160491, "prefill_tokens_per_s": 2255.211050258657, "prefill_tokens_per_s_q25": 2241.6427040395465, "prefill_tokens_per_s_q75": 2329.3518965941544, "handoff_plus_first_decode_ms": 72.596193, "handoff_plus_first_decode_ms_q25": 68.940962, "handoff_plus_first_decode_ms_q75": 74.4178035, "steady_tpot_ms": 44.69485, "steady_tpot_ms_q25": 44.68039125, "steady_tpot_ms_q75": 44.700368, "steady_tpot_p95_ms": 47.106267450000004, "steady_tpot_p95_ms_q25": 46.839246125, "steady_tpot_p95_ms_q75": 47.156936875, "decode_tokens_per_s": 21.282415866569966, "decode_tokens_per_s_q25": 21.25051559404212, "decode_tokens_per_s_q75": 21.39729160432636, "decode_ms": 704.8072030000001, "decode_ms_q25": 701.0435074999999, "decode_ms_q75": 705.8668170000001, "end_to_end_ms": 2506.423129, "end_to_end_ms_q25": 2455.4395645, "end_to_end_ms_q75": 2521.2039985, "output_tokens_per_s_e2e": 6.383598928239862, "output_tokens_per_s_e2e_q25": 6.346392419680106, "output_tokens_per_s_e2e_q75": 6.51895537797321, "all_tokens_per_s_e2e": 1634.2013256294047, "all_tokens_per_s_e2e_q25": 1624.676459438107, "all_tokens_per_s_e2e_q75": 1668.8525767611418, "energy_net_psys_j": 81.4949949933806, "energy_net_psys_j_q25": 78.42831116230036, "energy_net_psys_j_q75": 82.18184736906498, "prefill_energy_net_psys_j": 59.12704217341829, "prefill_energy_net_psys_j_q25": 56.45040936864072, "prefill_energy_net_psys_j_q75": 60.17508669734304, "decode_energy_net_psys_j": 21.645568523481565, "decode_energy_net_psys_j_q25": 21.616709645419277, "decode_energy_net_psys_j_q75": 22.006760671721942, "fastest_e2e": true, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "FP16", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 4096, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1809.143317, "ttft_ms_q25": 1753.3364430000001, "ttft_ms_q75": 1820.160491, "prefill_tokens_per_s": 2255.211050258657, "prefill_tokens_per_s_q25": 2241.6427040395465, "prefill_tokens_per_s_q75": 2329.3518965941544, "handoff_plus_first_decode_ms": 128.9693, "handoff_plus_first_decode_ms_q25": 125.140123, "handoff_plus_first_decode_ms_q75": 151.3195595, "steady_tpot_ms": 50.9610385, "steady_tpot_ms_q25": 50.9127735, "steady_tpot_ms_q75": 51.0554525, "steady_tpot_p95_ms": 51.880345350000006, "steady_tpot_p95_ms_q25": 51.671559775000006, "steady_tpot_p95_ms_q75": 54.092308575000004, "decode_tokens_per_s": 17.818719397930845, "decode_tokens_per_s_q25": 17.223284756227333, "decode_tokens_per_s_q75": 17.871784564224235, "decode_ms": 841.8113370000001, "decode_ms_q25": 839.3192180000001, "decode_ms_q75": 871.956162, "end_to_end_ms": 2645.970416, "end_to_end_ms_q25": 2622.800486, "end_to_end_ms_q75": 2659.479709, "output_tokens_per_s_e2e": 6.046930798337391, "output_tokens_per_s_e2e_q25": 6.016369595188279, "output_tokens_per_s_e2e_q75": 6.100825750060032, "all_tokens_per_s_e2e": 1548.0142843743722, "all_tokens_per_s_e2e_q25": 1540.1906163681995, "all_tokens_per_s_e2e_q75": 1561.8113920153683, "energy_net_psys_j": 78.1824820464143, "energy_net_psys_j_q25": 75.37543617265496, "energy_net_psys_j_q75": 79.17943603316526, "prefill_energy_net_psys_j": 59.12704217341829, "prefill_energy_net_psys_j_q25": 56.45040936864072, "prefill_energy_net_psys_j_q75": 60.17508669734304, "decode_energy_net_psys_j": 18.953258798648438, "decode_energy_net_psys_j_q25": 18.873936266840467, "decode_energy_net_psys_j_q75": 19.004349335822223, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.9472604507759544, "energy_reduction_vs_gpu": 0.04064682680495124 }, { "precision": "FP16", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 4096, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 2212.2164030000004, "ttft_ms_q25": 2208.340892, "ttft_ms_q75": 2226.989219, "prefill_tokens_per_s": 1844.3041984803508, "prefill_tokens_per_s_q25": 1832.1505594469477, "prefill_tokens_per_s_q75": 1847.5465361782622, "handoff_plus_first_decode_ms": 128.9693, "handoff_plus_first_decode_ms_q25": 125.140123, "handoff_plus_first_decode_ms_q75": 151.3195595, "steady_tpot_ms": 50.9610385, "steady_tpot_ms_q25": 50.9127735, "steady_tpot_ms_q75": 51.0554525, "steady_tpot_p95_ms": 51.880345350000006, "steady_tpot_p95_ms_q25": 51.671559775000006, "steady_tpot_p95_ms_q75": 54.092308575000004, "decode_tokens_per_s": 17.818719397930845, "decode_tokens_per_s_q25": 17.223284756227333, "decode_tokens_per_s_q75": 17.871784564224235, "decode_ms": 841.8113370000001, "decode_ms_q25": 839.3192180000001, "decode_ms_q75": 871.956162, "end_to_end_ms": 3054.0277400000004, "end_to_end_ms_q25": 3047.6601100000003, "end_to_end_ms_q75": 3098.945381, "output_tokens_per_s_e2e": 5.238983192732885, "output_tokens_per_s_e2e_q25": 5.164131723502363, "output_tokens_per_s_e2e_q75": 5.24995218264981, "all_tokens_per_s_e2e": 1341.1796973396185, "all_tokens_per_s_e2e_q25": 1322.017721216605, "all_tokens_per_s_e2e_q75": 1343.9877587583514, "energy_net_psys_j": 70.63349411528053, "energy_net_psys_j_q25": 69.99021321995122, "energy_net_psys_j_q75": 70.91537397998445, "prefill_energy_net_psys_j": 51.578054242284516, "prefill_energy_net_psys_j_q25": 51.06518641593698, "prefill_energy_net_psys_j_q75": 51.911024644162225, "decode_energy_net_psys_j": 18.953258798648438, "decode_energy_net_psys_j_q25": 18.873936266840467, "decode_energy_net_psys_j_q75": 19.004349335822223, "fastest_e2e": false, "lowest_energy": true, "e2e_speedup_vs_gpu": 0.8206942904192479, "energy_reduction_vs_gpu": 0.13327813418458756 }, { "precision": "FP16", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 4096, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 2212.2164030000004, "ttft_ms_q25": 2208.340892, "ttft_ms_q75": 2226.989219, "prefill_tokens_per_s": 1844.3041984803508, "prefill_tokens_per_s_q25": 1832.1505594469477, "prefill_tokens_per_s_q75": 1847.5465361782622, "handoff_plus_first_decode_ms": 72.596193, "handoff_plus_first_decode_ms_q25": 68.940962, "handoff_plus_first_decode_ms_q75": 74.4178035, "steady_tpot_ms": 44.69485, "steady_tpot_ms_q25": 44.68039125, "steady_tpot_ms_q75": 44.700368, "steady_tpot_p95_ms": 47.106267450000004, "steady_tpot_p95_ms_q25": 46.839246125, "steady_tpot_p95_ms_q75": 47.156936875, "decode_tokens_per_s": 21.282415866569966, "decode_tokens_per_s_q25": 21.25051559404212, "decode_tokens_per_s_q75": 21.39729160432636, "decode_ms": 704.8072030000001, "decode_ms_q25": 701.0435074999999, "decode_ms_q75": 705.8668170000001, "end_to_end_ms": 2917.0236060000007, "end_to_end_ms_q25": 2909.3843995, "end_to_end_ms_q75": 2932.856036, "output_tokens_per_s_e2e": 5.485043030536242, "output_tokens_per_s_e2e_q25": 5.455592120130046, "output_tokens_per_s_e2e_q75": 5.499483091115212, "all_tokens_per_s_e2e": 1404.171015817278, "all_tokens_per_s_e2e_q25": 1396.6315827532917, "all_tokens_per_s_e2e_q75": 1407.8676713254943, "energy_net_psys_j": 73.8895635695215, "energy_net_psys_j_q25": 73.01486646323396, "energy_net_psys_j_q75": 73.91778531588417, "prefill_energy_net_psys_j": 51.578054242284516, "prefill_energy_net_psys_j_q25": 51.06518641593698, "prefill_energy_net_psys_j_q75": 51.911024644162225, "decode_energy_net_psys_j": 21.645568523481565, "decode_energy_net_psys_j_q25": 21.616709645419277, "decode_energy_net_psys_j_q75": 22.006760671721942, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.8592399197060181, "energy_reduction_vs_gpu": 0.09332390810597446 }, { "precision": "FP16", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 8192, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 4599.011804, "ttft_ms_q25": 4485.462568, "ttft_ms_q75": 4602.603225499999, "prefill_tokens_per_s": 1777.773214865182, "prefill_tokens_per_s_q25": 1776.3870960761835, "prefill_tokens_per_s_q75": 1823.9463146042976, "handoff_plus_first_decode_ms": 79.860898, "handoff_plus_first_decode_ms_q25": 79.601507, "handoff_plus_first_decode_ms_q75": 82.1110245, "steady_tpot_ms": 48.571490000000004, "steady_tpot_ms_q25": 48.4244975, "steady_tpot_ms_q75": 48.717301250000006, "steady_tpot_p95_ms": 50.668234749999996, "steady_tpot_p95_ms_q25": 50.598343475, "steady_tpot_p95_ms_q75": 50.681439375, "decode_tokens_per_s": 19.545877697122307, "decode_tokens_per_s_q25": 19.53909174218755, "decode_tokens_per_s_q75": 19.56456280307617, "decode_ms": 767.4252459999998, "decode_ms_q25": 766.693017, "decode_ms_q75": 767.6918664999998, "end_to_end_ms": 5366.970291, "end_to_end_ms_q25": 5253.1544345, "end_to_end_ms_q75": 5369.562862999999, "output_tokens_per_s_e2e": 2.981197795491952, "output_tokens_per_s_e2e_q25": 2.9797590862020646, "output_tokens_per_s_e2e_q75": 3.047219439199628, "all_tokens_per_s_e2e": 1526.3732712918795, "all_tokens_per_s_e2e_q25": 1525.636652135457, "all_tokens_per_s_e2e_q75": 1560.1763528702095, "energy_net_psys_j": 177.1343408345819, "energy_net_psys_j_q25": 173.98962274979044, "energy_net_psys_j_q75": 178.29947360100846, "prefill_energy_net_psys_j": 152.23106387208279, "prefill_energy_net_psys_j_q25": 149.36397237921358, "prefill_energy_net_psys_j_q75": 153.26038140984252, "decode_energy_net_psys_j": 24.903276962499113, "decode_energy_net_psys_j_q25": 24.62565037057687, "decode_energy_net_psys_j_q75": 25.039092191165942, "fastest_e2e": true, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "FP16", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 8192, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 4599.011804, "ttft_ms_q25": 4485.462568, "ttft_ms_q75": 4602.603225499999, "prefill_tokens_per_s": 1777.773214865182, "prefill_tokens_per_s_q25": 1776.3870960761835, "prefill_tokens_per_s_q75": 1823.9463146042976, "handoff_plus_first_decode_ms": 154.256099, "handoff_plus_first_decode_ms_q25": 154.0629455, "handoff_plus_first_decode_ms_q75": 176.731884, "steady_tpot_ms": 55.3727985, "steady_tpot_ms_q25": 55.34510075, "steady_tpot_ms_q75": 55.6846465, "steady_tpot_p95_ms": 57.49918405, "steady_tpot_p95_ms_q25": 57.098385775, "steady_tpot_p95_ms_q75": 58.010706225, "decode_tokens_per_s": 16.07177991201729, "decode_tokens_per_s_q25": 15.622681807816047, "decode_tokens_per_s_q75": 16.075234784006504, "decode_ms": 933.31293, "decode_ms_q25": 933.1123865000004, "decode_ms_q75": 960.9365339999995, "end_to_end_ms": 5531.9236470000005, "end_to_end_ms_q25": 5446.1985585, "end_to_end_ms_q75": 5535.715612, "output_tokens_per_s_e2e": 2.8923031156941055, "output_tokens_per_s_e2e_q25": 2.890323244373117, "output_tokens_per_s_e2e_q75": 2.9385570412007462, "all_tokens_per_s_e2e": 1480.859195235382, "all_tokens_per_s_e2e_q25": 1479.8455011190358, "all_tokens_per_s_e2e_q75": 1504.541205094782, "energy_net_psys_j": 172.29413562317183, "energy_net_psys_j_q25": 169.7769538779724, "energy_net_psys_j_q75": 172.7336527943006, "prefill_energy_net_psys_j": 152.23106387208279, "prefill_energy_net_psys_j_q25": 149.36397237921358, "prefill_energy_net_psys_j_q75": 153.26038140984252, "decode_energy_net_psys_j": 20.063071751089048, "decode_energy_net_psys_j_q25": 19.473271384458094, "decode_energy_net_psys_j_q75": 20.412981498758846, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.9701815559060624, "energy_reduction_vs_gpu": 0.027325052774098357 }, { "precision": "FP16", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 8192, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 5483.789028, "ttft_ms_q25": 5470.867973, "ttft_ms_q75": 5499.028844500001, "prefill_tokens_per_s": 1490.9399246130151, "prefill_tokens_per_s_q25": 1486.819405236742, "prefill_tokens_per_s_q75": 1494.469551605921, "handoff_plus_first_decode_ms": 154.256099, "handoff_plus_first_decode_ms_q25": 154.0629455, "handoff_plus_first_decode_ms_q75": 176.731884, "steady_tpot_ms": 55.3727985, "steady_tpot_ms_q25": 55.34510075, "steady_tpot_ms_q75": 55.6846465, "steady_tpot_p95_ms": 57.49918405, "steady_tpot_p95_ms_q25": 57.098385775, "steady_tpot_p95_ms_q75": 58.010706225, "decode_tokens_per_s": 16.07177991201729, "decode_tokens_per_s_q25": 15.622681807816047, "decode_tokens_per_s_q75": 16.075234784006504, "decode_ms": 933.31293, "decode_ms_q25": 933.1123865000004, "decode_ms_q75": 960.9365339999995, "end_to_end_ms": 6446.5070559999995, "end_to_end_ms_q25": 6431.6039635, "end_to_end_ms_q75": 6447.0443235, "output_tokens_per_s_e2e": 2.481964242187281, "output_tokens_per_s_e2e_q25": 2.48175742375394, "output_tokens_per_s_e2e_q75": 2.4877287220074606, "all_tokens_per_s_e2e": 1270.7656919998878, "all_tokens_per_s_e2e_q25": 1270.6598009620172, "all_tokens_per_s_e2e_q75": 1273.7171056678199, "energy_net_psys_j": 145.28672479830834, "energy_net_psys_j_q25": 145.10051649871804, "energy_net_psys_j_q75": 149.32261254674575, "prefill_energy_net_psys_j": 126.03083718130063, "prefill_energy_net_psys_j_q25": 125.62724511425996, "prefill_energy_net_psys_j_q75": 129.3132231150276, "decode_energy_net_psys_j": 20.063071751089048, "decode_energy_net_psys_j_q25": 19.473271384458094, "decode_energy_net_psys_j_q75": 20.412981498758846, "fastest_e2e": false, "lowest_energy": true, "e2e_speedup_vs_gpu": 0.8325392719464666, "energy_reduction_vs_gpu": 0.17979357298094256 }, { "precision": "FP16", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 8192, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 5483.789028, "ttft_ms_q25": 5470.867973, "ttft_ms_q75": 5499.028844500001, "prefill_tokens_per_s": 1490.9399246130151, "prefill_tokens_per_s_q25": 1486.819405236742, "prefill_tokens_per_s_q75": 1494.469551605921, "handoff_plus_first_decode_ms": 79.860898, "handoff_plus_first_decode_ms_q25": 79.601507, "handoff_plus_first_decode_ms_q75": 82.1110245, "steady_tpot_ms": 48.571490000000004, "steady_tpot_ms_q25": 48.4244975, "steady_tpot_ms_q75": 48.717301250000006, "steady_tpot_p95_ms": 50.668234749999996, "steady_tpot_p95_ms_q25": 50.598343475, "steady_tpot_p95_ms_q75": 50.681439375, "decode_tokens_per_s": 19.545877697122307, "decode_tokens_per_s_q25": 19.53909174218755, "decode_tokens_per_s_q75": 19.56456280307617, "decode_ms": 767.4252459999998, "decode_ms_q25": 766.693017, "decode_ms_q75": 767.6918664999998, "end_to_end_ms": 6251.747515, "end_to_end_ms_q25": 6238.5598395, "end_to_end_ms_q75": 6265.988482000001, "output_tokens_per_s_e2e": 2.559284417934463, "output_tokens_per_s_e2e_q25": 2.553481017718692, "output_tokens_per_s_e2e_q75": 2.5647059431905914, "all_tokens_per_s_e2e": 1310.353621982445, "all_tokens_per_s_e2e_q25": 1307.3822810719703, "all_tokens_per_s_e2e_q75": 1313.1294429135828, "energy_net_psys_j": 151.2057446011334, "energy_net_psys_j_q25": 150.6663373054259, "energy_net_psys_j_q75": 154.0746887142713, "prefill_energy_net_psys_j": 126.03083718130063, "prefill_energy_net_psys_j_q25": 125.62724511425996, "prefill_energy_net_psys_j_q75": 129.3132231150276, "decode_energy_net_psys_j": 24.903276962499113, "decode_energy_net_psys_j_q25": 24.62565037057687, "decode_energy_net_psys_j_q75": 25.039092191165942, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.8584752148295931, "energy_reduction_vs_gpu": 0.14637814503547952 }, { "precision": "FP16", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 1024, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 607.063275, "ttft_ms_q25": 594.2362655, "ttft_ms_q75": 618.426575, "prefill_tokens_per_s": 1660.4529404286563, "prefill_tokens_per_s_q25": 1630.4933846355866, "prefill_tokens_per_s_q75": 1697.0857346633275, "handoff_plus_first_decode_ms": 110.586364, "handoff_plus_first_decode_ms_q25": 107.9654845, "handoff_plus_first_decode_ms_q75": 111.99283550000001, "steady_tpot_ms": 90.976238, "steady_tpot_ms_q25": 90.66058025000001, "steady_tpot_ms_q75": 91.04845574999999, "steady_tpot_p95_ms": 95.74199429999999, "steady_tpot_p95_ms_q25": 95.41095737499998, "steady_tpot_p95_ms_q75": 96.798083775, "decode_tokens_per_s": 10.791038897709017, "decode_tokens_per_s_q25": 10.707293359903568, "decode_tokens_per_s_q75": 10.807628278604277, "decode_ms": 1390.0422509999999, "decode_ms_q25": 1387.9118485, "decode_ms_q75": 1400.9999685, "end_to_end_ms": 1997.1055259999998, "end_to_end_ms_q25": 1995.236234, "end_to_end_ms_q75": 2006.3384234999999, "output_tokens_per_s_e2e": 8.011594676244464, "output_tokens_per_s_e2e_q25": 7.974895288985564, "output_tokens_per_s_e2e_q75": 8.019107597985064, "all_tokens_per_s_e2e": 512.7420592796457, "all_tokens_per_s_e2e_q25": 510.3932984950761, "all_tokens_per_s_e2e_q75": 513.2228862710441, "energy_net_psys_j": 65.03463419199078, "energy_net_psys_j_q25": 63.088387584242376, "energy_net_psys_j_q75": 65.397238369353, "prefill_energy_net_psys_j": 21.91163438025128, "prefill_energy_net_psys_j_q25": 20.994881327407033, "prefill_energy_net_psys_j_q75": 22.062885108962686, "decode_energy_net_psys_j": 43.1229998117395, "decode_energy_net_psys_j_q25": 42.09350625683534, "decode_energy_net_psys_j_q75": 43.334353260390316, "fastest_e2e": true, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "FP16", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 1024, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 607.063275, "ttft_ms_q25": 594.2362655, "ttft_ms_q75": 618.426575, "prefill_tokens_per_s": 1660.4529404286563, "prefill_tokens_per_s_q25": 1630.4933846355866, "prefill_tokens_per_s_q75": 1697.0857346633275, "handoff_plus_first_decode_ms": 103.587636, "handoff_plus_first_decode_ms_q25": 103.343637, "handoff_plus_first_decode_ms_q75": 142.3081965, "steady_tpot_ms": 97.750102, "steady_tpot_ms_q25": 97.70124425, "steady_tpot_ms_q75": 97.83212674999999, "steady_tpot_p95_ms": 98.4875902, "steady_tpot_p95_ms_q25": 98.29367895, "steady_tpot_p95_ms_q75": 99.01323395, "decode_tokens_per_s": 10.186762902220105, "decode_tokens_per_s_q25": 9.914916723458596, "decode_tokens_per_s_q75": 10.18754351163878, "decode_ms": 1472.499178, "decode_ms_q25": 1472.386358, "decode_ms_q75": 1514.0101544999998, "end_to_end_ms": 2102.289053, "end_to_end_ms_q25": 2090.812933, "end_to_end_ms_q75": 2119.60972, "output_tokens_per_s_e2e": 7.610751707605454, "output_tokens_per_s_e2e_q25": 7.5490635511970545, "output_tokens_per_s_e2e_q75": 7.652756398399472, "all_tokens_per_s_e2e": 487.08810928674905, "all_tokens_per_s_e2e_q25": 483.1400672766115, "all_tokens_per_s_e2e_q75": 489.7764094975662, "energy_net_psys_j": 51.82984807263372, "energy_net_psys_j_q25": 51.07475836656299, "energy_net_psys_j_q75": 52.30109249696879, "prefill_energy_net_psys_j": 21.91163438025128, "prefill_energy_net_psys_j_q25": 20.994881327407033, "prefill_energy_net_psys_j_q75": 22.062885108962686, "decode_energy_net_psys_j": 30.241540385929476, "decode_energy_net_psys_j_q25": 30.07987703915596, "decode_energy_net_psys_j_q75": 30.399870734779622, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.949967143267049, "energy_reduction_vs_gpu": 0.203042367861635 }, { "precision": "FP16", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 1024, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 738.013839, "ttft_ms_q25": 737.1676259999999, "ttft_ms_q75": 738.732179, "prefill_tokens_per_s": 1365.8280464846407, "prefill_tokens_per_s_q25": 1364.5012113635582, "prefill_tokens_per_s_q75": 1367.3977162352335, "handoff_plus_first_decode_ms": 103.587636, "handoff_plus_first_decode_ms_q25": 103.343637, "handoff_plus_first_decode_ms_q75": 142.3081965, "steady_tpot_ms": 97.750102, "steady_tpot_ms_q25": 97.70124425, "steady_tpot_ms_q75": 97.83212674999999, "steady_tpot_p95_ms": 98.4875902, "steady_tpot_p95_ms_q25": 98.29367895, "steady_tpot_p95_ms_q75": 99.01323395, "decode_tokens_per_s": 10.186762902220105, "decode_tokens_per_s_q25": 9.914916723458596, "decode_tokens_per_s_q75": 10.18754351163878, "decode_ms": 1472.499178, "decode_ms_q25": 1472.386358, "decode_ms_q75": 1514.0101544999998, "end_to_end_ms": 2211.949697, "end_to_end_ms_q25": 2210.272324, "end_to_end_ms_q75": 2252.7423335, "output_tokens_per_s_e2e": 7.233437551360373, "output_tokens_per_s_e2e_q25": 7.104784187649903, "output_tokens_per_s_e2e_q75": 7.238931167489643, "all_tokens_per_s_e2e": 462.94000328706386, "all_tokens_per_s_e2e_q25": 454.7061880095938, "all_tokens_per_s_e2e_q75": 463.2915947193371, "energy_net_psys_j": 51.049921114854904, "energy_net_psys_j_q25": 50.60905824675217, "energy_net_psys_j_q75": 51.057816044193146, "prefill_energy_net_psys_j": 20.491720031225135, "prefill_energy_net_psys_j_q25": 20.370850858746067, "prefill_energy_net_psys_j_q75": 20.65794530941352, "decode_energy_net_psys_j": 30.241540385929476, "decode_energy_net_psys_j_q25": 30.07987703915596, "decode_energy_net_psys_j_q75": 30.399870734779622, "fastest_e2e": false, "lowest_energy": true, "e2e_speedup_vs_gpu": 0.9028711316123569, "energy_reduction_vs_gpu": 0.21503485413404744 }, { "precision": "FP16", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 1024, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 738.013839, "ttft_ms_q25": 737.1676259999999, "ttft_ms_q75": 738.732179, "prefill_tokens_per_s": 1365.8280464846407, "prefill_tokens_per_s_q25": 1364.5012113635582, "prefill_tokens_per_s_q75": 1367.3977162352335, "handoff_plus_first_decode_ms": 110.586364, "handoff_plus_first_decode_ms_q25": 107.9654845, "handoff_plus_first_decode_ms_q75": 111.99283550000001, "steady_tpot_ms": 90.976238, "steady_tpot_ms_q25": 90.66058025000001, "steady_tpot_ms_q75": 91.04845574999999, "steady_tpot_p95_ms": 95.74199429999999, "steady_tpot_p95_ms_q25": 95.41095737499998, "steady_tpot_p95_ms_q75": 96.798083775, "decode_tokens_per_s": 10.791038897709017, "decode_tokens_per_s_q25": 10.707293359903568, "decode_tokens_per_s_q75": 10.807628278604277, "decode_ms": 1390.0422509999999, "decode_ms_q25": 1387.9118485, "decode_ms_q75": 1400.9999685, "end_to_end_ms": 2126.363664, "end_to_end_ms_q25": 2125.7978144999997, "end_to_end_ms_q75": 2138.1675944999997, "output_tokens_per_s_e2e": 7.524583057397467, "output_tokens_per_s_e2e_q25": 7.483271042601831, "output_tokens_per_s_e2e_q75": 7.526586500612541, "all_tokens_per_s_e2e": 481.5733156734379, "all_tokens_per_s_e2e_q25": 478.92934672651717, "all_tokens_per_s_e2e_q75": 481.7015360392026, "energy_net_psys_j": 63.372981498006496, "energy_net_psys_j_q25": 62.6305823937698, "energy_net_psys_j_q75": 63.70520411913638, "prefill_energy_net_psys_j": 20.491720031225135, "prefill_energy_net_psys_j_q25": 20.370850858746067, "prefill_energy_net_psys_j_q75": 20.65794530941352, "decode_energy_net_psys_j": 43.1229998117395, "decode_energy_net_psys_j_q25": 42.09350625683534, "decode_energy_net_psys_j_q75": 43.334353260390316, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.9392116502984035, "energy_reduction_vs_gpu": 0.025550273552379266 }, { "precision": "FP16", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 2048, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1295.2486940000001, "ttft_ms_q25": 1268.754579, "ttft_ms_q75": 1301.2946175000002, "prefill_tokens_per_s": 1568.8106920415082, "prefill_tokens_per_s_q25": 1561.555574394189, "prefill_tokens_per_s_q75": 1602.2692557587686, "handoff_plus_first_decode_ms": 113.84433100000001, "handoff_plus_first_decode_ms_q25": 113.2854395, "handoff_plus_first_decode_ms_q75": 119.05460550000001, "steady_tpot_ms": 99.86533, "steady_tpot_ms_q25": 99.2980655, "steady_tpot_ms_q75": 99.88143000000001, "steady_tpot_p95_ms": 104.039077, "steady_tpot_p95_ms_q25": 103.99117665, "steady_tpot_p95_ms_q75": 104.07479005, "decode_tokens_per_s": 9.858521320098015, "decode_tokens_per_s_q25": 9.84239684144253, "decode_tokens_per_s_q75": 9.85915772427815, "decode_ms": 1521.5263540000003, "decode_ms_q25": 1521.4281465, "decode_ms_q75": 1524.0231115000001, "end_to_end_ms": 2816.7750480000004, "end_to_end_ms_q25": 2792.7776905, "end_to_end_ms_q75": 2822.722764, "output_tokens_per_s_e2e": 5.680254804642816, "output_tokens_per_s_e2e_q25": 5.668311192750048, "output_tokens_per_s_e2e_q75": 5.729486267180118, "all_tokens_per_s_e2e": 727.0726149942805, "all_tokens_per_s_e2e_q25": 725.5438326720061, "all_tokens_per_s_e2e_q75": 733.3742421990551, "energy_net_psys_j": 91.30615653603783, "energy_net_psys_j_q25": 89.48061312461171, "energy_net_psys_j_q75": 91.31300907224337, "prefill_energy_net_psys_j": 46.848244522061265, "prefill_energy_net_psys_j_q25": 45.756798212515726, "prefill_energy_net_psys_j_q75": 47.06509762227614, "decode_energy_net_psys_j": 44.037910885957885, "decode_energy_net_psys_j_q25": 43.51381434808664, "decode_energy_net_psys_j_q75": 44.24791144996723, "fastest_e2e": true, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "FP16", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 2048, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1295.2486940000001, "ttft_ms_q25": 1268.754579, "ttft_ms_q75": 1301.2946175000002, "prefill_tokens_per_s": 1568.8106920415082, "prefill_tokens_per_s_q25": 1561.555574394189, "prefill_tokens_per_s_q75": 1602.2692557587686, "handoff_plus_first_decode_ms": 134.185059, "handoff_plus_first_decode_ms_q25": 133.730439, "handoff_plus_first_decode_ms_q75": 157.284578, "steady_tpot_ms": 99.4194085, "steady_tpot_ms_q25": 99.33907400000001, "steady_tpot_ms_q75": 99.550228, "steady_tpot_p95_ms": 100.32548745, "steady_tpot_p95_ms_q25": 100.00277105, "steady_tpot_p95_ms_q75": 100.393350875, "decode_tokens_per_s": 9.815708810668962, "decode_tokens_per_s_q25": 9.662051614427256, "decode_tokens_per_s_q75": 9.82897138692405, "decode_ms": 1528.162692, "decode_ms_q25": 1526.103467, "decode_ms_q75": 1552.858049, "end_to_end_ms": 2823.411386, "end_to_end_ms_q25": 2821.612628, "end_to_end_ms_q75": 2827.3980844999996, "output_tokens_per_s_e2e": 5.66690354772126, "output_tokens_per_s_e2e_q25": 5.6589243299162515, "output_tokens_per_s_e2e_q75": 5.6705184629467436, "all_tokens_per_s_e2e": 725.3636541083213, "all_tokens_per_s_e2e_q25": 724.3423142292802, "all_tokens_per_s_e2e_q75": 725.8263632571832, "energy_net_psys_j": 74.87595080459188, "energy_net_psys_j_q25": 74.50002106049851, "energy_net_psys_j_q75": 76.09583793255571, "prefill_energy_net_psys_j": 46.848244522061265, "prefill_energy_net_psys_j_q25": 45.756798212515726, "prefill_energy_net_psys_j_q75": 47.06509762227614, "decode_energy_net_psys_j": 29.45873941343495, "decode_energy_net_psys_j_q25": 28.743222847982786, "decode_energy_net_psys_j_q75": 29.74625687573174, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.9976495320402452, "energy_reduction_vs_gpu": 0.1799463076179434 }, { "precision": "FP16", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 2048, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1491.060119, "ttft_ms_q25": 1488.32563, "ttft_ms_q75": 1502.124392, "prefill_tokens_per_s": 1362.7887796789769, "prefill_tokens_per_s_q25": 1352.824214487296, "prefill_tokens_per_s_q75": 1365.2972295643517, "handoff_plus_first_decode_ms": 134.185059, "handoff_plus_first_decode_ms_q25": 133.730439, "handoff_plus_first_decode_ms_q75": 157.284578, "steady_tpot_ms": 99.4194085, "steady_tpot_ms_q25": 99.33907400000001, "steady_tpot_ms_q75": 99.550228, "steady_tpot_p95_ms": 100.32548745, "steady_tpot_p95_ms_q25": 100.00277105, "steady_tpot_p95_ms_q75": 100.393350875, "decode_tokens_per_s": 9.815708810668962, "decode_tokens_per_s_q25": 9.662051614427256, "decode_tokens_per_s_q75": 9.82897138692405, "decode_ms": 1528.162692, "decode_ms_q25": 1526.103467, "decode_ms_q75": 1552.858049, "end_to_end_ms": 3041.351357, "end_to_end_ms_q25": 3028.2278589999996, "end_to_end_ms_q75": 3052.2479519999997, "output_tokens_per_s_e2e": 5.260819327294844, "output_tokens_per_s_e2e_q25": 5.242104892929072, "output_tokens_per_s_e2e_q75": 5.28371749056015, "all_tokens_per_s_e2e": 673.3848738937401, "all_tokens_per_s_e2e_q25": 670.9894262949213, "all_tokens_per_s_e2e_q75": 676.3158387916992, "energy_net_psys_j": 74.0294116716735, "energy_net_psys_j_q25": 72.20316558915493, "energy_net_psys_j_q75": 74.23449914619869, "prefill_energy_net_psys_j": 43.995637333644986, "prefill_energy_net_psys_j_q25": 43.172425278875366, "prefill_energy_net_psys_j_q75": 44.48824227046694, "decode_energy_net_psys_j": 29.45873941343495, "decode_energy_net_psys_j_q25": 28.743222847982786, "decode_energy_net_psys_j_q75": 29.74625687573174, "fastest_e2e": false, "lowest_energy": true, "e2e_speedup_vs_gpu": 0.9261590383225164, "energy_reduction_vs_gpu": 0.18921774302859118 }, { "precision": "FP16", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 2048, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1491.060119, "ttft_ms_q25": 1488.32563, "ttft_ms_q75": 1502.124392, "prefill_tokens_per_s": 1362.7887796789769, "prefill_tokens_per_s_q25": 1352.824214487296, "prefill_tokens_per_s_q75": 1365.2972295643517, "handoff_plus_first_decode_ms": 113.84433100000001, "handoff_plus_first_decode_ms_q25": 113.2854395, "handoff_plus_first_decode_ms_q75": 119.05460550000001, "steady_tpot_ms": 99.86533, "steady_tpot_ms_q25": 99.2980655, "steady_tpot_ms_q75": 99.88143000000001, "steady_tpot_p95_ms": 104.039077, "steady_tpot_p95_ms_q25": 103.99117665, "steady_tpot_p95_ms_q75": 104.07479005, "decode_tokens_per_s": 9.858521320098015, "decode_tokens_per_s_q25": 9.84239684144253, "decode_tokens_per_s_q75": 9.85915772427815, "decode_ms": 1521.5263540000003, "decode_ms_q25": 1521.4281465, "decode_ms_q75": 1524.0231115000001, "end_to_end_ms": 3012.390058, "end_to_end_ms_q25": 3012.250534, "end_to_end_ms_q75": 3023.5525385, "output_tokens_per_s_e2e": 5.311397160373978, "output_tokens_per_s_e2e_q25": 5.291860443792936, "output_tokens_per_s_e2e_q75": 5.311643189612258, "all_tokens_per_s_e2e": 679.8588365278691, "all_tokens_per_s_e2e_q25": 677.3581368054959, "all_tokens_per_s_e2e_q75": 679.890328270369, "energy_net_psys_j": 87.9705650175043, "energy_net_psys_j_q25": 87.3888451277933, "energy_net_psys_j_q75": 88.00205661855358, "prefill_energy_net_psys_j": 43.995637333644986, "prefill_energy_net_psys_j_q25": 43.172425278875366, "prefill_energy_net_psys_j_q75": 44.48824227046694, "decode_energy_net_psys_j": 44.037910885957885, "decode_energy_net_psys_j_q25": 43.51381434808664, "decode_energy_net_psys_j_q75": 44.24791144996723, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.9350631869599673, "energy_reduction_vs_gpu": 0.03653194532634829 }, { "precision": "FP16", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 4096, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 3240.103964, "ttft_ms_q25": 3180.682677, "ttft_ms_q75": 3252.5380395, "prefill_tokens_per_s": 1259.2188538799617, "prefill_tokens_per_s_q25": 1254.4233390183695, "prefill_tokens_per_s_q75": 1283.1913430639618, "handoff_plus_first_decode_ms": 115.762957, "handoff_plus_first_decode_ms_q25": 115.3336545, "handoff_plus_first_decode_ms_q75": 119.953497, "steady_tpot_ms": 92.687543, "steady_tpot_ms_q25": 92.273866, "steady_tpot_ms_q75": 92.77256249999999, "steady_tpot_p95_ms": 100.3072394, "steady_tpot_p95_ms_q25": 98.29813765, "steady_tpot_p95_ms_q75": 100.356425, "decode_tokens_per_s": 10.48851653020552, "decode_tokens_per_s_q25": 10.485455162324818, "decode_tokens_per_s_q75": 10.528354143165284, "decode_ms": 1430.1355159999998, "decode_ms_q25": 1424.74451, "decode_ms_q75": 1430.5531849999995, "end_to_end_ms": 4671.074817999999, "end_to_end_ms_q25": 4611.2358619999995, "end_to_end_ms_q75": 4677.7002185, "output_tokens_per_s_e2e": 3.4253358431220065, "output_tokens_per_s_e2e_q25": 3.42049112828928, "output_tokens_per_s_e2e_q75": 3.470370043618033, "all_tokens_per_s_e2e": 876.8859758392337, "all_tokens_per_s_e2e_q25": 875.6457288420556, "all_tokens_per_s_e2e_q75": 888.4147311662165, "energy_net_psys_j": 156.04669809514434, "energy_net_psys_j_q25": 154.98019578291135, "energy_net_psys_j_q75": 158.8095087872635, "prefill_energy_net_psys_j": 114.60094196753823, "prefill_energy_net_psys_j_q25": 113.44999763618469, "prefill_energy_net_psys_j_q75": 116.04362185376917, "decode_energy_net_psys_j": 41.61464016584719, "decode_energy_net_psys_j_q25": 41.53019814672665, "decode_energy_net_psys_j_q75": 42.85032895261487, "fastest_e2e": true, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "FP16", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 4096, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 3240.103964, "ttft_ms_q25": 3180.682677, "ttft_ms_q75": 3252.5380395, "prefill_tokens_per_s": 1259.2188538799617, "prefill_tokens_per_s_q25": 1254.4233390183695, "prefill_tokens_per_s_q75": 1283.1913430639618, "handoff_plus_first_decode_ms": 205.440954, "handoff_plus_first_decode_ms_q25": 193.873883, "handoff_plus_first_decode_ms_q75": 213.41842350000002, "steady_tpot_ms": 102.098412, "steady_tpot_ms_q25": 101.98185075, "steady_tpot_ms_q75": 102.19718950000001, "steady_tpot_p95_ms": 103.19750105, "steady_tpot_p95_ms_q25": 102.936983825, "steady_tpot_p95_ms_q75": 103.788693575, "decode_tokens_per_s": 9.131076469404587, "decode_tokens_per_s_q25": 9.108240976025336, "decode_tokens_per_s_q75": 9.214446473562187, "decode_ms": 1642.7416909999997, "decode_ms_q25": 1628.0118475000002, "decode_ms_q75": 1646.8706005000001, "end_to_end_ms": 4853.3859680000005, "end_to_end_ms_q25": 4812.823434000001, "end_to_end_ms_q75": 4880.549887, "output_tokens_per_s_e2e": 3.2966675441626445, "output_tokens_per_s_e2e_q25": 3.2784206753191354, "output_tokens_per_s_e2e_q75": 3.3246880552262352, "all_tokens_per_s_e2e": 843.946891305637, "all_tokens_per_s_e2e_q25": 839.2756928816987, "all_tokens_per_s_e2e_q75": 851.1201421379162, "energy_net_psys_j": 145.01140496001767, "energy_net_psys_j_q25": 144.93403777621037, "energy_net_psys_j_q75": 146.639869330576, "prefill_energy_net_psys_j": 114.60094196753823, "prefill_energy_net_psys_j_q25": 113.44999763618469, "prefill_energy_net_psys_j_q75": 116.04362185376917, "decode_energy_net_psys_j": 30.782031961134198, "decode_energy_net_psys_j_q25": 30.596247476806816, "decode_energy_net_psys_j_q75": 31.669824624353048, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.9624362968035017, "energy_reduction_vs_gpu": 0.07071788938717727 }, { "precision": "FP16", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 4096, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 3530.077925, "ttft_ms_q25": 3529.8153009999996, "ttft_ms_q75": 3551.0869350000003, "prefill_tokens_per_s": 1155.7818514728679, "prefill_tokens_per_s_q25": 1148.9842086185654, "prefill_tokens_per_s_q75": 1155.8678499210394, "handoff_plus_first_decode_ms": 205.440954, "handoff_plus_first_decode_ms_q25": 193.873883, "handoff_plus_first_decode_ms_q75": 213.41842350000002, "steady_tpot_ms": 102.098412, "steady_tpot_ms_q25": 101.98185075, "steady_tpot_ms_q75": 102.19718950000001, "steady_tpot_p95_ms": 103.19750105, "steady_tpot_p95_ms_q25": 102.936983825, "steady_tpot_p95_ms_q75": 103.788693575, "decode_tokens_per_s": 9.131076469404587, "decode_tokens_per_s_q25": 9.108240976025336, "decode_tokens_per_s_q75": 9.214446473562187, "decode_ms": 1642.7416909999997, "decode_ms_q25": 1628.0118475000002, "decode_ms_q75": 1646.8706005000001, "end_to_end_ms": 5172.819616, "end_to_end_ms_q25": 5157.8271485000005, "end_to_end_ms_q75": 5197.9575355, "output_tokens_per_s_e2e": 3.09309065224516, "output_tokens_per_s_e2e_q25": 3.078204103742986, "output_tokens_per_s_e2e_q75": 3.1021076757263275, "all_tokens_per_s_e2e": 791.831206974761, "all_tokens_per_s_e2e_q25": 788.0202505582045, "all_tokens_per_s_e2e_q75": 794.1395649859398, "energy_net_psys_j": 131.26549072083276, "energy_net_psys_j_q25": 131.13414926132015, "energy_net_psys_j_q75": 131.75081211012662, "prefill_energy_net_psys_j": 100.59234480932811, "prefill_energy_net_psys_j_q25": 99.65010912129449, "prefill_energy_net_psys_j_q75": 101.0232231738072, "decode_energy_net_psys_j": 30.782031961134198, "decode_energy_net_psys_j_q25": 30.596247476806816, "decode_energy_net_psys_j_q75": 31.669824624353048, "fastest_e2e": false, "lowest_energy": true, "e2e_speedup_vs_gpu": 0.9030036159683477, "energy_reduction_vs_gpu": 0.15880635525656595 }, { "precision": "FP16", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 4096, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 3530.077925, "ttft_ms_q25": 3529.8153009999996, "ttft_ms_q75": 3551.0869350000003, "prefill_tokens_per_s": 1155.7818514728679, "prefill_tokens_per_s_q25": 1148.9842086185654, "prefill_tokens_per_s_q75": 1155.8678499210394, "handoff_plus_first_decode_ms": 115.762957, "handoff_plus_first_decode_ms_q25": 115.3336545, "handoff_plus_first_decode_ms_q75": 119.953497, "steady_tpot_ms": 92.687543, "steady_tpot_ms_q25": 92.273866, "steady_tpot_ms_q75": 92.77256249999999, "steady_tpot_p95_ms": 100.3072394, "steady_tpot_p95_ms_q25": 98.29813765, "steady_tpot_p95_ms_q75": 100.356425, "decode_tokens_per_s": 10.48851653020552, "decode_tokens_per_s_q25": 10.485455162324818, "decode_tokens_per_s_q75": 10.528354143165284, "decode_ms": 1430.1355159999998, "decode_ms_q25": 1424.74451, "decode_ms_q75": 1430.5531849999995, "end_to_end_ms": 4960.523530999999, "end_to_end_ms_q25": 4954.97748, "end_to_end_ms_q75": 4981.377495999999, "output_tokens_per_s_e2e": 3.225466001725535, "output_tokens_per_s_e2e_q25": 3.21201925188337, "output_tokens_per_s_e2e_q75": 3.229080275243806, "all_tokens_per_s_e2e": 825.719296441737, "all_tokens_per_s_e2e_q25": 822.2769284821427, "all_tokens_per_s_e2e_q75": 826.6445504624144, "energy_net_psys_j": 142.0381009369342, "energy_net_psys_j_q25": 141.18030726802112, "energy_net_psys_j_q75": 143.78911010730152, "prefill_energy_net_psys_j": 100.59234480932811, "prefill_energy_net_psys_j_q25": 99.65010912129449, "prefill_energy_net_psys_j_q75": 101.0232231738072, "decode_energy_net_psys_j": 41.61464016584719, "decode_energy_net_psys_j_q25": 41.53019814672665, "decode_energy_net_psys_j_q75": 42.85032895261487, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.9416495635609555, "energy_reduction_vs_gpu": 0.08977182682628027 }, { "precision": "FP16", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 8192, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 7706.322755, "ttft_ms_q25": 7599.4729475, "ttft_ms_q75": 7721.44232, "prefill_tokens_per_s": 1060.9469989685113, "prefill_tokens_per_s_q25": 1058.8735899593285, "prefill_tokens_per_s_q75": 1076.0768113839117, "handoff_plus_first_decode_ms": 131.406359, "handoff_plus_first_decode_ms_q25": 130.64529349999998, "handoff_plus_first_decode_ms_q75": 135.692465, "steady_tpot_ms": 99.441204, "steady_tpot_ms_q25": 98.57963075, "steady_tpot_ms_q75": 99.513441, "steady_tpot_p95_ms": 107.64871504999999, "steady_tpot_p95_ms_q25": 107.460514375, "steady_tpot_p95_ms_q75": 107.72995429999999, "decode_tokens_per_s": 9.70446882113418, "decode_tokens_per_s_q25": 9.695234573694833, "decode_tokens_per_s_q75": 9.779850182263372, "decode_ms": 1545.6796530000001, "decode_ms_q25": 1533.8569539999999, "decode_ms_q75": 1547.1532425, "end_to_end_ms": 9252.002408, "end_to_end_ms_q25": 9146.626189999999, "end_to_end_ms_q75": 9255.299274, "output_tokens_per_s_e2e": 1.7293553648629787, "output_tokens_per_s_e2e_q25": 1.7287395638475633, "output_tokens_per_s_e2e_q75": 1.7495110876326878, "all_tokens_per_s_e2e": 885.4299468098451, "all_tokens_per_s_e2e_q25": 885.1146566899524, "all_tokens_per_s_e2e_q75": 895.7496768679362, "energy_net_psys_j": 318.80381272461636, "energy_net_psys_j_q25": 316.7465021804097, "energy_net_psys_j_q75": 321.3544898898683, "prefill_energy_net_psys_j": 270.7895866055883, "prefill_energy_net_psys_j_q25": 269.2357969458508, "prefill_energy_net_psys_j_q75": 274.5292660496832, "decode_energy_net_psys_j": 47.00718435008983, "decode_energy_net_psys_j_q25": 46.32170295571602, "decode_energy_net_psys_j_q75": 47.510705234558955, "fastest_e2e": true, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "FP16", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 8192, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 7706.322755, "ttft_ms_q25": 7599.4729475, "ttft_ms_q75": 7721.44232, "prefill_tokens_per_s": 1060.9469989685113, "prefill_tokens_per_s_q25": 1058.8735899593285, "prefill_tokens_per_s_q75": 1076.0768113839117, "handoff_plus_first_decode_ms": 250.5723, "handoff_plus_first_decode_ms_q25": 249.529976, "handoff_plus_first_decode_ms_q75": 269.1960525, "steady_tpot_ms": 108.125211, "steady_tpot_ms_q25": 108.08177275, "steady_tpot_ms_q75": 108.156571, "steady_tpot_p95_ms": 110.94088165, "steady_tpot_p95_ms_q25": 110.91701512499999, "steady_tpot_p95_ms_q75": 110.98606692499999, "decode_tokens_per_s": 8.465157969907148, "decode_tokens_per_s_q25": 8.37940825708966, "decode_tokens_per_s_q75": 8.473290083584455, "decode_ms": 1771.969294999999, "decode_ms_q25": 1770.270304499999, "decode_ms_q75": 1790.2900224999994, "end_to_end_ms": 9474.894068999998, "end_to_end_ms_q25": 9388.063979499999, "end_to_end_ms_q75": 9491.712624499998, "output_tokens_per_s_e2e": 1.6886732330178629, "output_tokens_per_s_e2e_q25": 1.6856863316812385, "output_tokens_per_s_e2e_q75": 1.7044375552242785, "all_tokens_per_s_e2e": 864.6006953051458, "all_tokens_per_s_e2e_q25": 863.0714018207941, "all_tokens_per_s_e2e_q75": 872.6720282748306, "energy_net_psys_j": 301.62856917031655, "energy_net_psys_j_q25": 301.57297758855043, "energy_net_psys_j_q75": 307.59249950694516, "prefill_energy_net_psys_j": 270.7895866055883, "prefill_energy_net_psys_j_q25": 269.2357969458508, "prefill_energy_net_psys_j_q75": 274.5292660496832, "decode_energy_net_psys_j": 33.946561884203355, "decode_energy_net_psys_j_q25": 32.33718064269972, "decode_energy_net_psys_j_q75": 34.61702311699953, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.9764755511379007, "energy_reduction_vs_gpu": 0.0538740217926309 }, { "precision": "FP16", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 8192, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 8873.004211, "ttft_ms_q25": 8847.967757, "ttft_ms_q75": 8873.702487, "prefill_tokens_per_s": 921.4466493619024, "prefill_tokens_per_s_q25": 921.3741459785514, "prefill_tokens_per_s_q75": 924.0613990247903, "handoff_plus_first_decode_ms": 250.5723, "handoff_plus_first_decode_ms_q25": 249.529976, "handoff_plus_first_decode_ms_q75": 269.1960525, "steady_tpot_ms": 108.125211, "steady_tpot_ms_q25": 108.08177275, "steady_tpot_ms_q75": 108.156571, "steady_tpot_p95_ms": 110.94088165, "steady_tpot_p95_ms_q25": 110.91701512499999, "steady_tpot_p95_ms_q75": 110.98606692499999, "decode_tokens_per_s": 8.465157969907148, "decode_tokens_per_s_q25": 8.37940825708966, "decode_tokens_per_s_q75": 8.473290083584455, "decode_ms": 1771.969294999999, "decode_ms_q25": 1770.270304499999, "decode_ms_q75": 1790.2900224999994, "end_to_end_ms": 10646.370058, "end_to_end_ms_q25": 10618.9363375, "end_to_end_ms_q75": 10663.9925095, "output_tokens_per_s_e2e": 1.502859651959695, "output_tokens_per_s_e2e_q25": 1.5003802449201546, "output_tokens_per_s_e2e_q75": 1.5067523037517474, "all_tokens_per_s_e2e": 769.4641418033639, "all_tokens_per_s_e2e_q25": 768.1946853991192, "all_tokens_per_s_e2e_q75": 771.4571795208947, "energy_net_psys_j": 276.93145829420837, "energy_net_psys_j_q25": 275.7152032005781, "energy_net_psys_j_q75": 278.7892887981194, "prefill_energy_net_psys_j": 243.77114870575178, "prefill_energy_net_psys_j_q25": 242.70756132508222, "prefill_energy_net_psys_j_q75": 245.23585306178947, "decode_energy_net_psys_j": 33.946561884203355, "decode_energy_net_psys_j_q25": 32.33718064269972, "decode_energy_net_psys_j_q75": 34.61702311699953, "fastest_e2e": false, "lowest_energy": true, "e2e_speedup_vs_gpu": 0.8690288199260714, "energy_reduction_vs_gpu": 0.13134207546814203 }, { "precision": "FP16", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 8192, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 8873.004211, "ttft_ms_q25": 8847.967757, "ttft_ms_q75": 8873.702487, "prefill_tokens_per_s": 921.4466493619024, "prefill_tokens_per_s_q25": 921.3741459785514, "prefill_tokens_per_s_q75": 924.0613990247903, "handoff_plus_first_decode_ms": 131.406359, "handoff_plus_first_decode_ms_q25": 130.64529349999998, "handoff_plus_first_decode_ms_q75": 135.692465, "steady_tpot_ms": 99.441204, "steady_tpot_ms_q25": 98.57963075, "steady_tpot_ms_q75": 99.513441, "steady_tpot_p95_ms": 107.64871504999999, "steady_tpot_p95_ms_q25": 107.460514375, "steady_tpot_p95_ms_q75": 107.72995429999999, "decode_tokens_per_s": 9.70446882113418, "decode_tokens_per_s_q25": 9.695234573694833, "decode_tokens_per_s_q75": 9.779850182263372, "decode_ms": 1545.6796530000001, "decode_ms_q25": 1533.8569539999999, "decode_ms_q75": 1547.1532425, "end_to_end_ms": 10396.435018, "end_to_end_ms_q25": 10382.522987, "end_to_end_ms_q75": 10409.033030499999, "output_tokens_per_s_e2e": 1.5389890834981603, "output_tokens_per_s_e2e_q25": 1.5371287025624625, "output_tokens_per_s_e2e_q75": 1.5410540142713511, "all_tokens_per_s_e2e": 787.9624107510581, "all_tokens_per_s_e2e_q25": 787.0098957119808, "all_tokens_per_s_e2e_q75": 789.0196553069318, "energy_net_psys_j": 291.78537482477986, "energy_net_psys_j_q25": 289.53278516526734, "energy_net_psys_j_q75": 292.74655829634844, "prefill_energy_net_psys_j": 243.77114870575178, "prefill_energy_net_psys_j_q25": 242.70756132508222, "prefill_energy_net_psys_j_q75": 245.23585306178947, "decode_energy_net_psys_j": 47.00718435008983, "decode_energy_net_psys_j_q25": 46.32170295571602, "decode_energy_net_psys_j_q75": 47.510705234558955, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.8899206691506683, "energy_reduction_vs_gpu": 0.0847494189888347 }, { "precision": "FP16", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 1024, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1122.3871139999999, "ttft_ms_q25": 1110.2382564999998, "ttft_ms_q75": 1125.5228419999999, "prefill_tokens_per_s": 898.0858630919743, "prefill_tokens_per_s_q25": 895.5907303191804, "prefill_tokens_per_s_q75": 908.0219549834039, "handoff_plus_first_decode_ms": 194.39052800000002, "handoff_plus_first_decode_ms_q25": 194.27981200000002, "handoff_plus_first_decode_ms_q75": 198.3308715, "steady_tpot_ms": 186.323086, "steady_tpot_ms_q25": 186.13135925, "steady_tpot_ms_q75": 186.55247525, "steady_tpot_p95_ms": 187.25145110000003, "steady_tpot_p95_ms_q25": 186.6876767, "steady_tpot_p95_ms_q75": 187.75809557500003, "decode_tokens_per_s": 5.34787218891652, "decode_tokens_per_s_q25": 5.340328811433944, "decode_tokens_per_s_q75": 5.348010202163604, "decode_ms": 2804.853869, "decode_ms_q25": 2804.7814875000004, "decode_ms_q75": 2808.821415, "end_to_end_ms": 3927.0962200000004, "end_to_end_ms_q25": 3915.019744, "end_to_end_ms_q75": 3934.2718755, "output_tokens_per_s_e2e": 4.0742571874136555, "output_tokens_per_s_e2e_q25": 4.066839743501149, "output_tokens_per_s_e2e_q75": 4.08686374245765, "all_tokens_per_s_e2e": 260.75245999447395, "all_tokens_per_s_e2e_q25": 260.27774358407356, "all_tokens_per_s_e2e_q75": 261.5592795172896, "energy_net_psys_j": 132.8522017517532, "energy_net_psys_j_q25": 131.33742346136717, "energy_net_psys_j_q75": 134.39923450420244, "prefill_energy_net_psys_j": 46.053012388909565, "prefill_energy_net_psys_j_q25": 45.59526162958316, "prefill_energy_net_psys_j_q75": 47.34791628297139, "decode_energy_net_psys_j": 86.79918936284363, "decode_energy_net_psys_j_q25": 85.74216183178399, "decode_energy_net_psys_j_q75": 87.05131822123104, "fastest_e2e": true, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "FP16", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 1024, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1122.3871139999999, "ttft_ms_q25": 1110.2382564999998, "ttft_ms_q75": 1125.5228419999999, "prefill_tokens_per_s": 898.0858630919743, "prefill_tokens_per_s_q25": 895.5907303191804, "prefill_tokens_per_s_q75": 908.0219549834039, "handoff_plus_first_decode_ms": 213.00904400000002, "handoff_plus_first_decode_ms_q25": 212.45307200000002, "handoff_plus_first_decode_ms_q75": 246.69054150000002, "steady_tpot_ms": 205.83404, "steady_tpot_ms_q25": 205.77235474999998, "steady_tpot_ms_q75": 206.12990075, "steady_tpot_p95_ms": 206.5551994, "steady_tpot_p95_ms_q25": 206.36039689999998, "steady_tpot_p95_ms_q75": 207.17913145, "decode_tokens_per_s": 4.8466610673536, "decode_tokens_per_s_q25": 4.787817592065784, "decode_tokens_per_s_q75": 4.846979213151203, "decode_ms": 3094.914167, "decode_ms_q25": 3094.7110365, "decode_ms_q75": 3133.4247405, "end_to_end_ms": 4223.166476, "end_to_end_ms_q25": 4220.2338785, "end_to_end_ms_q75": 4246.5955945000005, "output_tokens_per_s_e2e": 3.788626399391791, "output_tokens_per_s_e2e_q25": 3.7678386562685864, "output_tokens_per_s_e2e_q75": 3.791260907947617, "all_tokens_per_s_e2e": 242.47208956107463, "all_tokens_per_s_e2e_q25": 241.14167400118953, "all_tokens_per_s_e2e_q75": 242.64069810864748, "energy_net_psys_j": 100.98954805108606, "energy_net_psys_j_q25": 98.63511666993068, "energy_net_psys_j_q75": 101.59815174569542, "prefill_energy_net_psys_j": 46.053012388909565, "prefill_energy_net_psys_j_q25": 45.59526162958316, "prefill_energy_net_psys_j_q75": 47.34791628297139, "decode_energy_net_psys_j": 52.346727874052846, "decode_energy_net_psys_j_q25": 51.74495114628571, "decode_energy_net_psys_j_q75": 54.25023546272402, "fastest_e2e": false, "lowest_energy": true, "e2e_speedup_vs_gpu": 0.9298937757527321, "energy_reduction_vs_gpu": 0.2398353454480604 }, { "precision": "FP16", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 1024, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1883.063757, "ttft_ms_q25": 1881.8492390000001, "ttft_ms_q75": 1888.3335175, "prefill_tokens_per_s": 535.2978603368659, "prefill_tokens_per_s_q25": 533.8081652392121, "prefill_tokens_per_s_q75": 535.6435568612792, "handoff_plus_first_decode_ms": 213.00904400000002, "handoff_plus_first_decode_ms_q25": 212.45307200000002, "handoff_plus_first_decode_ms_q75": 246.69054150000002, "steady_tpot_ms": 205.83404, "steady_tpot_ms_q25": 205.77235474999998, "steady_tpot_ms_q75": 206.12990075, "steady_tpot_p95_ms": 206.5551994, "steady_tpot_p95_ms_q25": 206.36039689999998, "steady_tpot_p95_ms_q75": 207.17913145, "decode_tokens_per_s": 4.8466610673536, "decode_tokens_per_s_q25": 4.787817592065784, "decode_tokens_per_s_q75": 4.846979213151203, "decode_ms": 3094.914167, "decode_ms_q25": 3094.7110365, "decode_ms_q75": 3133.4247405, "end_to_end_ms": 4988.111184, "end_to_end_ms_q25": 4983.044554, "end_to_end_ms_q75": 5020.340609500001, "output_tokens_per_s_e2e": 3.207626977386156, "output_tokens_per_s_e2e_q25": 3.1871661081684453, "output_tokens_per_s_e2e_q75": 3.2108917285201617, "all_tokens_per_s_e2e": 205.28812655271398, "all_tokens_per_s_e2e_q25": 203.9786309227805, "all_tokens_per_s_e2e_q75": 205.49707062529035, "energy_net_psys_j": 104.30911227310798, "energy_net_psys_j_q25": 102.39025491443726, "energy_net_psys_j_q75": 105.18933223341892, "prefill_energy_net_psys_j": 49.91580914233469, "prefill_energy_net_psys_j_q25": 49.62201613979134, "prefill_energy_net_psys_j_q75": 50.93909677069491, "decode_energy_net_psys_j": 52.346727874052846, "decode_energy_net_psys_j_q25": 51.74495114628571, "decode_energy_net_psys_j_q75": 54.25023546272402, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.7872912361289499, "energy_reduction_vs_gpu": 0.21484844889496568 }, { "precision": "FP16", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 1024, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1883.063757, "ttft_ms_q25": 1881.8492390000001, "ttft_ms_q75": 1888.3335175, "prefill_tokens_per_s": 535.2978603368659, "prefill_tokens_per_s_q25": 533.8081652392121, "prefill_tokens_per_s_q75": 535.6435568612792, "handoff_plus_first_decode_ms": 194.39052800000002, "handoff_plus_first_decode_ms_q25": 194.27981200000002, "handoff_plus_first_decode_ms_q75": 198.3308715, "steady_tpot_ms": 186.323086, "steady_tpot_ms_q25": 186.13135925, "steady_tpot_ms_q75": 186.55247525, "steady_tpot_p95_ms": 187.25145110000003, "steady_tpot_p95_ms_q25": 186.6876767, "steady_tpot_p95_ms_q75": 187.75809557500003, "decode_tokens_per_s": 5.34787218891652, "decode_tokens_per_s_q25": 5.340328811433944, "decode_tokens_per_s_q75": 5.348010202163604, "decode_ms": 2804.853869, "decode_ms_q25": 2804.7814875000004, "decode_ms_q75": 2808.821415, "end_to_end_ms": 4687.772863, "end_to_end_ms_q25": 4686.6307265, "end_to_end_ms_q75": 4697.0825509999995, "output_tokens_per_s_e2e": 3.413134652125742, "output_tokens_per_s_e2e_q25": 3.4063831496797774, "output_tokens_per_s_e2e_q75": 3.413966639140699, "all_tokens_per_s_e2e": 218.4406177360475, "all_tokens_per_s_e2e_q25": 218.00852157950575, "all_tokens_per_s_e2e_q75": 218.49386490500473, "energy_net_psys_j": 136.71499850517833, "energy_net_psys_j_q25": 135.36417797157532, "energy_net_psys_j_q75": 137.99041499192595, "prefill_energy_net_psys_j": 49.91580914233469, "prefill_energy_net_psys_j_q25": 49.62201613979134, "prefill_energy_net_psys_j_q75": 50.93909677069491, "decode_energy_net_psys_j": 86.79918936284363, "decode_energy_net_psys_j_q25": 85.74216183178399, "decode_energy_net_psys_j_q75": 87.05131822123104, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.8377317619196262, "energy_reduction_vs_gpu": -0.029075895638094984 }, { "precision": "FP16", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 2048, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 2327.862173, "ttft_ms_q25": 2297.254624, "ttft_ms_q75": 2350.933028, "prefill_tokens_per_s": 872.9039131132442, "prefill_tokens_per_s_q25": 864.4209279957477, "prefill_tokens_per_s_q75": 884.6911249767875, "handoff_plus_first_decode_ms": 199.299217, "handoff_plus_first_decode_ms_q25": 197.535943, "handoff_plus_first_decode_ms_q75": 201.2887685, "steady_tpot_ms": 185.658828, "steady_tpot_ms_q25": 184.875764, "steady_tpot_ms_q75": 185.846622, "steady_tpot_p95_ms": 186.86304305000002, "steady_tpot_p95_ms_q25": 186.3178438, "steady_tpot_p95_ms_q75": 187.170955575, "decode_tokens_per_s": 5.359118528725976, "decode_tokens_per_s_q25": 5.353027761500256, "decode_tokens_per_s_q75": 5.372636984930659, "decode_ms": 2798.967763, "decode_ms_q25": 2791.9427665000003, "decode_ms_q75": 2802.1561045, "end_to_end_ms": 5126.829936, "end_to_end_ms_q25": 5089.1973904999995, "end_to_end_ms_q75": 5153.089132499999, "output_tokens_per_s_e2e": 3.1208368913604625, "output_tokens_per_s_e2e_q25": 3.1050143082922745, "output_tokens_per_s_e2e_q75": 3.1440861299460954, "all_tokens_per_s_e2e": 399.4671220941392, "all_tokens_per_s_e2e_q25": 397.44183146141114, "all_tokens_per_s_e2e_q75": 402.4430246331002, "energy_net_psys_j": 187.82896060698425, "energy_net_psys_j_q25": 186.08959630468215, "energy_net_psys_j_q75": 188.02331934598294, "prefill_energy_net_psys_j": 98.78065362709816, "prefill_energy_net_psys_j_q25": 98.32382270062134, "prefill_energy_net_psys_j_q75": 100.00239527306141, "decode_energy_net_psys_j": 86.60482368795958, "decode_energy_net_psys_j_q25": 86.54403195809756, "decode_energy_net_psys_j_q75": 88.02092407292153, "fastest_e2e": true, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "FP16", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 2048, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 2327.862173, "ttft_ms_q25": 2297.254624, "ttft_ms_q75": 2350.933028, "prefill_tokens_per_s": 872.9039131132442, "prefill_tokens_per_s_q25": 864.4209279957477, "prefill_tokens_per_s_q75": 884.6911249767875, "handoff_plus_first_decode_ms": 262.7809, "handoff_plus_first_decode_ms_q25": 259.80020149999996, "handoff_plus_first_decode_ms_q75": 278.6357845, "steady_tpot_ms": 208.02285700000002, "steady_tpot_ms_q25": 207.901767, "steady_tpot_ms_q75": 208.03618675, "steady_tpot_p95_ms": 209.2143584, "steady_tpot_p95_ms_q25": 208.609963225, "steady_tpot_p95_ms_q75": 209.366780475, "decode_tokens_per_s": 4.728065618031263, "decode_tokens_per_s_q25": 4.699739044758417, "decode_tokens_per_s_q75": 4.728628149741116, "decode_ms": 3172.544802, "decode_ms_q25": 3172.1674315, "decode_ms_q75": 3191.7825215000003, "end_to_end_ms": 5500.406975, "end_to_end_ms_q25": 5489.0371455, "end_to_end_ms_q75": 5523.1004594999995, "output_tokens_per_s_e2e": 2.9088756655138233, "output_tokens_per_s_e2e_q25": 2.896972496103935, "output_tokens_per_s_e2e_q75": 2.9149135313507983, "all_tokens_per_s_e2e": 372.3360851857694, "all_tokens_per_s_e2e_q25": 370.8124795013037, "all_tokens_per_s_e2e_q75": 373.1089320129022, "energy_net_psys_j": 157.52080229855835, "energy_net_psys_j_q25": 154.61754982706316, "energy_net_psys_j_q75": 158.27159191513144, "prefill_energy_net_psys_j": 98.78065362709816, "prefill_energy_net_psys_j_q25": 98.32382270062134, "prefill_energy_net_psys_j_q75": 100.00239527306141, "decode_energy_net_psys_j": 56.2966653795337, "decode_energy_net_psys_j_q25": 55.071985480478574, "decode_energy_net_psys_j_q75": 58.26919664207003, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.932081927628637, "energy_reduction_vs_gpu": 0.16136041114470667 }, { "precision": "FP16", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 2048, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 3784.315485, "ttft_ms_q25": 3779.738423, "ttft_ms_q75": 3786.98877, "prefill_tokens_per_s": 536.9531182202691, "prefill_tokens_per_s_q25": 536.5743433596591, "prefill_tokens_per_s_q75": 537.6041281979026, "handoff_plus_first_decode_ms": 262.7809, "handoff_plus_first_decode_ms_q25": 259.80020149999996, "handoff_plus_first_decode_ms_q75": 278.6357845, "steady_tpot_ms": 208.02285700000002, "steady_tpot_ms_q25": 207.901767, "steady_tpot_ms_q75": 208.03618675, "steady_tpot_p95_ms": 209.2143584, "steady_tpot_p95_ms_q25": 208.609963225, "steady_tpot_p95_ms_q75": 209.366780475, "decode_tokens_per_s": 4.728065618031263, "decode_tokens_per_s_q25": 4.699739044758417, "decode_tokens_per_s_q75": 4.728628149741116, "decode_ms": 3172.544802, "decode_ms_q25": 3172.1674315, "decode_ms_q75": 3191.7825215000003, "end_to_end_ms": 6956.105546, "end_to_end_ms_q25": 6951.9058545, "end_to_end_ms_q75": 6978.393921, "output_tokens_per_s_e2e": 2.300137612086773, "output_tokens_per_s_e2e_q25": 2.292814564491957, "output_tokens_per_s_e2e_q75": 2.301527980094815, "all_tokens_per_s_e2e": 294.41761434710696, "all_tokens_per_s_e2e_q25": 293.4802642549705, "all_tokens_per_s_e2e_q75": 294.59558145213634, "energy_net_psys_j": 157.51794815752865, "energy_net_psys_j_q25": 157.41612531313348, "energy_net_psys_j_q75": 162.6573135078748, "prefill_energy_net_psys_j": 103.67064257610521, "prefill_energy_net_psys_j_q25": 102.34413983265489, "prefill_energy_net_psys_j_q75": 105.6127967648599, "decode_energy_net_psys_j": 56.2966653795337, "decode_energy_net_psys_j_q25": 55.071985480478574, "decode_energy_net_psys_j_q75": 58.26919664207003, "fastest_e2e": false, "lowest_energy": true, "e2e_speedup_vs_gpu": 0.7370258979103765, "energy_reduction_vs_gpu": 0.1613756065704839 }, { "precision": "FP16", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 2048, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 3784.315485, "ttft_ms_q25": 3779.738423, "ttft_ms_q75": 3786.98877, "prefill_tokens_per_s": 536.9531182202691, "prefill_tokens_per_s_q25": 536.5743433596591, "prefill_tokens_per_s_q75": 537.6041281979026, "handoff_plus_first_decode_ms": 199.299217, "handoff_plus_first_decode_ms_q25": 197.535943, "handoff_plus_first_decode_ms_q75": 201.2887685, "steady_tpot_ms": 185.658828, "steady_tpot_ms_q25": 184.875764, "steady_tpot_ms_q75": 185.846622, "steady_tpot_p95_ms": 186.86304305000002, "steady_tpot_p95_ms_q25": 186.3178438, "steady_tpot_p95_ms_q75": 187.170955575, "decode_tokens_per_s": 5.359118528725976, "decode_tokens_per_s_q25": 5.353027761500256, "decode_tokens_per_s_q75": 5.372636984930659, "decode_ms": 2798.967763, "decode_ms_q25": 2791.9427665000003, "decode_ms_q75": 2802.1561045, "end_to_end_ms": 6574.579825, "end_to_end_ms_q25": 6574.3544745, "end_to_end_ms_q75": 6582.1198779999995, "output_tokens_per_s_e2e": 2.4336155961114976, "output_tokens_per_s_e2e_q25": 2.430830992540381, "output_tokens_per_s_e2e_q75": 2.4336990165130516, "all_tokens_per_s_e2e": 311.5027963022717, "all_tokens_per_s_e2e_q25": 311.14636704516874, "all_tokens_per_s_e2e_q75": 311.5134741136706, "energy_net_psys_j": 190.15388280434075, "energy_net_psys_j_q25": 188.88817179075244, "energy_net_psys_j_q75": 193.5729291079194, "prefill_energy_net_psys_j": 103.67064257610521, "prefill_energy_net_psys_j_q25": 102.34413983265489, "prefill_energy_net_psys_j_q75": 105.6127967648599, "decode_energy_net_psys_j": 86.60482368795958, "decode_energy_net_psys_j_q25": 86.54403195809756, "decode_energy_net_psys_j_q75": 88.02092407292153, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.779795830678807, "energy_reduction_vs_gpu": -0.012377868619638521 }, { "precision": "FP16", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 4096, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 5528.934798, "ttft_ms_q25": 5448.807011000001, "ttft_ms_q75": 5530.436725, "prefill_tokens_per_s": 737.9359947373357, "prefill_tokens_per_s_q25": 737.7356443602816, "prefill_tokens_per_s_q75": 748.9497237643218, "handoff_plus_first_decode_ms": 209.129247, "handoff_plus_first_decode_ms_q25": 207.3933045, "handoff_plus_first_decode_ms_q75": 210.9861555, "steady_tpot_ms": 188.47727250000003, "steady_tpot_ms_q25": 188.40073825000002, "steady_tpot_ms_q75": 188.802173, "steady_tpot_p95_ms": 190.72192264999998, "steady_tpot_p95_ms_q25": 190.219103625, "steady_tpot_p95_ms_q75": 191.32966362500002, "decode_tokens_per_s": 5.254028638369279, "decode_tokens_per_s_q25": 5.253332518057384, "decode_tokens_per_s_q75": 5.254524002663129, "decode_ms": 2854.9520820000007, "decode_ms_q25": 2854.68296, "decode_ms_q75": 2855.3304425, "end_to_end_ms": 8384.643601, "end_to_end_ms_q25": 8303.8683315, "end_to_end_ms_q75": 8385.7671675, "output_tokens_per_s_e2e": 1.9082504589809577, "output_tokens_per_s_e2e_q25": 1.9079948164124154, "output_tokens_per_s_e2e_q75": 1.9269951629427777, "all_tokens_per_s_e2e": 488.51211749912517, "all_tokens_per_s_e2e_q25": 488.44667300157835, "all_tokens_per_s_e2e_q75": 493.3107617133511, "energy_net_psys_j": 320.48796452606746, "energy_net_psys_j_q25": 317.6113090734332, "energy_net_psys_j_q75": 321.2715341199631, "prefill_energy_net_psys_j": 231.97102930961674, "prefill_energy_net_psys_j_q25": 228.48211675690027, "prefill_energy_net_psys_j_q75": 232.25588411777665, "decode_energy_net_psys_j": 89.51436478792222, "decode_energy_net_psys_j_q25": 89.01565000218648, "decode_energy_net_psys_j_q75": 89.62790710226865, "fastest_e2e": true, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "FP16", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 4096, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 5528.934798, "ttft_ms_q25": 5448.807011000001, "ttft_ms_q75": 5530.436725, "prefill_tokens_per_s": 737.9359947373357, "prefill_tokens_per_s_q25": 737.7356443602816, "prefill_tokens_per_s_q75": 748.9497237643218, "handoff_plus_first_decode_ms": 330.0977, "handoff_plus_first_decode_ms_q25": 316.156475, "handoff_plus_first_decode_ms_q75": 332.4441515, "steady_tpot_ms": 214.40785449999998, "steady_tpot_ms_q25": 214.3305785, "steady_tpot_ms_q75": 214.41199475, "steady_tpot_p95_ms": 215.12930295, "steady_tpot_p95_ms_q25": 215.090931, "steady_tpot_p95_ms_q75": 215.477234925, "decode_tokens_per_s": 4.503460418454397, "decode_tokens_per_s_q25": 4.49964235404687, "decode_tokens_per_s_q75": 4.518838452798773, "decode_ms": 3330.77203, "decode_ms_q25": 3319.475541, "decode_ms_q75": 3333.6006774999996, "end_to_end_ms": 8837.11385, "end_to_end_ms_q25": 8768.282552, "end_to_end_ms_q75": 8852.740913499998, "output_tokens_per_s_e2e": 1.8105458718289569, "output_tokens_per_s_e2e_q25": 1.8073554860237997, "output_tokens_per_s_e2e_q75": 1.824871168771017, "all_tokens_per_s_e2e": 463.49974318821296, "all_tokens_per_s_e2e_q25": 462.6830044220927, "all_tokens_per_s_e2e_q75": 467.16701920538037, "energy_net_psys_j": 291.17361407593637, "energy_net_psys_j_q25": 290.45306624886246, "energy_net_psys_j_q75": 291.40285178421004, "prefill_energy_net_psys_j": 231.97102930961674, "prefill_energy_net_psys_j_q25": 228.48211675690027, "prefill_energy_net_psys_j_q75": 232.25588411777665, "decode_energy_net_psys_j": 59.09135056654719, "decode_energy_net_psys_j_q25": 58.426419839359525, "decode_energy_net_psys_j_q75": 62.635880219149875, "fastest_e2e": false, "lowest_energy": true, "e2e_speedup_vs_gpu": 0.9487988661592268, "energy_reduction_vs_gpu": 0.09146786679955576 }, { "precision": "FP16", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 4096, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 8327.051625, "ttft_ms_q25": 8321.200045, "ttft_ms_q75": 8330.5022765, "prefill_tokens_per_s": 489.9693413393484, "prefill_tokens_per_s_q25": 489.7664708099305, "prefill_tokens_per_s_q75": 490.31413684737333, "handoff_plus_first_decode_ms": 330.0977, "handoff_plus_first_decode_ms_q25": 316.156475, "handoff_plus_first_decode_ms_q75": 332.4441515, "steady_tpot_ms": 214.40785449999998, "steady_tpot_ms_q25": 214.3305785, "steady_tpot_ms_q75": 214.41199475, "steady_tpot_p95_ms": 215.12930295, "steady_tpot_p95_ms_q25": 215.090931, "steady_tpot_p95_ms_q75": 215.477234925, "decode_tokens_per_s": 4.503460418454397, "decode_tokens_per_s_q25": 4.49964235404687, "decode_tokens_per_s_q75": 4.518838452798773, "decode_ms": 3330.77203, "decode_ms_q25": 3319.475541, "decode_ms_q75": 3333.6006774999996, "end_to_end_ms": 11646.120495, "end_to_end_ms_q25": 11640.675586, "end_to_end_ms_q75": 11658.251374, "output_tokens_per_s_e2e": 1.373848055828483, "output_tokens_per_s_e2e_q25": 1.3724199976750873, "output_tokens_per_s_e2e_q75": 1.3744909719726897, "all_tokens_per_s_e2e": 351.7051022920916, "all_tokens_per_s_e2e_q25": 351.33951940482234, "all_tokens_per_s_e2e_q75": 351.86968882500855, "energy_net_psys_j": 297.0032745093148, "energy_net_psys_j_q25": 293.463646088685, "energy_net_psys_j_q75": 297.54450137030153, "prefill_energy_net_psys_j": 232.16252855588326, "prefill_energy_net_psys_j_q25": 231.49269659672274, "prefill_energy_net_psys_j_q75": 235.57845311031213, "decode_energy_net_psys_j": 59.09135056654719, "decode_energy_net_psys_j_q25": 58.426419839359525, "decode_energy_net_psys_j_q75": 62.635880219149875, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.7199516443780363, "energy_reduction_vs_gpu": 0.07327791560435493 }, { "precision": "FP16", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 4096, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 8327.051625, "ttft_ms_q25": 8321.200045, "ttft_ms_q75": 8330.5022765, "prefill_tokens_per_s": 489.9693413393484, "prefill_tokens_per_s_q25": 489.7664708099305, "prefill_tokens_per_s_q75": 490.31413684737333, "handoff_plus_first_decode_ms": 209.129247, "handoff_plus_first_decode_ms_q25": 207.3933045, "handoff_plus_first_decode_ms_q75": 210.9861555, "steady_tpot_ms": 188.47727250000003, "steady_tpot_ms_q25": 188.40073825000002, "steady_tpot_ms_q75": 188.802173, "steady_tpot_p95_ms": 190.72192264999998, "steady_tpot_p95_ms_q25": 190.219103625, "steady_tpot_p95_ms_q75": 191.32966362500002, "decode_tokens_per_s": 5.254028638369279, "decode_tokens_per_s_q25": 5.253332518057384, "decode_tokens_per_s_q75": 5.254524002663129, "decode_ms": 2854.9520820000007, "decode_ms_q25": 2854.68296, "decode_ms_q75": 2855.3304425, "end_to_end_ms": 11182.760428, "end_to_end_ms_q25": 11176.261365499999, "end_to_end_ms_q75": 11185.832719000002, "output_tokens_per_s_e2e": 1.4307737434791445, "output_tokens_per_s_e2e_q25": 1.430380876322689, "output_tokens_per_s_e2e_q75": 1.4316062310227453, "all_tokens_per_s_e2e": 366.278078330661, "all_tokens_per_s_e2e_q25": 366.1775043386084, "all_tokens_per_s_e2e_q75": 366.4911951418228, "energy_net_psys_j": 320.679463772334, "energy_net_psys_j_q25": 320.62188891325565, "energy_net_psys_j_q75": 324.5941031124986, "prefill_energy_net_psys_j": 232.16252855588326, "prefill_energy_net_psys_j_q25": 231.49269659672274, "prefill_energy_net_psys_j_q75": 235.57845311031213, "decode_energy_net_psys_j": 89.51436478792222, "decode_energy_net_psys_j_q25": 89.01565000218648, "decode_energy_net_psys_j_q75": 89.62790710226865, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.7497829945463266, "energy_reduction_vs_gpu": -0.0005975239867421767 }, { "precision": "FP16", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 8192, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 13534.217505999999, "ttft_ms_q25": 13441.503799999999, "ttft_ms_q75": 14256.6174665, "prefill_tokens_per_s": 604.0984634963498, "prefill_tokens_per_s_q25": 574.9643291451038, "prefill_tokens_per_s_q75": 608.2942150922063, "handoff_plus_first_decode_ms": 235.474582, "handoff_plus_first_decode_ms_q25": 235.0273355, "handoff_plus_first_decode_ms_q75": 249.6081695, "steady_tpot_ms": 196.93333750000002, "steady_tpot_ms_q25": 196.81365750000003, "steady_tpot_ms_q75": 197.04648200000003, "steady_tpot_p95_ms": 200.4805697, "steady_tpot_p95_ms_q25": 199.683263125, "steady_tpot_p95_ms_q75": 200.726778025, "decode_tokens_per_s": 4.994483062377571, "decode_tokens_per_s_q25": 4.9737249486687, "decode_tokens_per_s_q75": 5.002145834896865, "decode_ms": 3003.313819000001, "decode_ms_q25": 2998.7200885, "decode_ms_q75": 3015.9008465000015, "end_to_end_ms": 16528.343864, "end_to_end_ms_q25": 16452.810916000002, "end_to_end_ms_q75": 17255.337555, "output_tokens_per_s_e2e": 0.9680340711478799, "output_tokens_per_s_e2e_q25": 0.9288981742071483, "output_tokens_per_s_e2e_q75": 0.9724986999647753, "all_tokens_per_s_e2e": 495.6334444277145, "all_tokens_per_s_e2e_q25": 475.5958651940599, "all_tokens_per_s_e2e_q75": 497.919334381965, "energy_net_psys_j": 616.5810162520157, "energy_net_psys_j_q25": 611.4555356138947, "energy_net_psys_j_q75": 635.3122916265785, "prefill_energy_net_psys_j": 526.5659219765113, "prefill_energy_net_psys_j_q25": 520.2107352229555, "prefill_energy_net_psys_j_q75": 544.9391911780941, "decode_energy_net_psys_j": 90.73110662146432, "decode_energy_net_psys_j_q25": 90.37310044848437, "decode_energy_net_psys_j_q75": 91.60280656391917, "fastest_e2e": true, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "FP16", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 8192, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 13534.217505999999, "ttft_ms_q25": 13441.503799999999, "ttft_ms_q75": 14256.6174665, "prefill_tokens_per_s": 604.0984634963498, "prefill_tokens_per_s_q25": 574.9643291451038, "prefill_tokens_per_s_q75": 608.2942150922063, "handoff_plus_first_decode_ms": 402.618993, "handoff_plus_first_decode_ms_q25": 402.290937, "handoff_plus_first_decode_ms_q75": 415.835902, "steady_tpot_ms": 224.522828, "steady_tpot_ms_q25": 224.4906115, "steady_tpot_ms_q75": 224.76623325, "steady_tpot_p95_ms": 225.88628919999996, "steady_tpot_p95_ms_q25": 225.71495657499997, "steady_tpot_p95_ms_q75": 226.15309994999998, "decode_tokens_per_s": 4.22430750449732, "decode_tokens_per_s_q25": 4.21039922053839, "decode_tokens_per_s_q75": 4.226318357939814, "decode_ms": 3550.8778619999975, "decode_ms_q25": 3549.189182, "decode_ms_q75": 3562.6464129999986, "end_to_end_ms": 17085.095367999995, "end_to_end_ms_q25": 17004.150212999997, "end_to_end_ms_q75": 17805.806648499998, "output_tokens_per_s_e2e": 0.9364887731307399, "output_tokens_per_s_e2e_q25": 0.9000578568925386, "output_tokens_per_s_e2e_q75": 0.9409680800136279, "all_tokens_per_s_e2e": 479.4822518429388, "all_tokens_per_s_e2e_q25": 460.82962272897976, "all_tokens_per_s_e2e_q75": 481.77565696697746, "energy_net_psys_j": 587.2137486507721, "energy_net_psys_j_q25": 583.1809150293245, "energy_net_psys_j_q75": 606.9761207902717, "prefill_energy_net_psys_j": 526.5659219765113, "prefill_energy_net_psys_j_q25": 520.2107352229555, "prefill_energy_net_psys_j_q75": 544.9391911780941, "decode_energy_net_psys_j": 63.42603255009427, "decode_energy_net_psys_j_q25": 62.03692961217753, "decode_energy_net_psys_j_q75": 64.35928274428575, "fastest_e2e": false, "lowest_energy": true, "e2e_speedup_vs_gpu": 0.9674130291925218, "energy_reduction_vs_gpu": 0.047629211453439124 }, { "precision": "FP16", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 8192, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 19051.627731, "ttft_ms_q25": 19030.7399085, "ttft_ms_q75": 19067.34142, "prefill_tokens_per_s": 429.14968292690077, "prefill_tokens_per_s_q25": 428.7963053040074, "prefill_tokens_per_s_q75": 429.6212280163815, "handoff_plus_first_decode_ms": 402.618993, "handoff_plus_first_decode_ms_q25": 402.290937, "handoff_plus_first_decode_ms_q75": 415.835902, "steady_tpot_ms": 224.522828, "steady_tpot_ms_q25": 224.4906115, "steady_tpot_ms_q75": 224.76623325, "steady_tpot_p95_ms": 225.88628919999996, "steady_tpot_p95_ms_q25": 225.71495657499997, "steady_tpot_p95_ms_q75": 226.15309994999998, "decode_tokens_per_s": 4.22430750449732, "decode_tokens_per_s_q25": 4.21039922053839, "decode_tokens_per_s_q75": 4.226318357939814, "decode_ms": 3550.8778619999975, "decode_ms_q25": 3549.189182, "decode_ms_q75": 3562.6464129999986, "end_to_end_ms": 22599.128233000003, "end_to_end_ms_q25": 22591.697641500003, "end_to_end_ms_q75": 22616.530602, "output_tokens_per_s_e2e": 0.7079919116807464, "output_tokens_per_s_e2e_q25": 0.7074475636652305, "output_tokens_per_s_e2e_q75": 0.708224852544816, "all_tokens_per_s_e2e": 362.49185878054215, "all_tokens_per_s_e2e_q25": 362.21315259659804, "all_tokens_per_s_e2e_q75": 362.6111245029458, "energy_net_psys_j": 593.999688863384, "energy_net_psys_j_q25": 589.3421377157467, "energy_net_psys_j_q75": 598.0619848365291, "prefill_energy_net_psys_j": 528.7071559249067, "prefill_energy_net_psys_j_q25": 526.3719579093777, "prefill_energy_net_psys_j_q75": 533.7027020922433, "decode_energy_net_psys_j": 63.42603255009427, "decode_energy_net_psys_j_q25": 62.03692961217753, "decode_energy_net_psys_j_q75": 64.35928274428575, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.7313708605743808, "energy_reduction_vs_gpu": 0.036623455463964616 }, { "precision": "FP16", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 8192, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 19051.627731, "ttft_ms_q25": 19030.7399085, "ttft_ms_q75": 19067.34142, "prefill_tokens_per_s": 429.14968292690077, "prefill_tokens_per_s_q25": 428.7963053040074, "prefill_tokens_per_s_q75": 429.6212280163815, "handoff_plus_first_decode_ms": 235.474582, "handoff_plus_first_decode_ms_q25": 235.0273355, "handoff_plus_first_decode_ms_q75": 249.6081695, "steady_tpot_ms": 196.93333750000002, "steady_tpot_ms_q25": 196.81365750000003, "steady_tpot_ms_q75": 197.04648200000003, "steady_tpot_p95_ms": 200.4805697, "steady_tpot_p95_ms_q25": 199.683263125, "steady_tpot_p95_ms_q75": 200.726778025, "decode_tokens_per_s": 4.994483062377571, "decode_tokens_per_s_q25": 4.9737249486687, "decode_tokens_per_s_q75": 5.002145834896865, "decode_ms": 3003.313819000001, "decode_ms_q25": 2998.7200885, "decode_ms_q75": 3015.9008465000015, "end_to_end_ms": 22054.941550000003, "end_to_end_ms_q25": 22046.640755, "end_to_end_ms_q75": 22066.061508500003, "output_tokens_per_s_e2e": 0.7254610021852449, "output_tokens_per_s_e2e_q25": 0.7250955979215834, "output_tokens_per_s_e2e_q75": 0.7257342488576606, "all_tokens_per_s_e2e": 371.4360331188454, "all_tokens_per_s_e2e_q25": 371.2489461358507, "all_tokens_per_s_e2e_q75": 371.5759354151222, "energy_net_psys_j": 621.1816624312808, "energy_net_psys_j_q25": 617.6167583003169, "energy_net_psys_j_q75": 625.3055086561624, "prefill_energy_net_psys_j": 528.7071559249067, "prefill_energy_net_psys_j_q25": 526.3719579093777, "prefill_energy_net_psys_j_q75": 533.7027020922433, "decode_energy_net_psys_j": 90.73110662146432, "decode_energy_net_psys_j_q25": 90.37310044848437, "decode_energy_net_psys_j_q75": 91.60280656391917, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.7494168065024863, "energy_reduction_vs_gpu": -0.007461543670661097 }, { "precision": "INT8", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 1024, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 258.676058, "ttft_ms_q25": 249.456735, "ttft_ms_q75": 261.883168, "prefill_tokens_per_s": 3896.765737786216, "prefill_tokens_per_s_q25": 3849.621962995231, "prefill_tokens_per_s_q75": 4046.3075668165106, "handoff_plus_first_decode_ms": 32.335183, "handoff_plus_first_decode_ms_q25": 28.476062, "handoff_plus_first_decode_ms_q75": 36.0558015, "steady_tpot_ms": 18.722242, "steady_tpot_ms_q25": 18.680516750000002, "steady_tpot_ms_q75": 18.731333499999998, "steady_tpot_p95_ms": 20.45485955, "steady_tpot_p95_ms_q25": 20.2223636, "steady_tpot_p95_ms_q75": 23.408128325, "decode_tokens_per_s": 49.30111868083774, "decode_tokens_per_s_q25": 48.35031790623841, "decode_tokens_per_s_q75": 50.503898346549306, "decode_ms": 304.252731, "decode_ms_q25": 297.17532800000004, "decode_ms_q75": 310.35582550000004, "end_to_end_ms": 548.773983, "end_to_end_ms_q25": 546.632063, "end_to_end_ms_q75": 565.1615905000001, "output_tokens_per_s_e2e": 29.15590114628302, "output_tokens_per_s_e2e_q25": 28.33431035751181, "output_tokens_per_s_e2e_q75": 29.27059488057023, "all_tokens_per_s_e2e": 1865.9776733621134, "all_tokens_per_s_e2e_q25": 1813.3958628807559, "all_tokens_per_s_e2e_q75": 1873.3180723564947, "energy_net_psys_j": 19.008366268236916, "energy_net_psys_j_q25": 18.873820535060823, "energy_net_psys_j_q75": 19.184268749006918, "prefill_energy_net_psys_j": 7.7984988544351985, "prefill_energy_net_psys_j_q25": 7.4164487334728095, "prefill_energy_net_psys_j_q75": 7.816856745044378, "decode_energy_net_psys_j": 11.52495659412336, "decode_energy_net_psys_j_q25": 11.232866270786444, "decode_energy_net_psys_j_q75": 11.749462124924928, "fastest_e2e": true, "lowest_energy": true, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "INT8", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 1024, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 258.676058, "ttft_ms_q25": 249.456735, "ttft_ms_q75": 261.883168, "prefill_tokens_per_s": 3896.765737786216, "prefill_tokens_per_s_q25": 3849.621962995231, "prefill_tokens_per_s_q75": 4046.3075668165106, "handoff_plus_first_decode_ms": 428.263869, "handoff_plus_first_decode_ms_q25": 427.56293949999997, "handoff_plus_first_decode_ms_q75": 512.7273015000001, "steady_tpot_ms": 422.7833595, "steady_tpot_ms_q25": 422.41823225, "steady_tpot_ms_q75": 422.849783, "steady_tpot_p95_ms": 423.48913335, "steady_tpot_p95_ms_q25": 423.4693372, "steady_tpot_p95_ms_q75": 424.7582149, "decode_tokens_per_s": 2.3637284280508135, "decode_tokens_per_s_q25": 2.331620801672016, "decode_tokens_per_s_q75": 2.365490507573279, "decode_ms": 6345.906671, "decode_ms_q25": 6341.1830549999995, "decode_ms_q75": 6434.513251999999, "end_to_end_ms": 6610.996948999999, "end_to_end_ms_q25": 6603.066223, "end_to_end_ms_q75": 6687.177097, "output_tokens_per_s_e2e": 2.4202098599395376, "output_tokens_per_s_e2e_q25": 2.392949441978095, "output_tokens_per_s_e2e_q75": 2.4231201900260704, "all_tokens_per_s_e2e": 154.8934310361304, "all_tokens_per_s_e2e_q25": 153.14876428659807, "all_tokens_per_s_e2e_q75": 155.0796921616685, "energy_net_psys_j": 94.22866893801896, "energy_net_psys_j_q25": 93.15838990974521, "energy_net_psys_j_q75": 96.40623733088943, "prefill_energy_net_psys_j": 7.7984988544351985, "prefill_energy_net_psys_j_q25": 7.4164487334728095, "prefill_energy_net_psys_j_q75": 7.816856745044378, "decode_energy_net_psys_j": 86.3934543023654, "decode_energy_net_psys_j_q25": 85.34153316470083, "decode_energy_net_psys_j_q75": 88.97143070680744, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.08300926278343078, "energy_reduction_vs_gpu": -3.9572208157349946 }, { "precision": "INT8", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 1024, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 715.7900470000001, "ttft_ms_q25": 715.349371, "ttft_ms_q75": 716.878866, "prefill_tokens_per_s": 1408.2341661842079, "prefill_tokens_per_s_q25": 1406.0985378702533, "prefill_tokens_per_s_q75": 1409.1022140954415, "handoff_plus_first_decode_ms": 428.263869, "handoff_plus_first_decode_ms_q25": 427.56293949999997, "handoff_plus_first_decode_ms_q75": 512.7273015000001, "steady_tpot_ms": 422.7833595, "steady_tpot_ms_q25": 422.41823225, "steady_tpot_ms_q75": 422.849783, "steady_tpot_p95_ms": 423.48913335, "steady_tpot_p95_ms_q25": 423.4693372, "steady_tpot_p95_ms_q75": 424.7582149, "decode_tokens_per_s": 2.3637284280508135, "decode_tokens_per_s_q25": 2.331620801672016, "decode_tokens_per_s_q75": 2.365490507573279, "decode_ms": 6345.906671, "decode_ms_q25": 6341.1830549999995, "decode_ms_q75": 6434.513251999999, "end_to_end_ms": 7060.815366, "end_to_end_ms_q25": 7057.621244999999, "end_to_end_ms_q75": 7149.862622999999, "output_tokens_per_s_e2e": 2.2660272462363094, "output_tokens_per_s_e2e_q25": 2.2381523985161986, "output_tokens_per_s_e2e_q75": 2.267053263669979, "all_tokens_per_s_e2e": 145.0257437591238, "all_tokens_per_s_e2e_q25": 143.2417535050367, "all_tokens_per_s_e2e_q75": 145.09140887487865, "energy_net_psys_j": 97.75641667800228, "energy_net_psys_j_q25": 96.67973852350553, "energy_net_psys_j_q75": 100.58658423217739, "prefill_energy_net_psys_j": 11.362962375636888, "prefill_energy_net_psys_j_q25": 11.338205358804714, "prefill_energy_net_psys_j_q75": 11.615153525369948, "decode_energy_net_psys_j": 86.3934543023654, "decode_energy_net_psys_j_q25": 85.34153316470083, "decode_energy_net_psys_j_q75": 88.97143070680744, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.07772104984397635, "energy_reduction_vs_gpu": -4.1428100289372995 }, { "precision": "INT8", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 1024, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 715.7900470000001, "ttft_ms_q25": 715.349371, "ttft_ms_q75": 716.878866, "prefill_tokens_per_s": 1408.2341661842079, "prefill_tokens_per_s_q25": 1406.0985378702533, "prefill_tokens_per_s_q75": 1409.1022140954415, "handoff_plus_first_decode_ms": 32.335183, "handoff_plus_first_decode_ms_q25": 28.476062, "handoff_plus_first_decode_ms_q75": 36.0558015, "steady_tpot_ms": 18.722242, "steady_tpot_ms_q25": 18.680516750000002, "steady_tpot_ms_q75": 18.731333499999998, "steady_tpot_p95_ms": 20.45485955, "steady_tpot_p95_ms_q25": 20.2223636, "steady_tpot_p95_ms_q75": 23.408128325, "decode_tokens_per_s": 49.30111868083774, "decode_tokens_per_s_q25": 48.35031790623841, "decode_tokens_per_s_q75": 50.503898346549306, "decode_ms": 304.252731, "decode_ms_q25": 297.17532800000004, "decode_ms_q75": 310.35582550000004, "end_to_end_ms": 1020.042778, "end_to_end_ms_q25": 1014.054194, "end_to_end_ms_q75": 1025.7051965, "output_tokens_per_s_e2e": 15.685616667343336, "output_tokens_per_s_e2e_q25": 15.599499433575943, "output_tokens_per_s_e2e_q75": 15.778799722168692, "all_tokens_per_s_e2e": 1003.8794667099735, "all_tokens_per_s_e2e_q25": 998.3679637488603, "all_tokens_per_s_e2e_q75": 1009.8431822187963, "energy_net_psys_j": 22.88791896976025, "energy_net_psys_j_q25": 22.57107162959116, "energy_net_psys_j_q75": 23.364615650294876, "prefill_energy_net_psys_j": 11.362962375636888, "prefill_energy_net_psys_j_q25": 11.338205358804714, "prefill_energy_net_psys_j_q75": 11.615153525369948, "decode_energy_net_psys_j": 11.52495659412336, "decode_energy_net_psys_j_q25": 11.232866270786444, "decode_energy_net_psys_j_q75": 11.749462124924928, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.5379911458968244, "energy_reduction_vs_gpu": -0.20409711422733312 }, { "precision": "INT8", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 2048, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 563.861085, "ttft_ms_q25": 540.7457489999999, "ttft_ms_q75": 563.9430460000001, "prefill_tokens_per_s": 3603.724488275335, "prefill_tokens_per_s_q25": 3603.200814929086, "prefill_tokens_per_s_q75": 3764.652633735583, "handoff_plus_first_decode_ms": 30.489342, "handoff_plus_first_decode_ms_q25": 30.156737, "handoff_plus_first_decode_ms_q75": 37.794177000000005, "steady_tpot_ms": 19.3590285, "steady_tpot_ms_q25": 19.310399500000003, "steady_tpot_ms_q75": 19.42046825, "steady_tpot_p95_ms": 21.360242849999995, "steady_tpot_p95_ms_q25": 20.723510174999998, "steady_tpot_p95_ms_q75": 21.909633949999996, "decode_tokens_per_s": 48.71870171753366, "decode_tokens_per_s_q25": 47.52197164148824, "decode_tokens_per_s_q75": 49.14911932672539, "decode_ms": 307.8899779999999, "decode_ms_q25": 305.21707549999996, "decode_ms_q75": 315.84376699999996, "end_to_end_ms": 866.5691800000001, "end_to_end_ms_q25": 853.9985745, "end_to_end_ms_q75": 869.1601215000001, "output_tokens_per_s_e2e": 18.463615334207937, "output_tokens_per_s_e2e_q25": 18.408739407765353, "output_tokens_per_s_e2e_q75": 18.739454542103438, "all_tokens_per_s_e2e": 2363.342762778616, "all_tokens_per_s_e2e_q25": 2356.318644193965, "all_tokens_per_s_e2e_q75": 2398.65018138924, "energy_net_psys_j": 28.135222944952325, "energy_net_psys_j_q25": 28.004517460824047, "energy_net_psys_j_q75": 28.28943526997993, "prefill_energy_net_psys_j": 16.986121113617084, "prefill_energy_net_psys_j_q25": 16.25691982914457, "prefill_energy_net_psys_j_q75": 17.177019131899208, "decode_energy_net_psys_j": 11.14910183133524, "decode_energy_net_psys_j_q25": 11.112416138080722, "decode_energy_net_psys_j_q75": 11.747597631679474, "fastest_e2e": true, "lowest_energy": true, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "INT8", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 2048, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 563.861085, "ttft_ms_q25": 540.7457489999999, "ttft_ms_q75": 563.9430460000001, "prefill_tokens_per_s": 3603.724488275335, "prefill_tokens_per_s_q25": 3603.200814929086, "prefill_tokens_per_s_q75": 3764.652633735583, "handoff_plus_first_decode_ms": 455.564631, "handoff_plus_first_decode_ms_q25": 454.43201650000003, "handoff_plus_first_decode_ms_q75": 531.5957195, "steady_tpot_ms": 424.542054, "steady_tpot_ms_q25": 424.36665475, "steady_tpot_ms_q75": 424.65971775, "steady_tpot_p95_ms": 425.34586365, "steady_tpot_p95_ms_q25": 425.267551475, "steady_tpot_p95_ms_q75": 426.050509475, "decode_tokens_per_s": 2.343448505097232, "decode_tokens_per_s_q25": 2.3162625481922383, "decode_tokens_per_s_q75": 2.3454854728912924, "decode_ms": 6400.823388000001, "decode_ms_q25": 6395.269332, "decode_ms_q75": 6476.8420365, "end_to_end_ms": 6964.684473000001, "end_to_end_ms_q25": 6959.212378, "end_to_end_ms_q75": 7017.587785500001, "output_tokens_per_s_e2e": 2.2973043591604494, "output_tokens_per_s_e2e_q25": 2.280115311223644, "output_tokens_per_s_e2e_q75": 2.29911217296793, "all_tokens_per_s_e2e": 294.0549579725375, "all_tokens_per_s_e2e_q25": 291.85475983662644, "all_tokens_per_s_e2e_q75": 294.286358139895, "energy_net_psys_j": 101.01422013008192, "energy_net_psys_j_q25": 99.63349111381083, "energy_net_psys_j_q75": 102.53193960521517, "prefill_energy_net_psys_j": 16.986121113617084, "prefill_energy_net_psys_j_q25": 16.25691982914457, "prefill_energy_net_psys_j_q75": 17.177019131899208, "decode_energy_net_psys_j": 83.6463029799006, "decode_energy_net_psys_j_q25": 82.45647198191163, "decode_energy_net_psys_j_q75": 86.0841217577885, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.12442332217050601, "energy_reduction_vs_gpu": -2.5903117003096168 }, { "precision": "INT8", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 2048, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1425.810721, "ttft_ms_q25": 1424.4794295000002, "ttft_ms_q75": 1427.280946, "prefill_tokens_per_s": 1425.1541036069964, "prefill_tokens_per_s_q25": 1423.6875800901623, "prefill_tokens_per_s_q75": 1426.4872715981996, "handoff_plus_first_decode_ms": 455.564631, "handoff_plus_first_decode_ms_q25": 454.43201650000003, "handoff_plus_first_decode_ms_q75": 531.5957195, "steady_tpot_ms": 424.542054, "steady_tpot_ms_q25": 424.36665475, "steady_tpot_ms_q75": 424.65971775, "steady_tpot_p95_ms": 425.34586365, "steady_tpot_p95_ms_q25": 425.267551475, "steady_tpot_p95_ms_q75": 426.050509475, "decode_tokens_per_s": 2.343448505097232, "decode_tokens_per_s_q25": 2.3162625481922383, "decode_tokens_per_s_q75": 2.3454854728912924, "decode_ms": 6400.823388000001, "decode_ms_q25": 6395.269332, "decode_ms_q75": 6476.8420365, "end_to_end_ms": 7823.971526000001, "end_to_end_ms_q25": 7821.2189865, "end_to_end_ms_q75": 7901.321466000001, "output_tokens_per_s_e2e": 2.0449972174400264, "output_tokens_per_s_e2e_q25": 2.025171809922905, "output_tokens_per_s_e2e_q75": 2.0457171713666122, "all_tokens_per_s_e2e": 261.7596438323234, "all_tokens_per_s_e2e_q25": 259.22199167013184, "all_tokens_per_s_e2e_q75": 261.85179793492637, "energy_net_psys_j": 107.6461381745534, "energy_net_psys_j_q25": 105.79259181837244, "energy_net_psys_j_q75": 110.44493746447417, "prefill_energy_net_psys_j": 23.999835194652803, "prefill_energy_net_psys_j_q25": 23.33611983646081, "prefill_energy_net_psys_j_q75": 24.360815706685678, "decode_energy_net_psys_j": 83.6463029799006, "decode_energy_net_psys_j_q25": 82.45647198191163, "decode_energy_net_psys_j_q75": 86.0841217577885, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.11075822261370535, "energy_reduction_vs_gpu": -2.8260275521955993 }, { "precision": "INT8", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 2048, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1425.810721, "ttft_ms_q25": 1424.4794295000002, "ttft_ms_q75": 1427.280946, "prefill_tokens_per_s": 1425.1541036069964, "prefill_tokens_per_s_q25": 1423.6875800901623, "prefill_tokens_per_s_q75": 1426.4872715981996, "handoff_plus_first_decode_ms": 30.489342, "handoff_plus_first_decode_ms_q25": 30.156737, "handoff_plus_first_decode_ms_q75": 37.794177000000005, "steady_tpot_ms": 19.3590285, "steady_tpot_ms_q25": 19.310399500000003, "steady_tpot_ms_q75": 19.42046825, "steady_tpot_p95_ms": 21.360242849999995, "steady_tpot_p95_ms_q25": 20.723510174999998, "steady_tpot_p95_ms_q75": 21.909633949999996, "decode_tokens_per_s": 48.71870171753366, "decode_tokens_per_s_q25": 47.52197164148824, "decode_tokens_per_s_q75": 49.14911932672539, "decode_ms": 307.8899779999999, "decode_ms_q25": 305.21707549999996, "decode_ms_q75": 315.84376699999996, "end_to_end_ms": 1731.2953440000001, "end_to_end_ms_q25": 1731.16673, "end_to_end_ms_q75": 1740.4518105000002, "output_tokens_per_s_e2e": 9.241635204212736, "output_tokens_per_s_e2e_q25": 9.193269679046331, "output_tokens_per_s_e2e_q75": 9.242321846326025, "all_tokens_per_s_e2e": 1182.9293061392302, "all_tokens_per_s_e2e_q25": 1176.7385189179304, "all_tokens_per_s_e2e_q75": 1183.0171963297312, "energy_net_psys_j": 35.07556563947901, "energy_net_psys_j_q25": 34.448535974541535, "energy_net_psys_j_q75": 36.071727645110634, "prefill_energy_net_psys_j": 23.999835194652803, "prefill_energy_net_psys_j_q25": 23.33611983646081, "prefill_energy_net_psys_j_q75": 24.360815706685678, "decode_energy_net_psys_j": 11.14910183133524, "decode_energy_net_psys_j_q25": 11.112416138080722, "decode_energy_net_psys_j_q75": 11.747597631679474, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.5005322650483602, "energy_reduction_vs_gpu": -0.24667807708884126 }, { "precision": "INT8", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 4096, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1428.511305, "ttft_ms_q25": 1382.6912465, "ttft_ms_q75": 1429.152221, "prefill_tokens_per_s": 2856.12020410297, "prefill_tokens_per_s_q25": 2854.839925796016, "prefill_tokens_per_s_q75": 2954.011157255167, "handoff_plus_first_decode_ms": 46.114875, "handoff_plus_first_decode_ms_q25": 40.593903499999996, "handoff_plus_first_decode_ms_q75": 54.083431000000004, "steady_tpot_ms": 21.2911095, "steady_tpot_ms_q25": 21.28367175, "steady_tpot_ms_q75": 21.34767375, "steady_tpot_p95_ms": 30.2459046, "steady_tpot_p95_ms_q25": 27.58292205, "steady_tpot_p95_ms_q75": 30.443447999999997, "decode_tokens_per_s": 40.425453535386616, "decode_tokens_per_s_q25": 39.854156287275416, "decode_tokens_per_s_q75": 42.04361476601651, "decode_ms": 371.053351, "decode_ms_q25": 357.3016395000001, "decode_ms_q75": 376.4496425000001, "end_to_end_ms": 1773.3430650000003, "end_to_end_ms_q25": 1746.0300935, "end_to_end_ms_q75": 1786.4538605000002, "output_tokens_per_s_e2e": 9.022506877427013, "output_tokens_per_s_e2e_q25": 8.956773066656233, "output_tokens_per_s_e2e_q75": 9.165887931846811, "all_tokens_per_s_e2e": 2309.7617606213153, "all_tokens_per_s_e2e_q25": 2292.9339050639956, "all_tokens_per_s_e2e_q75": 2346.4673105527836, "energy_net_psys_j": 54.844003295893515, "energy_net_psys_j_q25": 53.97734479587888, "energy_net_psys_j_q75": 54.970716725069366, "prefill_energy_net_psys_j": 42.46399896559708, "prefill_energy_net_psys_j_q25": 40.855619776760605, "prefill_energy_net_psys_j_q75": 42.49913510894337, "decode_energy_net_psys_j": 12.633431188648146, "decode_energy_net_psys_j_q25": 12.471581616125999, "decode_energy_net_psys_j_q75": 13.24843844829413, "fastest_e2e": true, "lowest_energy": true, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "INT8", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 4096, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1428.511305, "ttft_ms_q25": 1382.6912465, "ttft_ms_q75": 1429.152221, "prefill_tokens_per_s": 2856.12020410297, "prefill_tokens_per_s_q25": 2854.839925796016, "prefill_tokens_per_s_q75": 2954.011157255167, "handoff_plus_first_decode_ms": 503.12422599999996, "handoff_plus_first_decode_ms_q25": 499.85916699999996, "handoff_plus_first_decode_ms_q75": 589.86765, "steady_tpot_ms": 428.0453165, "steady_tpot_ms_q25": 427.7648865, "steady_tpot_ms_q75": 428.14940275000004, "steady_tpot_p95_ms": 429.79907105, "steady_tpot_p95_ms_q25": 428.888505925, "steady_tpot_p95_ms_q75": 430.2993381, "decode_tokens_per_s": 2.3094296365368603, "decode_tokens_per_s_q25": 2.2784511735904087, "decode_tokens_per_s_q75": 2.31190794654563, "decode_ms": 6495.110204999999, "decode_ms_q25": 6488.155058, "decode_ms_q75": 6584.636778499999, "end_to_end_ms": 7923.621509999999, "end_to_end_ms_q25": 7917.307279000001, "end_to_end_ms_q75": 7967.328024999999, "output_tokens_per_s_e2e": 2.0192786820782915, "output_tokens_per_s_e2e_q25": 2.0082619234353514, "output_tokens_per_s_e2e_q75": 2.020890387710524, "all_tokens_per_s_e2e": 516.9353426120426, "all_tokens_per_s_e2e_q25": 514.11505239945, "all_tokens_per_s_e2e_q75": 517.3479392538942, "energy_net_psys_j": 126.08822509118554, "energy_net_psys_j_q25": 125.42869198447498, "energy_net_psys_j_q75": 126.11592997526176, "prefill_energy_net_psys_j": 42.46399896559708, "prefill_energy_net_psys_j_q25": 40.855619776760605, "prefill_energy_net_psys_j_q75": 42.49913510894337, "decode_energy_net_psys_j": 83.62422612558846, "decode_energy_net_psys_j_q25": 82.9295568755316, "decode_energy_net_psys_j_q75": 85.26031019850117, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.2238046154478674, "energy_reduction_vs_gpu": -1.2990339419775085 }, { "precision": "INT8", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 4096, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 3119.049644, "ttft_ms_q25": 3117.103102, "ttft_ms_q75": 3120.2194215, "prefill_tokens_per_s": 1308.0907538129586, "prefill_tokens_per_s_q25": 1307.6005313401383, "prefill_tokens_per_s_q75": 1308.9081296151264, "handoff_plus_first_decode_ms": 503.12422599999996, "handoff_plus_first_decode_ms_q25": 499.85916699999996, "handoff_plus_first_decode_ms_q75": 589.86765, "steady_tpot_ms": 428.0453165, "steady_tpot_ms_q25": 427.7648865, "steady_tpot_ms_q75": 428.14940275000004, "steady_tpot_p95_ms": 429.79907105, "steady_tpot_p95_ms_q25": 428.888505925, "steady_tpot_p95_ms_q75": 430.2993381, "decode_tokens_per_s": 2.3094296365368603, "decode_tokens_per_s_q25": 2.2784511735904087, "decode_tokens_per_s_q75": 2.31190794654563, "decode_ms": 6495.110204999999, "decode_ms_q25": 6488.155058, "decode_ms_q75": 6584.636778499999, "end_to_end_ms": 9610.266764999998, "end_to_end_ms_q25": 9606.4279375, "end_to_end_ms_q75": 9701.7398805, "output_tokens_per_s_e2e": 1.6648861463732743, "output_tokens_per_s_e2e_q25": 1.6493353437078206, "output_tokens_per_s_e2e_q75": 1.6655517179862893, "all_tokens_per_s_e2e": 426.2108534715582, "all_tokens_per_s_e2e_q25": 422.2298479892021, "all_tokens_per_s_e2e_q75": 426.38123980449006, "energy_net_psys_j": 137.66066928337548, "energy_net_psys_j_q25": 137.0624293163785, "energy_net_psys_j_q75": 138.7169977022757, "prefill_energy_net_psys_j": 54.03644315778702, "prefill_energy_net_psys_j_q25": 53.45668750377452, "prefill_energy_net_psys_j_q75": 54.13287244084691, "decode_energy_net_psys_j": 83.62422612558846, "decode_energy_net_psys_j_q25": 82.9295568755316, "decode_energy_net_psys_j_q75": 85.26031019850117, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.18452589385535131, "energy_reduction_vs_gpu": -1.5100404968738474 }, { "precision": "INT8", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 4096, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 3119.049644, "ttft_ms_q25": 3117.103102, "ttft_ms_q75": 3120.2194215, "prefill_tokens_per_s": 1308.0907538129586, "prefill_tokens_per_s_q25": 1307.6005313401383, "prefill_tokens_per_s_q75": 1308.9081296151264, "handoff_plus_first_decode_ms": 46.114875, "handoff_plus_first_decode_ms_q25": 40.593903499999996, "handoff_plus_first_decode_ms_q75": 54.083431000000004, "steady_tpot_ms": 21.2911095, "steady_tpot_ms_q25": 21.28367175, "steady_tpot_ms_q75": 21.34767375, "steady_tpot_p95_ms": 30.2459046, "steady_tpot_p95_ms_q25": 27.58292205, "steady_tpot_p95_ms_q75": 30.443447999999997, "decode_tokens_per_s": 40.425453535386616, "decode_tokens_per_s_q25": 39.854156287275416, "decode_tokens_per_s_q75": 42.04361476601651, "decode_ms": 371.053351, "decode_ms_q25": 357.3016395000001, "decode_ms_q75": 376.4496425000001, "end_to_end_ms": 3486.209911, "end_to_end_ms_q25": 3475.574519, "end_to_end_ms_q75": 3493.5527445000002, "output_tokens_per_s_e2e": 4.589511362903127, "output_tokens_per_s_e2e_q25": 4.57988524951776, "output_tokens_per_s_e2e_q75": 4.603598551925462, "all_tokens_per_s_e2e": 1174.9149089032005, "all_tokens_per_s_e2e_q25": 1172.4506238765466, "all_tokens_per_s_e2e_q75": 1178.5212292929182, "energy_net_psys_j": 66.66987434643517, "energy_net_psys_j_q25": 66.60445405697291, "energy_net_psys_j_q75": 66.70512595206864, "prefill_energy_net_psys_j": 54.03644315778702, "prefill_energy_net_psys_j_q25": 53.45668750377452, "prefill_energy_net_psys_j_q75": 54.13287244084691, "decode_energy_net_psys_j": 12.633431188648146, "decode_energy_net_psys_j_q25": 12.471581616125999, "decode_energy_net_psys_j_q75": 13.24843844829413, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.5086736341964351, "energy_reduction_vs_gpu": -0.21562742214019082 }, { "precision": "INT8", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 8192, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 3931.624254, "ttft_ms_q25": 3847.9403665, "ttft_ms_q75": 3935.3583015, "prefill_tokens_per_s": 2079.54765557309, "prefill_tokens_per_s_q25": 2077.5763563405662, "prefill_tokens_per_s_q75": 2125.778467163829, "handoff_plus_first_decode_ms": 52.534369, "handoff_plus_first_decode_ms_q25": 50.268923, "handoff_plus_first_decode_ms_q75": 56.2366885, "steady_tpot_ms": 25.031965, "steady_tpot_ms_q25": 25.030341999999997, "steady_tpot_ms_q75": 25.149005000000002, "steady_tpot_p95_ms": 27.1545193, "steady_tpot_p95_ms_q25": 26.919350299999998, "steady_tpot_p95_ms_q75": 28.1688526, "decode_tokens_per_s": 36.71694083202306, "decode_tokens_per_s_q25": 36.34434649602082, "decode_tokens_per_s_q75": 36.752512545544135, "decode_ms": 408.53076700000065, "decode_ms_q25": 408.13574400000016, "decode_ms_q75": 412.76231650000045, "end_to_end_ms": 4340.1550210000005, "end_to_end_ms_q25": 4260.702683, "end_to_end_ms_q75": 4343.4940455, "output_tokens_per_s_e2e": 3.686504265995894, "output_tokens_per_s_e2e_q25": 3.683672473777217, "output_tokens_per_s_e2e_q75": 3.7565554130960916, "all_tokens_per_s_e2e": 1887.4901841898977, "all_tokens_per_s_e2e_q25": 1886.0403065739351, "all_tokens_per_s_e2e_q75": 1923.356371505199, "energy_net_psys_j": 129.08153154364902, "energy_net_psys_j_q25": 125.66201705239604, "energy_net_psys_j_q75": 129.60250391913198, "prefill_energy_net_psys_j": 114.36186444242055, "prefill_energy_net_psys_j_q25": 110.84935359375453, "prefill_energy_net_psys_j_q75": 114.86505231196605, "decode_energy_net_psys_j": 14.755236113103393, "decode_energy_net_psys_j_q25": 14.73745160716593, "decode_energy_net_psys_j_q75": 14.830447964578973, "fastest_e2e": true, "lowest_energy": true, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "INT8", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 8192, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 3931.624254, "ttft_ms_q25": 3847.9403665, "ttft_ms_q75": 3935.3583015, "prefill_tokens_per_s": 2079.54765557309, "prefill_tokens_per_s_q25": 2077.5763563405662, "prefill_tokens_per_s_q75": 2125.778467163829, "handoff_plus_first_decode_ms": 569.7060819999999, "handoff_plus_first_decode_ms_q25": 551.0542674999999, "handoff_plus_first_decode_ms_q75": 681.593681, "steady_tpot_ms": 433.45848200000006, "steady_tpot_ms_q25": 432.96234975000004, "steady_tpot_ms_q75": 433.5642295, "steady_tpot_p95_ms": 434.4063293, "steady_tpot_p95_ms_q25": 433.880979275, "steady_tpot_p95_ms_q75": 434.483423825, "decode_tokens_per_s": 2.2639027953876485, "decode_tokens_per_s_q25": 2.225618068814971, "decode_tokens_per_s_q75": 2.267887124487505, "decode_ms": 6625.726171, "decode_ms_q25": 6614.106202999999, "decode_ms_q75": 6741.695744500001, "end_to_end_ms": 10557.350425, "end_to_end_ms_q25": 10549.4645045, "end_to_end_ms_q75": 10589.636111, "output_tokens_per_s_e2e": 1.5155317722628308, "output_tokens_per_s_e2e_q25": 1.510925262832859, "output_tokens_per_s_e2e_q75": 1.5166655078770992, "all_tokens_per_s_e2e": 775.9522673985693, "all_tokens_per_s_e2e_q25": 773.5937345704239, "all_tokens_per_s_e2e_q75": 776.5327400330748, "energy_net_psys_j": 199.4748840350694, "energy_net_psys_j_q25": 197.87124064255613, "energy_net_psys_j_q75": 200.5593455579097, "prefill_energy_net_psys_j": 114.36186444242055, "prefill_energy_net_psys_j_q25": 110.84935359375453, "prefill_energy_net_psys_j_q75": 114.86505231196605, "decode_energy_net_psys_j": 86.27556689923847, "decode_energy_net_psys_j_q25": 85.69429324594365, "decode_energy_net_psys_j_q75": 87.6031607020964, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.4111026769294721, "energy_reduction_vs_gpu": -0.5453402330264172 }, { "precision": "INT8", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 8192, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 8102.135766, "ttft_ms_q25": 8100.4667055, "ttft_ms_q75": 8123.1352045, "prefill_tokens_per_s": 1009.1166374068879, "prefill_tokens_per_s_q25": 1006.5146565095354, "prefill_tokens_per_s_q75": 1009.324603674625, "handoff_plus_first_decode_ms": 569.7060819999999, "handoff_plus_first_decode_ms_q25": 551.0542674999999, "handoff_plus_first_decode_ms_q75": 681.593681, "steady_tpot_ms": 433.45848200000006, "steady_tpot_ms_q25": 432.96234975000004, "steady_tpot_ms_q75": 433.5642295, "steady_tpot_p95_ms": 434.4063293, "steady_tpot_p95_ms_q25": 433.880979275, "steady_tpot_p95_ms_q75": 434.483423825, "decode_tokens_per_s": 2.2639027953876485, "decode_tokens_per_s_q25": 2.225618068814971, "decode_tokens_per_s_q75": 2.267887124487505, "decode_ms": 6625.726171, "decode_ms_q25": 6614.106202999999, "decode_ms_q75": 6741.695744500001, "end_to_end_ms": 14769.860814, "end_to_end_ms_q25": 14737.2414075, "end_to_end_ms_q75": 14863.161888499999, "output_tokens_per_s_e2e": 1.0832871210833606, "output_tokens_per_s_e2e_q25": 1.0765293834659087, "output_tokens_per_s_e2e_q75": 1.0856901874768, "all_tokens_per_s_e2e": 554.6430059946806, "all_tokens_per_s_e2e_q25": 551.1830443345452, "all_tokens_per_s_e2e_q75": 555.8733759881216, "energy_net_psys_j": 222.90180490131218, "energy_net_psys_j_q25": 219.5411328557679, "energy_net_psys_j_q75": 222.93832153994018, "prefill_energy_net_psys_j": 134.04408367361384, "prefill_energy_net_psys_j_q25": 132.55576244559433, "prefill_energy_net_psys_j_q75": 135.33516083784377, "decode_energy_net_psys_j": 86.27556689923847, "decode_energy_net_psys_j_q25": 85.69429324594365, "decode_energy_net_psys_j_q75": 87.6031607020964, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.29385212735966143, "energy_reduction_vs_gpu": -0.7268295645061955 }, { "precision": "INT8", "model": "llama32_1b", "model_label": "Llama 3.2 1B", "family": "Llama 3.2", "size_b": 1.0, "context_length": 8192, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 8102.135766, "ttft_ms_q25": 8100.4667055, "ttft_ms_q75": 8123.1352045, "prefill_tokens_per_s": 1009.1166374068879, "prefill_tokens_per_s_q25": 1006.5146565095354, "prefill_tokens_per_s_q75": 1009.324603674625, "handoff_plus_first_decode_ms": 52.534369, "handoff_plus_first_decode_ms_q25": 50.268923, "handoff_plus_first_decode_ms_q75": 56.2366885, "steady_tpot_ms": 25.031965, "steady_tpot_ms_q25": 25.030341999999997, "steady_tpot_ms_q75": 25.149005000000002, "steady_tpot_p95_ms": 27.1545193, "steady_tpot_p95_ms_q25": 26.919350299999998, "steady_tpot_p95_ms_q75": 28.1688526, "decode_tokens_per_s": 36.71694083202306, "decode_tokens_per_s_q25": 36.34434649602082, "decode_tokens_per_s_q75": 36.752512545544135, "decode_ms": 408.53076700000065, "decode_ms_q25": 408.13574400000016, "decode_ms_q75": 412.76231650000045, "end_to_end_ms": 8515.791511, "end_to_end_ms_q25": 8512.833998999999, "end_to_end_ms_q75": 8534.228460499999, "output_tokens_per_s_e2e": 1.878862344073656, "output_tokens_per_s_e2e_q25": 1.8748120873999137, "output_tokens_per_s_e2e_q75": 1.8795153215801375, "all_tokens_per_s_e2e": 961.9775201657119, "all_tokens_per_s_e2e_q25": 959.9037887487558, "all_tokens_per_s_e2e_q75": 962.3118446490304, "energy_net_psys_j": 148.94974348966838, "energy_net_psys_j_q25": 147.36842590423583, "energy_net_psys_j_q75": 150.16560880242275, "prefill_energy_net_psys_j": 134.04408367361384, "prefill_energy_net_psys_j_q25": 132.55576244559433, "prefill_energy_net_psys_j_q75": 135.33516083784377, "decode_energy_net_psys_j": 14.755236113103393, "decode_energy_net_psys_j_q25": 14.73745160716593, "decode_energy_net_psys_j_q75": 14.830447964578973, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.5096596147749443, "energy_reduction_vs_gpu": -0.15391986528530532 }, { "precision": "INT8", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 1024, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 497.33266599999996, "ttft_ms_q25": 478.1142695, "ttft_ms_q75": 501.8957445, "prefill_tokens_per_s": 2026.8123710981013, "prefill_tokens_per_s_q25": 2008.551253838245, "prefill_tokens_per_s_q75": 2111.694554771321, "handoff_plus_first_decode_ms": 53.068780000000004, "handoff_plus_first_decode_ms_q25": 52.72070050000001, "handoff_plus_first_decode_ms_q75": 59.843194, "steady_tpot_ms": 44.791843, "steady_tpot_ms_q25": 44.789614, "steady_tpot_ms_q75": 44.807373999999996, "steady_tpot_p95_ms": 46.701485, "steady_tpot_p95_ms_q25": 46.1616997, "steady_tpot_p95_ms_q75": 48.984910400000004, "decode_tokens_per_s": 21.909214204853633, "decode_tokens_per_s_q25": 21.562931642814615, "decode_tokens_per_s_q75": 21.967048969526545, "decode_ms": 684.64345, "decode_ms_q25": 682.8456565, "decode_ms_q75": 695.8176975, "end_to_end_ms": 1181.976116, "end_to_end_ms_q25": 1173.931967, "end_to_end_ms_q75": 1184.741401, "output_tokens_per_s_e2e": 13.53665254603165, "output_tokens_per_s_e2e_q25": 13.505130447944067, "output_tokens_per_s_e2e_q75": 13.630049910992803, "all_tokens_per_s_e2e": 866.3457629460256, "all_tokens_per_s_e2e_q25": 864.3283486684203, "all_tokens_per_s_e2e_q75": 872.3231943035394, "energy_net_psys_j": 41.03450894235209, "energy_net_psys_j_q25": 40.67057957181344, "energy_net_psys_j_q75": 41.47673791445965, "prefill_energy_net_psys_j": 16.793194167137834, "prefill_energy_net_psys_j_q25": 16.648150185324507, "prefill_energy_net_psys_j_q75": 16.88160164530715, "decode_energy_net_psys_j": 24.064499818875625, "decode_energy_net_psys_j_q25": 23.78897792650629, "decode_energy_net_psys_j_q75": 24.74018025096582, "fastest_e2e": true, "lowest_energy": true, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "INT8", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 1024, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 497.33266599999996, "ttft_ms_q25": 478.1142695, "ttft_ms_q75": 501.8957445, "prefill_tokens_per_s": 2026.8123710981013, "prefill_tokens_per_s_q25": 2008.551253838245, "prefill_tokens_per_s_q75": 2111.694554771321, "handoff_plus_first_decode_ms": 2837.053767, "handoff_plus_first_decode_ms_q25": 2836.1596645, "handoff_plus_first_decode_ms_q75": 2958.2273855000003, "steady_tpot_ms": 2828.334214, "steady_tpot_ms_q25": 2826.94664625, "steady_tpot_ms_q75": 2828.9700942500003, "steady_tpot_p95_ms": 2833.54747725, "steady_tpot_p95_ms_q25": 2833.211803225, "steady_tpot_p95_ms_q75": 2835.318217425, "decode_tokens_per_s": 0.3533523389066604, "decode_tokens_per_s_q25": 0.3528598698968898, "decode_tokens_per_s_q75": 0.35521912487418994, "decode_ms": 42450.546801000004, "decode_ms_q25": 42228.622377, "decode_ms_q75": 42509.875725000005, "end_to_end_ms": 42947.879467000006, "end_to_end_ms_q25": 42730.518121500005, "end_to_end_ms_q75": 42987.98999450001, "output_tokens_per_s_e2e": 0.37254458656786466, "output_tokens_per_s_e2e_q25": 0.37219730282298114, "output_tokens_per_s_e2e_q75": 0.37444933320951723, "all_tokens_per_s_e2e": 23.842853540343338, "all_tokens_per_s_e2e_q25": 23.820627380670793, "all_tokens_per_s_e2e_q75": 23.964757325409103, "energy_net_psys_j": 321.90318959507783, "energy_net_psys_j_q25": 313.6633753237112, "energy_net_psys_j_q75": 329.03898013965096, "prefill_energy_net_psys_j": 16.793194167137834, "prefill_energy_net_psys_j_q25": 16.648150185324507, "prefill_energy_net_psys_j_q75": 16.88160164530715, "decode_energy_net_psys_j": 305.4000833915667, "decode_energy_net_psys_j_q25": 296.92681766021735, "decode_energy_net_psys_j_q75": 312.3908299543265, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.027521175216769403, "energy_reduction_vs_gpu": -6.844694572738962 }, { "precision": "INT8", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 1024, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 3434.224864, "ttft_ms_q25": 3431.2598755, "ttft_ms_q75": 3434.5322255, "prefill_tokens_per_s": 293.5160159623141, "prefill_tokens_per_s_q25": 293.48975111918094, "prefill_tokens_per_s_q75": 293.769865657448, "handoff_plus_first_decode_ms": 2837.053767, "handoff_plus_first_decode_ms_q25": 2836.1596645, "handoff_plus_first_decode_ms_q75": 2958.2273855000003, "steady_tpot_ms": 2828.334214, "steady_tpot_ms_q25": 2826.94664625, "steady_tpot_ms_q75": 2828.9700942500003, "steady_tpot_p95_ms": 2833.54747725, "steady_tpot_p95_ms_q25": 2833.211803225, "steady_tpot_p95_ms_q75": 2835.318217425, "decode_tokens_per_s": 0.3533523389066604, "decode_tokens_per_s_q25": 0.3528598698968898, "decode_tokens_per_s_q75": 0.35521912487418994, "decode_ms": 42450.546801000004, "decode_ms_q25": 42228.622377, "decode_ms_q75": 42509.875725000005, "end_to_end_ms": 45884.77166500001, "end_to_end_ms_q25": 45659.8822525, "end_to_end_ms_q75": 45944.407950500005, "output_tokens_per_s_e2e": 0.3486995667498217, "output_tokens_per_s_e2e_q25": 0.3482475380801305, "output_tokens_per_s_e2e_q75": 0.3504255236786921, "all_tokens_per_s_e2e": 22.31677227198859, "all_tokens_per_s_e2e_q25": 22.28784243712835, "all_tokens_per_s_e2e_q75": 22.427233515436296, "energy_net_psys_j": 344.8710458081819, "energy_net_psys_j_q25": 336.6444824276074, "energy_net_psys_j_q75": 352.4744276629308, "prefill_energy_net_psys_j": 39.96436711816477, "prefill_energy_net_psys_j_q25": 39.71766476739, "prefill_energy_net_psys_j_q75": 40.330300059379056, "decode_energy_net_psys_j": 305.4000833915667, "decode_energy_net_psys_j_q25": 296.92681766021735, "decode_energy_net_psys_j_q75": 312.3908299543265, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.025759659972364818, "energy_reduction_vs_gpu": -7.404415081283874 }, { "precision": "INT8", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 1024, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 3434.224864, "ttft_ms_q25": 3431.2598755, "ttft_ms_q75": 3434.5322255, "prefill_tokens_per_s": 293.5160159623141, "prefill_tokens_per_s_q25": 293.48975111918094, "prefill_tokens_per_s_q75": 293.769865657448, "handoff_plus_first_decode_ms": 53.068780000000004, "handoff_plus_first_decode_ms_q25": 52.72070050000001, "handoff_plus_first_decode_ms_q75": 59.843194, "steady_tpot_ms": 44.791843, "steady_tpot_ms_q25": 44.789614, "steady_tpot_ms_q75": 44.807373999999996, "steady_tpot_p95_ms": 46.701485, "steady_tpot_p95_ms_q25": 46.1616997, "steady_tpot_p95_ms_q75": 48.984910400000004, "decode_tokens_per_s": 21.909214204853633, "decode_tokens_per_s_q25": 21.562931642814615, "decode_tokens_per_s_q75": 21.967048969526545, "decode_ms": 684.64345, "decode_ms_q25": 682.8456565, "decode_ms_q75": 695.8176975, "end_to_end_ms": 4118.868313999999, "end_to_end_ms_q25": 4114.105532, "end_to_end_ms_q75": 4130.349923, "output_tokens_per_s_e2e": 3.8845621613140997, "output_tokens_per_s_e2e_q25": 3.8737937310851502, "output_tokens_per_s_e2e_q75": 3.889064419671543, "all_tokens_per_s_e2e": 248.61197832410238, "all_tokens_per_s_e2e_q25": 247.92279878944962, "all_tokens_per_s_e2e_q75": 248.90012285897876, "energy_net_psys_j": 64.2096890347303, "energy_net_psys_j_q25": 64.11927798588535, "energy_net_psys_j_q75": 64.54825606720077, "prefill_energy_net_psys_j": 39.96436711816477, "prefill_energy_net_psys_j_q25": 39.71766476739, "prefill_energy_net_psys_j_q75": 40.330300059379056, "decode_energy_net_psys_j": 24.064499818875625, "decode_energy_net_psys_j_q25": 23.78897792650629, "decode_energy_net_psys_j_q75": 24.74018025096582, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.2869662309869128, "energy_reduction_vs_gpu": -0.564772935992391 }, { "precision": "INT8", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 2048, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1086.387235, "ttft_ms_q25": 1065.4610715, "ttft_ms_q75": 1086.429896, "prefill_tokens_per_s": 1870.4196206797296, "prefill_tokens_per_s_q25": 1870.3461775256214, "prefill_tokens_per_s_q75": 1907.8915198916916, "handoff_plus_first_decode_ms": 62.059429, "handoff_plus_first_decode_ms_q25": 59.75551, "handoff_plus_first_decode_ms_q75": 66.2599335, "steady_tpot_ms": 46.5672985, "steady_tpot_ms_q25": 46.55176075, "steady_tpot_ms_q75": 46.6040225, "steady_tpot_p95_ms": 47.932761899999996, "steady_tpot_p95_ms_q25": 47.827453725, "steady_tpot_p95_ms_q75": 49.535231825, "decode_tokens_per_s": 20.920751040234133, "decode_tokens_per_s_q25": 20.66714167113541, "decode_tokens_per_s_q75": 20.975931423823205, "decode_ms": 716.9914679999999, "decode_ms_q25": 715.1102615, "decode_ms_q75": 725.8990765, "end_to_end_ms": 1799.701612, "end_to_end_ms_q25": 1789.5216025, "end_to_end_ms_q75": 1801.5401575, "output_tokens_per_s_e2e": 8.890362654184253, "output_tokens_per_s_e2e_q25": 8.881298924960864, "output_tokens_per_s_e2e_q75": 8.941226398748022, "all_tokens_per_s_e2e": 1137.9664197355844, "all_tokens_per_s_e2e_q25": 1136.8062623949907, "all_tokens_per_s_e2e_q75": 1144.4769790397468, "energy_net_psys_j": 62.07486891919182, "energy_net_psys_j_q25": 61.8576837706784, "energy_net_psys_j_q75": 62.25892217557576, "prefill_energy_net_psys_j": 37.81409887432489, "prefill_energy_net_psys_j_q25": 37.07612097746711, "prefill_energy_net_psys_j_q75": 37.83269025366914, "decode_energy_net_psys_j": 24.591693798946324, "decode_energy_net_psys_j_q25": 24.426231921906627, "decode_energy_net_psys_j_q75": 24.947024670250986, "fastest_e2e": true, "lowest_energy": true, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "INT8", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 2048, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1086.387235, "ttft_ms_q25": 1065.4610715, "ttft_ms_q75": 1086.429896, "prefill_tokens_per_s": 1870.4196206797296, "prefill_tokens_per_s_q25": 1870.3461775256214, "prefill_tokens_per_s_q75": 1907.8915198916916, "handoff_plus_first_decode_ms": 2911.1904329999998, "handoff_plus_first_decode_ms_q25": 2909.095489, "handoff_plus_first_decode_ms_q75": 2997.206625, "steady_tpot_ms": 2830.3012484999995, "steady_tpot_ms_q25": 2816.40385725, "steady_tpot_ms_q75": 2831.04298025, "steady_tpot_p95_ms": 2834.8560459999994, "steady_tpot_p95_ms_q25": 2829.7039638999995, "steady_tpot_p95_ms_q75": 2835.052186, "decode_tokens_per_s": 0.35320610810309805, "decode_tokens_per_s_q25": 0.3523833832510136, "decode_tokens_per_s_q75": 0.3552997912762771, "decode_ms": 42468.12174500001, "decode_ms_q25": 42219.33490850001, "decode_ms_q75": 42567.505957500005, "end_to_end_ms": 43554.50898000001, "end_to_end_ms_q25": 43305.76480450001, "end_to_end_ms_q75": 43632.96702900001, "output_tokens_per_s_e2e": 0.3673557657910254, "output_tokens_per_s_e2e_q25": 0.366696395463441, "output_tokens_per_s_e2e_q75": 0.36947801219223875, "all_tokens_per_s_e2e": 47.02153802125125, "all_tokens_per_s_e2e_q25": 46.93713861932045, "all_tokens_per_s_e2e_q75": 47.29318556060656, "energy_net_psys_j": 354.8273616203003, "energy_net_psys_j_q25": 339.1192137041812, "energy_net_psys_j_q75": 356.4453942628092, "prefill_energy_net_psys_j": 37.81409887432489, "prefill_energy_net_psys_j_q25": 37.07612097746711, "prefill_energy_net_psys_j_q75": 37.83269025366914, "decode_energy_net_psys_j": 316.9760799872869, "decode_energy_net_psys_j_q25": 301.28652345051205, "decode_energy_net_psys_j_q75": 319.3506819059978, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.041320672741975185, "energy_reduction_vs_gpu": -4.71611938612725 }, { "precision": "INT8", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 2048, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 6953.09225, "ttft_ms_q25": 6939.7785735, "ttft_ms_q75": 6960.2044145, "prefill_tokens_per_s": 292.24407313163437, "prefill_tokens_per_s_q25": 291.9457534128883, "prefill_tokens_per_s_q75": 292.8058088953429, "handoff_plus_first_decode_ms": 2911.1904329999998, "handoff_plus_first_decode_ms_q25": 2909.095489, "handoff_plus_first_decode_ms_q75": 2997.206625, "steady_tpot_ms": 2830.3012484999995, "steady_tpot_ms_q25": 2816.40385725, "steady_tpot_ms_q75": 2831.04298025, "steady_tpot_p95_ms": 2834.8560459999994, "steady_tpot_p95_ms_q25": 2829.7039638999995, "steady_tpot_p95_ms_q75": 2835.052186, "decode_tokens_per_s": 0.35320610810309805, "decode_tokens_per_s_q25": 0.3523833832510136, "decode_tokens_per_s_q75": 0.3552997912762771, "decode_ms": 42468.12174500001, "decode_ms_q25": 42219.33490850001, "decode_ms_q75": 42567.505957500005, "end_to_end_ms": 49394.58664200001, "end_to_end_ms_q25": 49166.2256465, "end_to_end_ms_q75": 49507.284531000005, "output_tokens_per_s_e2e": 0.32392213575880535, "output_tokens_per_s_e2e_q25": 0.3231864373744046, "output_tokens_per_s_e2e_q75": 0.32543366843220445, "all_tokens_per_s_e2e": 41.462033377127085, "all_tokens_per_s_e2e_q25": 41.36786398392379, "all_tokens_per_s_e2e_q75": 41.65550955932217, "energy_net_psys_j": 397.6464616747197, "energy_net_psys_j_q25": 384.4032463141042, "energy_net_psys_j_q75": 400.91593869045266, "prefill_energy_net_psys_j": 82.46013188147681, "prefill_energy_net_psys_j_q25": 81.56525678445485, "prefill_energy_net_psys_j_q75": 84.01159796061407, "decode_energy_net_psys_j": 316.9760799872869, "decode_energy_net_psys_j_q25": 301.28652345051205, "decode_energy_net_psys_j_q75": 319.3506819059978, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.036435199367975304, "energy_reduction_vs_gpu": -5.4059170578736175 }, { "precision": "INT8", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 2048, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 6953.09225, "ttft_ms_q25": 6939.7785735, "ttft_ms_q75": 6960.2044145, "prefill_tokens_per_s": 292.24407313163437, "prefill_tokens_per_s_q25": 291.9457534128883, "prefill_tokens_per_s_q75": 292.8058088953429, "handoff_plus_first_decode_ms": 62.059429, "handoff_plus_first_decode_ms_q25": 59.75551, "handoff_plus_first_decode_ms_q75": 66.2599335, "steady_tpot_ms": 46.5672985, "steady_tpot_ms_q25": 46.55176075, "steady_tpot_ms_q75": 46.6040225, "steady_tpot_p95_ms": 47.932761899999996, "steady_tpot_p95_ms_q25": 47.827453725, "steady_tpot_p95_ms_q75": 49.535231825, "decode_tokens_per_s": 20.920751040234133, "decode_tokens_per_s_q25": 20.66714167113541, "decode_tokens_per_s_q75": 20.975931423823205, "decode_ms": 716.9914679999999, "decode_ms_q25": 715.1102615, "decode_ms_q75": 725.8990765, "end_to_end_ms": 7680.545634, "end_to_end_ms_q25": 7662.0009995, "end_to_end_ms_q75": 7684.2222845, "output_tokens_per_s_e2e": 2.0831853311529973, "output_tokens_per_s_e2e_q25": 2.0821890713876012, "output_tokens_per_s_e2e_q75": 2.088239577388888, "all_tokens_per_s_e2e": 266.64772238758366, "all_tokens_per_s_e2e_q25": 266.52020113761296, "all_tokens_per_s_e2e_q75": 267.29466590577766, "energy_net_psys_j": 107.76248742303247, "energy_net_psys_j_q25": 106.51228145470583, "energy_net_psys_j_q75": 108.79316075382536, "prefill_energy_net_psys_j": 82.46013188147681, "prefill_energy_net_psys_j_q25": 81.56525678445485, "prefill_energy_net_psys_j_q75": 84.01159796061407, "decode_energy_net_psys_j": 24.591693798946324, "decode_energy_net_psys_j_q25": 24.426231921906627, "decode_energy_net_psys_j_q75": 24.947024670250986, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.23431949991067524, "energy_reduction_vs_gpu": -0.7360082960999264 }, { "precision": "INT8", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 4096, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 2720.652013, "ttft_ms_q25": 2673.5443784999998, "ttft_ms_q75": 2721.772441, "prefill_tokens_per_s": 1499.6405201784987, "prefill_tokens_per_s_q25": 1499.0234414649467, "prefill_tokens_per_s_q75": 1526.5379955002797, "handoff_plus_first_decode_ms": 67.901695, "handoff_plus_first_decode_ms_q25": 67.59164899999999, "handoff_plus_first_decode_ms_q75": 72.556673, "steady_tpot_ms": 49.9552465, "steady_tpot_ms_q25": 49.91485, "steady_tpot_ms_q75": 50.310657000000006, "steady_tpot_p95_ms": 51.7227957, "steady_tpot_p95_ms_q25": 51.69764555, "steady_tpot_p95_ms_q75": 53.02946695, "decode_tokens_per_s": 19.294544304366095, "decode_tokens_per_s_q25": 19.155190280478728, "decode_tokens_per_s_q75": 19.37388034520471, "decode_ms": 777.4218330000003, "decode_ms_q25": 774.2512740000002, "decode_ms_q75": 783.1190245000005, "end_to_end_ms": 3493.973584, "end_to_end_ms_q25": 3454.613272, "end_to_end_ms_q75": 3496.0237150000003, "output_tokens_per_s_e2e": 4.579313384986371, "output_tokens_per_s_e2e_q25": 4.576629567336262, "output_tokens_per_s_e2e_q75": 4.632089330337967, "all_tokens_per_s_e2e": 1172.304226556511, "all_tokens_per_s_e2e_q25": 1171.617169238083, "all_tokens_per_s_e2e_q75": 1185.8148685665196, "energy_net_psys_j": 120.78508402481626, "energy_net_psys_j_q25": 118.99958443530213, "energy_net_psys_j_q75": 121.30392492664456, "prefill_energy_net_psys_j": 94.75144981319067, "prefill_energy_net_psys_j_q25": 92.4771215125366, "prefill_energy_net_psys_j_q75": 94.77767278372785, "decode_energy_net_psys_j": 27.011291633905472, "decode_energy_net_psys_j_q25": 26.522462922765527, "decode_energy_net_psys_j_q75": 27.01508085405664, "fastest_e2e": true, "lowest_energy": true, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "INT8", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 4096, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 2720.652013, "ttft_ms_q25": 2673.5443784999998, "ttft_ms_q75": 2721.772441, "prefill_tokens_per_s": 1499.6405201784987, "prefill_tokens_per_s_q25": 1499.0234414649467, "prefill_tokens_per_s_q75": 1526.5379955002797, "handoff_plus_first_decode_ms": 3007.338554, "handoff_plus_first_decode_ms_q25": 3000.3197135, "handoff_plus_first_decode_ms_q75": 3082.431552, "steady_tpot_ms": 2855.4519525, "steady_tpot_ms_q25": 2853.6767274999997, "steady_tpot_ms_q75": 2856.394367, "steady_tpot_p95_ms": 2862.3974482999997, "steady_tpot_p95_ms_q25": 2861.166444, "steady_tpot_p95_ms_q75": 2862.480444275, "decode_tokens_per_s": 0.3491382514618317, "decode_tokens_per_s_q25": 0.3486594938836961, "decode_tokens_per_s_q75": 0.34977202156965304, "decode_ms": 42962.923533, "decode_ms_q25": 42885.217577999996, "decode_ms_q75": 43021.9986715, "end_to_end_ms": 45683.575546, "end_to_end_ms_q25": 45606.990019000004, "end_to_end_ms_q75": 45695.54305, "output_tokens_per_s_e2e": 0.3502352827853673, "output_tokens_per_s_e2e_q25": 0.35014358139763313, "output_tokens_per_s_e2e_q75": 0.35082440458222774, "all_tokens_per_s_e2e": 89.66023239305403, "all_tokens_per_s_e2e_q25": 89.63675683779408, "all_tokens_per_s_e2e_q75": 89.8110475730503, "energy_net_psys_j": 413.19929953653985, "energy_net_psys_j_q25": 411.94730694220914, "energy_net_psys_j_q75": 419.2861992388679, "prefill_energy_net_psys_j": 94.75144981319067, "prefill_energy_net_psys_j_q25": 92.4771215125366, "prefill_energy_net_psys_j_q75": 94.77767278372785, "decode_energy_net_psys_j": 322.99650632465733, "decode_energy_net_psys_j_q25": 319.4439624591353, "decode_energy_net_psys_j_q75": 326.8090777263312, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.0764820516398027, "energy_reduction_vs_gpu": -2.4209464096712865 }, { "precision": "INT8", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 4096, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 14660.231373, "ttft_ms_q25": 14638.466973999999, "ttft_ms_q75": 14668.88678, "prefill_tokens_per_s": 278.30392960333535, "prefill_tokens_per_s_q25": 278.139812631316, "prefill_tokens_per_s_q75": 278.71832660813686, "handoff_plus_first_decode_ms": 3007.338554, "handoff_plus_first_decode_ms_q25": 3000.3197135, "handoff_plus_first_decode_ms_q75": 3082.431552, "steady_tpot_ms": 2855.4519525, "steady_tpot_ms_q25": 2853.6767274999997, "steady_tpot_ms_q75": 2856.394367, "steady_tpot_p95_ms": 2862.3974482999997, "steady_tpot_p95_ms_q25": 2861.166444, "steady_tpot_p95_ms_q75": 2862.480444275, "decode_tokens_per_s": 0.3491382514618317, "decode_tokens_per_s_q25": 0.3486594938836961, "decode_tokens_per_s_q75": 0.34977202156965304, "decode_ms": 42962.923533, "decode_ms_q25": 42885.217577999996, "decode_ms_q75": 43021.9986715, "end_to_end_ms": 57640.46572, "end_to_end_ms_q25": 57532.339959000004, "end_to_end_ms_q75": 57690.8854515, "output_tokens_per_s_e2e": 0.2775827675946127, "output_tokens_per_s_e2e_q25": 0.2773403822074658, "output_tokens_per_s_e2e_q75": 0.27810543644443214, "all_tokens_per_s_e2e": 71.06118850422085, "all_tokens_per_s_e2e_q25": 70.99913784511125, "all_tokens_per_s_e2e_q75": 71.19499172977463, "energy_net_psys_j": 502.74846344553436, "energy_net_psys_j_q25": 496.5750959106758, "energy_net_psys_j_q75": 509.4081391164607, "prefill_energy_net_psys_j": 179.75195712087702, "prefill_energy_net_psys_j_q25": 177.13113345154053, "prefill_energy_net_psys_j_q75": 182.5990613901294, "decode_energy_net_psys_j": 322.99650632465733, "decode_energy_net_psys_j_q25": 319.4439624591353, "decode_energy_net_psys_j_q75": 326.8090777263312, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.06061667858432425, "energy_reduction_vs_gpu": -3.162338980053536 }, { "precision": "INT8", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 4096, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 14660.231373, "ttft_ms_q25": 14638.466973999999, "ttft_ms_q75": 14668.88678, "prefill_tokens_per_s": 278.30392960333535, "prefill_tokens_per_s_q25": 278.139812631316, "prefill_tokens_per_s_q75": 278.71832660813686, "handoff_plus_first_decode_ms": 67.901695, "handoff_plus_first_decode_ms_q25": 67.59164899999999, "handoff_plus_first_decode_ms_q75": 72.556673, "steady_tpot_ms": 49.9552465, "steady_tpot_ms_q25": 49.91485, "steady_tpot_ms_q75": 50.310657000000006, "steady_tpot_p95_ms": 51.7227957, "steady_tpot_p95_ms_q25": 51.69764555, "steady_tpot_p95_ms_q75": 53.02946695, "decode_tokens_per_s": 19.294544304366095, "decode_tokens_per_s_q25": 19.155190280478728, "decode_tokens_per_s_q75": 19.37388034520471, "decode_ms": 777.4218330000003, "decode_ms_q25": 774.2512740000002, "decode_ms_q75": 783.1190245000005, "end_to_end_ms": 15449.047589000002, "end_to_end_ms_q25": 15418.4154395, "end_to_end_ms_q75": 15452.0058045, "output_tokens_per_s_e2e": 1.0356625486345377, "output_tokens_per_s_e2e_q25": 1.0354643137502522, "output_tokens_per_s_e2e_q75": 1.0377242211589968, "all_tokens_per_s_e2e": 265.12961245044164, "all_tokens_per_s_e2e_q25": 265.07886432006455, "all_tokens_per_s_e2e_q75": 265.65740061670317, "energy_net_psys_j": 206.7632487547825, "energy_net_psys_j_q25": 204.1462143055972, "energy_net_psys_j_q75": 209.12152431289493, "prefill_energy_net_psys_j": 179.75195712087702, "prefill_energy_net_psys_j_q25": 177.13113345154053, "prefill_energy_net_psys_j_q75": 182.5990613901294, "decode_energy_net_psys_j": 27.011291633905472, "decode_energy_net_psys_j_q25": 26.522462922765527, "decode_energy_net_psys_j_q75": 27.01508085405664, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.22616109917919935, "energy_reduction_vs_gpu": -0.7118276683261762 }, { "precision": "INT8", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 8192, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 7314.639501000001, "ttft_ms_q25": 7211.2596115, "ttft_ms_q75": 7314.9642140000005, "prefill_tokens_per_s": 1117.7584348322623, "prefill_tokens_per_s_q25": 1117.7088194721211, "prefill_tokens_per_s_q75": 1134.0155674794278, "handoff_plus_first_decode_ms": 82.083003, "handoff_plus_first_decode_ms_q25": 81.43185700000001, "handoff_plus_first_decode_ms_q75": 87.66125600000001, "steady_tpot_ms": 56.947846999999996, "steady_tpot_ms_q25": 56.93222374999999, "steady_tpot_ms_q75": 56.966063, "steady_tpot_p95_ms": 59.96024355, "steady_tpot_p95_ms_q25": 59.65237065, "steady_tpot_p95_ms_q75": 60.101113049999995, "decode_tokens_per_s": 16.93331856928576, "decode_tokens_per_s_q25": 16.824421064849883, "decode_tokens_per_s_q75": 16.945296621925877, "decode_ms": 885.8275440000007, "decode_ms_q25": 885.2018250000001, "decode_ms_q75": 891.5984915000004, "end_to_end_ms": 8199.215607, "end_to_end_ms_q25": 8102.232384, "end_to_end_ms_q75": 8200.166039, "output_tokens_per_s_e2e": 1.9514061791886723, "output_tokens_per_s_e2e_q25": 1.9511800296291266, "output_tokens_per_s_e2e_q75": 1.9750473747134585, "all_tokens_per_s_e2e": 999.1199637446002, "all_tokens_per_s_e2e_q25": 999.0041751701128, "all_tokens_per_s_e2e_q75": 1011.2242558532907, "energy_net_psys_j": 277.01937880869855, "energy_net_psys_j_q25": 272.58850604219083, "energy_net_psys_j_q75": 277.4082601770699, "prefill_energy_net_psys_j": 246.5728915708733, "prefill_energy_net_psys_j_q25": 242.75006325247682, "prefill_energy_net_psys_j_q75": 246.68237356609126, "decode_energy_net_psys_j": 30.446487237825238, "decode_energy_net_psys_j_q25": 29.83844278971398, "decode_energy_net_psys_j_q75": 30.725886610978648, "fastest_e2e": true, "lowest_energy": true, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "INT8", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 8192, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 7314.639501000001, "ttft_ms_q25": 7211.2596115, "ttft_ms_q75": 7314.9642140000005, "prefill_tokens_per_s": 1117.7584348322623, "prefill_tokens_per_s_q25": 1117.7088194721211, "prefill_tokens_per_s_q75": 1134.0155674794278, "handoff_plus_first_decode_ms": 3123.8754270000004, "handoff_plus_first_decode_ms_q25": 3108.1939005000004, "handoff_plus_first_decode_ms_q75": 3194.406264, "steady_tpot_ms": 2848.5759369999996, "steady_tpot_ms_q25": 2841.4786969999996, "steady_tpot_ms_q75": 2852.1456874999994, "steady_tpot_p95_ms": 2858.8260909, "steady_tpot_p95_ms_q25": 2857.403378, "steady_tpot_p95_ms_q75": 2860.3973753, "decode_tokens_per_s": 0.3491939275875777, "decode_tokens_per_s_q25": 0.34817720064221264, "decode_tokens_per_s_q75": 0.3500286445547708, "decode_ms": 42956.07344500001, "decode_ms_q25": 42853.879351, "decode_ms_q75": 43081.878658, "end_to_end_ms": 50270.71294600001, "end_to_end_ms_q25": 50168.843565, "end_to_end_ms_q75": 50293.13826950001, "output_tokens_per_s_e2e": 0.3182767671742977, "output_tokens_per_s_e2e_q25": 0.31813491326356447, "output_tokens_per_s_e2e_q75": 0.3189243528889761, "all_tokens_per_s_e2e": 162.95770479324042, "all_tokens_per_s_e2e_q25": 162.885075590945, "all_tokens_per_s_e2e_q75": 163.28926867915575, "energy_net_psys_j": 564.3997199908467, "energy_net_psys_j_q25": 560.1001275758556, "energy_net_psys_j_q75": 566.0037019722143, "prefill_energy_net_psys_j": 246.5728915708733, "prefill_energy_net_psys_j_q25": 242.75006325247682, "prefill_energy_net_psys_j_q75": 246.68237356609126, "decode_energy_net_psys_j": 317.8268284199735, "decode_energy_net_psys_j_q25": 313.4177540097644, "decode_energy_net_psys_j_q75": 323.2536387197376, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.16310123979756297, "energy_reduction_vs_gpu": -1.037401579694555 }, { "precision": "INT8", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 8192, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 31436.051444, "ttft_ms_q25": 31423.211943000002, "ttft_ms_q75": 31491.0653075, "prefill_tokens_per_s": 260.0835545317986, "prefill_tokens_per_s_q25": 259.62998941234116, "prefill_tokens_per_s_q75": 260.18986791807447, "handoff_plus_first_decode_ms": 3123.8754270000004, "handoff_plus_first_decode_ms_q25": 3108.1939005000004, "handoff_plus_first_decode_ms_q75": 3194.406264, "steady_tpot_ms": 2848.5759369999996, "steady_tpot_ms_q25": 2841.4786969999996, "steady_tpot_ms_q75": 2852.1456874999994, "steady_tpot_p95_ms": 2858.8260909, "steady_tpot_p95_ms_q25": 2857.403378, "steady_tpot_p95_ms_q75": 2860.3973753, "decode_tokens_per_s": 0.3491939275875777, "decode_tokens_per_s_q25": 0.34817720064221264, "decode_tokens_per_s_q75": 0.3500286445547708, "decode_ms": 42956.07344500001, "decode_ms_q25": 42853.879351, "decode_ms_q75": 43081.878658, "end_to_end_ms": 74502.152616, "end_to_end_ms_q25": 74332.1051575, "end_to_end_ms_q75": 74572.9439655, "output_tokens_per_s_e2e": 0.2147588953901428, "output_tokens_per_s_e2e_q25": 0.21455522026646848, "output_tokens_per_s_e2e_q75": 0.2152513197575947, "all_tokens_per_s_e2e": 109.95655443975312, "all_tokens_per_s_e2e_q25": 109.85227277643186, "all_tokens_per_s_e2e_q75": 110.20867571588849, "energy_net_psys_j": 702.089033212181, "energy_net_psys_j_q25": 699.998348089371, "energy_net_psys_j_q75": 710.8013081087704, "prefill_energy_net_psys_j": 388.8989833670057, "prefill_energy_net_psys_j_q25": 381.1537837798425, "prefill_energy_net_psys_j_q75": 395.2928689761961, "decode_energy_net_psys_j": 317.8268284199735, "decode_energy_net_psys_j_q25": 313.4177540097644, "decode_energy_net_psys_j_q75": 323.2536387197376, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.11005340542655871, "energy_reduction_vs_gpu": -1.5344401400055951 }, { "precision": "INT8", "model": "llama32_3b", "model_label": "Llama 3.2 3B", "family": "Llama 3.2", "size_b": 3.0, "context_length": 8192, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 31436.051444, "ttft_ms_q25": 31423.211943000002, "ttft_ms_q75": 31491.0653075, "prefill_tokens_per_s": 260.0835545317986, "prefill_tokens_per_s_q25": 259.62998941234116, "prefill_tokens_per_s_q75": 260.18986791807447, "handoff_plus_first_decode_ms": 82.083003, "handoff_plus_first_decode_ms_q25": 81.43185700000001, "handoff_plus_first_decode_ms_q75": 87.66125600000001, "steady_tpot_ms": 56.947846999999996, "steady_tpot_ms_q25": 56.93222374999999, "steady_tpot_ms_q75": 56.966063, "steady_tpot_p95_ms": 59.96024355, "steady_tpot_p95_ms_q25": 59.65237065, "steady_tpot_p95_ms_q75": 60.101113049999995, "decode_tokens_per_s": 16.93331856928576, "decode_tokens_per_s_q25": 16.824421064849883, "decode_tokens_per_s_q75": 16.945296621925877, "decode_ms": 885.8275440000007, "decode_ms_q25": 885.2018250000001, "decode_ms_q75": 891.5984915000004, "end_to_end_ms": 32333.420883, "end_to_end_ms_q25": 32314.8104345, "end_to_end_ms_q75": 32382.03808, "output_tokens_per_s_e2e": 0.4948440209248737, "output_tokens_per_s_e2e_q25": 0.4941021941241479, "output_tokens_per_s_e2e_q75": 0.4951291711645597, "all_tokens_per_s_e2e": 253.36013871353532, "all_tokens_per_s_e2e_q25": 252.98032339156373, "all_tokens_per_s_e2e_q75": 253.50613563625456, "energy_net_psys_j": 419.90426935113777, "energy_net_psys_j_q25": 411.2716259427099, "energy_net_psys_j_q75": 426.0187555871747, "prefill_energy_net_psys_j": 388.8989833670057, "prefill_energy_net_psys_j_q25": 381.1537837798425, "prefill_energy_net_psys_j_q75": 395.2928689761961, "decode_energy_net_psys_j": 30.446487237825238, "decode_energy_net_psys_j_q25": 29.83844278971398, "decode_energy_net_psys_j_q75": 30.725886610978648, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.2535833012123662, "energy_reduction_vs_gpu": -0.5157938450259516 }, { "precision": "INT8", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 1024, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 200.513376, "ttft_ms_q25": 186.1602465, "ttft_ms_q75": 209.012493, "prefill_tokens_per_s": 5027.096047697088, "prefill_tokens_per_s_q25": 4830.66571701444, "prefill_tokens_per_s_q75": 5447.070271358232, "handoff_plus_first_decode_ms": 23.977131, "handoff_plus_first_decode_ms_q25": 23.889815, "handoff_plus_first_decode_ms_q75": 36.138904, "steady_tpot_ms": 9.6898955, "steady_tpot_ms_q25": 9.58769475, "steady_tpot_ms_q75": 9.7572455, "steady_tpot_p95_ms": 12.9956709, "steady_tpot_p95_ms_q25": 12.935077375, "steady_tpot_p95_ms_q75": 13.069032825, "decode_tokens_per_s": 85.71541079027763, "decode_tokens_per_s_q25": 80.40469639304747, "decode_tokens_per_s_q75": 85.9918562621111, "decode_ms": 174.99770300000003, "decode_ms_q25": 174.43692550000003, "decode_ms_q75": 187.37369900000004, "end_to_end_ms": 374.38952400000005, "end_to_end_ms_q25": 372.97316800000004, "end_to_end_ms_q75": 383.44941850000004, "output_tokens_per_s_e2e": 42.73623852787077, "output_tokens_per_s_e2e_q25": 41.74980126494309, "output_tokens_per_s_e2e_q75": 42.8991469335344, "all_tokens_per_s_e2e": 2735.119265783729, "all_tokens_per_s_e2e_q25": 2671.9872809563576, "all_tokens_per_s_e2e_q75": 2745.5454037462014, "energy_net_psys_j": 11.999113096942306, "energy_net_psys_j_q25": 11.875437319028297, "energy_net_psys_j_q75": 12.063819293163622, "prefill_energy_net_psys_j": 4.832090152375949, "prefill_energy_net_psys_j_q25": 4.39745080108195, "prefill_energy_net_psys_j_q75": 4.881409094011685, "decode_energy_net_psys_j": 7.068385061294885, "decode_energy_net_psys_j_q25": 6.994028225016612, "decode_energy_net_psys_j_q75": 7.617049550445936, "fastest_e2e": true, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "INT8", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 1024, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 200.513376, "ttft_ms_q25": 186.1602465, "ttft_ms_q75": 209.012493, "prefill_tokens_per_s": 5027.096047697088, "prefill_tokens_per_s_q25": 4830.66571701444, "prefill_tokens_per_s_q75": 5447.070271358232, "handoff_plus_first_decode_ms": 125.945494, "handoff_plus_first_decode_ms_q25": 125.673336, "handoff_plus_first_decode_ms_q75": 188.992034, "steady_tpot_ms": 123.0124375, "steady_tpot_ms_q25": 122.7702555, "steady_tpot_ms_q75": 123.02086625000001, "steady_tpot_p95_ms": 123.3848382, "steady_tpot_p95_ms_q25": 123.28280985, "steady_tpot_p95_ms_q75": 131.042861875, "decode_tokens_per_s": 8.117983810938089, "decode_tokens_per_s_q25": 7.774275003566619, "decode_tokens_per_s_q75": 8.132856657330919, "decode_ms": 1847.7494350000002, "decode_ms_q25": 1844.3765575000002, "decode_ms_q75": 1933.2190665000003, "end_to_end_ms": 2065.261045, "end_to_end_ms_q25": 2053.3890505, "end_to_end_ms_q75": 2127.87843, "output_tokens_per_s_e2e": 7.747204663902426, "output_tokens_per_s_e2e_q25": 7.525743525382344, "output_tokens_per_s_e2e_q75": 7.792256832773566, "all_tokens_per_s_e2e": 495.8210984897553, "all_tokens_per_s_e2e_q25": 481.64758562447, "all_tokens_per_s_e2e_q75": 498.7044372975082, "energy_net_psys_j": 38.66509081544087, "energy_net_psys_j_q25": 38.021615858920086, "energy_net_psys_j_q75": 39.18531196540823, "prefill_energy_net_psys_j": 4.832090152375949, "prefill_energy_net_psys_j_q25": 4.39745080108195, "prefill_energy_net_psys_j_q75": 4.881409094011685, "decode_energy_net_psys_j": 33.73436277979345, "decode_energy_net_psys_j_q25": 33.1402067649084, "decode_energy_net_psys_j_q75": 34.738542222690555, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.1812795166530631, "energy_reduction_vs_gpu": -2.222329059078022 }, { "precision": "INT8", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 1024, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 303.182513, "ttft_ms_q25": 301.4690725, "ttft_ms_q75": 303.69026599999995, "prefill_tokens_per_s": 3324.730011720696, "prefill_tokens_per_s_q25": 3319.18052904099, "prefill_tokens_per_s_q75": 3343.734582262502, "handoff_plus_first_decode_ms": 125.945494, "handoff_plus_first_decode_ms_q25": 125.673336, "handoff_plus_first_decode_ms_q75": 188.992034, "steady_tpot_ms": 123.0124375, "steady_tpot_ms_q25": 122.7702555, "steady_tpot_ms_q75": 123.02086625000001, "steady_tpot_p95_ms": 123.3848382, "steady_tpot_p95_ms_q25": 123.28280985, "steady_tpot_p95_ms_q75": 131.042861875, "decode_tokens_per_s": 8.117983810938089, "decode_tokens_per_s_q25": 7.774275003566619, "decode_tokens_per_s_q75": 8.132856657330919, "decode_ms": 1847.7494350000002, "decode_ms_q25": 1844.3765575000002, "decode_ms_q75": 1933.2190665000003, "end_to_end_ms": 2147.505067, "end_to_end_ms_q25": 2146.353383, "end_to_end_ms_q75": 2234.688139, "output_tokens_per_s_e2e": 7.450506285580746, "output_tokens_per_s_e2e_q25": 7.170750017078461, "output_tokens_per_s_e2e_q75": 7.454506202644489, "all_tokens_per_s_e2e": 476.8324022771677, "all_tokens_per_s_e2e_q25": 458.9280010930215, "all_tokens_per_s_e2e_q75": 477.0883969692473, "energy_net_psys_j": 37.677082543824994, "energy_net_psys_j_q25": 37.067707074004005, "energy_net_psys_j_q75": 38.670625487477956, "prefill_energy_net_psys_j": 3.9214467655432665, "prefill_energy_net_psys_j_q25": 3.916863809851472, "prefill_energy_net_psys_j_q75": 3.9320832647874044, "decode_energy_net_psys_j": 33.73436277979345, "decode_energy_net_psys_j_q25": 33.1402067649084, "decode_energy_net_psys_j_q75": 34.738542222690555, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.17433696886359898, "energy_reduction_vs_gpu": -2.1399889508022154 }, { "precision": "INT8", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 1024, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 303.182513, "ttft_ms_q25": 301.4690725, "ttft_ms_q75": 303.69026599999995, "prefill_tokens_per_s": 3324.730011720696, "prefill_tokens_per_s_q25": 3319.18052904099, "prefill_tokens_per_s_q75": 3343.734582262502, "handoff_plus_first_decode_ms": 23.977131, "handoff_plus_first_decode_ms_q25": 23.889815, "handoff_plus_first_decode_ms_q75": 36.138904, "steady_tpot_ms": 9.6898955, "steady_tpot_ms_q25": 9.58769475, "steady_tpot_ms_q75": 9.7572455, "steady_tpot_p95_ms": 12.9956709, "steady_tpot_p95_ms_q25": 12.935077375, "steady_tpot_p95_ms_q75": 13.069032825, "decode_tokens_per_s": 85.71541079027763, "decode_tokens_per_s_q25": 80.40469639304747, "decode_tokens_per_s_q75": 85.9918562621111, "decode_ms": 174.99770300000003, "decode_ms_q25": 174.43692550000003, "decode_ms_q75": 187.37369900000004, "end_to_end_ms": 478.07416700000005, "end_to_end_ms_q25": 476.41375100000005, "end_to_end_ms_q75": 490.5031875, "output_tokens_per_s_e2e": 33.46761047643053, "output_tokens_per_s_e2e_q25": 32.64052163606426, "output_tokens_per_s_e2e_q75": 33.584661062113796, "all_tokens_per_s_e2e": 2141.927070491554, "all_tokens_per_s_e2e_q25": 2088.9933847081124, "all_tokens_per_s_e2e_q75": 2149.418307975283, "energy_net_psys_j": 11.011104825326427, "energy_net_psys_j_q25": 10.921528534112221, "energy_net_psys_j_q75": 11.54913281523334, "prefill_energy_net_psys_j": 3.9214467655432665, "prefill_energy_net_psys_j_q25": 3.916863809851472, "prefill_energy_net_psys_j_q75": 3.9320832647874044, "decode_energy_net_psys_j": 7.068385061294885, "decode_energy_net_psys_j_q25": 6.994028225016612, "decode_energy_net_psys_j_q75": 7.617049550445936, "fastest_e2e": false, "lowest_energy": true, "e2e_speedup_vs_gpu": 0.783120172230515, "energy_reduction_vs_gpu": 0.08234010827580662 }, { "precision": "INT8", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 2048, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 431.237221, "ttft_ms_q25": 402.07436699999994, "ttft_ms_q75": 438.01073199999996, "prefill_tokens_per_s": 4712.023686842189, "prefill_tokens_per_s_q25": 4640.265424769299, "prefill_tokens_per_s_q75": 5080.518769545417, "handoff_plus_first_decode_ms": 41.103348999999994, "handoff_plus_first_decode_ms_q25": 32.619724, "handoff_plus_first_decode_ms_q75": 46.111813, "steady_tpot_ms": 10.842264, "steady_tpot_ms_q25": 10.804337, "steady_tpot_ms_q75": 10.941298, "steady_tpot_p95_ms": 13.5699021, "steady_tpot_p95_ms_q25": 13.259286475, "steady_tpot_p95_ms_q75": 18.472543424999998, "decode_tokens_per_s": 70.3662392466489, "decode_tokens_per_s_q25": 67.74458303239297, "decode_tokens_per_s_q75": 75.50219641065138, "decode_ms": 213.170409, "decode_ms_q25": 199.59328649999998, "decode_ms_q75": 221.75201850000005, "end_to_end_ms": 630.800407, "end_to_end_ms_q25": 608.4411645, "end_to_end_ms_q75": 646.185628, "output_tokens_per_s_e2e": 25.364599994622388, "output_tokens_per_s_e2e_q25": 24.774731234407614, "output_tokens_per_s_e2e_q75": 26.33226888340149, "all_tokens_per_s_e2e": 3246.6687993116657, "all_tokens_per_s_e2e_q25": 3171.1655980041746, "all_tokens_per_s_e2e_q75": 3370.5304170753907, "energy_net_psys_j": 18.762792672588922, "energy_net_psys_j_q25": 18.227245520253454, "energy_net_psys_j_q75": 19.007359662895162, "prefill_energy_net_psys_j": 11.128946290697668, "prefill_energy_net_psys_j_q25": 10.176572668790886, "prefill_energy_net_psys_j_q75": 11.236914521531057, "decode_energy_net_psys_j": 8.122980362503737, "decode_energy_net_psys_j_q25": 7.770445141364107, "decode_energy_net_psys_j_q75": 8.295239841768808, "fastest_e2e": true, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "INT8", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 2048, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 431.237221, "ttft_ms_q25": 402.07436699999994, "ttft_ms_q75": 438.01073199999996, "prefill_tokens_per_s": 4712.023686842189, "prefill_tokens_per_s_q25": 4640.265424769299, "prefill_tokens_per_s_q75": 5080.518769545417, "handoff_plus_first_decode_ms": 135.361615, "handoff_plus_first_decode_ms_q25": 135.25535100000002, "handoff_plus_first_decode_ms_q75": 192.700142, "steady_tpot_ms": 122.9227825, "steady_tpot_ms_q25": 122.83192475000001, "steady_tpot_ms_q75": 122.94218475, "steady_tpot_p95_ms": 123.09534180000001, "steady_tpot_p95_ms_q25": 123.087232275, "steady_tpot_p95_ms_q75": 124.40221297500001, "decode_tokens_per_s": 8.08838783595787, "decode_tokens_per_s_q25": 7.833662975686522, "decode_tokens_per_s_q75": 8.092821854246818, "decode_ms": 1854.510479, "decode_ms_q25": 1853.4949579999998, "decode_ms_q75": 1916.8397810000001, "end_to_end_ms": 2297.2636799999996, "end_to_end_ms_q25": 2291.50569, "end_to_end_ms_q75": 2324.672138, "output_tokens_per_s_e2e": 6.964807801253361, "output_tokens_per_s_e2e_q25": 6.883647894393295, "output_tokens_per_s_e2e_q75": 6.982352735749647, "all_tokens_per_s_e2e": 891.4953985604302, "all_tokens_per_s_e2e_q25": 881.1069304823418, "all_tokens_per_s_e2e_q75": 893.7411501759548, "energy_net_psys_j": 44.82079071975753, "energy_net_psys_j_q25": 44.10535636020846, "energy_net_psys_j_q75": 44.86099473089841, "prefill_energy_net_psys_j": 11.128946290697668, "prefill_energy_net_psys_j_q25": 10.176572668790886, "prefill_energy_net_psys_j_q75": 11.236914521531057, "decode_energy_net_psys_j": 33.55631598967485, "decode_energy_net_psys_j_q25": 32.908645849818285, "decode_energy_net_psys_j_q75": 34.57645383127414, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.27458772473171217, "energy_reduction_vs_gpu": -1.3888123426976549 }, { "precision": "INT8", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 2048, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 579.501888, "ttft_ms_q25": 575.784952, "ttft_ms_q75": 579.6454225, "prefill_tokens_per_s": 3506.459671793166, "prefill_tokens_per_s_q25": 3505.5916008689364, "prefill_tokens_per_s_q75": 3529.2424280775795, "handoff_plus_first_decode_ms": 135.361615, "handoff_plus_first_decode_ms_q25": 135.25535100000002, "handoff_plus_first_decode_ms_q75": 192.700142, "steady_tpot_ms": 122.9227825, "steady_tpot_ms_q25": 122.83192475000001, "steady_tpot_ms_q75": 122.94218475, "steady_tpot_p95_ms": 123.09534180000001, "steady_tpot_p95_ms_q25": 123.087232275, "steady_tpot_p95_ms_q75": 124.40221297500001, "decode_tokens_per_s": 8.08838783595787, "decode_tokens_per_s_q25": 7.833662975686522, "decode_tokens_per_s_q75": 8.092821854246818, "decode_ms": 1854.510479, "decode_ms_q25": 1853.4949579999998, "decode_ms_q75": 1916.8397810000001, "end_to_end_ms": 2431.9813249999997, "end_to_end_ms_q25": 2429.27991, "end_to_end_ms_q75": 2495.4696825, "output_tokens_per_s_e2e": 6.5789978876585335, "output_tokens_per_s_e2e_q25": 6.415771385561058, "output_tokens_per_s_e2e_q75": 6.58632202887849, "all_tokens_per_s_e2e": 842.1117296202923, "all_tokens_per_s_e2e_q25": 821.2187373518154, "all_tokens_per_s_e2e_q75": 843.0492196964467, "energy_net_psys_j": 43.18449896937483, "energy_net_psys_j_q25": 42.46109409245022, "energy_net_psys_j_q75": 44.329350493740165, "prefill_energy_net_psys_j": 9.628182979699977, "prefill_energy_net_psys_j_q25": 9.552448242631936, "prefill_energy_net_psys_j_q75": 9.752896662466021, "decode_energy_net_psys_j": 33.55631598967485, "decode_energy_net_psys_j_q25": 32.908645849818285, "decode_energy_net_psys_j_q75": 34.57645383127414, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.25937715907419645, "energy_reduction_vs_gpu": -1.3016029502081663 }, { "precision": "INT8", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 2048, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 579.501888, "ttft_ms_q25": 575.784952, "ttft_ms_q75": 579.6454225, "prefill_tokens_per_s": 3506.459671793166, "prefill_tokens_per_s_q25": 3505.5916008689364, "prefill_tokens_per_s_q75": 3529.2424280775795, "handoff_plus_first_decode_ms": 41.103348999999994, "handoff_plus_first_decode_ms_q25": 32.619724, "handoff_plus_first_decode_ms_q75": 46.111813, "steady_tpot_ms": 10.842264, "steady_tpot_ms_q25": 10.804337, "steady_tpot_ms_q75": 10.941298, "steady_tpot_p95_ms": 13.5699021, "steady_tpot_p95_ms_q25": 13.259286475, "steady_tpot_p95_ms_q75": 18.472543424999998, "decode_tokens_per_s": 70.3662392466489, "decode_tokens_per_s_q25": 67.74458303239297, "decode_tokens_per_s_q75": 75.50219641065138, "decode_ms": 213.170409, "decode_ms_q25": 199.59328649999998, "decode_ms_q75": 221.75201850000005, "end_to_end_ms": 792.959366, "end_to_end_ms_q25": 779.238709, "end_to_end_ms_q75": 797.680505, "output_tokens_per_s_e2e": 20.177578683142762, "output_tokens_per_s_e2e_q25": 20.058858645168172, "output_tokens_per_s_e2e_q75": 20.53922872585386, "all_tokens_per_s_e2e": 2582.7300714422736, "all_tokens_per_s_e2e_q25": 2567.533906581526, "all_tokens_per_s_e2e_q75": 2629.021276909294, "energy_net_psys_j": 17.599693868067632, "energy_net_psys_j_q25": 17.322893383996043, "energy_net_psys_j_q75": 17.97240176716679, "prefill_energy_net_psys_j": 9.628182979699977, "prefill_energy_net_psys_j_q25": 9.552448242631936, "prefill_energy_net_psys_j_q75": 9.752896662466021, "decode_energy_net_psys_j": 8.122980362503737, "decode_energy_net_psys_j_q25": 7.770445141364107, "decode_energy_net_psys_j_q75": 8.295239841768808, "fastest_e2e": false, "lowest_energy": true, "e2e_speedup_vs_gpu": 0.7955015528500611, "energy_reduction_vs_gpu": 0.0619896422039824 }, { "precision": "INT8", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 4096, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1081.158091, "ttft_ms_q25": 1026.184426, "ttft_ms_q75": 1081.21848, "prefill_tokens_per_s": 3773.731181372623, "prefill_tokens_per_s_q25": 3773.5204199688396, "prefill_tokens_per_s_q75": 3987.336551180539, "handoff_plus_first_decode_ms": 35.639419, "handoff_plus_first_decode_ms_q25": 35.1314085, "handoff_plus_first_decode_ms_q75": 52.99225949999999, "steady_tpot_ms": 13.736344500000001, "steady_tpot_ms_q25": 13.726874500000001, "steady_tpot_ms_q75": 13.853233750000001, "steady_tpot_p95_ms": 17.34561505, "steady_tpot_p95_ms_q25": 17.285200749999998, "steady_tpot_p95_ms_q75": 17.7116704, "decode_tokens_per_s": 61.99981878692965, "decode_tokens_per_s_q25": 58.62843857291044, "decode_tokens_per_s_q75": 62.495546624094764, "decode_ms": 241.936191, "decode_ms_q25": 240.0322050000001, "decode_ms_q75": 256.697362, "end_to_end_ms": 1319.2863100000002, "end_to_end_ms_q25": 1280.977802, "end_to_end_ms_q75": 1321.250685, "output_tokens_per_s_e2e": 12.12776929368728, "output_tokens_per_s_e2e_q25": 12.109765049686093, "output_tokens_per_s_e2e_q75": 12.501639276033021, "all_tokens_per_s_e2e": 3104.7089391839436, "all_tokens_per_s_e2e_q25": 3100.09985271964, "all_tokens_per_s_e2e_q75": 3200.4196546644534, "energy_net_psys_j": 35.23845258394553, "energy_net_psys_j_q25": 34.750285630277006, "energy_net_psys_j_q75": 35.414115039388406, "prefill_energy_net_psys_j": 27.349379707607135, "prefill_energy_net_psys_j_q25": 25.705690942943104, "prefill_energy_net_psys_j_q75": 27.364781005554192, "decode_energy_net_psys_j": 8.240397787224154, "decode_energy_net_psys_j_q25": 8.049334033834215, "decode_energy_net_psys_j_q75": 9.220257142776784, "fastest_e2e": true, "lowest_energy": true, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "INT8", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 4096, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1081.158091, "ttft_ms_q25": 1026.184426, "ttft_ms_q75": 1081.21848, "prefill_tokens_per_s": 3773.731181372623, "prefill_tokens_per_s_q25": 3773.5204199688396, "prefill_tokens_per_s_q75": 3987.336551180539, "handoff_plus_first_decode_ms": 173.10389700000002, "handoff_plus_first_decode_ms_q25": 162.950516, "handoff_plus_first_decode_ms_q75": 224.01592950000003, "steady_tpot_ms": 125.98738949999999, "steady_tpot_ms_q25": 125.96643800000001, "steady_tpot_ms_q75": 126.00246949999999, "steady_tpot_p95_ms": 134.6569466, "steady_tpot_p95_ms_q25": 130.53024165, "steady_tpot_p95_ms_q75": 135.23860355, "decode_tokens_per_s": 7.639847760875907, "decode_tokens_per_s_q25": 7.456068285060912, "decode_tokens_per_s_q75": 7.739068646464693, "decode_ms": 1963.389909, "decode_ms_q25": 1938.5363634999999, "decode_ms_q75": 2013.007131, "end_to_end_ms": 3033.8351140000004, "end_to_end_ms_q25": 3014.3984005, "end_to_end_ms_q75": 3039.191557, "output_tokens_per_s_e2e": 5.273852862393891, "output_tokens_per_s_e2e_q25": 5.264574279087667, "output_tokens_per_s_e2e_q75": 5.308079132323302, "all_tokens_per_s_e2e": 1350.1063327728361, "all_tokens_per_s_e2e_q25": 1347.7310154464428, "all_tokens_per_s_e2e_q75": 1358.8682578747653, "energy_net_psys_j": 62.1721427445659, "energy_net_psys_j_q25": 60.38326964061885, "energy_net_psys_j_q75": 68.77355094114523, "prefill_energy_net_psys_j": 27.349379707607135, "prefill_energy_net_psys_j_q25": 25.705690942943104, "prefill_energy_net_psys_j_q75": 27.364781005554192, "decode_energy_net_psys_j": 38.11014056628683, "decode_energy_net_psys_j_q25": 34.67757869767575, "decode_energy_net_psys_j_q75": 43.05245870025506, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.434857617644411, "energy_reduction_vs_gpu": -0.7643266995467115 }, { "precision": "INT8", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 4096, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1298.635947, "ttft_ms_q25": 1293.3136195000002, "ttft_ms_q75": 1306.073258, "prefill_tokens_per_s": 3141.7580958122053, "prefill_tokens_per_s_q25": 3123.9689492329644, "prefill_tokens_per_s_q75": 3154.740687865078, "handoff_plus_first_decode_ms": 173.10389700000002, "handoff_plus_first_decode_ms_q25": 162.950516, "handoff_plus_first_decode_ms_q75": 224.01592950000003, "steady_tpot_ms": 125.98738949999999, "steady_tpot_ms_q25": 125.96643800000001, "steady_tpot_ms_q75": 126.00246949999999, "steady_tpot_p95_ms": 134.6569466, "steady_tpot_p95_ms_q25": 130.53024165, "steady_tpot_p95_ms_q75": 135.23860355, "decode_tokens_per_s": 7.639847760875907, "decode_tokens_per_s_q25": 7.456068285060912, "decode_tokens_per_s_q75": 7.739068646464693, "decode_ms": 1963.389909, "decode_ms_q25": 1938.5363634999999, "decode_ms_q75": 2013.007131, "end_to_end_ms": 3276.900478, "end_to_end_ms_q25": 3244.6096215, "end_to_end_ms_q75": 3313.7580615, "output_tokens_per_s_e2e": 4.882662780703467, "output_tokens_per_s_e2e_q25": 4.828952307701088, "output_tokens_per_s_e2e_q75": 4.931744261939837, "all_tokens_per_s_e2e": 1249.9616718600876, "all_tokens_per_s_e2e_q25": 1236.2117907714785, "all_tokens_per_s_e2e_q75": 1262.5265310565983, "energy_net_psys_j": 64.77302497724148, "energy_net_psys_j_q25": 59.503293836218525, "energy_net_psys_j_q75": 73.93139059507189, "prefill_energy_net_psys_j": 26.662884410954646, "prefill_energy_net_psys_j_q25": 24.825715138542776, "prefill_energy_net_psys_j_q75": 30.87893189481683, "decode_energy_net_psys_j": 38.11014056628683, "decode_energy_net_psys_j_q25": 34.67757869767575, "decode_energy_net_psys_j_q75": 43.05245870025506, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.4026018851830386, "energy_reduction_vs_gpu": -0.8381347711832203 }, { "precision": "INT8", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 4096, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1298.635947, "ttft_ms_q25": 1293.3136195000002, "ttft_ms_q75": 1306.073258, "prefill_tokens_per_s": 3141.7580958122053, "prefill_tokens_per_s_q25": 3123.9689492329644, "prefill_tokens_per_s_q75": 3154.740687865078, "handoff_plus_first_decode_ms": 35.639419, "handoff_plus_first_decode_ms_q25": 35.1314085, "handoff_plus_first_decode_ms_q75": 52.99225949999999, "steady_tpot_ms": 13.736344500000001, "steady_tpot_ms_q25": 13.726874500000001, "steady_tpot_ms_q75": 13.853233750000001, "steady_tpot_p95_ms": 17.34561505, "steady_tpot_p95_ms_q25": 17.285200749999998, "steady_tpot_p95_ms_q75": 17.7116704, "decode_tokens_per_s": 61.99981878692965, "decode_tokens_per_s_q25": 58.62843857291044, "decode_tokens_per_s_q75": 62.495546624094764, "decode_ms": 241.936191, "decode_ms_q25": 240.0322050000001, "decode_ms_q75": 256.697362, "end_to_end_ms": 1551.6387880000002, "end_to_end_ms_q25": 1546.105463, "end_to_end_ms_q75": 1555.5443065000002, "output_tokens_per_s_e2e": 10.311678287330878, "output_tokens_per_s_e2e_q25": 10.285853506262837, "output_tokens_per_s_e2e_q75": 10.348715090640408, "all_tokens_per_s_e2e": 2639.7896415567047, "all_tokens_per_s_e2e_q25": 2633.1784976032864, "all_tokens_per_s_e2e_q75": 2649.2710632039443, "energy_net_psys_j": 36.86300090928406, "energy_net_psys_j_q25": 34.045972281319564, "energy_net_psys_j_q75": 39.90812528420368, "prefill_energy_net_psys_j": 26.662884410954646, "prefill_energy_net_psys_j_q25": 24.825715138542776, "prefill_energy_net_psys_j_q75": 30.87893189481683, "decode_energy_net_psys_j": 8.240397787224154, "decode_energy_net_psys_j_q25": 8.049334033834215, "decode_energy_net_psys_j_q75": 9.220257142776784, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.8502534998499922, "energy_reduction_vs_gpu": -0.04610157955910554 }, { "precision": "INT8", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 8192, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 2767.602321, "ttft_ms_q25": 2663.1035624999995, "ttft_ms_q75": 2778.9175015, "prefill_tokens_per_s": 2954.1816531812337, "prefill_tokens_per_s_q25": 2942.2016133619118, "prefill_tokens_per_s_q75": 3074.836595538216, "handoff_plus_first_decode_ms": 49.816934, "handoff_plus_first_decode_ms_q25": 49.792398500000004, "handoff_plus_first_decode_ms_q75": 56.983838, "steady_tpot_ms": 16.926910499999998, "steady_tpot_ms_q25": 16.8871495, "steady_tpot_ms_q75": 16.975437, "steady_tpot_p95_ms": 20.88518585, "steady_tpot_p95_ms_q25": 20.627281875, "steady_tpot_p95_ms_q75": 21.275896875, "decode_tokens_per_s": 49.70044004073806, "decode_tokens_per_s_q25": 49.08083068974983, "decode_tokens_per_s_q75": 50.05398007053218, "decode_ms": 301.8081929999994, "decode_ms_q25": 299.6914199999999, "decode_ms_q75": 305.6670139999999, "end_to_end_ms": 3065.176968, "end_to_end_ms_q25": 2966.6538035000003, "end_to_end_ms_q75": 3078.6089215, "output_tokens_per_s_e2e": 5.219926995092832, "output_tokens_per_s_e2e_q25": 5.197251416241864, "output_tokens_per_s_e2e_q75": 5.399236723534921, "all_tokens_per_s_e2e": 2672.60262148753, "all_tokens_per_s_e2e_q25": 2660.992725115834, "all_tokens_per_s_e2e_q75": 2764.4092024498796, "energy_net_psys_j": 83.56347810145249, "energy_net_psys_j_q25": 80.89226014035711, "energy_net_psys_j_q75": 83.82630873998627, "prefill_energy_net_psys_j": 73.88001436422923, "prefill_energy_net_psys_j_q25": 70.81250135250667, "prefill_energy_net_psys_j_q75": 74.33757825423756, "decode_energy_net_psys_j": 9.683463737223263, "decode_energy_net_psys_j_q25": 9.488730485748711, "decode_energy_net_psys_j_q75": 10.079758787850423, "fastest_e2e": true, "lowest_energy": false, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "INT8", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 8192, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 2767.602321, "ttft_ms_q25": 2663.1035624999995, "ttft_ms_q75": 2778.9175015, "prefill_tokens_per_s": 2954.1816531812337, "prefill_tokens_per_s_q25": 2942.2016133619118, "prefill_tokens_per_s_q75": 3074.836595538216, "handoff_plus_first_decode_ms": 193.19836700000002, "handoff_plus_first_decode_ms_q25": 191.31255650000003, "handoff_plus_first_decode_ms_q75": 239.21696500000002, "steady_tpot_ms": 128.32535049999998, "steady_tpot_ms_q25": 128.0788365, "steady_tpot_ms_q75": 128.34406925, "steady_tpot_p95_ms": 128.76579915, "steady_tpot_p95_ms_q25": 128.42559725, "steady_tpot_p95_ms_q75": 133.27836625, "decode_tokens_per_s": 7.538911147524052, "decode_tokens_per_s_q25": 7.328415213156733, "decode_tokens_per_s_q75": 7.561856309187208, "decode_ms": 1989.6772499999997, "decode_ms_q25": 1983.6581779999997, "decode_ms_q75": 2048.5173345, "end_to_end_ms": 4745.241426999999, "end_to_end_ms_q25": 4705.601825, "end_to_end_ms_q75": 4762.5756795, "output_tokens_per_s_e2e": 3.371798937133405, "output_tokens_per_s_e2e_q25": 3.359571172387418, "output_tokens_per_s_e2e_q75": 3.400443997145498, "all_tokens_per_s_e2e": 1726.3610558123034, "all_tokens_per_s_e2e_q25": 1720.100440262358, "all_tokens_per_s_e2e_q75": 1741.027326538495, "energy_net_psys_j": 104.69055899941593, "energy_net_psys_j_q25": 103.3705781468208, "energy_net_psys_j_q75": 105.1172365495738, "prefill_energy_net_psys_j": 73.88001436422923, "prefill_energy_net_psys_j_q25": 70.81250135250667, "prefill_energy_net_psys_j_q75": 74.33757825423756, "decode_energy_net_psys_j": 30.8105446351867, "decode_energy_net_psys_j_q25": 30.779658295336247, "decode_energy_net_psys_j_q75": 32.55807679431413, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.6459475276767621, "energy_reduction_vs_gpu": -0.2528267297863491 }, { "precision": "INT8", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 8192, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 3803.3502559999997, "ttft_ms_q25": 3802.8790125, "ttft_ms_q75": 3813.0174985, "prefill_tokens_per_s": 2149.683686665959, "prefill_tokens_per_s_q25": 2144.247320628655, "prefill_tokens_per_s_q75": 2149.9501032435437, "handoff_plus_first_decode_ms": 193.19836700000002, "handoff_plus_first_decode_ms_q25": 191.31255650000003, "handoff_plus_first_decode_ms_q75": 239.21696500000002, "steady_tpot_ms": 128.32535049999998, "steady_tpot_ms_q25": 128.0788365, "steady_tpot_ms_q75": 128.34406925, "steady_tpot_p95_ms": 128.76579915, "steady_tpot_p95_ms_q25": 128.42559725, "steady_tpot_p95_ms_q75": 133.27836625, "decode_tokens_per_s": 7.538911147524052, "decode_tokens_per_s_q25": 7.328415213156733, "decode_tokens_per_s_q75": 7.561856309187208, "decode_ms": 1989.6772499999997, "decode_ms_q25": 1983.6581779999997, "decode_ms_q75": 2048.5173345, "end_to_end_ms": 5793.027505999999, "end_to_end_ms_q25": 5786.537190499999, "end_to_end_ms_q75": 5861.534833, "output_tokens_per_s_e2e": 2.7619409684190788, "output_tokens_per_s_e2e_q25": 2.7300334055429687, "output_tokens_per_s_e2e_q75": 2.7650423045613426, "all_tokens_per_s_e2e": 1414.1137758305683, "all_tokens_per_s_e2e_q25": 1397.777103638, "all_tokens_per_s_e2e_q75": 1415.7016599354074, "energy_net_psys_j": 92.38107921915228, "energy_net_psys_j_q25": 92.31603133301135, "energy_net_psys_j_q75": 96.454425338855, "prefill_energy_net_psys_j": 61.63230726366649, "prefill_energy_net_psys_j_q25": 61.536373037675105, "prefill_energy_net_psys_j_q75": 63.92723488439133, "decode_energy_net_psys_j": 30.8105446351867, "decode_energy_net_psys_j_q25": 30.779658295336247, "decode_energy_net_psys_j_q75": 32.55807679431413, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.529114865210861, "energy_reduction_vs_gpu": -0.10551979546608337 }, { "precision": "INT8", "model": "qwen25_0_5b", "model_label": "Qwen2.5 0.5B", "family": "Qwen2.5", "size_b": 0.5, "context_length": 8192, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 3803.3502559999997, "ttft_ms_q25": 3802.8790125, "ttft_ms_q75": 3813.0174985, "prefill_tokens_per_s": 2149.683686665959, "prefill_tokens_per_s_q25": 2144.247320628655, "prefill_tokens_per_s_q75": 2149.9501032435437, "handoff_plus_first_decode_ms": 49.816934, "handoff_plus_first_decode_ms_q25": 49.792398500000004, "handoff_plus_first_decode_ms_q75": 56.983838, "steady_tpot_ms": 16.926910499999998, "steady_tpot_ms_q25": 16.8871495, "steady_tpot_ms_q75": 16.975437, "steady_tpot_p95_ms": 20.88518585, "steady_tpot_p95_ms_q25": 20.627281875, "steady_tpot_p95_ms_q75": 21.275896875, "decode_tokens_per_s": 49.70044004073806, "decode_tokens_per_s_q25": 49.08083068974983, "decode_tokens_per_s_q75": 50.05398007053218, "decode_ms": 301.8081929999994, "decode_ms_q25": 299.6914199999999, "decode_ms_q75": 305.6670139999999, "end_to_end_ms": 4105.158448999999, "end_to_end_ms_q25": 4102.5704325, "end_to_end_ms_q75": 4118.6845125, "output_tokens_per_s_e2e": 3.8975353080214337, "output_tokens_per_s_e2e_q25": 3.8847774126769945, "output_tokens_per_s_e2e_q75": 3.8999955345939004, "all_tokens_per_s_e2e": 1995.538077706974, "all_tokens_per_s_e2e_q25": 1989.0060352906212, "all_tokens_per_s_e2e_q75": 1996.797713712077, "energy_net_psys_j": 71.12390254890698, "energy_net_psys_j_q25": 71.02510352342381, "energy_net_psys_j_q75": 73.91105944625036, "prefill_energy_net_psys_j": 61.63230726366649, "prefill_energy_net_psys_j_q25": 61.536373037675105, "prefill_energy_net_psys_j_q75": 63.92723488439133, "decode_energy_net_psys_j": 9.683463737223263, "decode_energy_net_psys_j_q25": 9.488730485748711, "decode_energy_net_psys_j_q75": 10.079758787850423, "fastest_e2e": false, "lowest_energy": true, "e2e_speedup_vs_gpu": 0.7466647161321303, "energy_reduction_vs_gpu": 0.1488637839780066 }, { "precision": "INT8", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 1024, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 339.972811, "ttft_ms_q25": 317.2443015, "ttft_ms_q75": 342.7688715, "prefill_tokens_per_s": 2964.9429818668646, "prefill_tokens_per_s_q25": 2940.9528162317993, "prefill_tokens_per_s_q75": 3193.754938851227, "handoff_plus_first_decode_ms": 35.874994, "handoff_plus_first_decode_ms_q25": 33.997557, "handoff_plus_first_decode_ms_q75": 41.853654500000005, "steady_tpot_ms": 23.400827999999997, "steady_tpot_ms_q25": 23.312720249999998, "steady_tpot_ms_q75": 23.736918, "steady_tpot_p95_ms": 24.7919254, "steady_tpot_p95_ms_q25": 24.539722849999997, "steady_tpot_p95_ms_q75": 30.054694724999997, "decode_tokens_per_s": 39.66048645855221, "decode_tokens_per_s_q25": 38.80590180853985, "decode_tokens_per_s_q75": 40.35164683844978, "decode_ms": 378.21018699999996, "decode_ms_q25": 371.841132, "decode_ms_q75": 386.72669300000007, "end_to_end_ms": 705.444888, "end_to_end_ms_q25": 689.0854334999999, "end_to_end_ms_q75": 723.1265095000001, "output_tokens_per_s_e2e": 22.680722863215365, "output_tokens_per_s_e2e_q25": 22.139378971008853, "output_tokens_per_s_e2e_q75": 23.232276192937274, "all_tokens_per_s_e2e": 1451.5662632457834, "all_tokens_per_s_e2e_q25": 1416.9202541445666, "all_tokens_per_s_e2e_q75": 1486.8656763479855, "energy_net_psys_j": 23.167475685392333, "energy_net_psys_j_q25": 23.12918756901884, "energy_net_psys_j_q75": 23.169200101865627, "prefill_energy_net_psys_j": 9.813751017035404, "prefill_energy_net_psys_j_q25": 9.419546391071115, "prefill_energy_net_psys_j_q75": 9.838721782282576, "decode_energy_net_psys_j": 13.303783137862586, "decode_energy_net_psys_j_q25": 13.290465786736267, "decode_energy_net_psys_j_q75": 13.72468294554734, "fastest_e2e": true, "lowest_energy": true, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "INT8", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 1024, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 339.972811, "ttft_ms_q25": 317.2443015, "ttft_ms_q75": 342.7688715, "prefill_tokens_per_s": 2964.9429818668646, "prefill_tokens_per_s_q25": 2940.9528162317993, "prefill_tokens_per_s_q75": 3193.754938851227, "handoff_plus_first_decode_ms": 508.60053000000005, "handoff_plus_first_decode_ms_q25": 505.48739, "handoff_plus_first_decode_ms_q75": 592.453497, "steady_tpot_ms": 504.70483099999996, "steady_tpot_ms_q25": 504.49935800000003, "steady_tpot_ms_q75": 504.81454199999996, "steady_tpot_p95_ms": 505.11252255000005, "steady_tpot_p95_ms_q25": 505.0448423750001, "steady_tpot_p95_ms_q75": 505.95412972500003, "decode_tokens_per_s": 1.9808261869857582, "decode_tokens_per_s_q25": 1.9584322120707864, "decode_tokens_per_s_q75": 1.9849727999164217, "decode_ms": 7572.597787, "decode_ms_q25": 7556.811589499999, "decode_ms_q75": 7660.189324000001, "end_to_end_ms": 7912.570597999999, "end_to_end_ms_q25": 7899.580461, "end_to_end_ms_q75": 7977.4336255, "output_tokens_per_s_e2e": 2.0220988617838302, "output_tokens_per_s_e2e_q25": 2.005790155485278, "output_tokens_per_s_e2e_q75": 2.025429495196115, "all_tokens_per_s_e2e": 129.41432715416514, "all_tokens_per_s_e2e_q25": 128.3705699510578, "all_tokens_per_s_e2e_q75": 129.62748769255137, "energy_net_psys_j": 107.2488111607607, "energy_net_psys_j_q25": 106.24865083453247, "energy_net_psys_j_q75": 109.9971724144817, "prefill_energy_net_psys_j": 9.813751017035404, "prefill_energy_net_psys_j_q25": 9.419546391071115, "prefill_energy_net_psys_j_q75": 9.838721782282576, "decode_energy_net_psys_j": 97.38511861323094, "decode_energy_net_psys_j_q25": 96.4099290522499, "decode_energy_net_psys_j_q75": 100.55265525816341, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.08915495656725135, "energy_reduction_vs_gpu": -3.62928342376048 }, { "precision": "INT8", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 1024, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 910.8172890000001, "ttft_ms_q25": 909.878027, "ttft_ms_q75": 912.985671, "prefill_tokens_per_s": 1106.698360004451, "prefill_tokens_per_s_q25": 1104.076129577348, "prefill_tokens_per_s_q75": 1107.8419790853409, "handoff_plus_first_decode_ms": 508.60053000000005, "handoff_plus_first_decode_ms_q25": 505.48739, "handoff_plus_first_decode_ms_q75": 592.453497, "steady_tpot_ms": 504.70483099999996, "steady_tpot_ms_q25": 504.49935800000003, "steady_tpot_ms_q75": 504.81454199999996, "steady_tpot_p95_ms": 505.11252255000005, "steady_tpot_p95_ms_q25": 505.0448423750001, "steady_tpot_p95_ms_q75": 505.95412972500003, "decode_tokens_per_s": 1.9808261869857582, "decode_tokens_per_s_q25": 1.9584322120707864, "decode_tokens_per_s_q75": 1.9849727999164217, "decode_ms": 7572.597787, "decode_ms_q25": 7556.811589499999, "decode_ms_q75": 7660.189324000001, "end_to_end_ms": 8481.536552, "end_to_end_ms_q25": 8468.8579985, "end_to_end_ms_q75": 8570.067351000002, "output_tokens_per_s_e2e": 1.8864506333144433, "output_tokens_per_s_e2e_q25": 1.8671624091760606, "output_tokens_per_s_e2e_q75": 1.889279034182134, "all_tokens_per_s_e2e": 120.73284053212437, "all_tokens_per_s_e2e_q25": 119.49839418726788, "all_tokens_per_s_e2e_q75": 120.91385818765657, "energy_net_psys_j": 112.08481262627237, "energy_net_psys_j_q25": 110.56533729627971, "energy_net_psys_j_q75": 115.20917096732981, "prefill_energy_net_psys_j": 14.61333740529138, "prefill_energy_net_psys_j_q25": 14.112229940154794, "prefill_energy_net_psys_j_q75": 14.6565157091664, "decode_energy_net_psys_j": 97.38511861323094, "decode_energy_net_psys_j_q25": 96.4099290522499, "decode_energy_net_psys_j_q75": 100.55265525816341, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.08317418473350228, "energy_reduction_vs_gpu": -3.8380243988750413 }, { "precision": "INT8", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 1024, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 910.8172890000001, "ttft_ms_q25": 909.878027, "ttft_ms_q75": 912.985671, "prefill_tokens_per_s": 1106.698360004451, "prefill_tokens_per_s_q25": 1104.076129577348, "prefill_tokens_per_s_q75": 1107.8419790853409, "handoff_plus_first_decode_ms": 35.874994, "handoff_plus_first_decode_ms_q25": 33.997557, "handoff_plus_first_decode_ms_q75": 41.853654500000005, "steady_tpot_ms": 23.400827999999997, "steady_tpot_ms_q25": 23.312720249999998, "steady_tpot_ms_q75": 23.736918, "steady_tpot_p95_ms": 24.7919254, "steady_tpot_p95_ms_q25": 24.539722849999997, "steady_tpot_p95_ms_q75": 30.054694724999997, "decode_tokens_per_s": 39.66048645855221, "decode_tokens_per_s_q25": 38.80590180853985, "decode_tokens_per_s_q75": 40.35164683844978, "decode_ms": 378.21018699999996, "decode_ms_q25": 371.841132, "decode_ms_q75": 386.72669300000007, "end_to_end_ms": 1289.027476, "end_to_end_ms_q25": 1281.719159, "end_to_end_ms_q75": 1299.712364, "output_tokens_per_s_e2e": 12.412458460272573, "output_tokens_per_s_e2e_q25": 12.311248137791932, "output_tokens_per_s_e2e_q75": 12.48363972944213, "all_tokens_per_s_e2e": 794.3973414574447, "all_tokens_per_s_e2e_q25": 787.9198808186836, "all_tokens_per_s_e2e_q75": 798.9529426842963, "energy_net_psys_j": 28.003477150904004, "energy_net_psys_j_q25": 27.445874030766078, "energy_net_psys_j_q75": 28.38119865471374, "prefill_energy_net_psys_j": 14.61333740529138, "prefill_energy_net_psys_j_q25": 14.112229940154794, "prefill_energy_net_psys_j_q75": 14.6565157091664, "decode_energy_net_psys_j": 13.303783137862586, "decode_energy_net_psys_j_q25": 13.290465786736267, "decode_energy_net_psys_j_q75": 13.72468294554734, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.5472690855194773, "energy_reduction_vs_gpu": -0.2087409751145608 }, { "precision": "INT8", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 2048, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 699.393099, "ttft_ms_q25": 672.2121945, "ttft_ms_q75": 711.764133, "prefill_tokens_per_s": 2905.376108093397, "prefill_tokens_per_s_q25": 2855.741028547521, "prefill_tokens_per_s_q75": 3027.805440987726, "handoff_plus_first_decode_ms": 42.768417, "handoff_plus_first_decode_ms_q25": 41.313729499999994, "handoff_plus_first_decode_ms_q75": 48.019420499999995, "steady_tpot_ms": 26.215950000000003, "steady_tpot_ms_q25": 26.068020500000003, "steady_tpot_ms_q75": 26.273407750000004, "steady_tpot_p95_ms": 29.713804399999997, "steady_tpot_p95_ms_q25": 29.385052249999998, "steady_tpot_p95_ms_q75": 30.03668225, "decode_tokens_per_s": 35.504392032945084, "decode_tokens_per_s_q25": 35.41831321350345, "decode_tokens_per_s_q75": 35.65444674854093, "decode_ms": 422.482942, "decode_ms_q25": 420.71233899999993, "decode_ms_q75": 423.51222400000006, "end_to_end_ms": 1118.3348349999999, "end_to_end_ms_q25": 1093.9538155, "end_to_end_ms_q75": 1132.4764719999998, "output_tokens_per_s_e2e": 14.306985259919943, "output_tokens_per_s_e2e_q25": 14.130532271324993, "output_tokens_per_s_e2e_q75": 14.63311442841865, "all_tokens_per_s_e2e": 1831.2941132697526, "all_tokens_per_s_e2e_q25": 1808.7081307295991, "all_tokens_per_s_e2e_q75": 1873.0386468375873, "energy_net_psys_j": 35.91370598660502, "energy_net_psys_j_q25": 35.260720895451456, "energy_net_psys_j_q75": 35.943466413730775, "prefill_energy_net_psys_j": 21.3310625709722, "prefill_energy_net_psys_j_q25": 20.44179675025776, "prefill_energy_net_psys_j_q75": 21.336014385710566, "decode_energy_net_psys_j": 14.632260640407594, "decode_energy_net_psys_j_q25": 14.607452028020205, "decode_energy_net_psys_j_q75": 14.843732757581087, "fastest_e2e": true, "lowest_energy": true, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "INT8", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 2048, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 699.393099, "ttft_ms_q25": 672.2121945, "ttft_ms_q75": 711.764133, "prefill_tokens_per_s": 2905.376108093397, "prefill_tokens_per_s_q25": 2855.741028547521, "prefill_tokens_per_s_q75": 3027.805440987726, "handoff_plus_first_decode_ms": 535.328571, "handoff_plus_first_decode_ms_q25": 533.273764, "handoff_plus_first_decode_ms_q75": 609.1082054999999, "steady_tpot_ms": 506.14783600000004, "steady_tpot_ms_q25": 505.94752200000005, "steady_tpot_ms_q75": 506.19152025000005, "steady_tpot_p95_ms": 506.6745119, "steady_tpot_p95_ms_q25": 506.631695675, "steady_tpot_p95_ms_q75": 507.505786525, "decode_tokens_per_s": 1.9677838729826396, "decode_tokens_per_s_q25": 1.953004612704816, "decode_tokens_per_s_q75": 1.968569280826768, "decode_ms": 7622.788359000001, "decode_ms_q25": 7619.748278000001, "decode_ms_q75": 7680.913268500002, "end_to_end_ms": 8340.843364, "end_to_end_ms_q25": 8331.512411, "end_to_end_ms_q75": 8362.456416, "output_tokens_per_s_e2e": 1.9182712468930618, "output_tokens_per_s_e2e_q25": 1.9133261905135255, "output_tokens_per_s_e2e_q75": 1.9204220410011144, "all_tokens_per_s_e2e": 245.53871960231191, "all_tokens_per_s_e2e_q25": 244.90575238573126, "all_tokens_per_s_e2e_q75": 245.81402124814264, "energy_net_psys_j": 120.36931267849182, "energy_net_psys_j_q25": 120.20880599990309, "energy_net_psys_j_q75": 120.92286109376958, "prefill_energy_net_psys_j": 21.3310625709722, "prefill_energy_net_psys_j_q25": 20.44179675025776, "prefill_energy_net_psys_j_q75": 21.336014385710566, "decode_energy_net_psys_j": 99.02834647804289, "decode_energy_net_psys_j_q25": 98.87279161419252, "decode_energy_net_psys_j_q75": 100.47611252877346, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.13407934739871227, "energy_reduction_vs_gpu": -2.351626053946836 }, { "precision": "INT8", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 2048, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1825.8328640000002, "ttft_ms_q25": 1822.8377095, "ttft_ms_q75": 1826.8214560000001, "prefill_tokens_per_s": 1112.9167625717575, "prefill_tokens_per_s_q25": 1112.3148287934919, "prefill_tokens_per_s_q75": 1114.7484362186804, "handoff_plus_first_decode_ms": 535.328571, "handoff_plus_first_decode_ms_q25": 533.273764, "handoff_plus_first_decode_ms_q75": 609.1082054999999, "steady_tpot_ms": 506.14783600000004, "steady_tpot_ms_q25": 505.94752200000005, "steady_tpot_ms_q75": 506.19152025000005, "steady_tpot_p95_ms": 506.6745119, "steady_tpot_p95_ms_q25": 506.631695675, "steady_tpot_p95_ms_q75": 507.505786525, "decode_tokens_per_s": 1.9677838729826396, "decode_tokens_per_s_q25": 1.953004612704816, "decode_tokens_per_s_q75": 1.968569280826768, "decode_ms": 7622.788359000001, "decode_ms_q25": 7619.748278000001, "decode_ms_q75": 7680.913268500002, "end_to_end_ms": 9442.630914000001, "end_to_end_ms_q25": 9442.5859875, "end_to_end_ms_q75": 9504.739570000002, "output_tokens_per_s_e2e": 1.6944430154818184, "output_tokens_per_s_e2e_q25": 1.6834425714615868, "output_tokens_per_s_e2e_q75": 1.694451077442433, "all_tokens_per_s_e2e": 216.88870598167276, "all_tokens_per_s_e2e_q25": 215.4806491470831, "all_tokens_per_s_e2e_q75": 216.88973791263143, "energy_net_psys_j": 130.28991509853228, "energy_net_psys_j_q25": 130.01844945518496, "energy_net_psys_j_q75": 132.09968109580248, "prefill_energy_net_psys_j": 31.261568620489392, "prefill_energy_net_psys_j_q25": 31.145657840992428, "prefill_energy_net_psys_j_q75": 31.623568567029025, "decode_energy_net_psys_j": 99.02834647804289, "decode_energy_net_psys_j_q25": 98.87279161419252, "decode_energy_net_psys_j_q75": 100.47611252877346, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.11843466563348509, "energy_reduction_vs_gpu": -2.6278604927914544 }, { "precision": "INT8", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 2048, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1825.8328640000002, "ttft_ms_q25": 1822.8377095, "ttft_ms_q75": 1826.8214560000001, "prefill_tokens_per_s": 1112.9167625717575, "prefill_tokens_per_s_q25": 1112.3148287934919, "prefill_tokens_per_s_q75": 1114.7484362186804, "handoff_plus_first_decode_ms": 42.768417, "handoff_plus_first_decode_ms_q25": 41.313729499999994, "handoff_plus_first_decode_ms_q75": 48.019420499999995, "steady_tpot_ms": 26.215950000000003, "steady_tpot_ms_q25": 26.068020500000003, "steady_tpot_ms_q75": 26.273407750000004, "steady_tpot_p95_ms": 29.713804399999997, "steady_tpot_p95_ms_q25": 29.385052249999998, "steady_tpot_p95_ms_q75": 30.03668225, "decode_tokens_per_s": 35.504392032945084, "decode_tokens_per_s_q25": 35.41831321350345, "decode_tokens_per_s_q75": 35.65444674854093, "decode_ms": 422.482942, "decode_ms_q25": 420.71233899999993, "decode_ms_q75": 423.51222400000006, "end_to_end_ms": 2248.315806, "end_to_end_ms_q25": 2243.5500485000002, "end_to_end_ms_q75": 2250.3336799999997, "output_tokens_per_s_e2e": 7.116437983178952, "output_tokens_per_s_e2e_q25": 7.110062390899245, "output_tokens_per_s_e2e_q75": 7.131586927107119, "all_tokens_per_s_e2e": 910.9040618469058, "all_tokens_per_s_e2e_q25": 910.0879860351033, "all_tokens_per_s_e2e_q75": 912.8431266697112, "energy_net_psys_j": 45.893829260896986, "energy_net_psys_j_q25": 45.75310986901263, "energy_net_psys_j_q75": 46.46730132461012, "prefill_energy_net_psys_j": 31.261568620489392, "prefill_energy_net_psys_j_q25": 31.145657840992428, "prefill_energy_net_psys_j_q75": 31.623568567029025, "decode_energy_net_psys_j": 14.632260640407594, "decode_energy_net_psys_j_q25": 14.607452028020205, "decode_energy_net_psys_j_q75": 14.843732757581087, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.49741003110663534, "energy_reduction_vs_gpu": -0.2778917686193212 }, { "precision": "INT8", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 4096, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1603.065692, "ttft_ms_q25": 1546.1392455, "ttft_ms_q75": 1605.3625309999998, "prefill_tokens_per_s": 2545.123397226319, "prefill_tokens_per_s_q25": 2541.487217331717, "prefill_tokens_per_s_q75": 2642.412939060362, "handoff_plus_first_decode_ms": 43.987549, "handoff_plus_first_decode_ms_q25": 43.735888, "handoff_plus_first_decode_ms_q75": 50.556247, "steady_tpot_ms": 25.1619285, "steady_tpot_ms_q25": 25.134072500000002, "steady_tpot_ms_q75": 25.167222, "steady_tpot_p95_ms": 27.58854755, "steady_tpot_p95_ms_q25": 27.237907925, "steady_tpot_p95_ms_q75": 27.624445225000002, "decode_tokens_per_s": 37.30816477446453, "decode_tokens_per_s_q25": 36.65430131309493, "decode_tokens_per_s_q75": 37.38747916500209, "decode_ms": 402.05676400000016, "decode_ms_q25": 401.20564000000013, "decode_ms_q75": 409.3591815000001, "end_to_end_ms": 2003.420208, "end_to_end_ms_q25": 1954.647303, "end_to_end_ms_q75": 2006.568171, "output_tokens_per_s_e2e": 7.986342523704842, "output_tokens_per_s_e2e_q25": 7.973832940712187, "output_tokens_per_s_e2e_q75": 8.190719646150514, "all_tokens_per_s_e2e": 2044.5036860684395, "all_tokens_per_s_e2e_q25": 2041.30123282232, "all_tokens_per_s_e2e_q75": 2096.8242294145316, "energy_net_psys_j": 65.44541461298002, "energy_net_psys_j_q25": 64.21131973265548, "energy_net_psys_j_q75": 65.50187665354747, "prefill_energy_net_psys_j": 51.26185070605367, "prefill_energy_net_psys_j_q25": 49.51459644977226, "prefill_energy_net_psys_j_q75": 51.29657245381291, "decode_energy_net_psys_j": 14.227044492542767, "decode_energy_net_psys_j_q25": 14.205304199734556, "decode_energy_net_psys_j_q75": 14.71846357569143, "fastest_e2e": true, "lowest_energy": true, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "INT8", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 4096, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1603.065692, "ttft_ms_q25": 1546.1392455, "ttft_ms_q75": 1605.3625309999998, "prefill_tokens_per_s": 2545.123397226319, "prefill_tokens_per_s_q25": 2541.487217331717, "prefill_tokens_per_s_q75": 2642.412939060362, "handoff_plus_first_decode_ms": 582.939636, "handoff_plus_first_decode_ms_q25": 580.6943795, "handoff_plus_first_decode_ms_q75": 628.354243, "steady_tpot_ms": 509.05022850000006, "steady_tpot_ms_q25": 509.02136825, "steady_tpot_ms_q75": 509.18104575, "steady_tpot_p95_ms": 509.9949101999999, "steady_tpot_p95_ms_q25": 509.78006782499995, "steady_tpot_p95_ms_q75": 510.52839697499996, "decode_tokens_per_s": 1.9455825050764004, "decode_tokens_per_s_q25": 1.9337595012668871, "decode_tokens_per_s_q75": 1.945695054301694, "decode_ms": 7709.773274000001, "decode_ms_q25": 7709.3273260000005, "decode_ms_q75": 7757.2007925, "end_to_end_ms": 9311.94707, "end_to_end_ms_q25": 9302.89409, "end_to_end_ms_q75": 9314.689857000001, "output_tokens_per_s_e2e": 1.7182228249070148, "output_tokens_per_s_e2e_q25": 1.7177170290072943, "output_tokens_per_s_e2e_q75": 1.7198965179135255, "all_tokens_per_s_e2e": 439.8650431761958, "all_tokens_per_s_e2e_q25": 439.73555942586734, "all_tokens_per_s_e2e_q75": 440.2935085858625, "energy_net_psys_j": 151.66995564329582, "energy_net_psys_j_q25": 151.24584492043118, "energy_net_psys_j_q75": 151.69562814205975, "prefill_energy_net_psys_j": 51.26185070605367, "prefill_energy_net_psys_j_q25": 49.51459644977226, "prefill_energy_net_psys_j_q75": 51.29657245381291, "decode_energy_net_psys_j": 100.39000643925154, "decode_energy_net_psys_j_q25": 99.97494496538219, "decode_energy_net_psys_j_q75": 102.14630994452824, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.21514514557909745, "energy_reduction_vs_gpu": -1.3175031671846815 }, { "precision": "INT8", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 4096, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 4026.3601529999996, "ttft_ms_q25": 4012.7952075, "ttft_ms_q75": 4027.595775, "prefill_tokens_per_s": 1013.3221681523034, "prefill_tokens_per_s_q25": 1013.0113874219435, "prefill_tokens_per_s_q75": 1016.7592445606155, "handoff_plus_first_decode_ms": 582.939636, "handoff_plus_first_decode_ms_q25": 580.6943795, "handoff_plus_first_decode_ms_q75": 628.354243, "steady_tpot_ms": 509.05022850000006, "steady_tpot_ms_q25": 509.02136825, "steady_tpot_ms_q75": 509.18104575, "steady_tpot_p95_ms": 509.9949101999999, "steady_tpot_p95_ms_q25": 509.78006782499995, "steady_tpot_p95_ms_q75": 510.52839697499996, "decode_tokens_per_s": 1.9455825050764004, "decode_tokens_per_s_q25": 1.9337595012668871, "decode_tokens_per_s_q75": 1.945695054301694, "decode_ms": 7709.773274000001, "decode_ms_q25": 7709.3273260000005, "decode_ms_q75": 7757.2007925, "end_to_end_ms": 11738.604671000001, "end_to_end_ms_q25": 11736.923101, "end_to_end_ms_q75": 11771.231622, "output_tokens_per_s_e2e": 1.363024009107973, "output_tokens_per_s_e2e_q25": 1.3592564853544513, "output_tokens_per_s_e2e_q75": 1.3632193199822265, "all_tokens_per_s_e2e": 348.9341463316411, "all_tokens_per_s_e2e_q25": 347.96966025073954, "all_tokens_per_s_e2e_q75": 348.98414591545, "energy_net_psys_j": 166.0139304845643, "energy_net_psys_j_q25": 165.5819915875751, "energy_net_psys_j_q75": 170.67634794726177, "prefill_energy_net_psys_j": 66.45404699305145, "prefill_energy_net_psys_j_q25": 65.6070466221929, "prefill_energy_net_psys_j_q75": 68.94509947660288, "decode_energy_net_psys_j": 100.39000643925154, "decode_energy_net_psys_j_q25": 99.97494496538219, "decode_energy_net_psys_j_q75": 102.14630994452824, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.17066936523975557, "energy_reduction_vs_gpu": -1.5366778018950495 }, { "precision": "INT8", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 4096, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 4026.3601529999996, "ttft_ms_q25": 4012.7952075, "ttft_ms_q75": 4027.595775, "prefill_tokens_per_s": 1013.3221681523034, "prefill_tokens_per_s_q25": 1013.0113874219435, "prefill_tokens_per_s_q75": 1016.7592445606155, "handoff_plus_first_decode_ms": 43.987549, "handoff_plus_first_decode_ms_q25": 43.735888, "handoff_plus_first_decode_ms_q75": 50.556247, "steady_tpot_ms": 25.1619285, "steady_tpot_ms_q25": 25.134072500000002, "steady_tpot_ms_q75": 25.167222, "steady_tpot_p95_ms": 27.58854755, "steady_tpot_p95_ms_q25": 27.237907925, "steady_tpot_p95_ms_q75": 27.624445225000002, "decode_tokens_per_s": 37.30816477446453, "decode_tokens_per_s_q25": 36.65430131309493, "decode_tokens_per_s_q75": 37.38747916500209, "decode_ms": 402.05676400000016, "decode_ms_q25": 401.20564000000013, "decode_ms_q75": 409.3591815000001, "end_to_end_ms": 4426.714669, "end_to_end_ms_q25": 4421.3032650000005, "end_to_end_ms_q75": 4428.801415, "output_tokens_per_s_e2e": 3.614418636928867, "output_tokens_per_s_e2e_q25": 3.6127164107929595, "output_tokens_per_s_e2e_q75": 3.6188478847649135, "all_tokens_per_s_e2e": 925.29117105379, "all_tokens_per_s_e2e_q25": 924.8554011629976, "all_tokens_per_s_e2e_q75": 926.4250584998179, "energy_net_psys_j": 80.6376108999778, "energy_net_psys_j_q25": 79.81235082192745, "energy_net_psys_j_q75": 83.64182275948609, "prefill_energy_net_psys_j": 66.45404699305145, "prefill_energy_net_psys_j_q25": 65.6070466221929, "prefill_energy_net_psys_j_q75": 68.94509947660288, "decode_energy_net_psys_j": 14.227044492542767, "decode_energy_net_psys_j_q25": 14.205304199734556, "decode_energy_net_psys_j_q75": 14.71846357569143, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.4525749585871942, "energy_reduction_vs_gpu": -0.2321353814753686 }, { "precision": "INT8", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 8192, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 4140.387078, "ttft_ms_q25": 4019.2779899999996, "ttft_ms_q75": 4158.200718, "prefill_tokens_per_s": 1974.6945988319019, "prefill_tokens_per_s_q25": 1966.2711367271445, "prefill_tokens_per_s_q75": 2036.0448040065307, "handoff_plus_first_decode_ms": 61.63259, "handoff_plus_first_decode_ms_q25": 60.214765, "handoff_plus_first_decode_ms_q75": 65.793271, "steady_tpot_ms": 29.060207000000002, "steady_tpot_ms_q25": 28.982426500000003, "steady_tpot_ms_q75": 30.7737385, "steady_tpot_p95_ms": 32.368322549999995, "steady_tpot_p95_ms_q25": 31.973692224999994, "steady_tpot_p95_ms_q75": 34.5626762, "decode_tokens_per_s": 31.061302329504077, "decode_tokens_per_s_q25": 30.204059251599833, "decode_tokens_per_s_q75": 31.43984592713797, "decode_ms": 482.91600400000016, "decode_ms_q25": 477.1707490000001, "decode_ms_q75": 497.02235200000064, "end_to_end_ms": 4611.812572, "end_to_end_ms_q25": 4496.4487389999995, "end_to_end_ms_q75": 4649.477815, "output_tokens_per_s_e2e": 3.4693517462400467, "output_tokens_per_s_e2e_q25": 3.4414725082996616, "output_tokens_per_s_e2e_q75": 3.5607075916662243, "all_tokens_per_s_e2e": 1776.308094074904, "all_tokens_per_s_e2e_q25": 1762.0339242494267, "all_tokens_per_s_e2e_q75": 1823.0822869331068, "energy_net_psys_j": 147.17802031892046, "energy_net_psys_j_q25": 145.08884473365845, "energy_net_psys_j_q75": 159.2070540992779, "prefill_energy_net_psys_j": 130.81313758436497, "prefill_energy_net_psys_j_q25": 128.31996768710655, "prefill_energy_net_psys_j_q75": 141.7355192197505, "decode_energy_net_psys_j": 17.172871358548306, "decode_energy_net_psys_j_q25": 16.7688770465519, "decode_energy_net_psys_j_q75": 17.875529191523803, "fastest_e2e": true, "lowest_energy": true, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "INT8", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 8192, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 4140.387078, "ttft_ms_q25": 4019.2779899999996, "ttft_ms_q75": 4158.200718, "prefill_tokens_per_s": 1974.6945988319019, "prefill_tokens_per_s_q25": 1966.2711367271445, "prefill_tokens_per_s_q75": 2036.0448040065307, "handoff_plus_first_decode_ms": 611.2510540000001, "handoff_plus_first_decode_ms_q25": 596.9570105, "handoff_plus_first_decode_ms_q75": 660.72153, "steady_tpot_ms": 515.2153075, "steady_tpot_ms_q25": 515.1769075, "steady_tpot_ms_q75": 515.3487475, "steady_tpot_p95_ms": 515.8123436500001, "steady_tpot_p95_ms_q25": 515.7713989000001, "steady_tpot_p95_ms_q75": 516.6305286250001, "decode_tokens_per_s": 1.9172556261856764, "decode_tokens_per_s_q25": 1.9042417474267752, "decode_tokens_per_s_q75": 1.9251361950512125, "decode_ms": 7823.682869999999, "decode_ms_q25": 7791.787093, "decode_ms_q75": 7877.519037499999, "end_to_end_ms": 11935.905674000001, "end_to_end_ms_q25": 11882.7148905, "end_to_end_ms_q75": 11949.987810999999, "output_tokens_per_s_e2e": 1.3404931671714548, "output_tokens_per_s_e2e_q25": 1.3389153589080698, "output_tokens_per_s_e2e_q75": 1.3465206185882077, "all_tokens_per_s_e2e": 686.3325015917849, "all_tokens_per_s_e2e_q25": 685.5246637609317, "all_tokens_per_s_e2e_q75": 689.4185567171623, "energy_net_psys_j": 231.65250603027113, "energy_net_psys_j_q25": 231.2544132075988, "energy_net_psys_j_q75": 245.3501960925922, "prefill_energy_net_psys_j": 130.81313758436497, "prefill_energy_net_psys_j_q25": 128.31996768710655, "prefill_energy_net_psys_j_q75": 141.7355192197505, "decode_energy_net_psys_j": 105.0295225950783, "decode_energy_net_psys_j_q25": 102.93444552049223, "decode_energy_net_psys_j_q75": 105.70975394742779, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.3863814525650883, "energy_reduction_vs_gpu": -0.5739612853081097 }, { "precision": "INT8", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 8192, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 8989.586643999999, "ttft_ms_q25": 8978.298316999999, "ttft_ms_q75": 9025.349912999998, "prefill_tokens_per_s": 909.4967681808797, "prefill_tokens_per_s_q25": 905.9070792627788, "prefill_tokens_per_s_q75": 910.6417091295481, "handoff_plus_first_decode_ms": 611.2510540000001, "handoff_plus_first_decode_ms_q25": 596.9570105, "handoff_plus_first_decode_ms_q75": 660.72153, "steady_tpot_ms": 515.2153075, "steady_tpot_ms_q25": 515.1769075, "steady_tpot_ms_q75": 515.3487475, "steady_tpot_p95_ms": 515.8123436500001, "steady_tpot_p95_ms_q25": 515.7713989000001, "steady_tpot_p95_ms_q75": 516.6305286250001, "decode_tokens_per_s": 1.9172556261856764, "decode_tokens_per_s_q25": 1.9042417474267752, "decode_tokens_per_s_q75": 1.9251361950512125, "decode_ms": 7823.682869999999, "decode_ms_q25": 7791.787093, "decode_ms_q75": 7877.519037499999, "end_to_end_ms": 16884.796051999998, "end_to_end_ms_q25": 16805.848679, "end_to_end_ms_q75": 16902.8689505, "output_tokens_per_s_e2e": 0.9475980610440838, "output_tokens_per_s_e2e_q25": 0.9465859516869999, "output_tokens_per_s_e2e_q75": 0.9520705198821406, "all_tokens_per_s_e2e": 485.1702072545709, "all_tokens_per_s_e2e_q25": 484.65200726374394, "all_tokens_per_s_e2e_q75": 487.460106179656, "energy_net_psys_j": 262.7768533889938, "energy_net_psys_j_q25": 262.45406954808317, "energy_net_psys_j_q75": 267.7749618664778, "prefill_energy_net_psys_j": 161.29191726126638, "prefill_energy_net_psys_j_q25": 159.51962402759094, "prefill_energy_net_psys_j_q75": 163.83750115272545, "decode_energy_net_psys_j": 105.0295225950783, "decode_energy_net_psys_j_q25": 102.93444552049223, "decode_energy_net_psys_j_q75": 105.70975394742779, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.27313404069537056, "energy_reduction_vs_gpu": -0.7854354394737877 }, { "precision": "INT8", "model": "qwen25_1_5b", "model_label": "Qwen2.5 1.5B", "family": "Qwen2.5", "size_b": 1.5, "context_length": 8192, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 8989.586643999999, "ttft_ms_q25": 8978.298316999999, "ttft_ms_q75": 9025.349912999998, "prefill_tokens_per_s": 909.4967681808797, "prefill_tokens_per_s_q25": 905.9070792627788, "prefill_tokens_per_s_q75": 910.6417091295481, "handoff_plus_first_decode_ms": 61.63259, "handoff_plus_first_decode_ms_q25": 60.214765, "handoff_plus_first_decode_ms_q75": 65.793271, "steady_tpot_ms": 29.060207000000002, "steady_tpot_ms_q25": 28.982426500000003, "steady_tpot_ms_q75": 30.7737385, "steady_tpot_p95_ms": 32.368322549999995, "steady_tpot_p95_ms_q25": 31.973692224999994, "steady_tpot_p95_ms_q75": 34.5626762, "decode_tokens_per_s": 31.061302329504077, "decode_tokens_per_s_q25": 30.204059251599833, "decode_tokens_per_s_q75": 31.43984592713797, "decode_ms": 482.91600400000016, "decode_ms_q25": 477.1707490000001, "decode_ms_q75": 497.02235200000064, "end_to_end_ms": 9478.13869, "end_to_end_ms_q25": 9475.320669, "end_to_end_ms_q75": 9505.338683, "output_tokens_per_s_e2e": 1.6880951548937506, "output_tokens_per_s_e2e_q25": 1.6832783712480628, "output_tokens_per_s_e2e_q75": 1.6885973545524116, "all_tokens_per_s_e2e": 864.3047193056003, "all_tokens_per_s_e2e_q25": 861.8385260790081, "all_tokens_per_s_e2e_q75": 864.5618455308347, "energy_net_psys_j": 177.65679999582187, "energy_net_psys_j_q25": 176.28850107414283, "energy_net_psys_j_q75": 181.30903603225283, "prefill_energy_net_psys_j": 161.29191726126638, "prefill_energy_net_psys_j_q25": 159.51962402759094, "prefill_energy_net_psys_j_q75": 163.83750115272545, "decode_energy_net_psys_j": 17.172871358548306, "decode_energy_net_psys_j_q25": 16.7688770465519, "decode_energy_net_psys_j_q75": 17.875529191523803, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.48657365362945537, "energy_reduction_vs_gpu": -0.20708784919688994 }, { "precision": "INT8", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 1024, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 489.558879, "ttft_ms_q25": 472.72711449999997, "ttft_ms_q75": 495.3317935, "prefill_tokens_per_s": 2058.9964624050867, "prefill_tokens_per_s_q25": 2035.2760495981352, "prefill_tokens_per_s_q75": 2135.0151003699957, "handoff_plus_first_decode_ms": 52.612127, "handoff_plus_first_decode_ms_q25": 52.0652285, "handoff_plus_first_decode_ms_q75": 59.6678235, "steady_tpot_ms": 43.007611000000004, "steady_tpot_ms_q25": 42.886047250000004, "steady_tpot_ms_q75": 43.379581, "steady_tpot_p95_ms": 45.50781405, "steady_tpot_p95_ms_q25": 45.48998077500001, "steady_tpot_p95_ms_q75": 45.900485325, "decode_tokens_per_s": 22.38918474131744, "decode_tokens_per_s_q25": 22.30778113768578, "decode_tokens_per_s_q75": 22.55167896479459, "decode_ms": 669.9663330000003, "decode_ms_q25": 665.1734815000002, "decode_ms_q75": 672.4200695000001, "end_to_end_ms": 1159.5252120000002, "end_to_end_ms_q25": 1145.147184, "end_to_end_ms_q75": 1160.505275, "output_tokens_per_s_e2e": 13.798751277173608, "output_tokens_per_s_e2e_q25": 13.78710787098456, "output_tokens_per_s_e2e_q75": 13.974206038364706, "all_tokens_per_s_e2e": 883.1200817391109, "all_tokens_per_s_e2e_q25": 882.3749037430118, "all_tokens_per_s_e2e_q75": 894.3491864553412, "energy_net_psys_j": 39.81794016999228, "energy_net_psys_j_q25": 39.57505361206672, "energy_net_psys_j_q75": 39.917907369158414, "prefill_energy_net_psys_j": 16.575855736145257, "prefill_energy_net_psys_j_q25": 16.144735711557786, "prefill_energy_net_psys_j_q75": 16.611346368933763, "decode_energy_net_psys_j": 23.171103168270015, "decode_energy_net_psys_j_q25": 22.963707243132955, "decode_energy_net_psys_j_q75": 23.73768102481212, "fastest_e2e": true, "lowest_energy": true, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "INT8", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 1024, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 489.558879, "ttft_ms_q25": 472.72711449999997, "ttft_ms_q75": 495.3317935, "prefill_tokens_per_s": 2058.9964624050867, "prefill_tokens_per_s_q25": 2035.2760495981352, "prefill_tokens_per_s_q75": 2135.0151003699957, "handoff_plus_first_decode_ms": 1343.034015, "handoff_plus_first_decode_ms_q25": 1335.152375, "handoff_plus_first_decode_ms_q75": 1489.3587885000002, "steady_tpot_ms": 1336.6938610000002, "steady_tpot_ms_q25": 1330.15157425, "steady_tpot_ms_q75": 1336.81788, "steady_tpot_p95_ms": 1338.9391349, "steady_tpot_p95_ms_q25": 1338.3699532, "steady_tpot_p95_ms_q75": 1340.4359989499999, "decode_tokens_per_s": 0.7489140064645603, "decode_tokens_per_s_q25": 0.7435102330863657, "decode_tokens_per_s_q75": 0.7512735981285256, "decode_ms": 20029.001822000002, "decode_ms_q25": 19966.291917, "decode_ms_q75": 20175.636749, "end_to_end_ms": 20518.560701000002, "end_to_end_ms_q25": 20461.623710499996, "end_to_end_ms_q75": 20648.363863500002, "output_tokens_per_s_e2e": 0.779781790406976, "output_tokens_per_s_e2e_q25": 0.7749104202650343, "output_tokens_per_s_e2e_q75": 0.7819576840632585, "all_tokens_per_s_e2e": 49.90603458604647, "all_tokens_per_s_e2e_q25": 49.594266896962196, "all_tokens_per_s_e2e_q75": 50.045291780048544, "energy_net_psys_j": 247.05295659878448, "energy_net_psys_j_q25": 239.63122704771592, "energy_net_psys_j_q75": 248.3044201335482, "prefill_energy_net_psys_j": 16.575855736145257, "prefill_energy_net_psys_j_q25": 16.144735711557786, "prefill_energy_net_psys_j_q75": 16.611346368933763, "decode_energy_net_psys_j": 230.47710086263922, "decode_energy_net_psys_j_q25": 223.01988067878216, "decode_energy_net_psys_j_q75": 232.15968442199042, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.056511040364711794, "energy_reduction_vs_gpu": -5.2045639614720525 }, { "precision": "INT8", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 1024, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1940.663878, "ttft_ms_q25": 1939.8918695, "ttft_ms_q75": 1941.3668505, "prefill_tokens_per_s": 519.4098841262609, "prefill_tokens_per_s_q25": 519.2218729330253, "prefill_tokens_per_s_q75": 519.6166732235179, "handoff_plus_first_decode_ms": 1343.034015, "handoff_plus_first_decode_ms_q25": 1335.152375, "handoff_plus_first_decode_ms_q75": 1489.3587885000002, "steady_tpot_ms": 1336.6938610000002, "steady_tpot_ms_q25": 1330.15157425, "steady_tpot_ms_q75": 1336.81788, "steady_tpot_p95_ms": 1338.9391349, "steady_tpot_p95_ms_q25": 1338.3699532, "steady_tpot_p95_ms_q75": 1340.4359989499999, "decode_tokens_per_s": 0.7489140064645603, "decode_tokens_per_s_q25": 0.7435102330863657, "decode_tokens_per_s_q75": 0.7512735981285256, "decode_ms": 20029.001822000002, "decode_ms_q25": 19966.291917, "decode_ms_q75": 20175.636749, "end_to_end_ms": 21968.121683, "end_to_end_ms_q25": 21906.183786499998, "end_to_end_ms_q75": 22116.231591000003, "output_tokens_per_s_e2e": 0.7283280851626736, "output_tokens_per_s_e2e_q25": 0.7234830011213556, "output_tokens_per_s_e2e_q75": 0.7303932106782905, "all_tokens_per_s_e2e": 46.61299745041111, "all_tokens_per_s_e2e_q25": 46.302912071766755, "all_tokens_per_s_e2e_q75": 46.745165483410595, "energy_net_psys_j": 263.66322710277404, "energy_net_psys_j_q25": 256.05902519237173, "energy_net_psys_j_q75": 265.27150792736074, "prefill_energy_net_psys_j": 33.037520770605816, "prefill_energy_net_psys_j_q25": 32.964841778825075, "prefill_energy_net_psys_j_q75": 33.111823505370324, "decode_energy_net_psys_j": 230.47710086263922, "decode_energy_net_psys_j_q25": 223.01988067878216, "decode_energy_net_psys_j_q75": 232.15968442199042, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.05278217358461271, "energy_reduction_vs_gpu": -5.621719405301551 }, { "precision": "INT8", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 1024, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1940.663878, "ttft_ms_q25": 1939.8918695, "ttft_ms_q75": 1941.3668505, "prefill_tokens_per_s": 519.4098841262609, "prefill_tokens_per_s_q25": 519.2218729330253, "prefill_tokens_per_s_q75": 519.6166732235179, "handoff_plus_first_decode_ms": 52.612127, "handoff_plus_first_decode_ms_q25": 52.0652285, "handoff_plus_first_decode_ms_q75": 59.6678235, "steady_tpot_ms": 43.007611000000004, "steady_tpot_ms_q25": 42.886047250000004, "steady_tpot_ms_q75": 43.379581, "steady_tpot_p95_ms": 45.50781405, "steady_tpot_p95_ms_q25": 45.48998077500001, "steady_tpot_p95_ms_q75": 45.900485325, "decode_tokens_per_s": 22.38918474131744, "decode_tokens_per_s_q25": 22.30778113768578, "decode_tokens_per_s_q75": 22.55167896479459, "decode_ms": 669.9663330000003, "decode_ms_q25": 665.1734815000002, "decode_ms_q75": 672.4200695000001, "end_to_end_ms": 2609.0861940000004, "end_to_end_ms_q25": 2605.065351, "end_to_end_ms_q75": 2613.0149115000004, "output_tokens_per_s_e2e": 6.132415263548781, "output_tokens_per_s_e2e_q25": 6.123208902778842, "output_tokens_per_s_e2e_q75": 6.141895100940912, "all_tokens_per_s_e2e": 392.474576867122, "all_tokens_per_s_e2e_q25": 391.8853697778459, "all_tokens_per_s_e2e_q75": 393.0812864602184, "energy_net_psys_j": 56.06326595531434, "energy_net_psys_j_q25": 56.002851756722535, "energy_net_psys_j_q75": 56.702522803637194, "prefill_energy_net_psys_j": 33.037520770605816, "prefill_energy_net_psys_j_q25": 32.964841778825075, "prefill_energy_net_psys_j_q75": 33.111823505370324, "decode_energy_net_psys_j": 23.171103168270015, "decode_energy_net_psys_j_q25": 22.963707243132955, "decode_energy_net_psys_j_q75": 23.73768102481212, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.4444181317836524, "energy_reduction_vs_gpu": -0.4079901098843106 }, { "precision": "INT8", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 2048, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1066.727625, "ttft_ms_q25": 1033.1412235, "ttft_ms_q75": 1073.7445535, "prefill_tokens_per_s": 1904.8911384478301, "prefill_tokens_per_s_q25": 1892.5234836586933, "prefill_tokens_per_s_q75": 1968.8980715159805, "handoff_plus_first_decode_ms": 62.479971, "handoff_plus_first_decode_ms_q25": 61.4363465, "handoff_plus_first_decode_ms_q75": 69.4866055, "steady_tpot_ms": 49.103414, "steady_tpot_ms_q25": 48.90559575, "steady_tpot_ms_q75": 49.2539075, "steady_tpot_p95_ms": 51.85995685, "steady_tpot_p95_ms_q25": 51.31604597500001, "steady_tpot_p95_ms_q75": 52.1106489, "decode_tokens_per_s": 19.680266234011846, "decode_tokens_per_s_q25": 19.67672433241007, "decode_tokens_per_s_q75": 19.762366803479843, "decode_ms": 762.1848110000001, "decode_ms_q25": 759.0314985, "decode_ms_q75": 762.3220325, "end_to_end_ms": 1822.605811, "end_to_end_ms_q25": 1792.3099435, "end_to_end_ms_q75": 1832.776052, "output_tokens_per_s_e2e": 8.778639848196995, "output_tokens_per_s_e2e_q25": 8.730195201032029, "output_tokens_per_s_e2e_q75": 8.929578772745742, "all_tokens_per_s_e2e": 1123.6659005692154, "all_tokens_per_s_e2e_q25": 1117.4649857320996, "all_tokens_per_s_e2e_q75": 1142.986082911455, "energy_net_psys_j": 59.92964746310727, "energy_net_psys_j_q25": 58.32317898069759, "energy_net_psys_j_q75": 60.48587625661966, "prefill_energy_net_psys_j": 36.1566927687641, "prefill_energy_net_psys_j_q25": 34.25054310827647, "prefill_energy_net_psys_j_q75": 36.174813476497434, "decode_energy_net_psys_j": 24.372317050499085, "decode_energy_net_psys_j_q25": 24.054515164687796, "decode_energy_net_psys_j_q75": 24.62886466593352, "fastest_e2e": true, "lowest_energy": true, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "INT8", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 2048, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1066.727625, "ttft_ms_q25": 1033.1412235, "ttft_ms_q75": 1073.7445535, "prefill_tokens_per_s": 1904.8911384478301, "prefill_tokens_per_s_q25": 1892.5234836586933, "prefill_tokens_per_s_q75": 1968.8980715159805, "handoff_plus_first_decode_ms": 1375.780532, "handoff_plus_first_decode_ms_q25": 1375.4221160000002, "handoff_plus_first_decode_ms_q75": 1451.828402, "steady_tpot_ms": 1336.4961575, "steady_tpot_ms_q25": 1336.03517975, "steady_tpot_ms_q75": 1337.2230605, "steady_tpot_p95_ms": 1337.7854578, "steady_tpot_p95_ms_q25": 1337.550065375, "steady_tpot_p95_ms_q75": 1340.5493260499998, "decode_tokens_per_s": 0.7468572382967064, "decode_tokens_per_s_q25": 0.743551630479671, "decode_tokens_per_s_q75": 0.7475004271214611, "decode_ms": 20084.159637, "decode_ms_q25": 20066.893023, "decode_ms_q75": 20173.8465155, "end_to_end_ms": 21164.921119, "end_to_end_ms_q25": 21140.637576499998, "end_to_end_ms_q75": 21214.0046675, "output_tokens_per_s_e2e": 0.7559678540751381, "output_tokens_per_s_e2e_q25": 0.7542227837235032, "output_tokens_per_s_e2e_q75": 0.7568372076332148, "all_tokens_per_s_e2e": 96.76388532161768, "all_tokens_per_s_e2e_q25": 96.5405163166084, "all_tokens_per_s_e2e_q75": 96.8751625770515, "energy_net_psys_j": 270.77864527522553, "energy_net_psys_j_q25": 270.6503646776142, "energy_net_psys_j_q75": 272.11536314097793, "prefill_energy_net_psys_j": 36.1566927687641, "prefill_energy_net_psys_j_q25": 34.25054310827647, "prefill_energy_net_psys_j_q75": 36.174813476497434, "decode_energy_net_psys_j": 234.6219525064614, "decode_energy_net_psys_j_q25": 234.47555120111676, "decode_energy_net_psys_j_q75": 237.8648200327014, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.08611446273540917, "energy_reduction_vs_gpu": -3.5182752900710295 }, { "precision": "INT8", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 2048, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 3904.601155, "ttft_ms_q25": 3901.976305, "ttft_ms_q75": 3908.4971225, "prefill_tokens_per_s": 520.4116680132468, "prefill_tokens_per_s_q25": 519.8934412148244, "prefill_tokens_per_s_q75": 520.7619833368458, "handoff_plus_first_decode_ms": 1375.780532, "handoff_plus_first_decode_ms_q25": 1375.4221160000002, "handoff_plus_first_decode_ms_q75": 1451.828402, "steady_tpot_ms": 1336.4961575, "steady_tpot_ms_q25": 1336.03517975, "steady_tpot_ms_q75": 1337.2230605, "steady_tpot_p95_ms": 1337.7854578, "steady_tpot_p95_ms_q25": 1337.550065375, "steady_tpot_p95_ms_q75": 1340.5493260499998, "decode_tokens_per_s": 0.7468572382967064, "decode_tokens_per_s_q25": 0.743551630479671, "decode_tokens_per_s_q75": 0.7475004271214611, "decode_ms": 20084.159637, "decode_ms_q25": 20066.893023, "decode_ms_q75": 20173.8465155, "end_to_end_ms": 23996.552727000002, "end_to_end_ms_q25": 23975.3901455, "end_to_end_ms_q75": 24079.718788000002, "output_tokens_per_s_e2e": 0.666762438006248, "output_tokens_per_s_e2e_q25": 0.6644675131271092, "output_tokens_per_s_e2e_q75": 0.667351495382352, "all_tokens_per_s_e2e": 85.34559206479975, "all_tokens_per_s_e2e_q25": 85.05184168026997, "all_tokens_per_s_e2e_q75": 85.42099140894105, "energy_net_psys_j": 303.472746758231, "energy_net_psys_j_q25": 303.4300304842504, "energy_net_psys_j_q75": 306.82081674932994, "prefill_energy_net_psys_j": 69.05816431449767, "prefill_energy_net_psys_j_q25": 68.95447928313362, "prefill_energy_net_psys_j_q75": 69.05968174799257, "decode_energy_net_psys_j": 234.6219525064614, "decode_energy_net_psys_j_q25": 234.47555120111676, "decode_energy_net_psys_j_q75": 237.8648200327014, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.07595281837916967, "energy_reduction_vs_gpu": -4.063816651767374 }, { "precision": "INT8", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 2048, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 3904.601155, "ttft_ms_q25": 3901.976305, "ttft_ms_q75": 3908.4971225, "prefill_tokens_per_s": 520.4116680132468, "prefill_tokens_per_s_q25": 519.8934412148244, "prefill_tokens_per_s_q75": 520.7619833368458, "handoff_plus_first_decode_ms": 62.479971, "handoff_plus_first_decode_ms_q25": 61.4363465, "handoff_plus_first_decode_ms_q75": 69.4866055, "steady_tpot_ms": 49.103414, "steady_tpot_ms_q25": 48.90559575, "steady_tpot_ms_q75": 49.2539075, "steady_tpot_p95_ms": 51.85995685, "steady_tpot_p95_ms_q25": 51.31604597500001, "steady_tpot_p95_ms_q75": 52.1106489, "decode_tokens_per_s": 19.680266234011846, "decode_tokens_per_s_q25": 19.67672433241007, "decode_tokens_per_s_q75": 19.762366803479843, "decode_ms": 762.1848110000001, "decode_ms_q25": 759.0314985, "decode_ms_q75": 762.3220325, "end_to_end_ms": 4661.810708999999, "end_to_end_ms_q25": 4661.145025, "end_to_end_ms_q75": 4668.194305, "output_tokens_per_s_e2e": 3.432142787160087, "output_tokens_per_s_e2e_q25": 3.4274558585364656, "output_tokens_per_s_e2e_q75": 3.4326330205399516, "all_tokens_per_s_e2e": 439.31427675649115, "all_tokens_per_s_e2e_q25": 438.7143498926676, "all_tokens_per_s_e2e_q75": 439.3770266291138, "energy_net_psys_j": 93.43351623198654, "energy_net_psys_j_q25": 93.11419691268037, "energy_net_psys_j_q75": 93.58486138256204, "prefill_energy_net_psys_j": 69.05816431449767, "prefill_energy_net_psys_j_q25": 68.95447928313362, "prefill_energy_net_psys_j_q75": 69.05968174799257, "decode_energy_net_psys_j": 24.372317050499085, "decode_energy_net_psys_j_q25": 24.054515164687796, "decode_energy_net_psys_j_q75": 24.62886466593352, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.39096521175373194, "energy_reduction_vs_gpu": -0.5590533264775881 }, { "precision": "INT8", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 4096, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 2492.958318, "ttft_ms_q25": 2438.228104, "ttft_ms_q75": 2499.2266405, "prefill_tokens_per_s": 1636.6097942917954, "prefill_tokens_per_s_q25": 1632.515274786158, "prefill_tokens_per_s_q75": 1674.189858722264, "handoff_plus_first_decode_ms": 65.289453, "handoff_plus_first_decode_ms_q25": 63.5090885, "handoff_plus_first_decode_ms_q75": 70.8923875, "steady_tpot_ms": 46.4175805, "steady_tpot_ms_q25": 46.3737525, "steady_tpot_ms_q75": 46.54891725, "steady_tpot_p95_ms": 48.02061165, "steady_tpot_p95_ms_q25": 47.922808375, "steady_tpot_p95_ms_q75": 48.0672597, "decode_tokens_per_s": 20.91488730532362, "decode_tokens_per_s_q25": 20.683897259594175, "decode_tokens_per_s_q75": 20.93192171329561, "decode_ms": 717.192485, "decode_ms_q25": 716.609308, "decode_ms_q75": 725.2922785000001, "end_to_end_ms": 3210.150803, "end_to_end_ms_q25": 3163.5203825, "end_to_end_ms_q75": 3215.8359485, "output_tokens_per_s_e2e": 4.984189523136244, "output_tokens_per_s_e2e_q25": 4.9753937270474395, "output_tokens_per_s_e2e_q75": 5.058755791483557, "all_tokens_per_s_e2e": 1275.9525179228785, "all_tokens_per_s_e2e_q25": 1273.7007941241445, "all_tokens_per_s_e2e_q75": 1295.0414826197905, "energy_net_psys_j": 112.90325488563047, "energy_net_psys_j_q25": 110.92819247846842, "energy_net_psys_j_q75": 113.06037754803806, "prefill_energy_net_psys_j": 87.98267846640633, "prefill_energy_net_psys_j_q25": 85.82492177695964, "prefill_energy_net_psys_j_q75": 88.11796715350407, "decode_energy_net_psys_j": 24.96424436984384, "decode_energy_net_psys_j_q25": 24.942410394533994, "decode_energy_net_psys_j_q75": 25.125104676818623, "fastest_e2e": true, "lowest_energy": true, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "INT8", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 4096, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 2492.958318, "ttft_ms_q25": 2438.228104, "ttft_ms_q75": 2499.2266405, "prefill_tokens_per_s": 1636.6097942917954, "prefill_tokens_per_s_q25": 1632.515274786158, "prefill_tokens_per_s_q75": 1674.189858722264, "handoff_plus_first_decode_ms": 1415.829474, "handoff_plus_first_decode_ms_q25": 1414.501694, "handoff_plus_first_decode_ms_q75": 1490.9803935, "steady_tpot_ms": 1338.565505, "steady_tpot_ms_q25": 1336.10523575, "steady_tpot_ms_q75": 1339.1301762500002, "steady_tpot_p95_ms": 1341.0676509999998, "steady_tpot_p95_ms_q25": 1340.696435675, "steady_tpot_p95_ms_q75": 1341.1681983999997, "decode_tokens_per_s": 0.7437284279239131, "decode_tokens_per_s_q25": 0.7411313284209509, "decode_tokens_per_s_q75": 0.7459565858680587, "decode_ms": 20168.652208000003, "decode_ms_q25": 20108.5882425, "decode_ms_q75": 20239.576472, "end_to_end_ms": 22674.147171000004, "end_to_end_ms_q25": 22607.814883000003, "end_to_end_ms_q75": 22684.0728985, "output_tokens_per_s_e2e": 0.7056494729144138, "output_tokens_per_s_e2e_q25": 0.7053408413464786, "output_tokens_per_s_e2e_q75": 0.7077259709383189, "all_tokens_per_s_e2e": 180.64626506608994, "all_tokens_per_s_e2e_q25": 180.5672553846985, "all_tokens_per_s_e2e_q75": 181.17784856020964, "energy_net_psys_j": 326.3680963287806, "energy_net_psys_j_q25": 318.3110116269386, "energy_net_psys_j_q75": 328.34558197099636, "prefill_energy_net_psys_j": 87.98267846640633, "prefill_energy_net_psys_j_q25": 85.82492177695964, "prefill_energy_net_psys_j_q75": 88.11796715350407, "decode_energy_net_psys_j": 242.0698117726104, "decode_energy_net_psys_j_q25": 232.17053011565034, "decode_energy_net_psys_j_q75": 242.385371506939, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.14157757638204577, "energy_reduction_vs_gpu": -1.8906881086766476 }, { "precision": "INT8", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 4096, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 8360.637991000001, "ttft_ms_q25": 8355.382377500002, "ttft_ms_q75": 8375.669299000001, "prefill_tokens_per_s": 488.00103585300656, "prefill_tokens_per_s_q25": 487.1268187622304, "prefill_tokens_per_s_q75": 488.3081862597645, "handoff_plus_first_decode_ms": 1415.829474, "handoff_plus_first_decode_ms_q25": 1414.501694, "handoff_plus_first_decode_ms_q75": 1490.9803935, "steady_tpot_ms": 1338.565505, "steady_tpot_ms_q25": 1336.10523575, "steady_tpot_ms_q75": 1339.1301762500002, "steady_tpot_p95_ms": 1341.0676509999998, "steady_tpot_p95_ms_q25": 1340.696435675, "steady_tpot_p95_ms_q75": 1341.1681983999997, "decode_tokens_per_s": 0.7437284279239131, "decode_tokens_per_s_q25": 0.7411313284209509, "decode_tokens_per_s_q75": 0.7459565858680587, "decode_ms": 20168.652208000003, "decode_ms_q25": 20108.5882425, "decode_ms_q75": 20239.576472, "end_to_end_ms": 28559.352815000002, "end_to_end_ms_q25": 28479.001928, "end_to_end_ms_q75": 28615.245771, "output_tokens_per_s_e2e": 0.5602367849034887, "output_tokens_per_s_e2e_q25": 0.5591446312131052, "output_tokens_per_s_e2e_q75": 0.5618219138073945, "all_tokens_per_s_e2e": 143.4206169352931, "all_tokens_per_s_e2e_q25": 143.14102559055493, "all_tokens_per_s_e2e_q75": 143.826409934693, "energy_net_psys_j": 388.6761570662737, "energy_net_psys_j_q25": 380.1838561448533, "energy_net_psys_j_q75": 389.2394782311085, "prefill_energy_net_psys_j": 147.73298762333283, "prefill_energy_net_psys_j_q25": 146.8541067241694, "prefill_energy_net_psys_j_q75": 148.5766471940377, "decode_energy_net_psys_j": 242.0698117726104, "decode_energy_net_psys_j_q25": 232.17053011565034, "decode_energy_net_psys_j_q75": 242.385371506939, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.11240278530800453, "energy_reduction_vs_gpu": -2.4425593616410577 }, { "precision": "INT8", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 4096, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 8360.637991000001, "ttft_ms_q25": 8355.382377500002, "ttft_ms_q75": 8375.669299000001, "prefill_tokens_per_s": 488.00103585300656, "prefill_tokens_per_s_q25": 487.1268187622304, "prefill_tokens_per_s_q75": 488.3081862597645, "handoff_plus_first_decode_ms": 65.289453, "handoff_plus_first_decode_ms_q25": 63.5090885, "handoff_plus_first_decode_ms_q75": 70.8923875, "steady_tpot_ms": 46.4175805, "steady_tpot_ms_q25": 46.3737525, "steady_tpot_ms_q75": 46.54891725, "steady_tpot_p95_ms": 48.02061165, "steady_tpot_p95_ms_q25": 47.922808375, "steady_tpot_p95_ms_q75": 48.0672597, "decode_tokens_per_s": 20.91488730532362, "decode_tokens_per_s_q25": 20.683897259594175, "decode_tokens_per_s_q75": 20.93192171329561, "decode_ms": 717.192485, "decode_ms_q25": 716.609308, "decode_ms_q75": 725.2922785000001, "end_to_end_ms": 9094.030063000002, "end_to_end_ms_q25": 9080.674656000001, "end_to_end_ms_q75": 9100.378400500002, "output_tokens_per_s_e2e": 1.7593959871649918, "output_tokens_per_s_e2e_q25": 1.7581695048246289, "output_tokens_per_s_e2e_q75": 1.761987431100899, "all_tokens_per_s_e2e": 450.4053727142379, "all_tokens_per_s_e2e_q25": 450.091393235105, "all_tokens_per_s_e2e_q75": 451.06878236183013, "energy_net_psys_j": 172.69723199317667, "energy_net_psys_j_q25": 171.97921140098805, "energy_net_psys_j_q75": 173.51905758857168, "prefill_energy_net_psys_j": 147.73298762333283, "prefill_energy_net_psys_j_q25": 146.8541067241694, "prefill_energy_net_psys_j_q75": 148.5766471940377, "decode_energy_net_psys_j": 24.96424436984384, "decode_energy_net_psys_j_q25": 24.942410394533994, "decode_energy_net_psys_j_q75": 25.125104676818623, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.35299540256204226, "energy_reduction_vs_gpu": -0.5296036608343728 }, { "precision": "INT8", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 8192, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 6571.898811999999, "ttft_ms_q25": 6458.2163755, "ttft_ms_q75": 6574.1596835, "prefill_tokens_per_s": 1244.08488838431, "prefill_tokens_per_s_q25": 1243.657191273974, "prefill_tokens_per_s_q75": 1266.3766126755118, "handoff_plus_first_decode_ms": 84.755449, "handoff_plus_first_decode_ms_q25": 82.5234705, "handoff_plus_first_decode_ms_q75": 86.53717, "steady_tpot_ms": 51.931095, "steady_tpot_ms_q25": 51.822685750000005, "steady_tpot_ms_q75": 51.97059125, "steady_tpot_p95_ms": 54.00788675, "steady_tpot_p95_ms_q25": 53.69206885, "steady_tpot_p95_ms_q75": 55.277183474999994, "decode_tokens_per_s": 18.343377692067843, "decode_tokens_per_s_q25": 18.31718661497328, "decode_tokens_per_s_q75": 18.36791021039229, "decode_ms": 817.7338030000001, "decode_ms_q25": 816.6430795000006, "decode_ms_q75": 818.9047249999999, "end_to_end_ms": 7387.4511680000005, "end_to_end_ms_q25": 7274.859455, "end_to_end_ms_q75": 7391.973685000001, "output_tokens_per_s_e2e": 2.1658349593303194, "output_tokens_per_s_e2e_q25": 2.164510680218256, "output_tokens_per_s_e2e_q75": 2.199882146189351, "all_tokens_per_s_e2e": 1108.9074991771236, "all_tokens_per_s_e2e_q25": 1108.229468271747, "all_tokens_per_s_e2e_q75": 1126.3396588489477, "energy_net_psys_j": 250.13882033965706, "energy_net_psys_j_q25": 247.2570084883078, "energy_net_psys_j_q75": 251.32788909997032, "prefill_energy_net_psys_j": 223.29775997963776, "prefill_energy_net_psys_j_q25": 220.62440282771084, "prefill_energy_net_psys_j_q75": 223.97924204847368, "decode_energy_net_psys_j": 26.8410603600193, "decode_energy_net_psys_j_q25": 26.63260566059695, "decode_energy_net_psys_j_q75": 27.34864705149664, "fastest_e2e": true, "lowest_energy": true, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "INT8", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 8192, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 6571.898811999999, "ttft_ms_q25": 6458.2163755, "ttft_ms_q75": 6574.1596835, "prefill_tokens_per_s": 1244.08488838431, "prefill_tokens_per_s_q25": 1243.657191273974, "prefill_tokens_per_s_q75": 1266.3766126755118, "handoff_plus_first_decode_ms": 1466.3667899999998, "handoff_plus_first_decode_ms_q25": 1463.1771775, "handoff_plus_first_decode_ms_q75": 1553.9912279999999, "steady_tpot_ms": 1349.0224505, "steady_tpot_ms_q25": 1341.510896, "steady_tpot_ms_q75": 1349.18147275, "steady_tpot_p95_ms": 1350.9115041999999, "steady_tpot_p95_ms_q25": 1350.0441313000001, "steady_tpot_p95_ms_q75": 1350.93285445, "decode_tokens_per_s": 0.738418513627556, "decode_tokens_per_s_q25": 0.737738155377583, "decode_tokens_per_s_q75": 0.7396567073285333, "decode_ms": 20313.683532, "decode_ms_q25": 20279.735026000002, "decode_ms_q75": 20332.434547499997, "end_to_end_ms": 26890.104087, "end_to_end_ms_q25": 26740.212273, "end_to_end_ms_q75": 26906.594230999995, "output_tokens_per_s_e2e": 0.5950144316375178, "output_tokens_per_s_e2e_q25": 0.5946499907167409, "output_tokens_per_s_e2e_q75": 0.5983685767164327, "all_tokens_per_s_e2e": 304.6473889984091, "all_tokens_per_s_e2e_q25": 304.4607952469713, "all_tokens_per_s_e2e_q75": 306.36471127881356, "energy_net_psys_j": 463.83781447519215, "energy_net_psys_j_q25": 452.63270597417926, "energy_net_psys_j_q75": 464.6260754693129, "prefill_energy_net_psys_j": 223.29775997963776, "prefill_energy_net_psys_j_q25": 220.62440282771084, "prefill_energy_net_psys_j_q75": 223.97924204847368, "decode_energy_net_psys_j": 240.5400544955544, "decode_energy_net_psys_j_q25": 232.00830314646842, "decode_energy_net_psys_j_q75": 240.64683342083927, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.27472750362358983, "energy_reduction_vs_gpu": -0.8543215876902224 }, { "precision": "INT8", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 8192, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 18627.111202999997, "ttft_ms_q25": 18620.885055, "ttft_ms_q75": 18627.8946935, "prefill_tokens_per_s": 438.9301116473289, "prefill_tokens_per_s_q25": 438.91165099385455, "prefill_tokens_per_s_q75": 439.0769230313261, "handoff_plus_first_decode_ms": 1466.3667899999998, "handoff_plus_first_decode_ms_q25": 1463.1771775, "handoff_plus_first_decode_ms_q75": 1553.9912279999999, "steady_tpot_ms": 1349.0224505, "steady_tpot_ms_q25": 1341.510896, "steady_tpot_ms_q75": 1349.18147275, "steady_tpot_p95_ms": 1350.9115041999999, "steady_tpot_p95_ms_q25": 1350.0441313000001, "steady_tpot_p95_ms_q75": 1350.93285445, "decode_tokens_per_s": 0.738418513627556, "decode_tokens_per_s_q25": 0.737738155377583, "decode_tokens_per_s_q75": 0.7396567073285333, "decode_ms": 20313.683532, "decode_ms_q25": 20279.735026000002, "decode_ms_q75": 20332.434547499997, "end_to_end_ms": 38940.794734999996, "end_to_end_ms_q25": 38900.620081, "end_to_end_ms_q75": 38960.329241, "output_tokens_per_s_e2e": 0.4108801607384555, "output_tokens_per_s_e2e_q25": 0.4106742508097466, "output_tokens_per_s_e2e_q75": 0.411304936319686, "all_tokens_per_s_e2e": 210.37064229808922, "all_tokens_per_s_e2e_q25": 210.26521641459027, "all_tokens_per_s_e2e_q75": 210.58812739567924, "energy_net_psys_j": 574.7364770321341, "energy_net_psys_j_q25": 560.7616380076554, "energy_net_psys_j_q75": 580.8076200823334, "prefill_energy_net_psys_j": 334.1964225365797, "prefill_energy_net_psys_j_q25": 328.753334861187, "prefill_energy_net_psys_j_q75": 340.16078666149417, "decode_energy_net_psys_j": 240.5400544955544, "decode_energy_net_psys_j_q25": 232.00830314646842, "decode_energy_net_psys_j_q75": 240.64683342083927, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.1897098202097082, "energy_reduction_vs_gpu": -1.2976700547788393 }, { "precision": "INT8", "model": "qwen25_3b", "model_label": "Qwen2.5 3B", "family": "Qwen2.5", "size_b": 3.0, "context_length": 8192, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 18627.111202999997, "ttft_ms_q25": 18620.885055, "ttft_ms_q75": 18627.8946935, "prefill_tokens_per_s": 438.9301116473289, "prefill_tokens_per_s_q25": 438.91165099385455, "prefill_tokens_per_s_q75": 439.0769230313261, "handoff_plus_first_decode_ms": 84.755449, "handoff_plus_first_decode_ms_q25": 82.5234705, "handoff_plus_first_decode_ms_q75": 86.53717, "steady_tpot_ms": 51.931095, "steady_tpot_ms_q25": 51.822685750000005, "steady_tpot_ms_q75": 51.97059125, "steady_tpot_p95_ms": 54.00788675, "steady_tpot_p95_ms_q25": 53.69206885, "steady_tpot_p95_ms_q75": 55.277183474999994, "decode_tokens_per_s": 18.343377692067843, "decode_tokens_per_s_q25": 18.31718661497328, "decode_tokens_per_s_q75": 18.36791021039229, "decode_ms": 817.7338030000001, "decode_ms_q25": 816.6430795000006, "decode_ms_q75": 818.9047249999999, "end_to_end_ms": 19444.23054, "end_to_end_ms_q25": 19438.311625, "end_to_end_ms_q75": 19445.708695, "output_tokens_per_s_e2e": 0.8228661950435813, "output_tokens_per_s_e2e_q25": 0.8228036500703717, "output_tokens_per_s_e2e_q75": 0.8231168319639813, "all_tokens_per_s_e2e": 421.3074918623136, "all_tokens_per_s_e2e_q25": 421.2754688360303, "all_tokens_per_s_e2e_q75": 421.43581796555844, "energy_net_psys_j": 361.037482896599, "energy_net_psys_j_q25": 355.38594052178394, "energy_net_psys_j_q75": 367.5094337129908, "prefill_energy_net_psys_j": 334.1964225365797, "prefill_energy_net_psys_j_q25": 328.753334861187, "prefill_energy_net_psys_j_q75": 340.16078666149417, "decode_energy_net_psys_j": 26.8410603600193, "decode_energy_net_psys_j_q25": 26.63260566059695, "decode_energy_net_psys_j_q75": 27.34864705149664, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.3799302396051513, "energy_reduction_vs_gpu": -0.44334846708861697 }, { "precision": "INT8", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 1024, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 851.337049, "ttft_ms_q25": 838.5334135, "ttft_ms_q75": 863.5570245, "prefill_tokens_per_s": 1184.0198910455265, "prefill_tokens_per_s_q25": 1167.4989113211343, "prefill_tokens_per_s_q75": 1202.3791164405643, "handoff_plus_first_decode_ms": 102.50777099999999, "handoff_plus_first_decode_ms_q25": 102.41931799999999, "handoff_plus_first_decode_ms_q75": 112.10647599999999, "steady_tpot_ms": 95.943353, "steady_tpot_ms_q25": 95.5408235, "steady_tpot_ms_q75": 96.12107925000001, "steady_tpot_p95_ms": 97.09120100000001, "steady_tpot_p95_ms_q25": 96.64610217500001, "steady_tpot_p95_ms_q75": 97.35144450000001, "decode_tokens_per_s": 10.329331942723378, "decode_tokens_per_s_q25": 10.311855500704429, "decode_tokens_per_s_q75": 10.343555809054653, "decode_ms": 1452.1752310000002, "decode_ms_q25": 1450.1810250000003, "decode_ms_q75": 1454.6405430000002, "end_to_end_ms": 2299.523868, "end_to_end_ms_q25": 2291.1797505000004, "end_to_end_ms_q75": 2313.7380495, "output_tokens_per_s_e2e": 6.957962134098623, "output_tokens_per_s_e2e_q25": 6.915477698495869, "output_tokens_per_s_e2e_q75": 6.983394562420749, "all_tokens_per_s_e2e": 445.3095765823119, "all_tokens_per_s_e2e_q25": 442.59057270373563, "all_tokens_per_s_e2e_q75": 446.9372519949279, "energy_net_psys_j": 79.48445138212134, "energy_net_psys_j_q25": 79.05736669684897, "energy_net_psys_j_q75": 79.7732347152897, "prefill_energy_net_psys_j": 32.52628369724302, "prefill_energy_net_psys_j_q25": 32.07452214529449, "prefill_energy_net_psys_j_q75": 32.70903795495218, "decode_energy_net_psys_j": 47.170225835796735, "decode_energy_net_psys_j_q25": 46.63711207506516, "decode_energy_net_psys_j_q75": 47.51595831228606, "fastest_e2e": true, "lowest_energy": true, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "INT8", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 1024, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 851.337049, "ttft_ms_q25": 838.5334135, "ttft_ms_q75": 863.5570245, "prefill_tokens_per_s": 1184.0198910455265, "prefill_tokens_per_s_q25": 1167.4989113211343, "prefill_tokens_per_s_q75": 1202.3791164405643, "handoff_plus_first_decode_ms": 3113.611299, "handoff_plus_first_decode_ms_q25": 3113.051727, "handoff_plus_first_decode_ms_q75": 3192.6054135, "steady_tpot_ms": 3107.214298, "steady_tpot_ms_q25": 3105.5753635, "steady_tpot_ms_q75": 3107.3147449999997, "steady_tpot_p95_ms": 3109.5429515, "steady_tpot_p95_ms_q25": 3109.005176525, "steady_tpot_p95_ms_q75": 3116.6707525499996, "decode_tokens_per_s": 0.32191372763738574, "decode_tokens_per_s_q25": 0.32154929644924435, "decode_tokens_per_s_q75": 0.3220333420635425, "decode_ms": 46596.335328999994, "decode_ms_q25": 46579.03424949999, "decode_ms_q75": 46649.205684, "end_to_end_ms": 47447.672377999996, "end_to_end_ms_q25": 47442.591274, "end_to_end_ms_q75": 47487.7390975, "output_tokens_per_s_e2e": 0.337213591270258, "output_tokens_per_s_e2e_q25": 0.33692931468435716, "output_tokens_per_s_e2e_q75": 0.33724971073186794, "all_tokens_per_s_e2e": 21.581669841296513, "all_tokens_per_s_e2e_q25": 21.56347613979886, "all_tokens_per_s_e2e_q75": 21.583981486839548, "energy_net_psys_j": 494.3333382928764, "energy_net_psys_j_q25": 493.6125967366713, "energy_net_psys_j_q75": 495.443663715468, "prefill_energy_net_psys_j": 32.52628369724302, "prefill_energy_net_psys_j_q25": 32.07452214529449, "prefill_energy_net_psys_j_q75": 32.70903795495218, "decode_energy_net_psys_j": 461.44154608021506, "decode_energy_net_psys_j_q25": 461.3553203336677, "decode_energy_net_psys_j_q75": 462.73462576051577, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.04846441885874717, "energy_reduction_vs_gpu": -5.219245773193677 }, { "precision": "INT8", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 1024, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 4747.19481, "ttft_ms_q25": 4745.7933875, "ttft_ms_q75": 4748.121032499999, "prefill_tokens_per_s": 212.3359247605851, "prefill_tokens_per_s_q25": 212.2945121781214, "prefill_tokens_per_s_q75": 212.39864562022592, "handoff_plus_first_decode_ms": 3113.611299, "handoff_plus_first_decode_ms_q25": 3113.051727, "handoff_plus_first_decode_ms_q75": 3192.6054135, "steady_tpot_ms": 3107.214298, "steady_tpot_ms_q25": 3105.5753635, "steady_tpot_ms_q75": 3107.3147449999997, "steady_tpot_p95_ms": 3109.5429515, "steady_tpot_p95_ms_q25": 3109.005176525, "steady_tpot_p95_ms_q75": 3116.6707525499996, "decode_tokens_per_s": 0.32191372763738574, "decode_tokens_per_s_q25": 0.32154929644924435, "decode_tokens_per_s_q75": 0.3220333420635425, "decode_ms": 46596.335328999994, "decode_ms_q25": 46579.03424949999, "decode_ms_q75": 46649.205684, "end_to_end_ms": 51343.530138999995, "end_to_end_ms_q25": 51327.15528199999, "end_to_end_ms_q75": 51394.9990715, "output_tokens_per_s_e2e": 0.3116264105074959, "output_tokens_per_s_e2e_q25": 0.3113146480229191, "output_tokens_per_s_e2e_q75": 0.3117258601332318, "all_tokens_per_s_e2e": 19.944090272479738, "all_tokens_per_s_e2e_q25": 19.92413747346682, "all_tokens_per_s_e2e_q75": 19.950455048526834, "energy_net_psys_j": 537.0834476199191, "energy_net_psys_j_q25": 536.3672046788387, "energy_net_psys_j_q75": 538.6695282595207, "prefill_energy_net_psys_j": 75.64190153970398, "prefill_energy_net_psys_j_q25": 75.01188434517098, "prefill_energy_net_psys_j_q75": 75.93490249900486, "decode_energy_net_psys_j": 461.44154608021506, "decode_energy_net_psys_j_q25": 461.3553203336677, "decode_energy_net_psys_j_q75": 462.73462576051577, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.04478702305382205, "energy_reduction_vs_gpu": -5.757088188706135 }, { "precision": "INT8", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 1024, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 4747.19481, "ttft_ms_q25": 4745.7933875, "ttft_ms_q75": 4748.121032499999, "prefill_tokens_per_s": 212.3359247605851, "prefill_tokens_per_s_q25": 212.2945121781214, "prefill_tokens_per_s_q75": 212.39864562022592, "handoff_plus_first_decode_ms": 102.50777099999999, "handoff_plus_first_decode_ms_q25": 102.41931799999999, "handoff_plus_first_decode_ms_q75": 112.10647599999999, "steady_tpot_ms": 95.943353, "steady_tpot_ms_q25": 95.5408235, "steady_tpot_ms_q75": 96.12107925000001, "steady_tpot_p95_ms": 97.09120100000001, "steady_tpot_p95_ms_q25": 96.64610217500001, "steady_tpot_p95_ms_q75": 97.35144450000001, "decode_tokens_per_s": 10.329331942723378, "decode_tokens_per_s_q25": 10.311855500704429, "decode_tokens_per_s_q75": 10.343555809054653, "decode_ms": 1452.1752310000002, "decode_ms_q25": 1450.1810250000003, "decode_ms_q75": 1454.6405430000002, "end_to_end_ms": 6201.222486, "end_to_end_ms_q25": 6198.3020575, "end_to_end_ms_q75": 6201.360153, "output_tokens_per_s_e2e": 2.58013642247507, "output_tokens_per_s_e2e_q25": 2.580079146047474, "output_tokens_per_s_e2e_q75": 2.581352667799944, "all_tokens_per_s_e2e": 165.12873103840448, "all_tokens_per_s_e2e_q25": 165.12506534703834, "all_tokens_per_s_e2e_q75": 165.20657073919642, "energy_net_psys_j": 122.33190177263936, "energy_net_psys_j_q25": 122.28772985602635, "energy_net_psys_j_q75": 122.57201457407004, "prefill_energy_net_psys_j": 75.64190153970398, "prefill_energy_net_psys_j_q25": 75.01188434517098, "prefill_energy_net_psys_j_q75": 75.93490249900486, "decode_energy_net_psys_j": 47.170225835796735, "decode_energy_net_psys_j_q25": 46.63711207506516, "decode_energy_net_psys_j_q75": 47.51595831228606, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.3708178303860973, "energy_reduction_vs_gpu": -0.5390670709234564 }, { "precision": "INT8", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 2048, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1814.903333, "ttft_ms_q25": 1789.113373, "ttft_ms_q75": 1822.5951925, "prefill_tokens_per_s": 1119.6188596122877, "prefill_tokens_per_s_q25": 1114.9136135404883, "prefill_tokens_per_s_q75": 1135.994145842691, "handoff_plus_first_decode_ms": 104.643907, "handoff_plus_first_decode_ms_q25": 104.42456150000001, "handoff_plus_first_decode_ms_q75": 113.77543600000001, "steady_tpot_ms": 94.52278799999999, "steady_tpot_ms_q25": 94.46632774999999, "steady_tpot_ms_q75": 94.7092935, "steady_tpot_p95_ms": 95.9783772, "steady_tpot_p95_ms_q25": 95.70110405, "steady_tpot_p95_ms_q75": 96.119991575, "decode_tokens_per_s": 10.44213945369756, "decode_tokens_per_s_q25": 10.393711447950494, "decode_tokens_per_s_q75": 10.470220632596908, "decode_ms": 1436.487232, "decode_ms_q25": 1432.6448719999999, "decode_ms_q75": 1443.2116695, "end_to_end_ms": 3243.705845, "end_to_end_ms_q25": 3228.4826825, "end_to_end_ms_q75": 3255.2400645, "output_tokens_per_s_e2e": 4.932629765014959, "output_tokens_per_s_e2e_q25": 4.915213797618445, "output_tokens_per_s_e2e_q75": 4.955998628935779, "all_tokens_per_s_e2e": 631.3766099219148, "all_tokens_per_s_e2e_q25": 629.1473660951609, "all_tokens_per_s_e2e_q75": 634.3678245037797, "energy_net_psys_j": 118.64620400500446, "energy_net_psys_j_q25": 117.51494607654138, "energy_net_psys_j_q75": 119.99973734789285, "prefill_energy_net_psys_j": 71.53034383558204, "prefill_energy_net_psys_j_q25": 70.20090348347841, "prefill_energy_net_psys_j_q75": 73.08926092298307, "decode_energy_net_psys_j": 47.11586016942242, "decode_energy_net_psys_j_q25": 46.91047642490979, "decode_energy_net_psys_j_q75": 47.314042593062965, "fastest_e2e": true, "lowest_energy": true, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "INT8", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 2048, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 1814.903333, "ttft_ms_q25": 1789.113373, "ttft_ms_q75": 1822.5951925, "prefill_tokens_per_s": 1119.6188596122877, "prefill_tokens_per_s_q25": 1114.9136135404883, "prefill_tokens_per_s_q75": 1135.994145842691, "handoff_plus_first_decode_ms": 3152.366479, "handoff_plus_first_decode_ms_q25": 3137.942583, "handoff_plus_first_decode_ms_q75": 3207.458847, "steady_tpot_ms": 3106.8281524999998, "steady_tpot_ms_q25": 3106.2449895, "steady_tpot_ms_q75": 3108.384433, "steady_tpot_p95_ms": 3111.3768621000004, "steady_tpot_p95_ms_q25": 3111.1016603000003, "steady_tpot_p95_ms_q75": 3117.6600318250003, "decode_tokens_per_s": 0.32192174323489836, "decode_tokens_per_s_q25": 0.32110542449324053, "decode_tokens_per_s_q75": 0.3220267201852633, "decode_ms": 46595.175117000006, "decode_ms_q25": 46579.990584, "decode_ms_q75": 46713.931934, "end_to_end_ms": 48425.462169000006, "end_to_end_ms_q25": 48402.585776500004, "end_to_end_ms_q75": 48510.737166499996, "output_tokens_per_s_e2e": 0.3304046937985146, "output_tokens_per_s_e2e_q25": 0.32982490837615175, "output_tokens_per_s_e2e_q75": 0.3305609259783978, "all_tokens_per_s_e2e": 42.29180080620987, "all_tokens_per_s_e2e_q25": 42.217588272147424, "all_tokens_per_s_e2e_q75": 42.31179852523492, "energy_net_psys_j": 554.6957757140775, "energy_net_psys_j_q25": 541.3838384201301, "energy_net_psys_j_q75": 558.1248563589727, "prefill_energy_net_psys_j": 71.53034383558204, "prefill_energy_net_psys_j_q25": 70.20090348347841, "prefill_energy_net_psys_j_q75": 73.08926092298307, "decode_energy_net_psys_j": 480.0475977036934, "decode_energy_net_psys_j_q25": 468.294577497147, "decode_energy_net_psys_j_q75": 486.3650357880932, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.06698347728060482, "energy_reduction_vs_gpu": -3.6752087887336087 }, { "precision": "INT8", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 2048, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 9483.739544, "ttft_ms_q25": 9479.5676275, "ttft_ms_q75": 9492.238798, "prefill_tokens_per_s": 214.26147255230862, "prefill_tokens_per_s_q25": 214.06979663537786, "prefill_tokens_per_s_q75": 214.35580961219705, "handoff_plus_first_decode_ms": 3152.366479, "handoff_plus_first_decode_ms_q25": 3137.942583, "handoff_plus_first_decode_ms_q75": 3207.458847, "steady_tpot_ms": 3106.8281524999998, "steady_tpot_ms_q25": 3106.2449895, "steady_tpot_ms_q75": 3108.384433, "steady_tpot_p95_ms": 3111.3768621000004, "steady_tpot_p95_ms_q25": 3111.1016603000003, "steady_tpot_p95_ms_q75": 3117.6600318250003, "decode_tokens_per_s": 0.32192174323489836, "decode_tokens_per_s_q25": 0.32110542449324053, "decode_tokens_per_s_q75": 0.3220267201852633, "decode_ms": 46595.175117000006, "decode_ms_q25": 46579.990584, "decode_ms_q75": 46713.931934, "end_to_end_ms": 56070.57082800001, "end_to_end_ms_q25": 56068.05746550001, "end_to_end_ms_q75": 56193.4995615, "output_tokens_per_s_e2e": 0.285354683637536, "output_tokens_per_s_e2e_q25": 0.28473180518235236, "output_tokens_per_s_e2e_q75": 0.2853674758037859, "all_tokens_per_s_e2e": 36.52539950560461, "all_tokens_per_s_e2e_q25": 36.4456710633411, "all_tokens_per_s_e2e_q75": 36.527036902884596, "energy_net_psys_j": 641.1556664126668, "energy_net_psys_j_q25": 622.671539129447, "energy_net_psys_j_q75": 644.4096714536097, "prefill_energy_net_psys_j": 154.98120262205958, "prefill_energy_net_psys_j_q25": 151.3135285888431, "prefill_energy_net_psys_j_q75": 158.04463566551652, "decode_energy_net_psys_j": 480.0475977036934, "decode_energy_net_psys_j_q25": 468.294577497147, "decode_energy_net_psys_j_q75": 486.3650357880932, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.057850415950825095, "energy_reduction_vs_gpu": -4.403929032450319 }, { "precision": "INT8", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 2048, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 9483.739544, "ttft_ms_q25": 9479.5676275, "ttft_ms_q75": 9492.238798, "prefill_tokens_per_s": 214.26147255230862, "prefill_tokens_per_s_q25": 214.06979663537786, "prefill_tokens_per_s_q75": 214.35580961219705, "handoff_plus_first_decode_ms": 104.643907, "handoff_plus_first_decode_ms_q25": 104.42456150000001, "handoff_plus_first_decode_ms_q75": 113.77543600000001, "steady_tpot_ms": 94.52278799999999, "steady_tpot_ms_q25": 94.46632774999999, "steady_tpot_ms_q75": 94.7092935, "steady_tpot_p95_ms": 95.9783772, "steady_tpot_p95_ms_q25": 95.70110405, "steady_tpot_p95_ms_q75": 96.119991575, "decode_tokens_per_s": 10.44213945369756, "decode_tokens_per_s_q25": 10.393711447950494, "decode_tokens_per_s_q75": 10.470220632596908, "decode_ms": 1436.487232, "decode_ms_q25": 1432.6448719999999, "decode_ms_q75": 1443.2116695, "end_to_end_ms": 10929.540564, "end_to_end_ms_q25": 10920.7117535, "end_to_end_ms_q75": 10931.6081075, "output_tokens_per_s_e2e": 1.4639224683150183, "output_tokens_per_s_e2e_q25": 1.4636456425264657, "output_tokens_per_s_e2e_q75": 1.4651069287905347, "all_tokens_per_s_e2e": 187.38207594432234, "all_tokens_per_s_e2e_q25": 187.3466422433876, "all_tokens_per_s_e2e_q75": 187.53368688518844, "energy_net_psys_j": 202.49342763876308, "energy_net_psys_j_q25": 198.62757118190606, "energy_net_psys_j_q75": 205.15329451406686, "prefill_energy_net_psys_j": 154.98120262205958, "prefill_energy_net_psys_j_q25": 151.3135285888431, "prefill_energy_net_psys_j_q75": 158.04463566551652, "decode_energy_net_psys_j": 47.11586016942242, "decode_energy_net_psys_j_q25": 46.91047642490979, "decode_energy_net_psys_j_q75": 47.314042593062965, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.29678336669376576, "energy_reduction_vs_gpu": -0.7066995892276668 }, { "precision": "INT8", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 4096, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 4301.698888, "ttft_ms_q25": 4259.214180999999, "ttft_ms_q75": 4308.775047499999, "prefill_tokens_per_s": 948.4624810401189, "prefill_tokens_per_s_q25": 946.9074061606327, "prefill_tokens_per_s_q75": 958.0185009024171, "handoff_plus_first_decode_ms": 113.064014, "handoff_plus_first_decode_ms_q25": 112.7966735, "handoff_plus_first_decode_ms_q75": 120.5083855, "steady_tpot_ms": 98.9142945, "steady_tpot_ms_q25": 98.7371235, "steady_tpot_ms_q75": 98.9985035, "steady_tpot_p95_ms": 99.9681923, "steady_tpot_p95_ms_q25": 99.924901175, "steady_tpot_p95_ms_q75": 101.56567347500001, "decode_tokens_per_s": 9.988896282958486, "decode_tokens_per_s_q25": 9.916204961946324, "decode_tokens_per_s_q75": 10.005481032816757, "decode_ms": 1501.6674090000006, "decode_ms_q25": 1499.1824145000005, "decode_ms_q75": 1512.7567610000006, "end_to_end_ms": 5803.3662970000005, "end_to_end_ms_q25": 5771.970942, "end_to_end_ms_q75": 5807.957462, "output_tokens_per_s_e2e": 2.757020525874967, "output_tokens_per_s_e2e_q25": 2.7548428346494895, "output_tokens_per_s_e2e_q75": 2.772098741785606, "all_tokens_per_s_e2e": 705.7972546239915, "all_tokens_per_s_e2e_q25": 705.2397656702693, "all_tokens_per_s_e2e_q75": 709.6572778971151, "energy_net_psys_j": 211.7978605549249, "energy_net_psys_j_q25": 210.96046435290813, "energy_net_psys_j_q75": 213.74874481832757, "prefill_energy_net_psys_j": 164.6064481677537, "prefill_energy_net_psys_j_q25": 162.7087105495131, "prefill_energy_net_psys_j_q75": 165.48876207032595, "decode_energy_net_psys_j": 49.31209521961884, "decode_energy_net_psys_j_q25": 48.25175380339502, "decode_energy_net_psys_j_q75": 49.320324164225454, "fastest_e2e": true, "lowest_energy": true, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "INT8", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 4096, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 4301.698888, "ttft_ms_q25": 4259.214180999999, "ttft_ms_q75": 4308.775047499999, "prefill_tokens_per_s": 948.4624810401189, "prefill_tokens_per_s_q25": 946.9074061606327, "prefill_tokens_per_s_q75": 958.0185009024171, "handoff_plus_first_decode_ms": 3243.338733, "handoff_plus_first_decode_ms_q25": 3224.3841645, "handoff_plus_first_decode_ms_q75": 3268.4665535, "steady_tpot_ms": 3112.18156, "steady_tpot_ms_q25": 3110.20444125, "steady_tpot_ms_q75": 3112.53361825, "steady_tpot_p95_ms": 3116.2414857000003, "steady_tpot_p95_ms_q25": 3115.6718375, "steady_tpot_p95_ms_q75": 3118.8060249500004, "decode_tokens_per_s": 0.32070852688149143, "decode_tokens_per_s_q25": 0.32045757980661094, "decode_tokens_per_s_q75": 0.3213179714875056, "decode_ms": 46771.44117699999, "decode_ms_q25": 46682.897607499996, "decode_ms_q75": 46808.0961215, "end_to_end_ms": 51061.48054000001, "end_to_end_ms_q25": 50985.842892500004, "end_to_end_ms_q75": 51067.3103025, "output_tokens_per_s_e2e": 0.31334774923860825, "output_tokens_per_s_e2e_q25": 0.3133119820436474, "output_tokens_per_s_e2e_q75": 0.31381329217813825, "all_tokens_per_s_e2e": 80.21702380508371, "all_tokens_per_s_e2e_q25": 80.20786740317374, "all_tokens_per_s_e2e_q75": 80.33620279760339, "energy_net_psys_j": 657.1732641951776, "energy_net_psys_j_q25": 639.5415244349463, "energy_net_psys_j_q75": 665.8276238963842, "prefill_energy_net_psys_j": 164.6064481677537, "prefill_energy_net_psys_j_q25": 162.7087105495131, "prefill_energy_net_psys_j_q75": 165.48876207032595, "decode_energy_net_psys_j": 496.36229126390504, "decode_energy_net_psys_j_q25": 476.83281388543315, "decode_energy_net_psys_j_q75": 502.2365994442989, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.11365448544825918, "energy_reduction_vs_gpu": -2.1028324010135826 }, { "precision": "INT8", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 4096, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 19791.634002, "ttft_ms_q25": 19780.841848, "ttft_ms_q75": 19794.6949645, "prefill_tokens_per_s": 206.14770865243895, "prefill_tokens_per_s_q25": 206.11583582766158, "prefill_tokens_per_s_q75": 206.26024139023977, "handoff_plus_first_decode_ms": 3243.338733, "handoff_plus_first_decode_ms_q25": 3224.3841645, "handoff_plus_first_decode_ms_q75": 3268.4665535, "steady_tpot_ms": 3112.18156, "steady_tpot_ms_q25": 3110.20444125, "steady_tpot_ms_q75": 3112.53361825, "steady_tpot_p95_ms": 3116.2414857000003, "steady_tpot_p95_ms_q25": 3115.6718375, "steady_tpot_p95_ms_q75": 3118.8060249500004, "decode_tokens_per_s": 0.32070852688149143, "decode_tokens_per_s_q25": 0.32045757980661094, "decode_tokens_per_s_q75": 0.3213179714875056, "decode_ms": 46771.44117699999, "decode_ms_q25": 46682.897607499996, "decode_ms_q75": 46808.0961215, "end_to_end_ms": 66563.07517899999, "end_to_end_ms_q25": 66477.592572, "end_to_end_ms_q75": 66588.9379695, "output_tokens_per_s_e2e": 0.24037350974204758, "output_tokens_per_s_e2e_q25": 0.24028018619152602, "output_tokens_per_s_e2e_q75": 0.24068300064808024, "all_tokens_per_s_e2e": 61.53561849396418, "all_tokens_per_s_e2e_q25": 61.51172766503066, "all_tokens_per_s_e2e_q75": 61.61484816590854, "energy_net_psys_j": 821.4738069699072, "energy_net_psys_j_q25": 803.8553449388226, "energy_net_psys_j_q75": 833.5423562237002, "prefill_energy_net_psys_j": 328.9335464007768, "prefill_energy_net_psys_j_q25": 327.02253105338946, "prefill_energy_net_psys_j_q75": 333.2167721267886, "decode_energy_net_psys_j": 496.36229126390504, "decode_energy_net_psys_j_q25": 476.83281388543315, "decode_energy_net_psys_j_q75": 502.2365994442989, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.08718597032053751, "energy_reduction_vs_gpu": -2.8785746221306936 }, { "precision": "INT8", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 4096, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 19791.634002, "ttft_ms_q25": 19780.841848, "ttft_ms_q75": 19794.6949645, "prefill_tokens_per_s": 206.14770865243895, "prefill_tokens_per_s_q25": 206.11583582766158, "prefill_tokens_per_s_q75": 206.26024139023977, "handoff_plus_first_decode_ms": 113.064014, "handoff_plus_first_decode_ms_q25": 112.7966735, "handoff_plus_first_decode_ms_q75": 120.5083855, "steady_tpot_ms": 98.9142945, "steady_tpot_ms_q25": 98.7371235, "steady_tpot_ms_q75": 98.9985035, "steady_tpot_p95_ms": 99.9681923, "steady_tpot_p95_ms_q25": 99.924901175, "steady_tpot_p95_ms_q75": 101.56567347500001, "decode_tokens_per_s": 9.988896282958486, "decode_tokens_per_s_q25": 9.916204961946324, "decode_tokens_per_s_q75": 10.005481032816757, "decode_ms": 1501.6674090000006, "decode_ms_q25": 1499.1824145000005, "decode_ms_q75": 1512.7567610000006, "end_to_end_ms": 21293.895807, "end_to_end_ms_q25": 21293.598609, "end_to_end_ms_q75": 21294.174577, "output_tokens_per_s_e2e": 0.7513890433680189, "output_tokens_per_s_e2e_q25": 0.7513792067819274, "output_tokens_per_s_e2e_q75": 0.7513995307657503, "all_tokens_per_s_e2e": 192.35559510221285, "all_tokens_per_s_e2e_q25": 192.3530769361734, "all_tokens_per_s_e2e_q75": 192.35827987603207, "energy_net_psys_j": 376.124958787948, "energy_net_psys_j_q25": 375.2742848567845, "energy_net_psys_j_q75": 381.47675487479023, "prefill_energy_net_psys_j": 328.9335464007768, "prefill_energy_net_psys_j_q25": 327.02253105338946, "prefill_energy_net_psys_j_q75": 333.2167721267886, "decode_energy_net_psys_j": 49.31209521961884, "decode_energy_net_psys_j_q25": 48.25175380339502, "decode_energy_net_psys_j_q75": 49.320324164225454, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.27253661563856457, "energy_reduction_vs_gpu": -0.7758676022622459 }, { "precision": "INT8", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 8192, "route": "gpu_full", "route_label": "Whole iGPU", "mode": "measured", "prefill_device": "iGPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 10684.807526, "ttft_ms_q25": 10595.7277625, "ttft_ms_q75": 10686.7457425, "prefill_tokens_per_s": 765.1986224463881, "prefill_tokens_per_s_q25": 765.0598663000288, "prefill_tokens_per_s_q75": 771.6862969477645, "handoff_plus_first_decode_ms": 144.399666, "handoff_plus_first_decode_ms_q25": 143.42664050000002, "handoff_plus_first_decode_ms_q75": 148.60709200000002, "steady_tpot_ms": 106.494876, "steady_tpot_ms_q25": 106.358266, "steady_tpot_ms_q75": 106.84022300000001, "steady_tpot_p95_ms": 108.0899779, "steady_tpot_p95_ms_q25": 108.086983825, "steady_tpot_p95_ms_q75": 108.29293767499999, "decode_tokens_per_s": 9.118984476622629, "decode_tokens_per_s_q25": 9.111874753983312, "decode_tokens_per_s_q75": 9.140002438818653, "decode_ms": 1644.9200059999985, "decode_ms_q25": 1641.1460959999995, "decode_ms_q75": 1646.2044899999992, "end_to_end_ms": 12322.179712000001, "end_to_end_ms_q25": 12236.8738585, "end_to_end_ms_q75": 12329.1763225, "output_tokens_per_s_e2e": 1.2984715670408815, "output_tokens_per_s_e2e_q25": 1.297735123100832, "output_tokens_per_s_e2e_q75": 1.3075870347094432, "all_tokens_per_s_e2e": 664.8174423249313, "all_tokens_per_s_e2e_q25": 664.440383027626, "all_tokens_per_s_e2e_q75": 669.4845617712349, "energy_net_psys_j": 438.76027407536543, "energy_net_psys_j_q25": 437.66146994311407, "energy_net_psys_j_q75": 443.1595887379537, "prefill_energy_net_psys_j": 386.72118574907455, "prefill_energy_net_psys_j_q25": 385.18148028533506, "prefill_energy_net_psys_j_q75": 391.27069725958825, "decode_energy_net_psys_j": 52.03908832629088, "decode_energy_net_psys_j_q25": 51.88889147836545, "decode_energy_net_psys_j_q75": 52.47998965777896, "fastest_e2e": true, "lowest_energy": true, "e2e_speedup_vs_gpu": 1.0, "energy_reduction_vs_gpu": 0.0 }, { "precision": "INT8", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 8192, "route": "gpu_npu_oracle", "route_label": "iGPU prefill + NPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "iGPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 10684.807526, "ttft_ms_q25": 10595.7277625, "ttft_ms_q75": 10686.7457425, "prefill_tokens_per_s": 765.1986224463881, "prefill_tokens_per_s_q25": 765.0598663000288, "prefill_tokens_per_s_q75": 771.6862969477645, "handoff_plus_first_decode_ms": 3322.44694, "handoff_plus_first_decode_ms_q25": 3316.6843965, "handoff_plus_first_decode_ms_q75": 3356.4823385, "steady_tpot_ms": 3127.600312, "steady_tpot_ms_q25": 3123.33519125, "steady_tpot_ms_q75": 3128.7201692500003, "steady_tpot_p95_ms": 3131.83723055, "steady_tpot_p95_ms_q25": 3131.3378924, "steady_tpot_p95_ms_q75": 3135.3172826750006, "decode_tokens_per_s": 0.3186405682687098, "decode_tokens_per_s_q25": 0.318304801402975, "decode_tokens_per_s_q75": 0.3192465858809851, "decode_ms": 47074.985089, "decode_ms_q25": 46985.793162, "decode_ms_q75": 47124.6950245, "end_to_end_ms": 57681.05295899999, "end_to_end_ms_q25": 57631.230859999996, "end_to_end_ms_q75": 57722.3610035, "output_tokens_per_s_e2e": 0.27738744664340453, "output_tokens_per_s_e2e_q25": 0.27718908090281824, "output_tokens_per_s_e2e_q75": 0.27762745509622627, "all_tokens_per_s_e2e": 142.02237268142312, "all_tokens_per_s_e2e_q25": 141.92080942224294, "all_tokens_per_s_e2e_q75": 142.14525700926785, "energy_net_psys_j": 897.8886979600086, "energy_net_psys_j_q25": 890.2210642062861, "energy_net_psys_j_q75": 905.4720977666332, "prefill_energy_net_psys_j": 386.72118574907455, "prefill_energy_net_psys_j_q25": 385.18148028533506, "prefill_energy_net_psys_j_q75": 391.27069725958825, "decode_energy_net_psys_j": 511.1675122109341, "decode_energy_net_psys_j_q25": 498.95036694669784, "decode_energy_net_psys_j_q75": 520.2906174812981, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.2136261229620526, "energy_reduction_vs_gpu": -1.046422046417491 }, { "precision": "INT8", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 8192, "route": "npu_full", "route_label": "Whole NPU", "mode": "measured", "prefill_device": "NPU", "decode_device": "NPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 41954.709656, "ttft_ms_q25": 41946.1006585, "ttft_ms_q75": 41962.1950545, "prefill_tokens_per_s": 194.87681042337374, "prefill_tokens_per_s_q25": 194.84205365208828, "prefill_tokens_per_s_q75": 194.91681505663374, "handoff_plus_first_decode_ms": 3322.44694, "handoff_plus_first_decode_ms_q25": 3316.6843965, "handoff_plus_first_decode_ms_q75": 3356.4823385, "steady_tpot_ms": 3127.600312, "steady_tpot_ms_q25": 3123.33519125, "steady_tpot_ms_q75": 3128.7201692500003, "steady_tpot_p95_ms": 3131.83723055, "steady_tpot_p95_ms_q25": 3131.3378924, "steady_tpot_p95_ms_q75": 3135.3172826750006, "decode_tokens_per_s": 0.3186405682687098, "decode_tokens_per_s_q25": 0.318304801402975, "decode_tokens_per_s_q75": 0.3192465858809851, "decode_ms": 47074.985089, "decode_ms_q25": 46985.793162, "decode_ms_q75": 47124.6950245, "end_to_end_ms": 89012.47675, "end_to_end_ms_q25": 88931.8938205, "end_to_end_ms_q75": 89078.2810815, "output_tokens_per_s_e2e": 0.1797500820580189, "output_tokens_per_s_e2e_q25": 0.17961739424219691, "output_tokens_per_s_e2e_q75": 0.17991310484344375, "all_tokens_per_s_e2e": 92.03204201370568, "all_tokens_per_s_e2e_q25": 91.96410585200482, "all_tokens_per_s_e2e_q75": 92.1155096798432, "energy_net_psys_j": 1275.755816067187, "energy_net_psys_j_q25": 1259.7978716065495, "energy_net_psys_j_q75": 1276.8386873869003, "prefill_energy_net_psys_j": 757.1067054634507, "prefill_energy_net_psys_j_q25": 752.8072707092012, "prefill_energy_net_psys_j_q75": 760.8475046598518, "decode_energy_net_psys_j": 511.1675122109341, "decode_energy_net_psys_j_q25": 498.95036694669784, "decode_energy_net_psys_j_q75": 520.2906174812981, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.1384320508978535, "energy_reduction_vs_gpu": -1.907637476423063 }, { "precision": "INT8", "model": "qwen25_7b", "model_label": "Qwen2.5 7B", "family": "Qwen2.5", "size_b": 7.0, "context_length": 8192, "route": "npu_gpu_oracle", "route_label": "NPU prefill + iGPU decode \u00b7 oracle", "mode": "overhead_free_oracle", "prefill_device": "NPU", "decode_device": "iGPU", "repeats": 3, "internal_overhead_ms": 0.0, "ttft_ms": 41954.709656, "ttft_ms_q25": 41946.1006585, "ttft_ms_q75": 41962.1950545, "prefill_tokens_per_s": 194.87681042337374, "prefill_tokens_per_s_q25": 194.84205365208828, "prefill_tokens_per_s_q75": 194.91681505663374, "handoff_plus_first_decode_ms": 144.399666, "handoff_plus_first_decode_ms_q25": 143.42664050000002, "handoff_plus_first_decode_ms_q75": 148.60709200000002, "steady_tpot_ms": 106.494876, "steady_tpot_ms_q25": 106.358266, "steady_tpot_ms_q75": 106.84022300000001, "steady_tpot_p95_ms": 108.0899779, "steady_tpot_p95_ms_q25": 108.086983825, "steady_tpot_p95_ms_q75": 108.29293767499999, "decode_tokens_per_s": 9.118984476622629, "decode_tokens_per_s_q25": 9.111874753983312, "decode_tokens_per_s_q75": 9.140002438818653, "decode_ms": 1644.9200059999985, "decode_ms_q25": 1641.1460959999995, "decode_ms_q75": 1646.2044899999992, "end_to_end_ms": 43592.081842, "end_to_end_ms_q25": 43588.5312385, "end_to_end_ms_q75": 43603.3411505, "output_tokens_per_s_e2e": 0.3670391347215805, "output_tokens_per_s_e2e_q25": 0.36694438189097073, "output_tokens_per_s_e2e_q75": 0.3670690351693355, "all_tokens_per_s_e2e": 187.92403697744922, "all_tokens_per_s_e2e_q25": 187.875523528177, "all_tokens_per_s_e2e_q75": 187.93934600669976, "energy_net_psys_j": 808.8454000938907, "energy_net_psys_j_q25": 805.1370635190547, "energy_net_psys_j_q75": 812.7363961382173, "prefill_energy_net_psys_j": 757.1067054634507, "prefill_energy_net_psys_j_q25": 752.8072707092012, "prefill_energy_net_psys_j_q75": 760.8475046598518, "decode_energy_net_psys_j": 52.03908832629088, "decode_energy_net_psys_j_q25": 51.88889147836545, "decode_energy_net_psys_j_q75": 52.47998965777896, "fastest_e2e": false, "lowest_energy": false, "e2e_speedup_vs_gpu": 0.2826701362110184, "energy_reduction_vs_gpu": -0.8434791112263644 } ]