[ { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 1.1579051027074456, "compile_target": "AUTO:GPU", "context_length": 1024, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 394921, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 2.4565200143075967, "dram": 0.4907827289910671, "package-0": 4.295934585025856, "psys": 9.931950112327097, "uncore": 0.002123975024632864 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_1b_ov_fp16/openvino_model.xml", "key": "llama32_1b_fp16", "size_b": 1.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "GPU", "key": "gpu_gpu", "label": "iGPU -> iGPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 1008, "prompt_sha256": "2669029615323cd09063f0eb8d822f5daea575e1396736877003e47194c143f8", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.026748363859951496, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 8.752053, "dram": 2.2796570000000003, "package-0": 22.433108999999998, "psys": 37.295436, "uncore": 8.877541 }, "net_j": { "core": 6.604306042694602, "dram": 1.8505633607260248, "package-0": 18.67715326585836, "psys": 28.611885485606145, "uncore": 8.87568399868514 } }, "first_decode": { "gpu_active_fraction": 0.8333333333333334, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 6, "gross_j": { "core": 1.818965, "dram": 0.127502, "package-0": 2.438226, "psys": 3.364981, "uncore": 0.34082 }, "logits": { "l2": 401.4866824038882, "sha256": "61d2cdecbae6d5fd94941833b71375ae3e9c75ee5788bfeb46933416f5518666", "sum": 39969.7027254026, "top1": 546 }, "net_j": { "core": 1.6800812192550942, "dram": 0.09975471500757785, "package-0": 2.19534760555331, "psys": 2.8034603228893857, "uncore": 0.34069991724882737 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.0565368, "watts": { "core": 32.17311556366827, "dram": 2.2552036903397434, "package-0": 43.126353100989085, "psys": 59.51841986104626, "uncore": 6.028286001330107 } }, "metrics": { "all_tokens_per_s_e2e": 1171.2164163915008, "decode_tokens_per_s": 25.96060966581099, "end_to_end_ms": 874.3046850000001, "handoff_estimate_ms": 19.738152999999993, "handoff_plus_first_decode_ms": 56.5368, "output_tokens_per_s_e2e": 18.3002565061172, "prefill_tokens_per_s": 3399.591231531468, "steady_tpot_ms": 36.798647, "steady_tpot_p95_ms": 39.3622204, "ttft_ms": 296.506236 }, "prefill": { "gpu_active_fraction": 0.8, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 30, "gross_j": { "core": 1.622066, "dram": 0.579466, "package-0": 7.125531, "psys": 11.968963, "uncore": 4.178822 }, "logits": { "l2": 628.2510355324671, "sha256": "c96d07293a33b871695e60a2f64ee2de15b94f1c3e0c78181714540acc5414c9", "sum": -187389.82698735525, "top1": 5809 }, "net_j": { "core": 0.8936924968989884, "dram": 0.43394586033305066, "package-0": 5.851759606091761, "psys": 9.024077856054117, "uncore": 4.178192228160088 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.296506236, "watts": { "core": 5.470596577941788, "dram": 1.9543130283438628, "package-0": 24.031639590878623, "psys": 40.36664847750454, "uncore": 14.093538322748802 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 518.3678398516904, "sha256": "f563a91e160dc6bc68c1aa9a1c95954b91970ab4b372530c63a6fe21c1c8c083", "sum": 92807.67067616689, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 52, "gross_j": { "core": 5.311022, "dram": 1.572689, "package-0": 12.869352, "psys": 21.961492, "uncore": 4.357899 }, "net_j": { "core": 4.030532326540519, "dram": 1.3168627853853963, "package-0": 10.63004605421329, "psys": 16.784347306662642, "uncore": 4.356791853276225 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.521261649, "token_latencies_s": [ 0.04130571, 0.038315726, 0.037117336, 0.037018289, 0.036638301, 0.036730731, 0.036755856, 0.036591956, 0.036983013, 0.036872688, 0.036731281, 0.036818849, 0.036778445, 0.036579627 ], "watts": { "core": 10.18878333019278, "dram": 3.0170817343211063, "package-0": 24.68885256509634, "psys": 42.1314171915993, "uncore": 8.360290860377491 } } }, { "energy": { "gross_j": { "core": 8.155985, "dram": 2.33764, "package-0": 22.12891, "psys": 37.627772, "uncore": 9.049781 }, "net_j": { "core": 5.891750015920376, "dram": 1.8852734741457975, "package-0": 18.169241314184383, "psys": 28.473249027060497, "uncore": 9.04782327992116 } }, "first_decode": { "gpu_active_fraction": 0.8, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2050, "gpu_sample_count": 5, "gross_j": { "core": 1.551327, "dram": 0.122925, "package-0": 2.043879, "psys": 2.893364, "uncore": 0.232544 }, "logits": { "l2": 401.4866824038882, "sha256": "61d2cdecbae6d5fd94941833b71375ae3e9c75ee5788bfeb46933416f5518666", "sum": 39969.7027254026, "top1": 546 }, "net_j": { "core": 1.4298406575613827, "dram": 0.09865351093013647, "package-0": 1.8314250500827818, "psys": 2.4021828635135023, "uncore": 0.23243895955430008 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.049454652, "watts": { "core": 31.36867690424755, "dram": 2.4856104537951254, "package-0": 41.32834662348852, "psys": 58.505396014110055, "uncore": 4.7021663401857525 } }, "metrics": { "all_tokens_per_s_e2e": 1110.9608818598306, "decode_tokens_per_s": 25.240822816698234, "end_to_end_ms": 921.7246230000001, "handoff_estimate_ms": 11.256723000000003, "handoff_plus_first_decode_ms": 49.454652, "output_tokens_per_s_e2e": 17.358763779059853, "prefill_tokens_per_s": 3078.3398461903835, "steady_tpot_ms": 38.197929, "steady_tpot_p95_ms": 42.97029485, "ttft_ms": 327.449226 }, "prefill": { "gpu_active_fraction": 0.7272727272727273, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 33, "gross_j": { "core": 1.901972, "dram": 0.625486, "package-0": 7.700236, "psys": 12.976041, "uncore": 4.367176 }, "logits": { "l2": 628.2510355324671, "sha256": "c96d07293a33b871695e60a2f64ee2de15b94f1c3e0c78181714540acc5414c9", "sum": -187389.82698735525, "top1": 5809 }, "net_j": { "core": 1.0975864226614687, "dram": 0.4647795752577073, "package-0": 6.293535545186653, "psys": 9.723831623047879, "uncore": 4.366480506022141 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.327449226, "watts": { "core": 5.8084486050976345, "dram": 1.9101770605498394, "package-0": 23.515816769711957, "psys": 39.627642912797725, "uncore": 13.336956246157076 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 518.3678398516904, "sha256": "f563a91e160dc6bc68c1aa9a1c95954b91970ab4b372530c63a6fe21c1c8c083", "sum": 92807.67067616689, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 54, "gross_j": { "core": 4.702686, "dram": 1.589229, "package-0": 12.384795, "psys": 21.758367, "uncore": 4.450061 }, "net_j": { "core": 3.3643229356975244, "dram": 1.3218403879579537, "package-0": 10.044280718914948, "psys": 16.347234540499116, "uncore": 4.448903814344718 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.544820745, "token_latencies_s": [ 0.04185664, 0.045038511, 0.038284758, 0.038300555, 0.038141586, 0.038000403, 0.03829248, 0.038045776, 0.038221264, 0.038348972, 0.037997244, 0.038174594, 0.038014085, 0.038076784 ], "watts": { "core": 8.631620662682366, "dram": 2.916975931230372, "package-0": 22.73187119554341, "psys": 39.9367446993965, "uncore": 8.167936042890585 } } }, { "energy": { "gross_j": { "core": 8.685586, "dram": 2.333246, "package-0": 22.631656, "psys": 38.226282, "uncore": 9.028542 }, "net_j": { "core": 6.4339668060840545, "dram": 1.8833999551061995, "package-0": 18.694049666196953, "psys": 29.122765897675617, "uncore": 9.026595187881634 } }, "first_decode": { "gpu_active_fraction": 0.8333333333333334, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 6, "gross_j": { "core": 1.628902, "dram": 0.12439, "package-0": 2.120417, "psys": 3.002007, "uncore": 0.227905 }, "logits": { "l2": 401.4866824038882, "sha256": "61d2cdecbae6d5fd94941833b71375ae3e9c75ee5788bfeb46933416f5518666", "sum": 39969.7027254026, "top1": 546 }, "net_j": { "core": 1.503730968431162, "dram": 0.09938235499561084, "package-0": 1.9015193071548229, "psys": 2.495928305827166, "uncore": 0.2277967736703502 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.050954615, "watts": { "core": 31.967703023563224, "dram": 2.4411920294167664, "package-0": 41.613836155959575, "psys": 58.915311203901744, "uncore": 4.472705759821755 } }, "metrics": { "all_tokens_per_s_e2e": 1117.1855798031909, "decode_tokens_per_s": 25.393661298258756, "end_to_end_ms": 916.588988, "handoff_estimate_ms": 12.7774825, "handoff_plus_first_decode_ms": 50.954615000000004, "output_tokens_per_s_e2e": 17.456024684424857, "prefill_tokens_per_s": 3093.0644642042917, "steady_tpot_ms": 38.1771325, "steady_tpot_p95_ms": 40.4363914, "ttft_ms": 325.890395 }, "prefill": { "gpu_active_fraction": 0.7272727272727273, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 33, "gross_j": { "core": 2.134638, "dram": 0.620421, "package-0": 7.909281, "psys": 13.265957, "uncore": 4.354664 }, "logits": { "l2": 628.2510355324671, "sha256": "c96d07293a33b871695e60a2f64ee2de15b94f1c3e0c78181714540acc5414c9", "sum": -187389.82698735525, "top1": 5809 }, "net_j": { "core": 1.3340817222118915, "dram": 0.4604796225899232, "package-0": 6.509277181191763, "psys": 10.029229854773428, "uncore": 4.353971816940252 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.325890395, "watts": { "core": 6.550171569186627, "dram": 1.903771972168741, "package-0": 24.26975793502598, "psys": 40.70680573448628, "uncore": 13.362357611061228 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 518.3678398516904, "sha256": "f563a91e160dc6bc68c1aa9a1c95954b91970ab4b372530c63a6fe21c1c8c083", "sum": 92807.67067616689, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 53, "gross_j": { "core": 4.922046, "dram": 1.588435, "package-0": 12.601958, "psys": 21.958318, "uncore": 4.445973 }, "net_j": { "core": 3.596154115441001, "dram": 1.3235379775206655, "package-0": 10.283253177850366, "psys": 16.597607737075023, "uncore": 4.444826597271033 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.539743978, "token_latencies_s": [ 0.042332392, 0.039415468, 0.038536746, 0.038111865, 0.038129457, 0.038193244, 0.038175865, 0.038246715, 0.038066327, 0.038120735, 0.038244376, 0.0381784, 0.038130421, 0.037837393 ], "watts": { "core": 9.119223558988926, "dram": 2.9429415885025403, "package-0": 23.348028905660158, "psys": 40.68284018909424, "uncore": 8.237188706531526 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 3.0791938668116927, "compile_target": "NPU", "context_length": 1024, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 394990, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 2.4565200143075967, "dram": 0.4907827289910671, "package-0": 4.295934585025856, "psys": 9.931950112327097, "uncore": 0.002123975024632864 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_1b_ov_fp16/openvino_model.xml", "key": "llama32_1b_fp16", "size_b": 1.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "gpu_npu", "label": "iGPU -> NPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 1008, "prompt_sha256": "2669029615323cd09063f0eb8d822f5daea575e1396736877003e47194c143f8", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "GPU", "NPUW_ONLINE_PIPELINE": "NONE" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 1024, "NPUW_LLM_MIN_RESPONSE_LEN": 1024, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.026207742281258106, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 8.934059, "dram": 2.517695, "package-0": 20.726571, "psys": 36.344755, "uncore": 4.616504 }, "net_j": { "core": 6.246924785246469, "dram": 1.9808383672834422, "package-0": 16.027340893780103, "psys": 25.48040917313246, "uncore": 4.615354077636267 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 0, "gpu_sample_count": 17, "gross_j": { "core": 3.006828, "dram": 0.215576, "package-0": 3.596793, "psys": 5.600449, "uncore": 0.00531 }, "logits": { "l2": 403.29190860783467, "sha256": "1dfee3a1ff1e7c218e8d6915056b5ffa93b645fba18d446b92fe98015d2630a1", "sum": 41609.541811217554, "top1": 546 }, "net_j": { "core": 2.5687796953541797, "dram": 0.12805929297875357, "package-0": 2.8307390493382787, "psys": 3.8293769618354565, "uncore": 0.0049312513420483005 }, "npu_busy_delta_us": 52717, "npu_utilization": 0.29563033167988134, "seconds": 0.178320674, "watts": { "core": 16.861914732332156, "dram": 1.2089232009071476, "package-0": 20.17036454225156, "psys": 31.406616374722763, "uncore": 0.029777814769811825 } }, "metrics": { "all_tokens_per_s_e2e": 936.1186653200733, "decode_tokens_per_s": 20.525507423692204, "end_to_end_ms": 1093.8784130000001, "handoff_estimate_ms": 140.52945250000002, "handoff_plus_first_decode_ms": 178.32067400000003, "output_tokens_per_s_e2e": 14.626854145626146, "prefill_tokens_per_s": 2776.2445850767785, "steady_tpot_ms": 37.7912215, "steady_tpot_p95_ms": 46.6162753, "ttft_ms": 363.08040200000005 }, "prefill": { "gpu_active_fraction": 0.9722222222222222, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 36, "gross_j": { "core": 2.232966, "dram": 0.675841, "package-0": 8.485574, "psys": 14.76516, "uncore": 4.611194 }, "logits": { "l2": 628.3565558181504, "sha256": "d9e1e7e1f931cd6721a36966da9356a1de9f71a7b58989f1fd1d69ea34d47bf0", "sum": -187423.86018045805, "top1": 5809 }, "net_j": { "core": 1.3410517256841517, "dram": 0.4976474094632663, "package-0": 6.925804343903108, "psys": 11.159063560572331, "uncore": 4.610422826294219 }, "npu_busy_delta_us": 9371, "npu_utilization": 0.025809710324161204, "seconds": 0.363080402, "watts": { "core": 6.150059291825945, "dram": 1.8614086474433285, "package-0": 23.371060385682835, "psys": 40.6663645811431, "uncore": 12.700200767101718 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 513.951448933101, "sha256": "6251fce17c6ac65a0e5a8579de91fe30d8b29d6fafc4fb59eee5beae93676e4d", "sum": 91180.61728955619, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 55, "gross_j": { "core": 3.694265, "dram": 1.626278, "package-0": 8.644204, "psys": 15.979146, "uncore": 0.0 }, "net_j": { "core": 2.337093364208137, "dram": 1.3551316648414224, "package-0": 6.270797500538715, "psys": 10.491968650724674, "uncore": 0.0 }, "npu_busy_delta_us": 541473, "npu_utilization": 0.9800818309403341, "seconds": 0.552477337, "token_latencies_s": [ 0.049497386, 0.045064908, 0.040802834, 0.03939486, 0.038527777, 0.037805866, 0.037604214, 0.037743968, 0.037820727, 0.037722436, 0.03753061, 0.037776577, 0.037596288, 0.037560723 ], "watts": { "core": 6.686726771563483, "dram": 2.9436103367259023, "package-0": 15.646259893552882, "psys": 28.92271760280368, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 7.549419, "dram": 2.438226, "package-0": 19.059827, "psys": 33.292578, "uncore": 4.604053 }, "net_j": { "core": 5.158998414563909, "dram": 1.960649130475725, "package-0": 14.879486353100308, "psys": 23.627874328239827, "uncore": 4.603285310161245 } }, "first_decode": { "gpu_active_fraction": 0.16666666666666666, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 6, "gross_j": { "core": 1.763668, "dram": 0.132019, "package-0": 2.054377, "psys": 2.92498, "uncore": 0.0 }, "logits": { "l2": 403.29190860783467, "sha256": "1dfee3a1ff1e7c218e8d6915056b5ffa93b645fba18d446b92fe98015d2630a1", "sum": 41609.541811217554, "top1": 546 }, "net_j": { "core": 1.626144816908459, "dram": 0.10454354581924957, "package-0": 1.8138780034337878, "psys": 2.3689603519620004, "uncore": 0.0 }, "npu_busy_delta_us": 50833, "npu_utilization": 0.908008955873119, "seconds": 0.055982928, "watts": { "core": 31.503675549088822, "dram": 2.358201057293752, "package-0": 36.69649075875417, "psys": 52.247713803036525, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 1052.3154418836605, "decode_tokens_per_s": 24.523744588437246, "end_to_end_ms": 973.092249, "handoff_estimate_ms": 17.964838499999995, "handoff_plus_first_decode_ms": 55.982928, "output_tokens_per_s_e2e": 16.442428779432195, "prefill_tokens_per_s": 2788.843510420912, "steady_tpot_ms": 38.0180895, "steady_tpot_p95_ms": 46.445875050000005, "ttft_ms": 361.44014400000003 }, "prefill": { "gpu_active_fraction": 0.9722222222222222, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 36, "gross_j": { "core": 1.973444, "dram": 0.673033, "package-0": 8.240091, "psys": 14.31344, "uncore": 4.604053 }, "logits": { "l2": 628.3565558181504, "sha256": "d9e1e7e1f931cd6721a36966da9356a1de9f71a7b58989f1fd1d69ea34d47bf0", "sum": -187423.86018045805, "top1": 5809 }, "net_j": { "core": 1.08555905228978, "dram": 0.4956444197607557, "package-0": 6.687367784973674, "psys": 10.723634521199678, "uncore": 4.603285310161245 }, "npu_busy_delta_us": 8154, "npu_utilization": 0.022559751968226304, "seconds": 0.361440144, "watts": { "core": 5.459946917241157, "dram": 1.8620870182034897, "package-0": 22.79794078435294, "psys": 39.601135174403865, "uncore": 12.73807870107533 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 513.951448933101, "sha256": "6251fce17c6ac65a0e5a8579de91fe30d8b29d6fafc4fb59eee5beae93676e4d", "sum": 91180.61728955619, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 55, "gross_j": { "core": 3.812307, "dram": 1.633174, "package-0": 8.765359, "psys": 16.054158, "uncore": 0.0 }, "net_j": { "core": 2.44729454536567, "dram": 1.3604611648957197, "package-0": 6.378240564692846, "psys": 10.535279455078147, "uncore": 0.0 }, "npu_busy_delta_us": 543663, "npu_utilization": 0.9783933003719586, "seconds": 0.555669177, "token_latencies_s": [ 0.050866189, 0.044065706, 0.040956959, 0.039648773, 0.038754758, 0.038050713, 0.037901184, 0.037985466, 0.037876823, 0.037906122, 0.037800944, 0.0378264, 0.037925677, 0.038080741 ], "watts": { "core": 6.860749449127715, "dram": 2.9391120969086972, "package-0": 15.77442003769808, "psys": 28.891575535419705, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 7.59489, "dram": 2.436823, "package-0": 19.117382, "psys": 32.888831, "uncore": 4.608082 }, "net_j": { "core": 5.2074352153945425, "dram": 1.9598386612985248, "package-0": 14.942227912251631, "psys": 23.236118350385535, "uncore": 4.60731365773134 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 6, "gross_j": { "core": 1.781306, "dram": 0.134277, "package-0": 2.080683, "psys": 2.952568, "uncore": 0.0 }, "logits": { "l2": 403.29190860783467, "sha256": "1dfee3a1ff1e7c218e8d6915056b5ffa93b645fba18d446b92fe98015d2630a1", "sum": 41609.541811217554, "top1": 546 }, "net_j": { "core": 1.6432835144764308, "dram": 0.10670179128488577, "package-0": 1.8393108289537736, "psys": 2.394529623509969, "uncore": 0.0 }, "npu_busy_delta_us": 51225, "npu_utilization": 0.911700997526367, "seconds": 0.056186184, "watts": { "core": 31.70363020204398, "dram": 2.3898579764733623, "package-0": 37.031932974839506, "psys": 52.549715780662375, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 1053.6226741846665, "decode_tokens_per_s": 24.58461776556627, "end_to_end_ms": 971.8849309999999, "handoff_estimate_ms": 18.176638500000003, "handoff_plus_first_decode_ms": 56.186184, "output_tokens_per_s_e2e": 16.462854284135414, "prefill_tokens_per_s": 2786.4753927491456, "steady_tpot_ms": 38.0095455, "steady_tpot_p95_ms": 45.8224811, "ttft_ms": 361.747318 }, "prefill": { "gpu_active_fraction": 0.9722222222222222, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 36, "gross_j": { "core": 2.171137, "dram": 0.674376, "package-0": 8.437417, "psys": 14.375024, "uncore": 4.608082 }, "logits": { "l2": 628.3565558181504, "sha256": "d9e1e7e1f931cd6721a36966da9356a1de9f71a7b58989f1fd1d69ea34d47bf0", "sum": -187423.86018045805, "top1": 5809 }, "net_j": { "core": 1.2824974732109051, "dram": 0.49683666406676064, "package-0": 6.883374185563453, "psys": 10.782167684355873, "uncore": 4.60731365773134 }, "npu_busy_delta_us": 8071, "npu_utilization": 0.02231115366555392, "seconds": 0.361747318, "watts": { "core": 6.001805381733335, "dram": 1.8642183824013867, "package-0": 23.324062350062814, "psys": 39.73774865692301, "uncore": 12.738399901557804 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 513.951448933101, "sha256": "6251fce17c6ac65a0e5a8579de91fe30d8b29d6fafc4fb59eee5beae93676e4d", "sum": 91180.61728955619, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 55, "gross_j": { "core": 3.642447, "dram": 1.62817, "package-0": 8.599282, "psys": 15.561239, "uncore": 0.0 }, "net_j": { "core": 2.2816542277072065, "dram": 1.3563002059468785, "package-0": 6.219542897734405, "psys": 10.059421042519695, "uncore": 0.0 }, "npu_busy_delta_us": 543178, "npu_utilization": 0.9805516721575241, "seconds": 0.553951429, "token_latencies_s": [ 0.050091308, 0.043523882, 0.041000158, 0.039627643, 0.039086622, 0.038120256, 0.038038178, 0.037786302, 0.037980913, 0.037874564, 0.037727016, 0.037719199, 0.037585589, 0.037760515 ], "watts": { "core": 6.5753905655147245, "dram": 2.9391927067309718, "package-0": 15.523530674022325, "psys": 28.091341921603743, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 14.522015229798853, "compile_target": "NPU", "context_length": 1024, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 395128, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 2.4565200143075967, "dram": 0.4907827289910671, "package-0": 4.295934585025856, "psys": 9.931950112327097, "uncore": 0.002123975024632864 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_1b_ov_fp16/openvino_model.xml", "key": "llama32_1b_fp16", "size_b": 1.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU+GPU", "key": "npu_gpu_combined", "label": "NPU + iGPU (subgraph combination)", "mode": "combined", "prefill_device": "NPU+GPU" }, "prompt_length": 1008, "prompt_sha256": "2669029615323cd09063f0eb8d822f5daea575e1396736877003e47194c143f8", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU,GPU", "NPUW_SUBMODEL_DEVICE": "0:GPU,last:NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "NPU,GPU", "NPUW_SUBMODEL_DEVICE": "0:GPU,last:NPU" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 1024, "NPUW_LLM_MIN_RESPONSE_LEN": 1024, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.026272007264196873, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 47.860779, "dram": 8.310891, "package-0": 102.183393, "psys": 183.774858, "uncore": 28.635181 }, "net_j": { "core": 33.41361216364959, "dram": 5.417603169704745, "package-0": 76.85777634647256, "psys": 125.22350993163995, "uncore": 28.62265963213676 } }, "first_decode": { "gpu_active_fraction": 0.40404040404040403, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 99, "gross_j": { "core": 12.692472, "dram": 0.972531, "package-0": 17.322466, "psys": 28.4229, "uncore": 1.863399 }, "logits": { "l2": 430.57070801613617, "sha256": "0cacbe9345c91be7436afb7ddea70f4d375bcbddb0707a37cae2ac55ada07d2d", "sum": -86575.56225838233, "top1": 90 }, "net_j": { "core": 10.24428880786052, "dram": 0.48341386636779454, "package-0": 13.041110755859878, "psys": 18.524656473162462, "uncore": 1.861282233221979 }, "npu_busy_delta_us": 10004, "npu_utilization": 0.01003806671904195, "seconds": 0.996606247, "watts": { "core": 12.735693799037566, "dram": 0.9758427693259282, "package-0": 17.381454362888416, "psys": 28.519688779354, "uncore": 1.869744450839269 } }, "metrics": { "all_tokens_per_s_e2e": 173.6991077156562, "decode_tokens_per_s": 2.73146209932149, "end_to_end_ms": 5895.251929999999, "handoff_estimate_ms": 677.2658710000001, "handoff_plus_first_decode_ms": 996.606247, "output_tokens_per_s_e2e": 2.714048558057128, "prefill_tokens_per_s": 2496.9807155726944, "steady_tpot_ms": 319.340376, "steady_tpot_p95_ms": 332.3457439, "ttft_ms": 403.68753899999996 }, "prefill": { "gpu_active_fraction": 0.1, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 1900, "gpu_sample_count": 40, "gross_j": { "core": 0.957029, "dram": 0.582091, "package-0": 5.999923, "psys": 10.481174, "uncore": 0.234924 }, "logits": { "l2": 633.1706927261538, "sha256": "ad53eb322905ce268ba906538fffea0ad7f5b54a7f40119f6d9c48fbf51f50d8", "sum": -190262.31765024643, "top1": 5809 }, "net_j": { "core": 0.0, "dram": 0.3839681279498922, "package-0": 4.265707739665926, "psys": 6.4717695016839, "uncore": 0.23406657774940848 }, "npu_busy_delta_us": 347381, "npu_utilization": 0.8605195019408315, "seconds": 0.403687539, "watts": { "core": 2.370717219487917, "dram": 1.4419345255043903, "package-0": 14.862789708255027, "psys": 25.96358070889079, "uncore": 0.5819451365329362 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 569.9173451531581, "sha256": "a40d57a3dc57a3450a26a1b67cdc6755d927ed0f29960e651636fd033560248b", "sum": 139928.31610714644, "top1": 802 }, "gpu_active_fraction": 0.9977528089887641, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 445, "gross_j": { "core": 34.211278, "dram": 6.756269, "package-0": 78.861004, "psys": 144.870784, "uncore": 26.536858 }, "net_j": { "core": 23.16932335578907, "dram": 4.550221175387058, "package-0": 59.55095785094676, "psys": 100.22708395679359, "uncore": 26.527310821165372 }, "npu_busy_delta_us": 137087, "npu_utilization": 0.030497948058312328, "seconds": 4.494958144, "token_latencies_s": [ 0.337700947, 0.31774869, 0.317453958, 0.318008498, 0.319460646, 0.319301666, 0.318089272, 0.318683853, 0.319202785, 0.319379086, 0.319390041, 0.319846872, 0.329462173, 0.321176748 ], "watts": { "core": 7.6110337191162065, "dram": 1.5030771774857266, "package-0": 17.544324434981878, "psys": 32.2296180206656, "uncore": 5.903694127924676 } } }, { "energy": { "gross_j": { "core": 42.552443, "dram": 8.022929, "package-0": 96.140501, "psys": 170.289236, "uncore": 28.966662 }, "net_j": { "core": 29.467158532994233, "dram": 5.408648805685504, "package-0": 73.2571026277884, "psys": 117.38415410570833, "uncore": 28.955348101787788 } }, "first_decode": { "gpu_active_fraction": 0.9722222222222222, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 36, "gross_j": { "core": 4.382435, "dram": 0.513548, "package-0": 7.701823, "psys": 12.754606, "uncore": 1.913691 }, "logits": { "l2": 634.5086714166829, "sha256": "1fbf009fd116a1af69f35aa003ae823419111749aab8664994f7bf095dbdd55c", "sum": 167224.29535845853, "top1": 802 }, "net_j": { "core": 3.501240428766107, "dram": 0.337496075357252, "package-0": 6.160799840142815, "psys": 9.191850366105704, "uncore": 1.9129290948454554 }, "npu_busy_delta_us": 8042, "npu_utilization": 0.022418810328573957, "seconds": 0.358716626, "watts": { "core": 12.216983218391444, "dram": 1.4316258650358737, "package-0": 21.470493536588965, "psys": 35.5562164548236, "uncore": 5.334826604886723 } }, "metrics": { "all_tokens_per_s_e2e": 192.2370507873706, "decode_tokens_per_s": 3.0536580641770157, "end_to_end_ms": 5326.756708999999, "handoff_estimate_ms": 34.05667150000002, "handoff_plus_first_decode_ms": 358.71662599999996, "output_tokens_per_s_e2e": 3.0037039185526657, "prefill_tokens_per_s": 2431.168893413972, "steady_tpot_ms": 324.65995449999997, "steady_tpot_p95_ms": 331.10038959999997, "ttft_ms": 414.615374 }, "prefill": { "gpu_active_fraction": 0.0975609756097561, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 1900, "gpu_sample_count": 41, "gross_j": { "core": 1.672237, "dram": 0.597777, "package-0": 6.67552, "psys": 10.899813, "uncore": 0.236938 }, "logits": { "l2": 633.1706927261538, "sha256": "ad53eb322905ce268ba906538fffea0ad7f5b54a7f40119f6d9c48fbf51f50d8", "sum": -190262.31765024643, "top1": 5809 }, "net_j": { "core": 0.6537260355293704, "dram": 0.39429093526662806, "package-0": 4.89435947534997, "psys": 6.7818737896281585, "uncore": 0.23605736730079518 }, "npu_busy_delta_us": 348123, "npu_utilization": 0.8396287784543176, "seconds": 0.414615374, "watts": { "core": 4.033224778587202, "dram": 1.4417627456332578, "package-0": 16.100512471590115, "psys": 26.288974513521055, "uncore": 0.5714645786386108 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 1087.0447466317994, "sha256": "1ffb883bcf5cae00e56811d75cd2b9c78794a471985aba335602ec78b832ac9a", "sum": 326622.44280824065, "top1": 8940 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 449, "gross_j": { "core": 36.497771, "dram": 6.911604, "package-0": 81.763158, "psys": 146.634817, "uncore": 26.816033 }, "net_j": { "core": 25.312192068698756, "dram": 4.676861795061624, "package-0": 62.20194331229561, "psys": 101.41042994997447, "uncore": 26.806361639641537 }, "npu_busy_delta_us": 136077, "npu_utilization": 0.029884539373416927, "seconds": 4.553424709, "token_latencies_s": [ 0.334018198, 0.321224467, 0.32096755, 0.321251807, 0.321796775, 0.325829801, 0.326582775, 0.326650077, 0.329529262, 0.328230902, 0.328368279, 0.322585459, 0.322820395, 0.323490108 ], "watts": { "core": 8.015455032749506, "dram": 1.5178913546849644, "package-0": 17.956409345781502, "psys": 32.20319350184297, "uncore": 5.889200923207799 } } }, { "energy": { "gross_j": { "core": 44.61212999999999, "dram": 8.165994, "package-0": 98.58049700000001, "psys": 173.889752, "uncore": 28.972948000000002 }, "net_j": { "core": 31.391813381566674, "dram": 5.524736029240304, "package-0": 75.46095591939219, "psys": 120.43872194174978, "uncore": 28.961517349261662 } }, "first_decode": { "gpu_active_fraction": 0.972972972972973, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 37, "gross_j": { "core": 4.788135, "dram": 0.538634, "package-0": 8.207682, "psys": 13.459682, "uncore": 1.955073 }, "logits": { "l2": 1085.5057909602467, "sha256": "eda78e61e9193b4d2f161191c2f713a7b36c6702e1f3ac7fda9b3c8bd416522c", "sum": 325965.88127466664, "top1": 8940 }, "net_j": { "core": 3.876696525697236, "dram": 0.3565397125036909, "package-0": 6.613768642455436, "psys": 9.774647314955068, "uncore": 1.9542849451318804 }, "npu_busy_delta_us": 7865, "npu_utilization": 0.02119784325245778, "seconds": 0.371028312, "watts": { "core": 12.905039440763753, "dram": 1.4517328801582126, "package-0": 22.121443929055204, "psys": 36.27669793565511, "uncore": 5.26933642735059 } }, "metrics": { "all_tokens_per_s_e2e": 190.27354391373697, "decode_tokens_per_s": 3.0181301006750814, "end_to_end_ms": 5381.725588, "handoff_estimate_ms": 46.12672950000002, "handoff_plus_first_decode_ms": 371.028312, "output_tokens_per_s_e2e": 2.97302412365214, "prefill_tokens_per_s": 2448.0225791614002, "steady_tpot_ms": 324.9015825, "steady_tpot_p95_ms": 343.38017314999996, "ttft_ms": 411.760908 }, "prefill": { "gpu_active_fraction": 0.0975609756097561, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 41, "gross_j": { "core": 2.226191, "dram": 0.594908, "package-0": 7.315777, "psys": 11.5029, "uncore": 0.315917 }, "logits": { "l2": 633.1706927261538, "sha256": "ad53eb322905ce268ba906538fffea0ad7f5b54a7f40119f6d9c48fbf51f50d8", "sum": -190262.31765024643, "top1": 5809 }, "net_j": { "core": 1.214692088388531, "dram": 0.3928228578799203, "package-0": 5.54687907456115, "psys": 7.413311203537493, "uncore": 0.3150424301152879 }, "npu_busy_delta_us": 351326, "npu_utilization": 0.8532281554032323, "seconds": 0.411760908, "watts": { "core": 5.406513723736008, "dram": 1.4447898973449904, "package-0": 17.767050873124653, "psys": 27.9358719502338, "uncore": 0.7672340765287025 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 944.2549601773937, "sha256": "466639dd7dc3f26c0c878b29f045f1c72f6ae69ea88632a94899e0eb977e183b", "sum": 267898.8189940732, "top1": 3765 }, "gpu_active_fraction": 0.9978021978021978, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 455, "gross_j": { "core": 37.597804, "dram": 7.032452, "package-0": 83.057038, "psys": 148.92717, "uncore": 26.701958 }, "net_j": { "core": 26.30042476748091, "dram": 4.775373458856693, "package-0": 63.300308202375604, "psys": 103.25076342325721, "uncore": 26.692189974014493 }, "npu_busy_delta_us": 136120, "npu_utilization": 0.029598148160331315, "seconds": 4.598936368, "token_latencies_s": [ 0.361038547, 0.324808595, 0.332903266, 0.324700847, 0.32499457, 0.324450015, 0.325823728, 0.325840706, 0.333871818, 0.323532067, 0.323924733, 0.324029129, 0.323600632, 0.325364959 ], "watts": { "core": 8.175325986593426, "dram": 1.5291474891743924, "package-0": 18.06005375023706, "psys": 32.382959467814054, "uncore": 5.8061159936449025 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 4.770954256877303, "compile_target": "AUTO:NPU,GPU", "context_length": 1024, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 394836, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 2.4565200143075967, "dram": 0.4907827289910671, "package-0": 4.295934585025856, "psys": 9.931950112327097, "uncore": 0.002123975024632864 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_1b_ov_fp16/openvino_model.xml", "key": "llama32_1b_fp16", "size_b": 1.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": null, "key": "openvino_auto", "label": "OpenVINO AUTO", "mode": "stock", "prefill_device": null }, "prompt_length": 1008, "prompt_sha256": "2669029615323cd09063f0eb8d822f5daea575e1396736877003e47194c143f8", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.028868702240288258, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 8.555276000000001, "dram": 2.280024, "package-0": 22.186527, "psys": 37.365445, "uncore": 8.826454 }, "net_j": { "core": 6.38960574697493, "dram": 1.8473495049078017, "package-0": 18.399227207796013, "psys": 28.60942885214928, "uncore": 8.82458150170883 } }, "first_decode": { "gpu_active_fraction": 0.8333333333333334, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 6, "gross_j": { "core": 1.774104, "dram": 0.126953, "package-0": 2.380609, "psys": 3.3153, "uncore": 0.337157 }, "logits": { "l2": 401.4866824038882, "sha256": "61d2cdecbae6d5fd94941833b71375ae3e9c75ee5788bfeb46933416f5518666", "sum": 39969.7027254026, "top1": 546 }, "net_j": { "core": 1.639020976499471, "dram": 0.09996505993526558, "package-0": 2.1443773270944773, "psys": 2.7691461552865793, "uncore": 0.33704020348847313 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.054989588, "watts": { "core": 32.26254395650318, "dram": 2.3086734165020477, "package-0": 43.29199556832469, "psys": 60.28959518663788, "uncore": 6.131287981281111 } }, "metrics": { "all_tokens_per_s_e2e": 1161.5233164593224, "decode_tokens_per_s": 25.521630132804763, "end_to_end_ms": 881.6008989999999, "handoff_estimate_ms": 17.576560999999998, "handoff_plus_first_decode_ms": 54.989588, "output_tokens_per_s_e2e": 18.148801819676912, "prefill_tokens_per_s": 3430.156452532469, "steady_tpot_ms": 37.413027, "steady_tpot_p95_ms": 40.821983800000005, "ttft_ms": 293.864147 }, "prefill": { "gpu_active_fraction": 0.8275862068965517, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 29, "gross_j": { "core": 1.728512, "dram": 0.571471, "package-0": 7.179242, "psys": 12.130828, "uncore": 4.134572 }, "logits": { "l2": 628.2510355324671, "sha256": "c96d07293a33b871695e60a2f64ee2de15b94f1c3e0c78181714540acc5414c9", "sum": -187389.82698735525, "top1": 5809 }, "net_j": { "core": 1.0066288414070703, "dram": 0.4272475519827078, "package-0": 5.916820847603578, "psys": 9.212183952194442, "uncore": 4.133947839891137 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.293864147, "watts": { "core": 5.88201050603155, "dram": 1.9446775179416491, "package-0": 24.430479435111216, "psys": 41.280394780517405, "uncore": 14.069671452638964 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 518.3678398516904, "sha256": "f563a91e160dc6bc68c1aa9a1c95954b91970ab4b372530c63a6fe21c1c8c083", "sum": 92807.67067616689, "top1": 42 }, "gpu_active_fraction": 0.9811320754716981, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 53, "gross_j": { "core": 5.05266, "dram": 1.5816, "package-0": 12.626676, "psys": 21.919317, "uncore": 4.354725 }, "net_j": { "core": 3.743955929068389, "dram": 1.3201368929898283, "package-0": 10.338029033097959, "psys": 16.628098744668257, "uncore": 4.35359345832922 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.532747164, "token_latencies_s": [ 0.043741516, 0.038761713, 0.037647569, 0.037422258, 0.037285907, 0.037350122, 0.037403796, 0.03716556, 0.037422477, 0.037353281, 0.037133919, 0.037311275, 0.037469203, 0.039249928 ], "watts": { "core": 9.484161233376364, "dram": 2.968762870786488, "package-0": 23.701066571984605, "psys": 41.14393934155227, "uncore": 8.174093255238802 } } }, { "energy": { "gross_j": { "core": 8.692056, "dram": 2.336909, "package-0": 22.539309, "psys": 38.068506, "uncore": 8.915444 }, "net_j": { "core": 6.447430771586349, "dram": 1.8884602640862582, "package-0": 18.613933634441782, "psys": 28.993267182636274, "uncore": 8.9135032350573 } }, "first_decode": { "gpu_active_fraction": 0.8, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2250, "gpu_sample_count": 5, "gross_j": { "core": 1.521297, "dram": 0.121155, "package-0": 2.035578, "psys": 2.853935, "uncore": 0.256774 }, "logits": { "l2": 401.4866824038882, "sha256": "61d2cdecbae6d5fd94941833b71375ae3e9c75ee5788bfeb46933416f5518666", "sum": 39969.7027254026, "top1": 546 }, "net_j": { "core": 1.4026985031858825, "dram": 0.0974604672538147, "package-0": 1.828174282045985, "psys": 2.3744297051543524, "uncore": 0.25667145646535544 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.048279068, "watts": { "core": 31.5104881477828, "dram": 2.5094726352215413, "package-0": 42.162744318096614, "psys": 59.11329937023639, "uncore": 5.318536803568785 } }, "metrics": { "all_tokens_per_s_e2e": 1120.6665873700208, "decode_tokens_per_s": 25.2723554572565, "end_to_end_ms": 913.7418850000001, "handoff_estimate_ms": 10.164394, "handoff_plus_first_decode_ms": 48.279068, "output_tokens_per_s_e2e": 17.510415427656575, "prefill_tokens_per_s": 3147.9540954465865, "steady_tpot_ms": 38.114674, "steady_tpot_p95_ms": 42.4972871, "ttft_ms": 320.207973 }, "prefill": { "gpu_active_fraction": 0.78125, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 32, "gross_j": { "core": 2.14355, "dram": 0.625548, "package-0": 7.826579, "psys": 13.158414, "uncore": 4.251515 }, "logits": { "l2": 628.2510355324671, "sha256": "c96d07293a33b871695e60a2f64ee2de15b94f1c3e0c78181714540acc5414c9", "sum": -187389.82698735525, "top1": 5809 }, "net_j": { "core": 1.3569527055846333, "dram": 0.4683954571663621, "package-0": 6.450986494388275, "psys": 9.978124386594619, "uncore": 4.25083488626266 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.320207973, "watts": { "core": 6.694243056839812, "dram": 1.953567845732561, "package-0": 24.44217402419271, "psys": 41.09333654849375, "uncore": 13.277355214387496 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 518.3678398516904, "sha256": "f563a91e160dc6bc68c1aa9a1c95954b91970ab4b372530c63a6fe21c1c8c083", "sum": 92807.67067616689, "top1": 42 }, "gpu_active_fraction": 0.9814814814814815, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 54, "gross_j": { "core": 5.027209, "dram": 1.590206, "package-0": 12.677152, "psys": 22.056157, "uncore": 4.407155 }, "net_j": { "core": 3.687779562815834, "dram": 1.3226043396660814, "package-0": 10.334772858007522, "psys": 16.640713090887303, "uncore": 4.405996892329284 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.545254844, "token_latencies_s": [ 0.048495088, 0.039267702, 0.038694144, 0.038090107, 0.037870312, 0.038040618, 0.037815981, 0.037915328, 0.038186964, 0.038660971, 0.038083697, 0.038139241, 0.037812673, 0.038164603 ], "watts": { "core": 9.219925426283787, "dram": 2.9164454337245647, "package-0": 23.249957592307055, "psys": 40.45109776227866, "uncore": 8.08274341529738 } } }, { "energy": { "gross_j": { "core": 8.220743, "dram": 2.34588, "package-0": 22.060674, "psys": 37.247219, "uncore": 8.915016000000001 }, "net_j": { "core": 5.984695352368028, "dram": 1.8991449600788228, "package-0": 18.150299011284325, "psys": 28.20666017869245, "uncore": 8.913082651470456 } }, "first_decode": { "gpu_active_fraction": 0.8, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2300, "gpu_sample_count": 5, "gross_j": { "core": 1.503842, "dram": 0.121887, "package-0": 2.022394, "psys": 2.825432, "uncore": 0.262389 }, "logits": { "l2": 401.4866824038882, "sha256": "61d2cdecbae6d5fd94941833b71375ae3e9c75ee5788bfeb46933416f5518666", "sum": 39969.7027254026, "top1": 546 }, "net_j": { "core": 1.3853916534514252, "dram": 0.09822206586941742, "package-0": 1.8152493655648732, "psys": 2.346525691133677, "uncore": 0.2622865845601652 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.048218759, "watts": { "core": 31.187903446457423, "dram": 2.5277921399843573, "package-0": 41.94205827653092, "psys": 58.59611608834645, "uncore": 5.441637351139626 } }, "metrics": { "all_tokens_per_s_e2e": 1124.9655155224127, "decode_tokens_per_s": 25.46414952552676, "end_to_end_ms": 910.2501239999998, "handoff_estimate_ms": 10.166610499999999, "handoff_plus_first_decode_ms": 48.218759, "output_tokens_per_s_e2e": 17.5775861800377, "prefill_tokens_per_s": 3138.361924210023, "steady_tpot_ms": 38.0521485, "steady_tpot_p95_ms": 41.2908715, "ttft_ms": 321.186665 }, "prefill": { "gpu_active_fraction": 0.78125, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 32, "gross_j": { "core": 1.862788, "dram": 0.633299, "package-0": 7.559124, "psys": 12.792265, "uncore": 4.260182 }, "logits": { "l2": 628.2510355324671, "sha256": "c96d07293a33b871695e60a2f64ee2de15b94f1c3e0c78181714540acc5414c9", "sum": -187389.82698735525, "top1": 5809 }, "net_j": { "core": 1.073786529098791, "dram": 0.4756661320357603, "package-0": 6.179327097577386, "psys": 9.602255066475285, "uncore": 4.2594998075452954 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.321186665, "watts": { "core": 5.7997052897572825, "dram": 1.9717474883336144, "package-0": 23.53498704561723, "psys": 39.828132341671164, "uncore": 13.263881923615978 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 518.3678398516904, "sha256": "f563a91e160dc6bc68c1aa9a1c95954b91970ab4b372530c63a6fe21c1c8c083", "sum": 92807.67067616689, "top1": 42 }, "gpu_active_fraction": 0.9811320754716981, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 53, "gross_j": { "core": 4.854113, "dram": 1.590694, "package-0": 12.479156, "psys": 21.629522, "uncore": 4.392445 }, "net_j": { "core": 3.525517169817812, "dram": 1.325256762173645, "package-0": 10.155722548142066, "psys": 16.257879421083487, "uncore": 4.391296259364995 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.5408447, "token_latencies_s": [ 0.043849577, 0.038705866, 0.038219717, 0.037993932, 0.037977967, 0.037863963, 0.038114856, 0.03816807, 0.037983366, 0.038002681, 0.039913107, 0.037998039, 0.038101616, 0.037929621 ], "watts": { "core": 8.975058829272063, "dram": 2.941128941450291, "package-0": 23.07345528208005, "psys": 39.99211233834778, "uncore": 8.121453348807893 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 1.1849790308624506, "compile_target": "AUTO:GPU", "context_length": 2048, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 401967, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_1b_ov_fp16/openvino_model.xml", "key": "llama32_1b_fp16", "size_b": 1.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "GPU", "key": "gpu_gpu", "label": "iGPU -> iGPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 2032, "prompt_sha256": "cd1a97358e68d07c27508cfacc4ba6882a7a2962b1281e2e2b0c7e5e320da4fa", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.02615249902009964, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 10.018405999999999, "dram": 2.998649, "package-0": 30.825787, "psys": 51.754994, "uncore": 14.380579 }, "net_j": { "core": 5.285439166381808, "dram": 2.3789948263692744, "package-0": 23.639646245287466, "psys": 39.010278270589, "uncore": 14.378128194818315 } }, "first_decode": { "gpu_active_fraction": 0.8333333333333334, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 6, "gross_j": { "core": 1.936213, "dram": 0.131591, "package-0": 2.537774, "psys": 3.501883, "uncore": 0.315856 }, "logits": { "l2": 388.14070680904894, "sha256": "3f0cac1ef1eb2e21f28a1dc9e144e620a998951178cf91d34bb243e46b451e57", "sum": 39520.489759358345, "top1": 3862 }, "net_j": { "core": 1.7153050901533535, "dram": 0.10266907573207325, "package-0": 2.202365916699789, "psys": 2.907032325497897, "uncore": 0.3157416103857983 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.058568969, "watts": { "core": 33.0586833447589, "dram": 2.2467699576545392, "package-0": 43.32966830951045, "psys": 59.79075711576893, "uncore": 5.392889876548792 } }, "metrics": { "all_tokens_per_s_e2e": 1632.0752488184387, "decode_tokens_per_s": 24.502581425361424, "end_to_end_ms": 1254.8441019999998, "handoff_estimate_ms": 19.5501405, "handoff_plus_first_decode_ms": 58.568968999999996, "output_tokens_per_s_e2e": 12.750587881394052, "prefill_tokens_per_s": 3161.8402922538776, "steady_tpot_ms": 39.0188285, "steady_tpot_p95_ms": 42.345227900000005, "ttft_ms": 642.663706 }, "prefill": { "gpu_active_fraction": 0.828125, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 64, "gross_j": { "core": 3.399588, "dram": 1.254635, "package-0": 15.718648, "psys": 26.331964, "uncore": 9.458228 }, "logits": { "l2": 454.23152425564587, "sha256": "7b34e1c17c4e4a4835f5c0f10762b904fd00cbdbf0f7a022a5035089aee9fa90", "sum": -65261.15840908815, "top1": 8 }, "net_j": { "core": 0.9756167665852598, "dram": 0.9372814362950087, "package-0": 12.038292932864183, "psys": 19.80480558285045, "uncore": 9.456972829307293 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.642663706, "watts": { "core": 5.289839722176563, "dram": 1.952241877496035, "package-0": 24.458589855391647, "psys": 40.973161786111504, "uncore": 14.71722755104518 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 453.8179678889412, "sha256": "2feedb4df01c626caf4b4dd96c8d01409d1b11d9e4fa214b89bf17963116df4e", "sum": 5239.512074235128, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 55, "gross_j": { "core": 4.682605, "dram": 1.612423, "package-0": 12.569365, "psys": 21.921147, "uncore": 4.606495 }, "net_j": { "core": 2.5945173096431953, "dram": 1.3390443143421926, "package-0": 9.398987395723493, "psys": 16.29844036224065, "uncore": 4.605413755125223 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.553611427, "token_latencies_s": [ 0.04218409, 0.042644484, 0.038901968, 0.039041332, 0.03937528, 0.038623992, 0.039055993, 0.038714656, 0.038832269, 0.040446649, 0.038846011, 0.038996325, 0.039128344, 0.038795583 ], "watts": { "core": 8.458288199314932, "dram": 2.9125536818083058, "package-0": 22.70430917243332, "psys": 39.59663029137584, "uncore": 8.320809100640187 } } }, { "energy": { "gross_j": { "core": 9.735937, "dram": 3.029229, "package-0": 30.717878, "psys": 51.886525, "uncore": 14.47268 }, "net_j": { "core": 4.914578354441195, "dram": 2.3980023059800053, "package-0": 23.39753051523866, "psys": 38.90379185876414, "uncore": 14.470183424138245 } }, "first_decode": { "gpu_active_fraction": 0.8, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2100, "gpu_sample_count": 5, "gross_j": { "core": 1.442135, "dram": 0.124756, "package-0": 1.959651, "psys": 2.789605, "uncore": 0.254516 }, "logits": { "l2": 388.14070680904894, "sha256": "3f0cac1ef1eb2e21f28a1dc9e144e620a998951178cf91d34bb243e46b451e57", "sum": 39520.489759358345, "top1": 3862 }, "net_j": { "core": 1.2522959150342932, "dram": 0.09990169920885955, "package-0": 1.6714152203576336, "psys": 2.2784150303103425, "uncore": 0.2544176983153536 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.05033174, "watts": { "core": 28.65259575766703, "dram": 2.478674490490494, "package-0": 38.93469607845864, "psys": 55.42437038735398, "uncore": 5.056769346738261 } }, "metrics": { "all_tokens_per_s_e2e": 1602.153789107206, "decode_tokens_per_s": 25.44030908225242, "end_to_end_ms": 1278.279285, "handoff_estimate_ms": 12.049381499999997, "handoff_plus_first_decode_ms": 50.331739999999996, "output_tokens_per_s_e2e": 12.516826477400047, "prefill_tokens_per_s": 2950.6414876831186, "steady_tpot_ms": 38.2823585, "steady_tpot_p95_ms": 39.660728899999995, "ttft_ms": 688.6638069999999 }, "prefill": { "gpu_active_fraction": 0.7647058823529411, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 68, "gross_j": { "core": 3.684988, "dram": 1.30322, "package-0": 16.338703, "psys": 27.476248, "uncore": 9.666845 }, "logits": { "l2": 454.23152425564587, "sha256": "7b34e1c17c4e4a4835f5c0f10762b904fd00cbdbf0f7a022a5035089aee9fa90", "sum": -65261.15840908815, "top1": 8 }, "net_j": { "core": 1.0875155706764739, "dram": 0.9631511408039959, "package-0": 12.394918236705006, "psys": 20.481893530500358, "uncore": 9.665499987640342 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.688663807, "watts": { "core": 5.350924446070099, "dram": 1.8923892714460602, "package-0": 23.725223881266057, "psys": 39.897912044621215, "uncore": 14.03710330315065 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 453.8179678889412, "sha256": "2feedb4df01c626caf4b4dd96c8d01409d1b11d9e4fa214b89bf17963116df4e", "sum": 5239.512074235128, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 53, "gross_j": { "core": 4.608814, "dram": 1.601253, "package-0": 12.419524, "psys": 21.620672, "uncore": 4.551319 }, "net_j": { "core": 2.574766868730428, "dram": 1.3349494659671497, "package-0": 9.33119705817602, "psys": 16.143483297953438, "uncore": 4.55026573818255 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.539283738, "token_latencies_s": [ 0.041746276, 0.038537742, 0.038375251, 0.038293233, 0.0380478, 0.038220112, 0.038149965, 0.038396353, 0.038101786, 0.03805424, 0.038533271, 0.038417281, 0.038271484, 0.038116643 ], "watts": { "core": 8.546176484186883, "dram": 2.9692217420433327, "package-0": 23.029665322487435, "psys": 40.091459238476055, "uncore": 8.43956284845363 } } }, { "energy": { "gross_j": { "core": 9.396459, "dram": 3.032829, "package-0": 30.241073, "psys": 51.64977, "uncore": 14.374719 }, "net_j": { "core": 4.544882830167648, "dram": 2.397646137324883, "package-0": 22.874845755180708, "psys": 38.58566849713154, "uncore": 14.372206777025626 } }, "first_decode": { "gpu_active_fraction": 0.8333333333333334, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 6, "gross_j": { "core": 1.625606, "dram": 0.126465, "package-0": 2.132929, "psys": 3.011039, "uncore": 0.239502 }, "logits": { "l2": 388.14070680904894, "sha256": "3f0cac1ef1eb2e21f28a1dc9e144e620a998951178cf91d34bb243e46b451e57", "sum": 39520.489759358345, "top1": 3862 }, "net_j": { "core": 1.4276382132655119, "dram": 0.10054646526125459, "package-0": 1.8323512798545583, "psys": 2.4779604354956857, "uncore": 0.23939948914484477 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.05248689, "watts": { "core": 30.97165787494744, "dram": 2.4094588191451236, "package-0": 40.63736677863748, "psys": 57.367449281144296, "uncore": 4.5630823239860465 } }, "metrics": { "all_tokens_per_s_e2e": 1592.1749452598515, "decode_tokens_per_s": 24.565094490356536, "end_to_end_ms": 1286.29081, "handoff_estimate_ms": 12.980355500000005, "handoff_plus_first_decode_ms": 52.48689, "output_tokens_per_s_e2e": 12.43886675984259, "prefill_tokens_per_s": 3007.3928910139794, "steady_tpot_ms": 39.506534499999994, "steady_tpot_p95_ms": 42.16395955, "ttft_ms": 675.668286 }, "prefill": { "gpu_active_fraction": 0.7910447761194029, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 67, "gross_j": { "core": 3.690603, "dram": 1.288509, "package-0": 16.16335, "psys": 27.319694, "uncore": 9.536963 }, "logits": { "l2": 454.23152425564587, "sha256": "7b34e1c17c4e4a4835f5c0f10762b904fd00cbdbf0f7a022a5035089aee9fa90", "sum": -65261.15840908815, "top1": 8 }, "net_j": { "core": 1.142146515988371, "dram": 0.9548574549654291, "package-0": 12.293986946093433, "psys": 20.457327416992882, "uncore": 9.535643368873082 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.675668286, "watts": { "core": 5.462152178028969, "dram": 1.907014176480084, "package-0": 23.922019628430515, "psys": 40.43358933084511, "uncore": 14.114859610267398 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 453.8179678889412, "sha256": "2feedb4df01c626caf4b4dd96c8d01409d1b11d9e4fa214b89bf17963116df4e", "sum": 5239.512074235128, "top1": 42 }, "gpu_active_fraction": 0.9818181818181818, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 55, "gross_j": { "core": 4.08025, "dram": 1.617855, "package-0": 11.944794, "psys": 21.319037, "uncore": 4.598254 }, "net_j": { "core": 1.9750981009137654, "dram": 1.3422422170981994, "package-0": 8.748507529232718, "psys": 15.650380644642972, "uncore": 4.5971639190077 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.558135634, "token_latencies_s": [ 0.044164889, 0.040065096, 0.038947973, 0.040946505, 0.040507038, 0.038612673, 0.03844799, 0.04072849, 0.041086536, 0.040085377, 0.038408862, 0.038505675, 0.038718139, 0.038889417 ], "watts": { "core": 7.3104990103534595, "dram": 2.898677134096047, "package-0": 21.401238825041585, "psys": 38.19687491947522, "uncore": 8.238595996900639 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 3.8732490157708526, "compile_target": "NPU", "context_length": 2048, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 402038, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_1b_ov_fp16/openvino_model.xml", "key": "llama32_1b_fp16", "size_b": 1.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "gpu_npu", "label": "iGPU -> NPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 2032, "prompt_sha256": "cd1a97358e68d07c27508cfacc4ba6882a7a2962b1281e2e2b0c7e5e320da4fa", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "GPU", "NPUW_ONLINE_PIPELINE": "NONE" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 2048, "NPUW_LLM_MIN_RESPONSE_LEN": 2048, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.02616929356008768, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 14.178613, "dram": 3.661306, "package-0": 34.775546, "psys": 61.267604, "uncore": 10.533786000000001 }, "net_j": { "core": 7.855560562014698, "dram": 2.8334730632776717, "package-0": 25.175152332323574, "psys": 44.241178779378494, "uncore": 10.531650949636026 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 0, "gpu_sample_count": 19, "gross_j": { "core": 3.304191, "dram": 0.23059, "package-0": 3.934072, "psys": 6.120712, "uncore": 0.005432 }, "logits": { "l2": 1026.4886340623634, "sha256": "2b5d51c7401796ac587631de9e5f269ea1f06106fe3f8dad834e85e69bf2f811", "sum": -244601.09724290296, "top1": 370 }, "net_j": { "core": 2.5832979539685184, "dram": 0.13620852630323968, "package-0": 2.839528327925412, "psys": 4.179524312454586, "uncore": 0.005058710164097304 }, "npu_busy_delta_us": 56471, "npu_utilization": 0.2954597524260638, "seconds": 0.191129247, "watts": { "core": 17.287730956215192, "dram": 1.2064610917448966, "package-0": 20.583307169101126, "psys": 32.02394241630638, "uncore": 0.028420558785542643 } }, "metrics": { "all_tokens_per_s_e2e": 1221.6501600116544, "decode_tokens_per_s": 19.37041679877237, "end_to_end_ms": 1676.421014, "handoff_estimate_ms": 151.2270395, "handoff_plus_first_decode_ms": 191.129247, "output_tokens_per_s_e2e": 9.54414187509105, "prefill_tokens_per_s": 2252.6610209608502, "steady_tpot_ms": 39.902207499999996, "steady_tpot_p95_ms": 48.3112845, "ttft_ms": 902.044285 }, "prefill": { "gpu_active_fraction": 0.9888888888888889, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 90, "gross_j": { "core": 6.016403, "dram": 1.742061, "package-0": 20.765633, "psys": 36.900967, "uncore": 10.528354 }, "logits": { "l2": 366.74491757063316, "sha256": "1f0df7529225c1c78b67b5a87bd8d4ea0b140bf8c9c4e37ddaa073fb1bd6884e", "sum": -50010.3206056105, "top1": 90 }, "net_j": { "core": 2.6141112917668807, "dram": 1.2966227936848753, "package-0": 15.599877996121418, "psys": 27.739432137349148, "uncore": 10.52659223947193 }, "npu_busy_delta_us": 9726, "npu_utilization": 0.010782175733201392, "seconds": 0.902044285, "watts": { "core": 6.669742384100355, "dram": 1.9312366687185432, "package-0": 23.020635843837756, "psys": 40.90815452591665, "uncore": 11.671659778876599 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 813.1332002581323, "sha256": "5e3c7b11b2975536d5ab3fd0541e09bbe1892c78109e321d44e4f8d6ce291c4f", "sum": -91797.23669979721, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 58, "gross_j": { "core": 4.858019, "dram": 1.688655, "package-0": 10.075841, "psys": 18.245925, "uncore": 0.0 }, "net_j": { "core": 2.658151316279299, "dram": 1.4006417432895568, "package-0": 6.735746008276745, "psys": 12.32222232957476, "uncore": 0.0 }, "npu_busy_delta_us": 571127, "npu_utilization": 0.9792189724361298, "seconds": 0.583247482, "token_latencies_s": [ 0.053266943, 0.045642853, 0.043085412, 0.041374964, 0.040419602, 0.039826308, 0.039855048, 0.039860939, 0.039777561, 0.039593654, 0.039838664, 0.039907398, 0.039897017, 0.040861054 ], "watts": { "core": 8.329258419327388, "dram": 2.895263249503407, "package-0": 17.275412772377816, "psys": 31.283332655690746, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 12.49124, "dram": 3.5721950000000002, "package-0": 32.787330000000004, "psys": 57.997777, "uncore": 10.48325 }, "net_j": { "core": 6.587447668510164, "dram": 2.7992528503800553, "package-0": 23.823505791613226, "psys": 42.1003161397483, "uncore": 10.481335872016974 } }, "first_decode": { "gpu_active_fraction": 0.125, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 8, "gross_j": { "core": 2.360773, "dram": 0.15509, "package-0": 2.765984, "psys": 3.938162, "uncore": 0.007996 }, "logits": { "l2": 957.03468159783, "sha256": "a85018f448b20deb48b5ea1269f1000c3280c7556b9fea050e8873cb46a40981", "sum": -197233.62805647403, "top1": 83 }, "net_j": { "core": 2.050858500705465, "dram": 0.11451506683282053, "package-0": 2.2954357599325212, "psys": 3.103638469034935, "uncore": 0.00783552136419905 }, "npu_busy_delta_us": 54586, "npu_utilization": 0.6643287736399075, "seconds": 0.082167147, "watts": { "core": 28.73134928245714, "dram": 1.8874940370024045, "package-0": 33.662894489935255, "psys": 47.92866910664429, "uncore": 0.09731383274144835 } }, "metrics": { "all_tokens_per_s_e2e": 1308.4061208293747, "decode_tokens_per_s": 22.47622076528829, "end_to_end_ms": 1565.263237, "handoff_estimate_ms": 42.030404499999996, "handoff_plus_first_decode_ms": 82.167147, "output_tokens_per_s_e2e": 10.22192281897949, "prefill_tokens_per_s": 2263.0802880466854, "steady_tpot_ms": 40.136742500000004, "steady_tpot_p95_ms": 48.466140249999995, "ttft_ms": 897.891255 }, "prefill": { "gpu_active_fraction": 0.9887640449438202, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 89, "gross_j": { "core": 6.207138, "dram": 1.73437, "package-0": 20.904243, "psys": 37.129239, "uncore": 10.475254 }, "logits": { "l2": 369.2233804425175, "sha256": "ea02272bdbf413a702ab91b2f85a27d9af2a58670dcde9962d3526c27e15f262", "sum": -46712.414185937494, "top1": 90 }, "net_j": { "core": 2.8205105111051174, "dram": 1.2909826001305618, "package-0": 15.762271235880824, "psys": 28.009884027808333, "uncore": 10.473500350652774 }, "npu_busy_delta_us": 7980, "npu_utilization": 0.008887490501285704, "seconds": 0.897891255, "watts": { "core": 6.913017545760594, "dram": 1.9316036216434695, "package-0": 23.28148635326669, "psys": 41.35159886371763, "uncore": 11.666506318741238 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 871.4172103989001, "sha256": "160a3b8ab6ee81bb0777c0eee19112601ae899ac4cee1eb1b9d3e5204371b1fc", "sum": -145708.02170770615, "top1": 75 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 58, "gross_j": { "core": 3.923329, "dram": 1.682735, "package-0": 9.117103, "psys": 16.930376, "uncore": 0.0 }, "net_j": { "core": 1.716078656699581, "dram": 1.3937551834166728, "package-0": 5.7657987957998795, "psys": 10.986793642905036, "uncore": 0.0 }, "npu_busy_delta_us": 572066, "npu_utilization": 0.9775483143436434, "seconds": 0.585204835, "token_latencies_s": [ 0.052923031, 0.046066276, 0.043457531, 0.041763888, 0.040699016, 0.040042184, 0.040006913, 0.040116214, 0.04002152, 0.040157271, 0.04023111, 0.03985848, 0.039904702, 0.039931882 ], "watts": { "core": 6.704197855781557, "dram": 2.8754632555282975, "package-0": 15.579336421579635, "psys": 28.93068373230375, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 14.188255999999999, "dram": 3.587881, "package-0": 34.45908, "psys": 59.431549, "uncore": 10.424778 }, "net_j": { "core": 8.204117462763234, "dram": 2.804419684817169, "package-0": 25.373264836700248, "psys": 43.31773556678963, "uncore": 10.42301598782402 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 11, "gross_j": { "core": 2.4096, "dram": 0.163329, "package-0": 2.839531, "psys": 4.119862, "uncore": 0.0 }, "logits": { "l2": 837.2293686109759, "sha256": "894545dfa5886dde4385e1e4378f9e7553287f9507144db59a71f97471aef493", "sum": -125513.81106831506, "top1": 62 }, "net_j": { "core": 2.0252021703778436, "dram": 0.11300248645503379, "package-0": 2.2558935044126684, "psys": 3.084773177754483, "uncore": 0.0 }, "npu_busy_delta_us": 54364, "npu_utilization": 0.5334259806593735, "seconds": 0.101914796, "watts": { "core": 23.64327943118289, "dram": 1.6026034139341259, "package-0": 27.861813116909932, "psys": 40.424571913974106, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 1290.838768948975, "decode_tokens_per_s": 21.91725837821502, "end_to_end_ms": 1586.565301, "handoff_estimate_ms": 62.070555999999996, "handoff_plus_first_decode_ms": 101.914796, "output_tokens_per_s_e2e": 10.084677882413867, "prefill_tokens_per_s": 2252.339299326418, "steady_tpot_ms": 39.844240000000006, "steady_tpot_p95_ms": 48.58107759999999, "ttft_ms": 902.173132 }, "prefill": { "gpu_active_fraction": 0.9888888888888889, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 90, "gross_j": { "core": 6.945905, "dram": 1.736751, "package-0": 21.591986, "psys": 37.487758, "uncore": 10.424778 }, "logits": { "l2": 369.6101676549024, "sha256": "e185f24df452e75d08c9dbbd0c65b619de2470e823b0c7928125bccf1bc0ffdb", "sum": -39722.19562444929, "top1": 90 }, "net_j": { "core": 3.5431273122124165, "dram": 1.2912491677880942, "package-0": 16.425493125435846, "psys": 28.32491451392336, "uncore": 10.42301598782402 }, "npu_busy_delta_us": 9348, "npu_utilization": 0.010361647524657163, "seconds": 0.902173132, "watts": { "core": 7.699082087051114, "dram": 1.925075064195106, "package-0": 23.933306406646565, "psys": 41.55273158810941, "uncore": 11.555185618185755 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 906.6597247212674, "sha256": "2f8dbeac505661241a401ff006629454b54aeca37d59486d1e910a10ea22d139", "sum": -173918.8459377624, "top1": 62 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 57, "gross_j": { "core": 4.832751, "dram": 1.687801, "package-0": 10.027563, "psys": 17.823929, "uncore": 0.0 }, "net_j": { "core": 2.6357879801729744, "dram": 1.4001680305740412, "package-0": 6.691878206851732, "psys": 11.908047875111784, "uncore": 0.0 }, "npu_busy_delta_us": 571930, "npu_utilization": 0.9818922184982455, "seconds": 0.582477373, "token_latencies_s": [ 0.053674813, 0.045838297, 0.042991903, 0.041648133, 0.040481749, 0.039860386, 0.039828094, 0.039559582, 0.039788167, 0.03990437, 0.039754667, 0.039558386, 0.039766189, 0.039798849 ], "watts": { "core": 8.296890530029223, "dram": 2.89762500353812, "package-0": 17.21536915391905, "psys": 30.600208396421262, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 3.2246395656839013, "compile_target": "AUTO:NPU,GPU", "context_length": 2048, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 401877, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_1b_ov_fp16/openvino_model.xml", "key": "llama32_1b_fp16", "size_b": 1.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": null, "key": "openvino_auto", "label": "OpenVINO AUTO", "mode": "stock", "prefill_device": null }, "prompt_length": 2032, "prompt_sha256": "cd1a97358e68d07c27508cfacc4ba6882a7a2962b1281e2e2b0c7e5e320da4fa", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.02877042442560196, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 11.224703, "dram": 2.95507, "package-0": 31.846659, "psys": 53.059556, "uncore": 14.227747 }, "net_j": { "core": 6.496222690010083, "dram": 2.3360032154613775, "package-0": 24.667330207303156, "psys": 40.32692137537289, "uncore": 14.225298518011304 } }, "first_decode": { "gpu_active_fraction": 0.7142857142857143, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 7, "gross_j": { "core": 2.061884, "dram": 0.133605, "package-0": 2.627556, "psys": 3.645803, "uncore": 0.276366 }, "logits": { "l2": 388.14070680904894, "sha256": "3f0cac1ef1eb2e21f28a1dc9e144e620a998951178cf91d34bb243e46b451e57", "sum": 39520.489759358345, "top1": 3862 }, "net_j": { "core": 1.8320066260658108, "dram": 0.10350876666707012, "package-0": 2.2785294330523658, "psys": 3.0267997645230493, "uncore": 0.2762469658552458 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.060947029, "watts": { "core": 33.830754900292185, "dram": 2.192149513965644, "package-0": 43.1121261054415, "psys": 59.819207922341874, "uncore": 4.534527843842889 } }, "metrics": { "all_tokens_per_s_e2e": 1633.6238106579526, "decode_tokens_per_s": 24.215759902290376, "end_to_end_ms": 1253.6545970000002, "handoff_estimate_ms": 21.786800999999993, "handoff_plus_first_decode_ms": 60.947029, "output_tokens_per_s_e2e": 12.762686020765255, "prefill_tokens_per_s": 3203.9189630380047, "steady_tpot_ms": 39.160228000000004, "steady_tpot_p95_ms": 42.22771325, "ttft_ms": 634.223282 }, "prefill": { "gpu_active_fraction": 0.8253968253968254, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 63, "gross_j": { "core": 3.766897, "dram": 1.209347, "package-0": 15.950277, "psys": 26.54284, "uncore": 9.389808 }, "logits": { "l2": 454.23152425564587, "sha256": "7b34e1c17c4e4a4835f5c0f10762b904fd00cbdbf0f7a022a5035089aee9fa90", "sum": -65261.15840908815, "top1": 8 }, "net_j": { "core": 1.374760993598723, "dram": 0.8961613994312079, "package-0": 12.318257877382536, "psys": 20.101406020128554, "uncore": 9.388569314092011 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.634223282, "watts": { "core": 5.9393861860782335, "dram": 1.9068158396619692, "package-0": 25.14930853642172, "psys": 41.85093917760023, "uncore": 14.805208617365139 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 453.8179678889412, "sha256": "2feedb4df01c626caf4b4dd96c8d01409d1b11d9e4fa214b89bf17963116df4e", "sum": 5239.512074235128, "top1": 42 }, "gpu_active_fraction": 0.9821428571428571, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 56, "gross_j": { "core": 5.395922, "dram": 1.612118, "package-0": 13.268826, "psys": 22.870913, "uncore": 4.561573 }, "net_j": { "core": 3.289455070345549, "dram": 1.3363330493630994, "package-0": 10.070542896868254, "psys": 17.198715590721285, "uncore": 4.560482238064046 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.558484286, "token_latencies_s": [ 0.044595803, 0.039527306, 0.039287647, 0.038826702, 0.040952588, 0.040452623, 0.038780344, 0.038644735, 0.038873274, 0.039032809, 0.038856582, 0.038941791, 0.040882787, 0.040789719 ], "watts": { "core": 9.661725737436415, "dram": 2.8865950939217653, "package-0": 23.75863803623653, "psys": 40.95175741435275, "uncore": 8.167773228985713 } } }, { "energy": { "gross_j": { "core": 11.064912, "dram": 3.010735, "package-0": 31.979288, "psys": 53.449387, "uncore": 14.415795999999999 }, "net_j": { "core": 6.288190086378872, "dram": 2.385352280428749, "package-0": 24.726713193520734, "psys": 40.58684960059669, "uncore": 14.413322537744948 } }, "first_decode": { "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 5, "gross_j": { "core": 1.690059, "dram": 0.124084, "package-0": 2.213617, "psys": 3.085869, "uncore": 0.255371 }, "logits": { "l2": 388.14070680904894, "sha256": "3f0cac1ef1eb2e21f28a1dc9e144e620a998951178cf91d34bb243e46b451e57", "sum": 39520.489759358345, "top1": 3862 }, "net_j": { "core": 1.499058843808815, "dram": 0.09907768829138602, "package-0": 1.923618346974861, "psys": 2.571552551036241, "uncore": 0.2552720970942854 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.050639573, "watts": { "core": 33.37427430519605, "dram": 2.450336617174872, "package-0": 43.71318454837682, "psys": 60.937895349157074, "uncore": 5.042913770224722 } }, "metrics": { "all_tokens_per_s_e2e": 1617.1253345520813, "decode_tokens_per_s": 25.248461695390443, "end_to_end_ms": 1266.4448180000002, "handoff_estimate_ms": 12.310220000000003, "handoff_plus_first_decode_ms": 50.639573, "output_tokens_per_s_e2e": 12.633791676188135, "prefill_tokens_per_s": 3022.238957969607, "steady_tpot_ms": 38.329353, "steady_tpot_p95_ms": 41.38146835, "ttft_ms": 672.3492180000001 }, "prefill": { "gpu_active_fraction": 0.7910447761194029, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 67, "gross_j": { "core": 4.691333, "dram": 1.283139, "package-0": 17.299028, "psys": 28.634265, "uncore": 9.649389 }, "logits": { "l2": 454.23152425564587, "sha256": "7b34e1c17c4e4a4835f5c0f10762b904fd00cbdbf0f7a022a5035089aee9fa90", "sum": -65261.15840908815, "top1": 8 }, "net_j": { "core": 2.1553952334429822, "dram": 0.9511264428966679, "package-0": 13.448672319028802, "psys": 21.80560824443612, "uncore": 9.648075851263236 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.672349218, "watts": { "core": 6.9775242900632035, "dram": 1.9084412767176, "package-0": 25.729230490456224, "psys": 42.58838150385117, "uncore": 14.351751651773318 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 453.8179678889412, "sha256": "2feedb4df01c626caf4b4dd96c8d01409d1b11d9e4fa214b89bf17963116df4e", "sum": 5239.512074235128, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 54, "gross_j": { "core": 4.68352, "dram": 1.603512, "package-0": 12.466643, "psys": 21.729253, "uncore": 4.511036 }, "net_j": { "core": 2.6337360091270745, "dram": 1.335148149240695, "package-0": 9.354422527517071, "psys": 16.20968880512433, "uncore": 4.509974589387426 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.543456027, "token_latencies_s": [ 0.043760027, 0.040100706, 0.038354925, 0.038688037, 0.038334887, 0.038244146, 0.038476487, 0.038410052, 0.038095727, 0.038323819, 0.038280581, 0.037913214, 0.03821631, 0.038232035 ], "watts": { "core": 8.618029366339146, "dram": 2.9505827892860967, "package-0": 22.939561584805094, "psys": 39.9834612561947, "uncore": 8.300645822076788 } } }, { "energy": { "gross_j": { "core": 12.121491, "dram": 3.017693, "package-0": 33.008766, "psys": 54.76048, "uncore": 14.337121 }, "net_j": { "core": 7.316822382260864, "dram": 2.388651414340159, "package-0": 25.713759256749945, "psys": 41.82268899919216, "uncore": 14.334633066498967 } }, "first_decode": { "gpu_active_fraction": 0.8333333333333334, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 6, "gross_j": { "core": 1.616695, "dram": 0.124817, "package-0": 2.132258, "psys": 2.997795, "uncore": 0.251159 }, "logits": { "l2": 388.14070680904894, "sha256": "3f0cac1ef1eb2e21f28a1dc9e144e620a998951178cf91d34bb243e46b451e57", "sum": 39520.489759358345, "top1": 3862 }, "net_j": { "core": 1.4246279458082924, "dram": 0.09967100681418523, "package-0": 1.8406394583358323, "psys": 2.480605657097421, "uncore": 0.25105954463760444 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.050922438, "watts": { "core": 31.748185348077797, "dram": 2.451119877646078, "package-0": 41.872661320732526, "psys": 58.86982473227224, "uncore": 4.932187260947718 } }, "metrics": { "all_tokens_per_s_e2e": 1607.7192075447776, "decode_tokens_per_s": 25.110921889340272, "end_to_end_ms": 1273.854284, "handoff_estimate_ms": 12.282993499999998, "handoff_plus_first_decode_ms": 50.922438, "output_tokens_per_s_e2e": 12.560306308943575, "prefill_tokens_per_s": 3003.6748453983278, "steady_tpot_ms": 38.6394445, "steady_tpot_p95_ms": 40.783110050000005, "ttft_ms": 676.5046500000001 }, "prefill": { "gpu_active_fraction": 0.7910447761194029, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 67, "gross_j": { "core": 5.221911, "dram": 1.284237, "package-0": 17.708512, "psys": 29.153307, "uncore": 9.505408 }, "logits": { "l2": 454.23152425564587, "sha256": "7b34e1c17c4e4a4835f5c0f10762b904fd00cbdbf0f7a022a5035089aee9fa90", "sum": -65261.15840908815, "top1": 8 }, "net_j": { "core": 2.6702999543447987, "dram": 0.9501724503201123, "package-0": 13.83435932368807, "psys": 22.282445958271417, "uncore": 9.504086735391104 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.67650465, "watts": { "core": 7.718958029335053, "dram": 1.898341718715459, "package-0": 26.17648230503663, "psys": 43.09402307877706, "uncore": 14.050765209078753 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 453.8179678889412, "sha256": "2feedb4df01c626caf4b4dd96c8d01409d1b11d9e4fa214b89bf17963116df4e", "sum": 5239.512074235128, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 54, "gross_j": { "core": 5.282885, "dram": 1.608639, "package-0": 13.167996, "psys": 22.609378, "uncore": 4.580554 }, "net_j": { "core": 3.221894482107773, "dram": 1.3388079572058613, "package-0": 10.038760474726045, "psys": 17.059637383823322, "uncore": 4.579486786470259 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.546427196, "token_latencies_s": [ 0.042838841, 0.039676178, 0.039318594, 0.038500786, 0.038632513, 0.038396205, 0.038521508, 0.038862444, 0.038765446, 0.038699566, 0.038519141, 0.038460859, 0.038561309, 0.038646376 ], "watts": { "core": 9.668049172281682, "dram": 2.943921920020979, "package-0": 24.098353991882938, "psys": 41.37674362752618, "uncore": 8.382734302997614 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 1.166511282324791, "compile_target": "AUTO:GPU", "context_length": 4096, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 402375, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_1b_ov_fp16/openvino_model.xml", "key": "llama32_1b_fp16", "size_b": 1.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "GPU", "key": "gpu_gpu", "label": "iGPU -> iGPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 4080, "prompt_sha256": "66f8de7a9ef70b33481de3e8dae7cef6e4434d9723c6203a6a3dbe07ef228a16", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.026095177978277206, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 18.209975, "dram": 4.7343020000000005, "package-0": 57.792576000000004, "psys": 95.677123, "uncore": 29.015367 }, "net_j": { "core": 9.586043665621498, "dram": 3.6052311164909274, "package-0": 44.69872036855377, "psys": 72.45499675437996, "uncore": 29.010901391779242 } }, "first_decode": { "gpu_active_fraction": 0.7142857142857143, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 7, "gross_j": { "core": 2.146906, "dram": 0.139648, "package-0": 2.79876, "psys": 3.904348, "uncore": 0.342651 }, "logits": { "l2": 386.3595092072282, "sha256": "a08f57e55d540bd7c46befedb70a9eeed4a0b576f2871d968449003d95edde09", "sum": 34339.569468914415, "top1": 3862 }, "net_j": { "core": 1.8948035148991353, "dram": 0.106641988640312, "package-0": 2.4159886841987666, "psys": 3.2254980058553375, "uncore": 0.34252045734070813 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.066839538, "watts": { "core": 32.120299814160894, "dram": 2.089302292903341, "package-0": 41.872820844452875, "psys": 58.413749059725696, "uncore": 5.126471700028806 } }, "metrics": { "all_tokens_per_s_e2e": 1791.4238235718765, "decode_tokens_per_s": 23.23151177517207, "end_to_end_ms": 2286.449441, "handoff_estimate_ms": 26.066698499999998, "handoff_plus_first_decode_ms": 66.839538, "output_tokens_per_s_e2e": 6.9977493108276425, "prefill_tokens_per_s": 2486.630225771123, "steady_tpot_ms": 40.7728395, "steady_tpot_p95_ms": 43.9999003, "ttft_ms": 1640.774715 }, "prefill": { "gpu_active_fraction": 0.8588957055214724, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 163, "gross_j": { "core": 10.768588, "dram": 2.932793, "package-0": 41.390213, "psys": 68.230477, "uncore": 23.778137 }, "logits": { "l2": 388.37710963475644, "sha256": "cc21758ed0d374d7022ec1be971528d81a755ceb6dd836aac0bc64de1d25bf88", "sum": -1062.9958727225894, "top1": 72 }, "net_j": { "core": 4.579984734337649, "dram": 2.122562551040601, "package-0": 31.993955703997393, "psys": 51.56609033606972, "uncore": 23.77493244322112 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.640774715, "watts": { "core": 6.563111865116716, "dram": 1.7874440489534238, "package-0": 25.22601830806492, "psys": 41.584305496809165, "uncore": 14.492018180570268 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 450.3147795636656, "sha256": "4f4fcbcd85ba65dc12597e879daaf0bd341940cd416489da4580479054b6afca", "sum": 3555.901248254231, "top1": 42 }, "gpu_active_fraction": 0.9827586206896551, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 58, "gross_j": { "core": 5.294481, "dram": 1.661861, "package-0": 13.603603, "psys": 23.542298, "uncore": 4.894579 }, "net_j": { "core": 3.1112554163847137, "dram": 1.3760265768100144, "package-0": 10.28877598035761, "psys": 17.663408412454903, "uncore": 4.893448491217411 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.578835188, "token_latencies_s": [ 0.046730757, 0.041800779, 0.041028553, 0.041001933, 0.041178261, 0.042529439, 0.040810324, 0.040505382, 0.040519514, 0.04018004, 0.040700433, 0.040497106, 0.040735355, 0.040589902 ], "watts": { "core": 9.146784973963953, "dram": 2.8710434929536452, "package-0": 23.50168628656349, "psys": 40.67185010182898, "uncore": 8.455911287825854 } } }, { "energy": { "gross_j": { "core": 17.817156, "dram": 4.819262, "package-0": 57.560399, "psys": 97.004818, "uncore": 28.963671 }, "net_j": { "core": 8.98013209621303, "dram": 3.662292399967974, "package-0": 44.14300144413635, "psys": 73.20888586969251, "uncore": 28.95909504907209 } }, "first_decode": { "gpu_active_fraction": 0.8333333333333334, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 6, "gross_j": { "core": 1.945186, "dram": 0.134643, "package-0": 2.493219, "psys": 3.485038, "uncore": 0.257018 }, "logits": { "l2": 386.3595092072282, "sha256": "a08f57e55d540bd7c46befedb70a9eeed4a0b576f2871d968449003d95edde09", "sum": 34339.569468914415, "top1": 3862 }, "net_j": { "core": 1.719660818573979, "dram": 0.10511656872715963, "package-0": 2.150800437936988, "psys": 2.877754148635842, "uncore": 0.2569012194894515 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.059793139, "watts": { "core": 32.53192644728018, "dram": 2.2518135400116726, "package-0": 41.697409463651, "psys": 58.28491459530164, "uncore": 4.298453038232363 } }, "metrics": { "all_tokens_per_s_e2e": 1748.226123800929, "decode_tokens_per_s": 23.853103467420855, "end_to_end_ms": 2342.9463410000003, "handoff_estimate_ms": 19.824571000000006, "handoff_plus_first_decode_ms": 59.793139000000004, "output_tokens_per_s_e2e": 6.829008296097379, "prefill_tokens_per_s": 2380.2615383773136, "steady_tpot_ms": 39.968568, "steady_tpot_p95_ms": 43.72544909999999, "ttft_ms": 1714.097352 }, "prefill": { "gpu_active_fraction": 0.8352941176470589, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 170, "gross_j": { "core": 10.838352, "dram": 3.028801, "package-0": 41.799819, "psys": 70.456912, "uncore": 23.866699 }, "logits": { "l2": 388.37710963475644, "sha256": "cc21758ed0d374d7022ec1be971528d81a755ceb6dd836aac0bc64de1d25bf88", "sum": -1062.9958727225894, "top1": 72 }, "net_j": { "core": 4.373193576193849, "dram": 2.182363122590927, "package-0": 31.98366349580651, "psys": 53.0478303069371, "uncore": 23.863351238580478 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.714097352, "watts": { "core": 6.323066765930107, "dram": 1.7669947371810653, "package-0": 24.385907224714035, "psys": 41.10438180059682, "uncore": 13.92377099944321 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 450.3147795636656, "sha256": "4f4fcbcd85ba65dc12597e879daaf0bd341940cd416489da4580479054b6afca", "sum": 3555.901248254231, "top1": 42 }, "gpu_active_fraction": 0.9824561403508771, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 57, "gross_j": { "core": 5.033618, "dram": 1.655818, "package-0": 13.267361, "psys": 23.062868, "uncore": 4.839954 }, "net_j": { "core": 2.887277701445201, "dram": 1.3748127086498876, "package-0": 10.008537510392856, "psys": 17.28330141411956, "uncore": 4.838842591002161 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.56905585, "token_latencies_s": [ 0.045728922, 0.042646656, 0.040019732, 0.039950196, 0.04007741, 0.039750512, 0.039966918, 0.039837886, 0.039718784, 0.039806412, 0.039970218, 0.039930786, 0.040127735, 0.041494791 ], "watts": { "core": 8.845560589527372, "dram": 2.9097636022896523, "package-0": 23.314690464916577, "psys": 40.52830315337238, "uncore": 8.505235470296983 } } }, { "energy": { "gross_j": { "core": 17.309892, "dram": 4.800769, "package-0": 56.924414999999996, "psys": 95.3703, "uncore": 28.884141999999997 }, "net_j": { "core": 8.487423931997183, "dram": 3.6457050935237088, "package-0": 43.52911780706399, "psys": 71.61356315581945, "uncore": 28.879573586314375 } }, "first_decode": { "gpu_active_fraction": 0.8333333333333334, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 6, "gross_j": { "core": 1.722103, "dram": 0.13507, "package-0": 2.239802, "psys": 3.226737, "uncore": 0.258605 }, "logits": { "l2": 386.3595092072282, "sha256": "a08f57e55d540bd7c46befedb70a9eeed4a0b576f2871d968449003d95edde09", "sum": 34339.569468914415, "top1": 3862 }, "net_j": { "core": 1.4944962651770213, "dram": 0.10527104553073696, "package-0": 1.8942229814569767, "psys": 2.6138480372432618, "uncore": 0.25848714162813724 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.060345018, "watts": { "core": 28.537616808731418, "dram": 2.238295794360356, "package-0": 37.11660173835726, "psys": 53.47147298887209, "uncore": 4.285440763312059 } }, "metrics": { "all_tokens_per_s_e2e": 1751.1104518795883, "decode_tokens_per_s": 23.812829937474554, "end_to_end_ms": 2339.0871750000006, "handoff_estimate_ms": 20.376389500000002, "handoff_plus_first_decode_ms": 60.345018, "output_tokens_per_s_e2e": 6.840275202654642, "prefill_tokens_per_s": 2387.117089941522, "steady_tpot_ms": 39.9686285, "steady_tpot_p95_ms": 43.48767765, "ttft_ms": 1709.174643 }, "prefill": { "gpu_active_fraction": 0.834319526627219, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 169, "gross_j": { "core": 10.529025, "dram": 3.011589, "package-0": 41.391861, "psys": 69.304388, "uncore": 23.789246 }, "logits": { "l2": 388.37710963475644, "sha256": "cc21758ed0d374d7022ec1be971528d81a755ceb6dd836aac0bc64de1d25bf88", "sum": -1062.9958727225894, "top1": 72 }, "net_j": { "core": 4.082433837570317, "dram": 2.1675820039829317, "package-0": 31.603896471651325, "psys": 51.94530337567546, "uncore": 23.785907853002463 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.709174643, "watts": { "core": 6.1602979210591995, "dram": 1.7620136200440926, "package-0": 24.21745558274117, "psys": 40.54845318694562, "uncore": 13.91855776554485 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 450.3147795636656, "sha256": "4f4fcbcd85ba65dc12597e879daaf0bd341940cd416489da4580479054b6afca", "sum": 3555.901248254231, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 57, "gross_j": { "core": 5.058764, "dram": 1.65411, "package-0": 13.292752, "psys": 22.839175, "uncore": 4.836291 }, "net_j": { "core": 2.910493829249845, "dram": 1.37285204401004, "package-0": 10.030998353955686, "psys": 17.054411742900733, "uncore": 4.835178591683776 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.569567514, "token_latencies_s": [ 0.046172866, 0.040434646, 0.039540705, 0.039966414, 0.03973317, 0.039857281, 0.039970843, 0.040040039, 0.042041807, 0.039726003, 0.041269873, 0.039839921, 0.04138198, 0.039562882 ], "watts": { "core": 8.881763576143845, "dram": 2.904150885262743, "package-0": 23.338325436868224, "psys": 40.09915319713968, "uncore": 8.49116370074435 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 4.908063311129808, "compile_target": "NPU", "context_length": 4096, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 402443, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_1b_ov_fp16/openvino_model.xml", "key": "llama32_1b_fp16", "size_b": 1.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "gpu_npu", "label": "iGPU -> NPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 4080, "prompt_sha256": "66f8de7a9ef70b33481de3e8dae7cef6e4434d9723c6203a6a3dbe07ef228a16", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "GPU", "NPUW_ONLINE_PIPELINE": "NONE" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 4096, "NPUW_LLM_MIN_RESPONSE_LEN": 4096, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.025964446365833282, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 21.981327, "dram": 6.124497, "package-0": 62.326867, "psys": 110.383691, "uncore": 24.162231 }, "net_j": { "core": 11.094065291328032, "dram": 4.699104134520901, "package-0": 45.7965599384356, "psys": 81.0669726652864, "uncore": 24.158231171876977 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 0, "gpu_sample_count": 21, "gross_j": { "core": 3.358756, "dram": 0.274353, "package-0": 4.08312, "psys": 6.517379, "uncore": 0.0 }, "logits": { "l2": 939.2982638865852, "sha256": "d408c999cd01a994cd8f761c9d707a0e2e7e44d9d66fb15053e70ed4832f1885", "sum": -187687.95846249163, "top1": 285 }, "net_j": { "core": 2.534318931138132, "dram": 0.16641523314571613, "package-0": 2.8313637488971537, "psys": 4.297372723006728, "uncore": 0.0 }, "npu_busy_delta_us": 61033, "npu_utilization": 0.2792228107354196, "seconds": 0.218581712, "watts": { "core": 15.366134564816658, "dram": 1.2551507511296278, "package-0": 18.680062310061878, "psys": 29.816671030557213, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 1419.0084209097367, "decode_tokens_per_s": 17.887835209400944, "end_to_end_ms": 2886.5226869999997, "handoff_estimate_ms": 175.831959, "handoff_plus_first_decode_ms": 218.581712, "output_tokens_per_s_e2e": 5.543001644178659, "prefill_tokens_per_s": 1992.2225409376806, "steady_tpot_ms": 42.749753, "steady_tpot_p95_ms": 50.35086325, "ttft_ms": 2047.9639779999998 }, "prefill": { "gpu_active_fraction": 0.9901477832512315, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 203, "gross_j": { "core": 13.782923, "dram": 4.066274, "package-0": 47.835998, "psys": 85.236843, "uncore": 24.162231 }, "logits": { "l2": 371.2406598563232, "sha256": "bdb308b75faeea2b2633af4eb1c7004c83d886cd7bf96bf3bb48fc7235b4a04e", "sum": -41810.29560299311, "top1": 90 }, "net_j": { "core": 6.058500843598317, "dram": 3.054969548402806, "package-0": 36.1078818276143, "psys": 64.43687375913271, "uncore": 24.158231171876977 }, "npu_busy_delta_us": 9847, "npu_utilization": 0.004808190039366015, "seconds": 2.047963978, "watts": { "core": 6.730061245247157, "dram": 1.9855202746149083, "package-0": 23.357831736237696, "psys": 41.6202842997466, "uncore": 11.798171871946861 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 1007.3695891492324, "sha256": "f7e5903d64b65e5e0eaea38dd2f8fdc85c5016fc0f999d164beb841f65c04032", "sum": -228969.46392723545, "top1": 83 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 61, "gross_j": { "core": 4.839648, "dram": 1.78387, "package-0": 10.407749, "psys": 18.629469, "uncore": 0.0 }, "net_j": { "core": 2.5012455165915837, "dram": 1.4777193529723793, "package-0": 6.857314361924146, "psys": 12.332726183146963, "uncore": 0.0 }, "npu_busy_delta_us": 609812, "npu_utilization": 0.9836042352390698, "seconds": 0.619976997, "token_latencies_s": [ 0.054577069, 0.048075214, 0.045824008, 0.044201008, 0.043082338, 0.042682271, 0.042702843, 0.042517596, 0.042668934, 0.042728472, 0.042929054, 0.042771034, 0.042558382, 0.042633422 ], "watts": { "core": 7.806173492594921, "dram": 2.8773164304997594, "package-0": 16.787314771938227, "psys": 30.04864549837484, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 20.939216000000002, "dram": 6.061569, "package-0": 61.011257, "psys": 106.939178, "uncore": 24.059997000000003 }, "net_j": { "core": 10.351644628989332, "dram": 4.675412489867736, "package-0": 44.93597466710354, "psys": 78.4294520433472, "uncore": 24.055617264842965 } }, "first_decode": { "gpu_active_fraction": 0.07692307692307693, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 13, "gross_j": { "core": 2.491632, "dram": 0.206604, "package-0": 3.046379, "psys": 4.602405, "uncore": 0.023377 }, "logits": { "l2": 898.7402652849972, "sha256": "2c1f8e1f8b3933228503d4bde8c6fbdd7d411e07685d7be43122a51b53e1c935", "sum": -192974.0822976362, "top1": 16 }, "net_j": { "core": 2.0054836631892807, "dram": 0.14295600572954212, "package-0": 2.308252049441899, "psys": 3.2933270848602314, "uncore": 0.02312526469214614 }, "npu_busy_delta_us": 58612, "npu_utilization": 0.4547381849865347, "seconds": 0.128891749, "watts": { "core": 19.331198616910694, "dram": 1.6029264991974002, "package-0": 23.635174661180212, "psys": 35.707522286783465, "uncore": 0.18136925118457348 } }, "metrics": { "all_tokens_per_s_e2e": 1459.1746779203918, "decode_tokens_per_s": 19.842784979637216, "end_to_end_ms": 2807.066256, "handoff_estimate_ms": 85.7579085, "handoff_plus_first_decode_ms": 128.891749, "output_tokens_per_s_e2e": 5.69990108562653, "prefill_tokens_per_s": 1989.1532817151763, "steady_tpot_ms": 43.133840500000005, "steady_tpot_p95_ms": 51.68753355, "ttft_ms": 2051.123982 }, "prefill": { "gpu_active_fraction": 0.9901477832512315, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 203, "gross_j": { "core": 14.413537, "dram": 4.069325, "package-0": 48.354673, "psys": 84.738796, "uncore": 24.036498 }, "logits": { "l2": 369.66770690126174, "sha256": "080bfd9ddf05d67e276fff0c1594529574c430e3233356ed92f30ba9ca1508c0", "sum": -18263.070847075433, "top1": 36802 }, "net_j": { "core": 6.6771960768748135, "dram": 3.0564601078156697, "package-0": 36.60846036966127, "psys": 63.90673245146563, "uncore": 24.032492000150818 }, "npu_busy_delta_us": 7723, "npu_utilization": 0.003765252645756447, "seconds": 2.051123982, "watts": { "core": 7.027140790361058, "dram": 1.9839488181655907, "package-0": 23.574719726523092, "psys": 41.3133466058806, "uncore": 11.718695803343204 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 873.3404111901898, "sha256": "c166bc38f102527b2e6731e4c39fd240f13a426ac388431ea0b4312f82ce49bd", "sum": -176208.4787269421, "top1": 16 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 62, "gross_j": { "core": 4.034047, "dram": 1.78564, "package-0": 9.610205, "psys": 17.597977, "uncore": 0.000122 }, "net_j": { "core": 1.6689648889252378, "dram": 1.4759963763225246, "package-0": 6.019262248000365, "psys": 11.229392507021341, "uncore": 0.0 }, "npu_busy_delta_us": 613424, "npu_utilization": 0.9782688564051516, "seconds": 0.627050525, "token_latencies_s": [ 0.056709221, 0.048983548, 0.04656247, 0.044554815, 0.043137564, 0.042737479, 0.042900163, 0.043371374, 0.042844431, 0.042893608, 0.04320063, 0.043130117, 0.042918209, 0.04307913 ], "watts": { "core": 6.433368347789838, "dram": 2.8476812135672795, "package-0": 15.326045696237955, "psys": 28.06468745082384, "uncore": 0.00019456167427656646 } } }, { "energy": { "gross_j": { "core": 23.035463999999997, "dram": 6.062302, "package-0": 62.961752000000004, "psys": 109.02706400000001, "uncore": 23.844848 }, "net_j": { "core": 12.44004145686111, "dram": 4.675117590930733, "package-0": 46.8745491043812, "psys": 80.49619676760318, "uncore": 23.840585124055476 } }, "first_decode": { "gpu_active_fraction": 0.07692307692307693, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 13, "gross_j": { "core": 2.88659, "dram": 0.206787, "package-0": 3.42327, "psys": 4.981249, "uncore": 0.002746 }, "logits": { "l2": 855.424098241804, "sha256": "6e333ff040eb0721a0fb72967308ed20c3ecf95f25bba81031549aff4eb53398", "sum": -173123.48829822987, "top1": 7952 }, "net_j": { "core": 2.3964403006442345, "dram": 0.1426151353775177, "package-0": 2.6790677154135487, "psys": 3.6613963995603687, "uncore": 0.0024921927232513775 }, "npu_busy_delta_us": 58583, "npu_utilization": 0.45080274793066133, "seconds": 0.129952624, "watts": { "core": 22.212633428625498, "dram": 1.5912491309140477, "package-0": 26.34244615176066, "psys": 38.33126909388148, "uncore": 0.02113077762862257 } }, "metrics": { "all_tokens_per_s_e2e": 1458.0934344386096, "decode_tokens_per_s": 19.82939927041136, "end_to_end_ms": 2809.1478249999996, "handoff_estimate_ms": 87.09321399999999, "handoff_plus_first_decode_ms": 129.952624, "output_tokens_per_s_e2e": 5.695677478275819, "prefill_tokens_per_s": 1987.630646140281, "steady_tpot_ms": 42.859410000000004, "steady_tpot_p95_ms": 52.703164599999994, "ttft_ms": 2052.695257 }, "prefill": { "gpu_active_fraction": 0.9901477832512315, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 203, "gross_j": { "core": 15.602255, "dram": 4.066762, "package-0": 49.407527, "psys": 85.858179, "uncore": 23.842102 }, "logits": { "l2": 384.14249037604066, "sha256": "2bf92929ffc495c30c149948ed262893f16b023751064a74e2d7ac66c13a72fe", "sum": 25864.579894389957, "top1": 36802 }, "net_j": { "core": 7.859987609635066, "dram": 3.0531211969992422, "package-0": 37.65231611776977, "psys": 65.01015693265377, "uncore": 23.838092931332223 }, "npu_busy_delta_us": 7793, "npu_utilization": 0.0037964719670027475, "seconds": 2.052695257, "watts": { "core": 7.600862790905743, "dram": 1.9811815641565542, "package-0": 24.069586964510634, "psys": 41.82704602995047, "uncore": 11.615022696961393 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 852.712668795079, "sha256": "4502f07f983dd39c0355d7b49af58e80332f12949df77a2d9ec746487eb958b2", "sum": -171446.30799585953, "top1": 7952 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 62, "gross_j": { "core": 4.546619, "dram": 1.788753, "package-0": 10.130955, "psys": 18.187636, "uncore": 0.0 }, "net_j": { "core": 2.183613546581808, "dram": 1.4793812585539732, "package-0": 6.543165271197879, "psys": 11.824643435389044, "uncore": 0.0 }, "npu_busy_delta_us": 612505, "npu_utilization": 0.9776616995196411, "seconds": 0.626499944, "token_latencies_s": [ 0.060635632, 0.048431836, 0.045668249, 0.044093972, 0.04302695, 0.042891686, 0.042752533, 0.042605508, 0.042652026, 0.042923348, 0.042827134, 0.0426689, 0.042623286, 0.042675365 ], "watts": { "core": 7.257173833043471, "dram": 2.8551526893671997, "package-0": 16.170719721564733, "psys": 29.030546888604356, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 1.496285448782146, "compile_target": "AUTO:NPU,GPU", "context_length": 4096, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 402290, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_1b_ov_fp16/openvino_model.xml", "key": "llama32_1b_fp16", "size_b": 1.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": null, "key": "openvino_auto", "label": "OpenVINO AUTO", "mode": "stock", "prefill_device": null }, "prompt_length": 4080, "prompt_sha256": "66f8de7a9ef70b33481de3e8dae7cef6e4434d9723c6203a6a3dbe07ef228a16", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.025952915661036968, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 15.143148, "dram": 4.665576, "package-0": 54.364118, "psys": 90.98621700000001, "uncore": 28.793262 }, "net_j": { "core": 6.569871967329371, "dram": 3.5431370600948524, "package-0": 41.34717313950183, "psys": 67.9004930244886, "uncore": 28.78882262189668 } }, "first_decode": { "gpu_active_fraction": 0.8571428571428571, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 7, "gross_j": { "core": 1.816523, "dram": 0.139892, "package-0": 2.443475, "psys": 3.472342, "uncore": 0.31671 }, "logits": { "l2": 386.3595092072282, "sha256": "a08f57e55d540bd7c46befedb70a9eeed4a0b576f2871d968449003d95edde09", "sum": 34339.569468914415, "top1": 3862 }, "net_j": { "core": 1.5665191241805552, "dram": 0.10716074484414066, "package-0": 2.0638900369063977, "psys": 2.7991430445503096, "uncore": 0.3165805440338294 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.066283137, "watts": { "core": 27.405507376634873, "dram": 2.1105217153497122, "package-0": 36.86420273077901, "psys": 52.386506691739704, "uncore": 4.778138367229058 } }, "metrics": { "all_tokens_per_s_e2e": 1802.0084721850667, "decode_tokens_per_s": 23.255637931745145, "end_to_end_ms": 2273.0192799999995, "handoff_estimate_ms": 25.405990000000003, "handoff_plus_first_decode_ms": 66.28313700000001, "output_tokens_per_s_e2e": 7.039095594472917, "prefill_tokens_per_s": 2506.1203420634615, "steady_tpot_ms": 40.877147, "steady_tpot_p95_ms": 43.18287125, "ttft_ms": 1628.014398 }, "prefill": { "gpu_active_fraction": 0.8571428571428571, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 161, "gross_j": { "core": 9.233069, "dram": 2.865471, "package-0": 39.549703, "psys": 65.546158, "uncore": 23.605043 }, "logits": { "l2": 388.37710963475644, "sha256": "cc21758ed0d374d7022ec1be971528d81a755ceb6dd836aac0bc64de1d25bf88", "sum": -1062.9958727225894, "top1": 72 }, "net_j": { "core": 3.0925945473521095, "dram": 2.06154171918834, "package-0": 30.226520465946386, "psys": 49.011370393453014, "uncore": 23.601863365082707 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.628014398, "watts": { "core": 5.671368147199888, "dram": 1.7601017555619922, "package-0": 24.293214512467724, "psys": 40.26141174213375, "uncore": 14.499283930780077 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 450.3147795636656, "sha256": "4f4fcbcd85ba65dc12597e879daaf0bd341940cd416489da4580479054b6afca", "sum": 3555.901248254231, "top1": 42 }, "gpu_active_fraction": 0.9824561403508771, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 57, "gross_j": { "core": 4.093556, "dram": 1.660213, "package-0": 12.37094, "psys": 21.967717, "uncore": 4.871509 }, "net_j": { "core": 1.9107582957967066, "dram": 1.3744345960623716, "package-0": 9.056762636649045, "psys": 16.089979586485278, "uncore": 4.870378712780146 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.578721745, "token_latencies_s": [ 0.046900673, 0.041089277, 0.040752755, 0.04089594, 0.040669917, 0.040930185, 0.040825164, 0.040834051, 0.041152592, 0.040858354, 0.041180978, 0.0406794, 0.041161525, 0.040757978 ], "watts": { "core": 7.073444250829041, "dram": 2.868758629416975, "package-0": 21.37631790559382, "psys": 37.95903158952495, "uncore": 8.417705127012292 } } }, { "energy": { "gross_j": { "core": 18.963147, "dram": 4.823778, "package-0": 58.784762, "psys": 96.56927, "uncore": 29.017687000000002 }, "net_j": { "core": 10.08583415514328, "dram": 3.6615336513034142, "package-0": 45.30619308765672, "psys": 72.66484965943805, "uncore": 29.013090186820357 } }, "first_decode": { "gpu_active_fraction": 0.6666666666666666, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 6, "gross_j": { "core": 1.931086, "dram": 0.135131, "package-0": 2.487115, "psys": 3.469595, "uncore": 0.258667 }, "logits": { "l2": 386.3595092072282, "sha256": "a08f57e55d540bd7c46befedb70a9eeed4a0b576f2871d968449003d95edde09", "sum": 34339.569468914415, "top1": 3862 }, "net_j": { "core": 1.6982450661889696, "dram": 0.10464676854431859, "package-0": 2.1335888117453394, "psys": 2.8426116265873365, "uncore": 0.2585464312770081 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.061732753, "watts": { "core": 31.28138477802861, "dram": 2.188967661947621, "package-0": 40.28841869404399, "psys": 56.20347111362424, "uncore": 4.190109584129513 } }, "metrics": { "all_tokens_per_s_e2e": 1740.2919459129414, "decode_tokens_per_s": 23.802358204040285, "end_to_end_ms": 2353.628085, "handoff_estimate_ms": 21.885813000000002, "handoff_plus_first_decode_ms": 61.732753, "output_tokens_per_s_e2e": 6.798015413722427, "prefill_tokens_per_s": 2367.3604687284605, "steady_tpot_ms": 39.84694, "steady_tpot_p95_ms": 43.49010445, "ttft_ms": 1723.438426 }, "prefill": { "gpu_active_fraction": 0.8362573099415205, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 171, "gross_j": { "core": 11.666108, "dram": 3.037406, "package-0": 42.719556, "psys": 70.018619, "uncore": 23.925781 }, "logits": { "l2": 388.37710963475644, "sha256": "cc21758ed0d374d7022ec1be971528d81a755ceb6dd836aac0bc64de1d25bf88", "sum": -1062.9958727225894, "top1": 72 }, "net_j": { "core": 5.165717316376147, "dram": 2.186355409759852, "package-0": 32.849906780964005, "psys": 52.51466549573151, "uncore": 23.922414994758377 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.723438426, "watts": { "core": 6.769088946842363, "dram": 1.7624105127153524, "package-0": 24.787399048046986, "psys": 40.62728203322536, "uncore": 13.882585324229042 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 450.3147795636656, "sha256": "4f4fcbcd85ba65dc12597e879daaf0bd341940cd416489da4580479054b6afca", "sum": 3555.901248254231, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 56, "gross_j": { "core": 5.365953, "dram": 1.651241, "package-0": 13.578091, "psys": 23.081056, "uncore": 4.833239 }, "net_j": { "core": 3.2218717725781625, "dram": 1.3705314729992435, "package-0": 10.322697494947377, "psys": 17.307572537119196, "uncore": 4.83212876078497 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.568456906, "token_latencies_s": [ 0.04644419, 0.040632549, 0.039936178, 0.039789846, 0.041899443, 0.03980609, 0.039836461, 0.039904495, 0.039776595, 0.03963153, 0.039857419, 0.039720383, 0.039661209, 0.041537716 ], "watts": { "core": 9.43950709959358, "dram": 2.904777798583029, "package-0": 23.885875704358142, "psys": 40.603000432191074, "uncore": 8.502384171932286 } } }, { "energy": { "gross_j": { "core": 20.296885, "dram": 4.785206, "package-0": 59.998078, "psys": 98.63902999999999, "uncore": 28.968310000000002 }, "net_j": { "core": 11.486442752495272, "dram": 3.6317165499112005, "package-0": 46.6210398065094, "psys": 74.91467573297432, "uncore": 28.963747813474683 } }, "first_decode": { "gpu_active_fraction": 0.6666666666666666, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 6, "gross_j": { "core": 1.948421, "dram": 0.134827, "package-0": 2.496271, "psys": 3.514701, "uncore": 0.254577 }, "logits": { "l2": 386.3595092072282, "sha256": "a08f57e55d540bd7c46befedb70a9eeed4a0b576f2871d968449003d95edde09", "sum": 34339.569468914415, "top1": 3862 }, "net_j": { "core": 1.7227560862980686, "dram": 0.10528227455967387, "package-0": 2.1536402801429517, "psys": 2.907040883980436, "uncore": 0.25446014713386605 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.059830186, "watts": { "core": 32.56585229402429, "dram": 2.2534945821495524, "package-0": 41.722601363799875, "psys": 58.7446109560816, "uncore": 4.254992621951735 } }, "metrics": { "all_tokens_per_s_e2e": 1753.5006315521923, "decode_tokens_per_s": 24.010657466465695, "end_to_end_ms": 2335.8987879999995, "handoff_estimate_ms": 20.0771685, "handoff_plus_first_decode_ms": 59.830186, "output_tokens_per_s_e2e": 6.849611842000751, "prefill_tokens_per_s": 2384.324882993946, "steady_tpot_ms": 39.7530175, "steady_tpot_p95_ms": 43.07203225000001, "ttft_ms": 1711.176203 }, "prefill": { "gpu_active_fraction": 0.8402366863905325, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 169, "gross_j": { "core": 12.681852, "dram": 3.00103, "package-0": 43.607433, "psys": 71.210145, "uncore": 23.875489 }, "logits": { "l2": 388.37710963475644, "sha256": "cc21758ed0d374d7022ec1be971528d81a755ceb6dd836aac0bc64de1d25bf88", "sum": -1062.9958727225894, "top1": 72 }, "net_j": { "core": 6.227711439935663, "dram": 2.156034614265201, "package-0": 33.808006098065576, "psys": 53.8307317044288, "uncore": 23.87214694380481 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.711176203, "watts": { "core": 7.411190021089839, "dram": 1.7537819861792456, "package-0": 25.483894016027293, "psys": 41.61473545223209, "uncore": 13.952677087340259 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 450.3147795636656, "sha256": "4f4fcbcd85ba65dc12597e879daaf0bd341940cd416489da4580479054b6afca", "sum": 3555.901248254231, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 56, "gross_j": { "core": 5.666612, "dram": 1.649349, "package-0": 13.894374, "psys": 23.914184, "uncore": 4.838244 }, "net_j": { "core": 3.5359752262615407, "dram": 1.3703996610863258, "package-0": 10.659393428300875, "psys": 18.176903144565085, "uncore": 4.837140722536006 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.564892399, "token_latencies_s": [ 0.045324669, 0.040117195, 0.039953954, 0.039629823, 0.039756715, 0.039680659, 0.039402416, 0.03974932, 0.039635683, 0.039542059, 0.039846411, 0.041859074, 0.039652525, 0.040717917 ], "watts": { "core": 10.031312175613111, "dram": 2.9197578209934454, "package-0": 24.59649665068338, "psys": 42.33405165715462, "uncore": 8.564894851771585 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 1.156899374909699, "compile_target": "AUTO:GPU", "context_length": 8192, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 402857, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_1b_ov_fp16/openvino_model.xml", "key": "llama32_1b_fp16", "size_b": 1.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "GPU", "key": "gpu_gpu", "label": "iGPU -> iGPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 8176, "prompt_sha256": "998ace73fce48442f97bcb84c31be293d0d94180b17f70a4d127bba1092ad744", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.026081707328557968, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 34.007787, "dram": 8.636512999999999, "package-0": 124.026293, "psys": 203.921414, "uncore": 69.746404 }, "net_j": { "core": 15.122852288488216, "dram": 6.164040864845366, "package-0": 95.35298854822662, "psys": 153.06892909409794, "uncore": 69.73662508172869 } }, "first_decode": { "gpu_active_fraction": 0.625, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 8, "gross_j": { "core": 2.245722, "dram": 0.152954, "package-0": 2.92498, "psys": 4.14507, "uncore": 0.342589 }, "logits": { "l2": 434.858175530361, "sha256": "9f8607463d2448c1a2e372ef90cbd5de694c07a24236a1d6b9f47ec97f4c074c", "sum": 63194.81769055477, "top1": 546 }, "net_j": { "core": 1.954787633673432, "dram": 0.11486400260843758, "package-0": 2.4832496054500868, "psys": 3.361655299433866, "uncore": 0.34243834957763514 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.077134974, "watts": { "core": 29.11418625745567, "dram": 1.9829396714387952, "package-0": 37.9202824389362, "psys": 53.73788030316831, "uncore": 4.441422382536876 } }, "metrics": { "all_tokens_per_s_e2e": 1636.1312634918727, "decode_tokens_per_s": 21.094972489912774, "end_to_end_ms": 5006.933235, "handoff_estimate_ms": 32.444108, "handoff_plus_first_decode_ms": 77.134974, "output_tokens_per_s_e2e": 3.195568874007564, "prefill_tokens_per_s": 1903.2262817325463, "steady_tpot_ms": 44.69086599999999, "steady_tpot_p95_ms": 47.949569, "ttft_ms": 4295.863334 }, "prefill": { "gpu_active_fraction": 0.8938679245283019, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 424, "gross_j": { "core": 26.235285, "dram": 6.716657, "package-0": 106.385836, "psys": 174.396465, "uncore": 63.859395 }, "logits": { "l2": 408.920230973876, "sha256": "b215d51e686e0c73b412e329e624a7ec911f718cf36cfb8e899b127b30d3aab8", "sum": -40438.595054788515, "top1": 198 }, "net_j": { "core": 10.03233305627045, "dram": 4.595318070313336, "package-0": 81.78462973706243, "psys": 130.76590012188592, "uncore": 63.85100485487998 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 4.295863334, "watts": { "core": 6.107104197742619, "dram": 1.563517383535088, "package-0": 24.764716130049962, "psys": 40.59637177461475, "uncore": 14.865322761685416 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 565.7297862327463, "sha256": "e36bc34d8a2d387ad8383ae4653516b9609d7b60cccdd5ffcf42ce7e972bfcad", "sum": 109975.08116664726, "top1": 42 }, "gpu_active_fraction": 0.9841269841269841, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 63, "gross_j": { "core": 5.52678, "dram": 1.766902, "package-0": 14.715477, "psys": 25.379879, "uncore": 5.54442 }, "net_j": { "core": 3.1357315985443353, "dram": 1.4538587919235921, "package-0": 11.085109205714096, "psys": 18.94137367277815, "uncore": 5.543181877271076 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.633934927, "token_latencies_s": [ 0.051501117, 0.046037197, 0.045284153, 0.045013893, 0.04470893, 0.044451329, 0.044612654, 0.044750672, 0.044902173, 0.044556705, 0.04426657, 0.044647506, 0.044672802, 0.044499663 ], "watts": { "core": 8.718213438963902, "dram": 2.787197746559877, "package-0": 23.212914091417463, "psys": 40.035464081631204, "uncore": 8.746039638860283 } } }, { "energy": { "gross_j": { "core": 35.521393, "dram": 8.999671, "package-0": 127.029032, "psys": 209.3271, "uncore": 70.442874 }, "net_j": { "core": 15.976562026294566, "dram": 6.440803272523199, "package-0": 97.35379628812373, "psys": 156.69767674481108, "uncore": 70.43275337699346 } }, "first_decode": { "gpu_active_fraction": 0.625, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 8, "gross_j": { "core": 2.280878, "dram": 0.152221, "package-0": 2.919853, "psys": 4.096059, "uncore": 0.306701 }, "logits": { "l2": 434.858175530361, "sha256": "9f8607463d2448c1a2e372ef90cbd5de694c07a24236a1d6b9f47ec97f4c074c", "sum": 63194.81769055477, "top1": 546 }, "net_j": { "core": 1.995239470711877, "dram": 0.11482434943276595, "package-0": 2.486163361219168, "psys": 3.3269046862011815, "uncore": 0.306553091845923 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.075730897, "watts": { "core": 30.118196011860256, "dram": 2.010025049617463, "package-0": 38.555637337822624, "psys": 54.08702606546441, "uncore": 4.049879403910929 } }, "metrics": { "all_tokens_per_s_e2e": 1580.8902175759986, "decode_tokens_per_s": 21.236507844865102, "end_to_end_ms": 5181.8904999999995, "handoff_estimate_ms": 31.548724, "handoff_plus_first_decode_ms": 75.730897, "output_tokens_per_s_e2e": 3.087676206203122, "prefill_tokens_per_s": 1826.810630307582, "steady_tpot_ms": 44.182173000000006, "steady_tpot_p95_ms": 48.84928345, "ttft_ms": 4475.55968 }, "prefill": { "gpu_active_fraction": 0.8733031674208145, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 442, "gross_j": { "core": 27.806874, "dram": 7.082318, "package-0": 109.471888, "psys": 180.056301, "uncore": 64.575946 }, "logits": { "l2": 408.920230973876, "sha256": "b215d51e686e0c73b412e329e624a7ec911f718cf36cfb8e899b127b30d3aab8", "sum": -40438.595054788515, "top1": 198 }, "net_j": { "core": 10.926151132866629, "dram": 4.8722432738844255, "package-0": 83.84161108724595, "psys": 134.60066570578104, "uncore": 64.56720489436209 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 4.47555968, "watts": { "core": 6.213049537527338, "dram": 1.5824429806285143, "package-0": 24.45993257317038, "psys": 40.2310132975369, "uncore": 14.428574439208461 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 565.7297862327463, "sha256": "e36bc34d8a2d387ad8383ae4653516b9609d7b60cccdd5ffcf42ce7e972bfcad", "sum": 109975.08116664726, "top1": 42 }, "gpu_active_fraction": 0.9841269841269841, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 63, "gross_j": { "core": 5.433641, "dram": 1.765132, "package-0": 14.637291, "psys": 25.17474, "uncore": 5.560227 }, "net_j": { "core": 3.05517142271606, "dram": 1.4537356492060076, "package-0": 11.026021839658608, "psys": 18.770106352828847, "uncore": 5.558995390785442 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.630599923, "token_latencies_s": [ 0.050734353, 0.047834246, 0.044262776, 0.044190022, 0.044211449, 0.044067744, 0.044131718, 0.046623565, 0.043943286, 0.04389234, 0.044337456, 0.044174324, 0.044100972, 0.044071358 ], "watts": { "core": 8.616621730859297, "dram": 2.799131328152731, "package-0": 23.211691702030226, "psys": 39.92188879477551, "uncore": 8.817360734121118 } } }, { "energy": { "gross_j": { "core": 36.630277, "dram": 9.026283000000001, "package-0": 128.287086, "psys": 212.735784, "uncore": 70.520754 }, "net_j": { "core": 17.053544178739912, "dram": 6.463238587226493, "package-0": 98.56341319255087, "psys": 160.02045691479918, "uncore": 70.51061685771158 } }, "first_decode": { "gpu_active_fraction": 0.625, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 8, "gross_j": { "core": 2.391046, "dram": 0.153137, "package-0": 3.039054, "psys": 4.273427, "uncore": 0.313232 }, "logits": { "l2": 434.858175530361, "sha256": "9f8607463d2448c1a2e372ef90cbd5de694c07a24236a1d6b9f47ec97f4c074c", "sum": 63194.81769055477, "top1": 546 }, "net_j": { "core": 2.11127555492739, "dram": 0.11650861656904325, "package-0": 2.6142739692125434, "psys": 3.5200739938866157, "uncore": 0.31308713043286174 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.074175101, "watts": { "core": 32.23515664643315, "dram": 2.064533757763269, "package-0": 40.97134967163712, "psys": 57.612688656804124, "uncore": 4.222872578225408 } }, "metrics": { "all_tokens_per_s_e2e": 1578.3140308760924, "decode_tokens_per_s": 21.482071330361432, "end_to_end_ms": 5190.3485869999995, "handoff_estimate_ms": 30.237168999999994, "handoff_plus_first_decode_ms": 74.175101, "output_tokens_per_s_e2e": 3.082644591554868, "prefill_tokens_per_s": 1820.0874304553242, "steady_tpot_ms": 43.937931999999996, "steady_tpot_p95_ms": 47.9535477, "ttft_ms": 4492.091898 }, "prefill": { "gpu_active_fraction": 0.8741573033707866, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 445, "gross_j": { "core": 28.49242, "dram": 7.113324, "package-0": 110.328758, "psys": 182.669272, "uncore": 64.676592 }, "logits": { "l2": 408.920230973876, "sha256": "b215d51e686e0c73b412e329e624a7ec911f718cf36cfb8e899b127b30d3aab8", "sum": -40438.595054788515, "top1": 198 }, "net_j": { "core": 11.549341626429264, "dram": 4.895085504483314, "package-0": 84.60380570458587, "psys": 137.04572866170543, "uncore": 64.66781860569336 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 4.492091898, "watts": { "core": 6.342795438509526, "dram": 1.58352147763652, "package-0": 24.560663607332103, "psys": 40.66463379374079, "uncore": 14.397878197637889 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 565.7297862327463, "sha256": "e36bc34d8a2d387ad8383ae4653516b9609d7b60cccdd5ffcf42ce7e972bfcad", "sum": 109975.08116664726, "top1": 42 }, "gpu_active_fraction": 0.9838709677419355, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 62, "gross_j": { "core": 5.746811, "dram": 1.759822, "package-0": 14.919274, "psys": 25.793085, "uncore": 5.53093 }, "net_j": { "core": 3.392926997383257, "dram": 1.4516444661741361, "package-0": 11.34533351875246, "psys": 19.454654259207153, "uncore": 5.529711121585361 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.624081588, "token_latencies_s": [ 0.050485663, 0.043985239, 0.044190389, 0.043998546, 0.046590101, 0.043720321, 0.043863455, 0.043949691, 0.043804734, 0.043866955, 0.04387884, 0.043837604, 0.043961435, 0.043926173 ], "watts": { "core": 9.208429010727361, "dram": 2.8198588675556313, "package-0": 23.90596724350086, "psys": 41.32966826125946, "uncore": 8.862511098468746 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 7.078193558380008, "compile_target": "NPU", "context_length": 8192, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 402928, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_1b_ov_fp16/openvino_model.xml", "key": "llama32_1b_fp16", "size_b": 1.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "gpu_npu", "label": "iGPU -> NPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 8176, "prompt_sha256": "998ace73fce48442f97bcb84c31be293d0d94180b17f70a4d127bba1092ad744", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "GPU", "NPUW_ONLINE_PIPELINE": "NONE" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 8192, "NPUW_LLM_MIN_RESPONSE_LEN": 8192, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.025981368497014046, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 41.879715, "dram": 13.022060999999999, "package-0": 136.576677, "psys": 243.282398, "uncore": 61.252041000000006 }, "net_j": { "core": 18.448433684824945, "dram": 9.954367569880533, "package-0": 101.00058033308278, "psys": 180.18771974942854, "uncore": 61.2412469156494 } }, "first_decode": { "gpu_active_fraction": 0.038461538461538464, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 26, "gross_j": { "core": 4.216603, "dram": 0.339415, "package-0": 5.105456, "psys": 8.046245, "uncore": 0.005799 }, "logits": { "l2": 854.470127280065, "sha256": "6266d336e326e5bcea5486fb86eccb6fe7396351155b33ffd895286994487164", "sum": -172627.30543726124, "top1": 1506 }, "net_j": { "core": 3.2114983896765588, "dram": 0.20782369826119576, "package-0": 3.5793892733770414, "psys": 5.339745464464475, "uncore": 0.0052785409150939485 }, "npu_busy_delta_us": 68633, "npu_utilization": 0.25755228363331284, "seconds": 0.266481815, "watts": { "core": 15.823229814011887, "dram": 1.2736891633674892, "package-0": 19.158740719324506, "psys": 30.19434928420913, "uncore": 0.021761334821289776 } }, "metrics": { "all_tokens_per_s_e2e": 1318.6744538164178, "decode_tokens_per_s": 15.755154612359009, "end_to_end_ms": 6212.2990070000005, "handoff_estimate_ms": 219.001846, "handoff_plus_first_decode_ms": 266.481815, "output_tokens_per_s_e2e": 2.575536042610191, "prefill_tokens_per_s": 1554.3047645783913, "steady_tpot_ms": 47.479969, "steady_tpot_p95_ms": 55.725221499999996, "ttft_ms": 5260.229645 }, "prefill": { "gpu_active_fraction": 0.9942196531791907, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 519, "gross_j": { "core": 32.901466, "dram": 10.709385, "package-0": 120.433529, "psys": 215.489011, "uncore": 61.246242 }, "logits": { "l2": 382.2709395892454, "sha256": "725790162612ec31b6bdfcc457776f6461f44d6aa1009d0b7216add0abf45935", "sum": 21197.831756433472, "top1": 36802 }, "net_j": { "core": 13.061158819753803, "dram": 8.11183264382666, "package-0": 90.30966694348331, "psys": 162.0639429854989, "uncore": 61.235968374734306 }, "npu_busy_delta_us": 9715, "npu_utilization": 0.0018468775425488102, "seconds": 5.260229645, "watts": { "core": 6.254758484028125, "dram": 2.035915867319515, "package-0": 22.89510860319103, "psys": 40.96570407431328, "uncore": 11.643263913052984 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 846.521889761484, "sha256": "0dd3730743829ee15aaf4b29d9da089dfab4f667b1d4f80f5652bc78e4e652bd", "sum": -167734.4824058395, "top1": 1506 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 68, "gross_j": { "core": 4.761646, "dram": 1.973261, "package-0": 11.037692, "psys": 19.747142, "uncore": 0.0 }, "net_j": { "core": 2.175776475394582, "dram": 1.6347112277926783, "package-0": 7.111524116222437, "psys": 12.78403129946518, "uncore": 0.0 }, "npu_busy_delta_us": 670982, "npu_utilization": 0.9786963064543527, "seconds": 0.685587547, "token_latencies_s": [ 0.060509241, 0.053149211, 0.04993172, 0.047973488, 0.047831694, 0.047155661, 0.047147659, 0.047046709, 0.046944158, 0.047338623, 0.048452045, 0.047373583, 0.047586355, 0.047104668 ], "watts": { "core": 6.945350773706512, "dram": 2.878204262365343, "package-0": 16.099609813945467, "psys": 28.803239041329903, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 42.762159, "dram": 12.936490000000001, "package-0": 136.692398, "psys": 239.878658, "uncore": 60.60037 }, "net_j": { "core": 19.62691710141337, "dram": 9.907554935721011, "package-0": 101.5657828087265, "psys": 177.58114109489182, "uncore": 60.59006664014917 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 16, "gross_j": { "core": 3.144035, "dram": 0.268737, "package-0": 3.838186, "psys": 5.772202, "uncore": 0.0 }, "logits": { "l2": 825.823560961018, "sha256": "dc26d19b3ff8d298bcd6d704495a510b3d83d3dd2060440d27fe5c4ef784ad90", "sum": -153551.2782026939, "top1": 7952 }, "net_j": { "core": 2.499851326154392, "dram": 0.18439854673895054, "package-0": 2.8601114191131716, "psys": 4.0375737870206, "uncore": 0.0 }, "npu_busy_delta_us": 65432, "npu_utilization": 0.38311060033340905, "seconds": 0.170791411, "watts": { "core": 18.408624775633477, "dram": 1.573480764790918, "package-0": 22.472945082700907, "psys": 33.79679321227693, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 1335.5482612176588, "decode_tokens_per_s": 17.47525852897468, "end_to_end_ms": 6133.810539, "handoff_estimate_ms": 123.2905085, "handoff_plus_first_decode_ms": 170.791411, "output_tokens_per_s_e2e": 2.60849269769074, "prefill_tokens_per_s": 1549.8191785152774, "steady_tpot_ms": 47.500902499999995, "steady_tpot_p95_ms": 56.091384250000004, "ttft_ms": 5275.4541389999995 }, "prefill": { "gpu_active_fraction": 0.9923224568138196, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 521, "gross_j": { "core": 34.969088, "dram": 10.702182, "package-0": 121.937248, "psys": 215.072203, "uncore": 60.60037 }, "logits": { "l2": 611.459166441154, "sha256": "bd78e88b7ae6c11ade7484e9b0e711a1ea87e31cb1727a506e055cb563fe9aa9", "sum": 160544.48878543824, "top1": 70047 }, "net_j": { "core": 15.071357730267412, "dram": 8.097111641205075, "package-0": 91.72619952994485, "psys": 161.49250872726898, "uncore": 60.59006664014917 }, "npu_busy_delta_us": 9630, "npu_utilization": 0.0018254352604087723, "seconds": 5.275454139, "watts": { "core": 6.628640317708959, "dram": 2.028675014134172, "package-0": 23.114076018318695, "psys": 40.768471743509174, "uncore": 11.487232834041324 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 824.9906646159244, "sha256": "5240a7a1fab300f6d868051f3c76151567b4ee8890b9bf6f9032d2a22ee47201", "sum": -152912.27035018802, "top1": 7952 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 68, "gross_j": { "core": 4.649036, "dram": 1.965571, "package-0": 10.916964, "psys": 19.034253, "uncore": 0.0 }, "net_j": { "core": 2.055708044991565, "dram": 1.6260447477769857, "package-0": 6.979471859668472, "psys": 12.05105858060223, "uncore": 0.0 }, "npu_busy_delta_us": 672336, "npu_utilization": 0.9778508370210225, "seconds": 0.687564989, "token_latencies_s": [ 0.060806988, 0.053552213, 0.050631662, 0.048428059, 0.047484321, 0.047619957, 0.047401296, 0.047517484, 0.047354344, 0.047559459, 0.047451612, 0.047266378, 0.047218905, 0.047243619 ], "watts": { "core": 6.761595011929847, "dram": 2.858742128302289, "package-0": 15.87771945147719, "psys": 27.683569269115303, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 42.853589, "dram": 12.872036999999999, "package-0": 136.347795, "psys": 240.07922000000002, "uncore": 60.456632 }, "net_j": { "core": 19.970343132448978, "dram": 9.876094009796086, "package-0": 101.60378949358383, "psys": 178.46026643441846, "uncore": 60.446119090590884 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 0, "gpu_sample_count": 16, "gross_j": { "core": 3.050591, "dram": 0.258605, "package-0": 3.713125, "psys": 5.865219, "uncore": 0.002808 }, "logits": { "l2": 818.7195203975049, "sha256": "b893964f6d8b73b805ad2828c353b169a2e47afcb291c2a885575f97ed567554", "sum": -147686.32956599817, "top1": 7952 }, "net_j": { "core": 2.444403295540639, "dram": 0.17924109269268468, "package-0": 2.7927403192793614, "psys": 4.232904588429796, "uncore": 0.002494106406533452 }, "npu_busy_delta_us": 65427, "npu_utilization": 0.4070929407052678, "seconds": 0.160717599, "watts": { "core": 18.98106379749986, "dram": 1.609064605301875, "package-0": 23.103412588934955, "psys": 36.4939436408579, "uncore": 0.01747163980467379 } }, "metrics": { "all_tokens_per_s_e2e": 1350.2556529500794, "decode_tokens_per_s": 17.752097188973398, "end_to_end_ms": 6066.999225, "handoff_estimate_ms": 113.50485349999998, "handoff_plus_first_decode_ms": 160.71759899999998, "output_tokens_per_s_e2e": 2.6372180721681238, "prefill_tokens_per_s": 1565.6750600848916, "steady_tpot_ms": 47.2127455, "steady_tpot_p95_ms": 55.732401749999994, "ttft_ms": 5222.028637 }, "prefill": { "gpu_active_fraction": 0.9941972920696325, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 517, "gross_j": { "core": 34.930513, "dram": 10.64664, "package-0": 121.491327, "psys": 214.722046, "uncore": 60.453824 }, "logits": { "l2": 966.8558789071615, "sha256": "bb371fe043ec74e5e598b48b757dabc067e49e45408d2d4baa79743863d3df6c", "sum": 294894.52650701255, "top1": 128009 }, "net_j": { "core": 15.234290733239558, "dram": 8.067951671642046, "package-0": 91.58623141815724, "psys": 161.68496322347465, "uncore": 60.44362498418435 }, "npu_busy_delta_us": 7442, "npu_utilization": 0.0014251166581643545, "seconds": 5.222028637, "watts": { "core": 6.689069598834526, "dram": 2.0387938749635777, "package-0": 23.265159087636768, "psys": 41.118511775024565, "uncore": 11.576693312568672 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 818.2795658820971, "sha256": "3a7e9eb144765f50cf35ea63cec5d07510d7a98b269c3dd4c9b6806d6afb579b", "sum": -147559.66908771917, "top1": 7952 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 68, "gross_j": { "core": 4.872485, "dram": 1.966792, "package-0": 11.143343, "psys": 19.491955, "uncore": 0.0 }, "net_j": { "core": 2.291649103668782, "dram": 1.6289012454613547, "package-0": 7.224817756147224, "psys": 12.542398622513998, "uncore": 0.0 }, "npu_busy_delta_us": 671696, "npu_utilization": 0.9816486165177716, "seconds": 0.684252989, "token_latencies_s": [ 0.060789379, 0.053009414, 0.050029215, 0.048535803, 0.047775461, 0.047343773, 0.047067618, 0.046939714, 0.046996144, 0.047171086, 0.047254405, 0.047109545, 0.047063552, 0.047137655 ], "watts": { "core": 7.120882302788158, "dram": 2.8743637683985335, "package-0": 16.28541369806132, "psys": 28.486474028394785, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 1.4913084590807557, "compile_target": "AUTO:NPU,GPU", "context_length": 8192, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 402771, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_1b_ov_fp16/openvino_model.xml", "key": "llama32_1b_fp16", "size_b": 1.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": null, "key": "openvino_auto", "label": "OpenVINO AUTO", "mode": "stock", "prefill_device": null }, "prompt_length": 8176, "prompt_sha256": "998ace73fce48442f97bcb84c31be293d0d94180b17f70a4d127bba1092ad744", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.02621446456760168, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 32.331826, "dram": 8.649026, "package-0": 122.510796, "psys": 201.54513799999998, "uncore": 69.86578800000001 }, "net_j": { "core": 13.418726848390946, "dram": 6.1728664921111465, "package-0": 93.79472901936649, "psys": 150.6168131832479, "uncore": 69.85599449773558 } }, "first_decode": { "gpu_active_fraction": 0.75, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 8, "gross_j": { "core": 1.910822, "dram": 0.155578, "package-0": 2.633355, "psys": 3.814504, "uncore": 0.374877 }, "logits": { "l2": 434.858175530361, "sha256": "9f8607463d2448c1a2e372ef90cbd5de694c07a24236a1d6b9f47ec97f4c074c", "sum": 63194.81769055477, "top1": 546 }, "net_j": { "core": 1.6116990326341636, "dram": 0.11641592646918844, "package-0": 2.1791917189074645, "psys": 3.009039410614499, "uncore": 0.3747221093903355 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.079306005, "watts": { "core": 24.094291472631866, "dram": 1.9617429978978262, "package-0": 33.20498870167524, "psys": 48.09855193184929, "uncore": 4.726968657669744 } }, "metrics": { "all_tokens_per_s_e2e": 1633.6948187509818, "decode_tokens_per_s": 21.039996389312194, "end_to_end_ms": 5014.400429, "handoff_estimate_ms": 34.614231, "handoff_plus_first_decode_ms": 79.306005, "output_tokens_per_s_e2e": 3.1908101928730113, "prefill_tokens_per_s": 1900.7444322758727, "steady_tpot_ms": 44.691774, "steady_tpot_p95_ms": 48.0945765, "ttft_ms": 4301.4725499999995 }, "prefill": { "gpu_active_fraction": 0.8941176470588236, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 425, "gross_j": { "core": 24.506651, "dram": 6.732283, "package-0": 104.753883, "psys": 172.209276, "uncore": 63.932515 }, "logits": { "l2": 408.920230973876, "sha256": "b215d51e686e0c73b412e329e624a7ec911f718cf36cfb8e899b127b30d3aab8", "sum": -40438.595054788515, "top1": 198 }, "net_j": { "core": 8.282542457416227, "dram": 4.608174185114325, "package-0": 80.12055432788401, "psys": 128.52174160410203, "uncore": 63.924113899658046 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 4.30147255, "watts": { "core": 5.697270112766383, "dram": 1.5651112315014077, "package-0": 24.353028360020573, "psys": 40.034958725936775, "uncore": 14.862936879604174 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 565.7297862327463, "sha256": "e36bc34d8a2d387ad8383ae4653516b9609d7b60cccdd5ffcf42ce7e972bfcad", "sum": 109975.08116664726, "top1": 42 }, "gpu_active_fraction": 0.9841269841269841, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 63, "gross_j": { "core": 5.914353, "dram": 1.761165, "package-0": 15.123558, "psys": 25.521358, "uncore": 5.558396 }, "net_j": { "core": 3.524485358340555, "dram": 1.4482763805276333, "package-0": 11.494982972575015, "psys": 19.086032168531354, "uncore": 5.5571584886871985 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.633621874, "token_latencies_s": [ 0.051816951, 0.046090221, 0.045064518, 0.044570015, 0.044172358, 0.044642264, 0.044328, 0.044415373, 0.044644226, 0.044833058, 0.044665584, 0.044875997, 0.044742084, 0.044717964 ], "watts": { "core": 9.334199532385462, "dram": 2.7795205188891567, "package-0": 23.86842787564496, "psys": 40.278530535705585, "uncore": 8.772418106260012 } } }, { "energy": { "gross_j": { "core": 37.808863, "dram": 8.978554, "package-0": 129.486546, "psys": 212.124396, "uncore": 70.544985 }, "net_j": { "core": 18.13163891029228, "dram": 6.402352969803206, "package-0": 99.61029566030305, "psys": 159.13847064215634, "uncore": 70.5347958217413 } }, "first_decode": { "gpu_active_fraction": 0.625, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 8, "gross_j": { "core": 1.915644, "dram": 0.151672, "package-0": 2.554986, "psys": 3.702872, "uncore": 0.302245 }, "logits": { "l2": 434.858175530361, "sha256": "9f8607463d2448c1a2e372ef90cbd5de694c07a24236a1d6b9f47ec97f4c074c", "sum": 63194.81769055477, "top1": 546 }, "net_j": { "core": 1.630622994058136, "dram": 0.11435619743620133, "package-0": 2.1222339569897573, "psys": 2.9353805247084037, "uncore": 0.3020974116092913 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.075567174, "watts": { "core": 25.350213572893434, "dram": 2.0071148882714605, "package-0": 33.8107919716569, "psys": 49.00106493330027, "uncore": 3.99968642469017 } }, "metrics": { "all_tokens_per_s_e2e": 1570.2536063848963, "decode_tokens_per_s": 21.04278621303582, "end_to_end_ms": 5216.99168, "handoff_estimate_ms": 31.057063999999997, "handoff_plus_first_decode_ms": 75.567174, "output_tokens_per_s_e2e": 3.0669015749705006, "prefill_tokens_per_s": 1815.2115043771373, "steady_tpot_ms": 44.510110000000005, "steady_tpot_p95_ms": 49.04146014999999, "ttft_ms": 4504.1583200000005 }, "prefill": { "gpu_active_fraction": 0.8677130044843049, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 446, "gross_j": { "core": 30.426741, "dram": 7.063519, "package-0": 112.203692, "psys": 183.083149, "uncore": 64.674151 }, "logits": { "l2": 408.920230973876, "sha256": "b215d51e686e0c73b412e329e624a7ec911f718cf36cfb8e899b127b30d3aab8", "sum": -40438.595054788515, "top1": 198 }, "net_j": { "core": 13.438151016632144, "dram": 4.83932198840855, "package-0": 86.4096386849775, "psys": 137.33705408895148, "uncore": 64.66535403906103 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 4.50415832, "watts": { "core": 6.755255663393288, "dram": 1.5682217404826924, "package-0": 24.911134118393957, "psys": 40.647582965067706, "uncore": 14.358765035594928 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 565.7297862327463, "sha256": "e36bc34d8a2d387ad8383ae4653516b9609d7b60cccdd5ffcf42ce7e972bfcad", "sum": 109975.08116664726, "top1": 42 }, "gpu_active_fraction": 0.984375, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 64, "gross_j": { "core": 5.466478, "dram": 1.763363, "package-0": 14.727868, "psys": 25.338375, "uncore": 5.568589 }, "net_j": { "core": 3.062864899602003, "dram": 1.4486747839584553, "package-0": 11.078423018335801, "psys": 18.866036028496463, "uncore": 5.567344371070979 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.637266186, "token_latencies_s": [ 0.051881342, 0.046605175, 0.04463742, 0.047512293, 0.044406685, 0.046715937, 0.044653599, 0.044435373, 0.044415668, 0.044457714, 0.044289617, 0.044246308, 0.044562506, 0.044420155 ], "watts": { "core": 8.57801358379307, "dram": 2.76707447961157, "package-0": 23.111014397992868, "psys": 39.76105363293197, "uncore": 8.738246469584377 } } }, { "energy": { "gross_j": { "core": 37.494595000000004, "dram": 8.947732, "package-0": 129.174597, "psys": 212.203011, "uncore": 70.579043 }, "net_j": { "core": 17.921079053578353, "dram": 6.385108750101654, "package-0": 99.45580842609655, "psys": 159.4963461676301, "uncore": 70.56890752346031 } }, "first_decode": { "gpu_active_fraction": 0.75, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 8, "gross_j": { "core": 2.299616, "dram": 0.1521, "package-0": 3.002068, "psys": 4.203603, "uncore": 0.323913 }, "logits": { "l2": 434.858175530361, "sha256": "9f8607463d2448c1a2e372ef90cbd5de694c07a24236a1d6b9f47ec97f4c074c", "sum": 63194.81769055477, "top1": 546 }, "net_j": { "core": 2.000508289390609, "dram": 0.11293992392937363, "package-0": 2.5479278834896677, "psys": 3.3981794933075946, "uncore": 0.3237581172905256 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.07930196, "watts": { "core": 28.998224003542912, "dram": 1.9179853814458054, "package-0": 37.85616395862095, "psys": 53.00755492045846, "uncore": 4.084552260751185 } }, "metrics": { "all_tokens_per_s_e2e": 1578.5734241658304, "decode_tokens_per_s": 21.289175594032894, "end_to_end_ms": 5189.495702, "handoff_estimate_ms": 35.1541915, "handoff_plus_first_decode_ms": 79.30196000000001, "output_tokens_per_s_e2e": 3.0831512190738875, "prefill_tokens_per_s": 1823.0010919172291, "steady_tpot_ms": 44.147768500000005, "steady_tpot_p95_ms": 46.964878549999995, "ttft_ms": 4484.912289000001 }, "prefill": { "gpu_active_fraction": 0.8738738738738738, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 444, "gross_j": { "core": 29.68596, "dram": 7.041608, "package-0": 111.49648, "psys": 182.702902, "uncore": 64.729814 }, "logits": { "l2": 408.920230973876, "sha256": "b215d51e686e0c73b412e329e624a7ec911f718cf36cfb8e899b127b30d3aab8", "sum": -40438.595054788515, "top1": 198 }, "net_j": { "core": 12.769961365833694, "dram": 4.826914864958608, "package-0": 85.81264331468881, "psys": 137.15227774034435, "uncore": 64.72105462801127 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 4.484912289, "watts": { "core": 6.619072589849705, "dram": 1.570065933568138, "package-0": 24.860347943361976, "psys": 40.737229677402944, "uncore": 14.432793737964671 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 565.7297862327463, "sha256": "e36bc34d8a2d387ad8383ae4653516b9609d7b60cccdd5ffcf42ce7e972bfcad", "sum": 109975.08116664726, "top1": 42 }, "gpu_active_fraction": 0.9838709677419355, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 62, "gross_j": { "core": 5.509019, "dram": 1.754024, "package-0": 14.676049, "psys": 25.296506, "uncore": 5.525316 }, "net_j": { "core": 3.1506093983540486, "dram": 1.4452539612136726, "package-0": 11.095237227918076, "psys": 18.945888933978146, "uncore": 5.5240947781585135 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.625281453, "token_latencies_s": [ 0.050768492, 0.044916779, 0.044437505, 0.044250153, 0.044552905, 0.04395325, 0.043859333, 0.044011696, 0.043881639, 0.04417409, 0.044045581, 0.04429722, 0.043991104, 0.044121447 ], "watts": { "core": 8.810462830088133, "dram": 2.805175160057082, "package-0": 23.471108777633933, "psys": 40.456191173800896, "uncore": 8.83652629306438 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 2.5536438301205635, "compile_target": "AUTO:GPU", "context_length": 1024, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 395389, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 2.4565200143075967, "dram": 0.4907827289910671, "package-0": 4.295934585025856, "psys": 9.931950112327097, "uncore": 0.002123975024632864 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_3b_ov_fp16/openvino_model.xml", "key": "llama32_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "GPU", "key": "gpu_gpu", "label": "iGPU -> iGPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 1008, "prompt_sha256": "2669029615323cd09063f0eb8d822f5daea575e1396736877003e47194c143f8", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.043110777623951435, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 15.565939, "dram": 5.705856000000001, "package-0": 49.701655, "psys": 84.284452, "uncore": 22.461978000000002 }, "net_j": { "core": 10.873495316169034, "dram": 4.768363000401306, "package-0": 41.495562237781044, "psys": 65.31244432195557, "uncore": 22.45792078364072 } }, "first_decode": { "gpu_active_fraction": 0.9090909090909091, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 11, "gross_j": { "core": 2.392267, "dram": 0.300597, "package-0": 3.819387, "psys": 5.67479, "uncore": 0.807432 }, "logits": { "l2": 534.553078595872, "sha256": "3565942ee06dde1770dddad80ce01950d2e4f39473f435d47f4fdea8aaa08c44", "sum": -125913.45951230347, "top1": 546 }, "net_j": { "core": 2.1297754015669206, "dram": 0.24815438106513033, "package-0": 3.3603446435691176, "psys": 4.613510854966381, "uncore": 0.8072050425170565 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.106855062, "watts": { "core": 22.387961367707597, "dram": 2.813128310196479, "package-0": 35.743622515515455, "psys": 53.107357702904146, "uncore": 7.556328964555746 } }, "metrics": { "all_tokens_per_s_e2e": 536.0696183352625, "decode_tokens_per_s": 11.257175278493813, "end_to_end_ms": 1910.199655, "handoff_estimate_ms": 19.85353849999999, "handoff_plus_first_decode_ms": 106.855062, "output_tokens_per_s_e2e": 8.376087786488476, "prefill_tokens_per_s": 1744.8013915164981, "steady_tpot_ms": 87.0015235, "steady_tpot_p95_ms": 89.6039883, "ttft_ms": 577.716183 }, "prefill": { "gpu_active_fraction": 0.896551724137931, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 58, "gross_j": { "core": 3.293998, "dram": 1.395382, "package-0": 17.105181, "psys": 27.834462, "uncore": 10.765536 }, "logits": { "l2": 878.279080814622, "sha256": "952e315a56fbaeb7978e914a4c86da9699720487343b981aaec4300879317855", "sum": -272259.80647715216, "top1": 198 }, "net_j": { "core": 1.87482663387111, "dram": 1.1118488751249571, "package-0": 14.623350069121175, "psys": 22.096613691359966, "uncore": 10.764308945255983 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.577716183, "watts": { "core": 5.7017582282267485, "dram": 2.415341721524875, "package-0": 29.60827739180711, "psys": 48.180166696143935, "uncore": 18.634645032957298 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 534.7418300276124, "sha256": "a801c56c0f265ea4dad1913c86c55ea9541f3f9ec2db193cbf0ca5420b0121b9", "sum": -97346.80173273385, "top1": 42 }, "gpu_active_fraction": 0.9918032786885246, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 122, "gross_j": { "core": 9.879674, "dram": 4.009877, "package-0": 28.777087, "psys": 50.7752, "uncore": 10.88901 }, "net_j": { "core": 6.868893280731003, "dram": 3.408359744211218, "package-0": 23.51186752509075, "psys": 38.60231977562922, "uncore": 10.88640679586768 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.22562841, "token_latencies_s": [ 0.093123425, 0.087629737, 0.086805589, 0.086817869, 0.087708907, 0.086866376, 0.086873926, 0.086878485, 0.087124562, 0.087206075, 0.086816568, 0.087515447, 0.087364025, 0.086851441 ], "watts": { "core": 8.060904854514591, "dram": 3.271690642353828, "package-0": 23.479454918966837, "psys": 41.42789085641382, "uncore": 8.884430151223405 } } }, { "energy": { "gross_j": { "core": 16.063496, "dram": 5.754318, "package-0": 50.259697, "psys": 84.860806, "uncore": 22.399052 }, "net_j": { "core": 11.299604254683455, "dram": 4.802550549392244, "package-0": 41.928656671068275, "psys": 65.59992683783813, "uncore": 22.3949330076742 } }, "first_decode": { "gpu_active_fraction": 0.9, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 10, "gross_j": { "core": 2.546929, "dram": 0.295104, "package-0": 3.85137, "psys": 5.615343, "uncore": 0.704527 }, "logits": { "l2": 534.553078595872, "sha256": "3565942ee06dde1770dddad80ce01950d2e4f39473f435d47f4fdea8aaa08c44", "sum": -125913.45951230347, "top1": 546 }, "net_j": { "core": 2.3090300484081165, "dram": 0.24757469187047199, "package-0": 3.435334987896495, "psys": 4.653494325768257, "uncore": 0.7043213059984756 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.096843889, "watts": { "core": 26.299325918231144, "dram": 3.047213438526823, "package-0": 39.76884901844452, "psys": 57.98345211023072, "uncore": 7.27487306917218 } }, "metrics": { "all_tokens_per_s_e2e": 528.0297347486922, "decode_tokens_per_s": 11.351113110507946, "end_to_end_ms": 1939.284727, "handoff_estimate_ms": 9.935859500000005, "handoff_plus_first_decode_ms": 96.843889, "output_tokens_per_s_e2e": 8.250464605448316, "prefill_tokens_per_s": 1631.5209299832452, "steady_tpot_ms": 86.9080295, "steady_tpot_p95_ms": 89.6742428, "ttft_ms": 617.828421 }, "prefill": { "gpu_active_fraction": 0.8548387096774194, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 62, "gross_j": { "core": 3.793508, "dram": 1.447872, "package-0": 17.802566, "psys": 29.081834, "uncore": 10.830417 }, "logits": { "l2": 878.279080814622, "sha256": "952e315a56fbaeb7978e914a4c86da9699720487343b981aaec4300879317855", "sum": -272259.80647715216, "top1": 198 }, "net_j": { "core": 2.2758001184054404, "dram": 1.144652481493378, "package-0": 15.148415518614183, "psys": 22.945592944650176, "uncore": 10.829104747864289 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.617828421, "watts": { "core": 6.140067162756826, "dram": 2.343485587238791, "package-0": 28.814741107547718, "psys": 47.07105243382774, "uncore": 17.529813507883283 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 534.7418300276124, "sha256": "a801c56c0f265ea4dad1913c86c55ea9541f3f9ec2db193cbf0ca5420b0121b9", "sum": -97346.80173273385, "top1": 42 }, "gpu_active_fraction": 0.9917355371900827, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 121, "gross_j": { "core": 9.723059, "dram": 4.011342, "package-0": 28.605761, "psys": 50.163629, "uncore": 10.864108 }, "net_j": { "core": 6.714774087869898, "dram": 3.4103233760283933, "package-0": 23.344906164557596, "psys": 38.00083956741969, "uncore": 10.861506953811437 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.224612417, "token_latencies_s": [ 0.092988572, 0.087408365, 0.087751093, 0.086844468, 0.087103207, 0.08682583, 0.086671677, 0.086737236, 0.086731682, 0.087889604, 0.086971591, 0.086789989, 0.087064941, 0.086788371 ], "watts": { "core": 7.939703097098353, "dram": 3.275601279486292, "package-0": 23.359032296991646, "psys": 40.96286163983915, "uncore": 8.871466473134742 } } }, { "energy": { "gross_j": { "core": 15.508871, "dram": 5.743332, "package-0": 49.502925, "psys": 83.542205, "uncore": 22.272587 }, "net_j": { "core": 10.77822681355638, "dram": 4.798207006030458, "package-0": 41.23002762825894, "psys": 64.41574896248571, "uncore": 22.268496754431506 } }, "first_decode": { "gpu_active_fraction": 0.9, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 10, "gross_j": { "core": 2.402521, "dram": 0.291564, "package-0": 3.743216, "psys": 5.430955, "uncore": 0.756041 }, "logits": { "l2": 534.553078595872, "sha256": "3565942ee06dde1770dddad80ce01950d2e4f39473f435d47f4fdea8aaa08c44", "sum": -125913.45951230347, "top1": 546 }, "net_j": { "core": 2.1729654466803114, "dram": 0.24570160105469913, "package-0": 3.3417718289991143, "psys": 4.502839499088268, "uncore": 0.7558425199309685 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.093447459, "watts": { "core": 25.70985905566464, "dram": 3.1200848382618944, "package-0": 40.056905132112796, "psys": 58.11773865354648, "uncore": 8.090546367879302 } }, "metrics": { "all_tokens_per_s_e2e": 531.7407937505551, "decode_tokens_per_s": 11.383173770453885, "end_to_end_ms": 1925.750313, "handoff_estimate_ms": 6.480631499999986, "handoff_plus_first_decode_ms": 93.447459, "output_tokens_per_s_e2e": 8.308449902352423, "prefill_tokens_per_s": 1657.851428200273, "steady_tpot_ms": 86.96682750000001, "steady_tpot_p95_ms": 89.9530712, "ttft_ms": 608.015883 }, "prefill": { "gpu_active_fraction": 0.8688524590163934, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 61, "gross_j": { "core": 3.908254, "dram": 1.439144, "package-0": 17.70839, "psys": 28.708239, "uncore": 10.653354 }, "logits": { "l2": 878.279080814622, "sha256": "952e315a56fbaeb7978e914a4c86da9699720487343b981aaec4300879317855", "sum": -272259.80647715216, "top1": 198 }, "net_j": { "core": 2.414650814393594, "dram": 1.1407403056713465, "package-0": 15.096393539975267, "psys": 22.669455582541488, "uncore": 10.652062589449928 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.608015883, "watts": { "core": 6.427881424275228, "dram": 2.3669513251843783, "package-0": 29.12488060776531, "psys": 47.21626490800077, "uncore": 17.521506095261003 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 534.7418300276124, "sha256": "a801c56c0f265ea4dad1913c86c55ea9541f3f9ec2db193cbf0ca5420b0121b9", "sum": -97346.80173273385, "top1": 42 }, "gpu_active_fraction": 0.9917355371900827, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 121, "gross_j": { "core": 9.198096, "dram": 4.012624, "package-0": 28.051319, "psys": 49.403011, "uncore": 10.863192 }, "net_j": { "core": 6.190610552482475, "dram": 3.411765099304412, "package-0": 22.791862259284553, "psys": 37.24345388085595, "uncore": 10.860591645050611 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.224286971, "token_latencies_s": [ 0.092954025, 0.088337173, 0.087070661, 0.086864319, 0.086993783, 0.086939872, 0.08706385, 0.086786729, 0.086742508, 0.086824781, 0.086553325, 0.0866952, 0.087206589, 0.087199271 ], "watts": { "core": 7.5130228597360444, "dram": 3.2775191560868127, "package-0": 22.912372396716457, "psys": 40.35247631496684, "uncore": 8.87307653950358 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 5.555001899600029, "compile_target": "NPU", "context_length": 1024, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 395459, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 2.4565200143075967, "dram": 0.4907827289910671, "package-0": 4.295934585025856, "psys": 9.931950112327097, "uncore": 0.002123975024632864 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_3b_ov_fp16/openvino_model.xml", "key": "llama32_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "gpu_npu", "label": "iGPU -> NPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 1008, "prompt_sha256": "2669029615323cd09063f0eb8d822f5daea575e1396736877003e47194c143f8", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "GPU", "NPUW_ONLINE_PIPELINE": "NONE" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 1024, "NPUW_LLM_MIN_RESPONSE_LEN": 1024, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.04405000526458025, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 16.173786, "dram": 6.314864, "package-0": 46.298099, "psys": 81.830112, "uncore": 11.602082 }, "net_j": { "core": 10.151623549275305, "dram": 5.11170941351996, "package-0": 35.76660900682593, "psys": 57.48192160034395, "uncore": 11.600541322463972 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 35, "gross_j": { "core": 4.724963, "dram": 0.507506, "package-0": 6.044906, "psys": 10.00858, "uncore": 0.0 }, "logits": { "l2": 531.4957347674297, "sha256": "ea56709a94624ac035abbd9e592a608814486168ca89335794227ce270636dc5", "sum": -124549.59498099051, "top1": 546 }, "net_j": { "core": 3.8510704566173164, "dram": 0.3329129314362225, "package-0": 4.516652544360215, "psys": 6.475347177202146, "uncore": 0.0 }, "npu_busy_delta_us": 123236, "npu_utilization": 0.3464175347135818, "seconds": 0.355744117, "watts": { "core": 13.281914652154317, "dram": 1.4266040554087362, "package-0": 16.99228662156625, "psys": 28.134210860330263, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 417.70319469682715, "decode_tokens_per_s": 8.689975332919893, "end_to_end_ms": 2451.50148, "handoff_estimate_ms": 258.53080500000004, "handoff_plus_first_decode_ms": 355.744117, "output_tokens_per_s_e2e": 6.526612417137924, "prefill_tokens_per_s": 1389.6268198671048, "steady_tpot_ms": 97.213312, "steady_tpot_p95_ms": 100.73025975, "ttft_ms": 725.374601 }, "prefill": { "gpu_active_fraction": 0.9861111111111112, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 72, "gross_j": { "core": 4.163258, "dram": 1.577083, "package-0": 19.398937, "psys": 32.816627, "uncore": 11.602082 }, "logits": { "l2": 877.5494660304123, "sha256": "5e812874ce95b07d0da2ea5f5b3e9c6c0c235c65f52b97c6d2e966cef93ec2be", "sum": -271925.10265737865, "top1": 198 }, "net_j": { "core": 2.3813607747731127, "dram": 1.2210816737804135, "package-0": 16.28277516446477, "psys": 25.612242650118823, "uncore": 11.600541322463972 }, "npu_busy_delta_us": 14204, "npu_utilization": 0.019581606497413053, "seconds": 0.725374601, "watts": { "core": 5.7394593004229, "dram": 2.1741635257504695, "package-0": 26.743336440587615, "psys": 45.24093751664183, "uncore": 15.994607453866447 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 528.2898786352387, "sha256": "4e0d1a2576ffa7766379d7748d45a9bb599bb9599bebd7b0b95e95ac55cbc585", "sum": -93911.72267914936, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 136, "gross_j": { "core": 7.285565, "dram": 4.230275, "package-0": 20.854256, "psys": 39.004905, "uncore": 0.0 }, "net_j": { "core": 3.9191923178848764, "dram": 3.5577148083033237, "package-0": 14.967181298000943, "psys": 25.394331773022984, "uncore": 0.0 }, "npu_busy_delta_us": 1354569, "npu_utilization": 0.9884603320776447, "seconds": 1.370382762, "token_latencies_s": [ 0.106353153, 0.097702548, 0.097038749, 0.096922185, 0.097211904, 0.09721472, 0.097331065, 0.097288579, 0.09718452, 0.097081743, 0.097358517, 0.09741837, 0.097206233, 0.097040792 ], "watts": { "core": 5.316445304206184, "dram": 3.0869295187471133, "package-0": 15.217832986722874, "psys": 28.462781407929008, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 13.560694000000002, "dram": 6.131576, "package-0": 43.219311, "psys": 75.729237, "uncore": 11.600984 }, "net_j": { "core": 8.106232875038629, "dram": 5.041841212376557, "package-0": 33.68061070326938, "psys": 53.676318453755876, "uncore": 11.59944846284036 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 13, "gross_j": { "core": 2.486749, "dram": 0.33319, "package-0": 3.307914, "psys": 5.069506, "uncore": 0.0 }, "logits": { "l2": 531.4957347674297, "sha256": "ea56709a94624ac035abbd9e592a608814486168ca89335794227ce270636dc5", "sum": -124549.59498099051, "top1": 546 }, "net_j": { "core": 2.165719917559779, "dram": 0.26905230592551505, "package-0": 2.746501949610992, "psys": 3.7715540953414393, "uncore": 0.0 }, "npu_busy_delta_us": 119789, "npu_utilization": 0.9166274711223015, "seconds": 0.130684497, "watts": { "core": 19.028645762014143, "dram": 2.5495755628917482, "package-0": 25.31221434781204, "psys": 38.79194637754162, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 461.17782069054414, "decode_tokens_per_s": 10.017047478861741, "end_to_end_ms": 2220.401663, "handoff_estimate_ms": 33.56858300000001, "handoff_plus_first_decode_ms": 130.68449700000002, "output_tokens_per_s_e2e": 7.205903448289752, "prefill_tokens_per_s": 1394.278745642604, "steady_tpot_ms": 97.115914, "steady_tpot_p95_ms": 99.6845494, "ttft_ms": 722.954433 }, "prefill": { "gpu_active_fraction": 0.9861111111111112, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 72, "gross_j": { "core": 3.874623, "dram": 1.572689, "package-0": 19.126233, "psys": 32.137735, "uncore": 11.600984 }, "logits": { "l2": 877.5494660304123, "sha256": "5e812874ce95b07d0da2ea5f5b3e9c6c0c235c65f52b97c6d2e966cef93ec2be", "sum": -271925.10265737865, "top1": 198 }, "net_j": { "core": 2.0986709659031, "dram": 1.2178754504360705, "package-0": 16.020468047877543, "psys": 24.957387637958277, "uncore": 11.59944846284036 }, "npu_busy_delta_us": 11317, "npu_utilization": 0.015653821988529118, "seconds": 0.722954433, "watts": { "core": 5.359429063767841, "dram": 2.1753639347280966, "package-0": 26.455654916773987, "psys": 44.45333417023255, "uncore": 16.046632360853096 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 528.2898786352387, "sha256": "4e0d1a2576ffa7766379d7748d45a9bb599bb9599bebd7b0b95e95ac55cbc585", "sum": -93911.72267914936, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 135, "gross_j": { "core": 7.199322, "dram": 4.225697, "package-0": 20.785164, "psys": 38.521996, "uncore": 0.0 }, "net_j": { "core": 3.8418419915757505, "dram": 3.554913456014971, "package-0": 14.913640705780843, "psys": 24.947376720456163, "uncore": 0.0 }, "npu_busy_delta_us": 1354409, "npu_utilization": 0.9909613185216983, "seconds": 1.366762733, "token_latencies_s": [ 0.103065033, 0.097864289, 0.097126703, 0.097512483, 0.097448986, 0.097454217, 0.097148124, 0.097019804, 0.097105125, 0.096955413, 0.097014231, 0.096994465, 0.096999023, 0.097008555 ], "watts": { "core": 5.26742632512208, "dram": 3.0917560875578833, "package-0": 15.207587606941285, "psys": 28.184845159953596, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 13.908657, "dram": 6.135421, "package-0": 43.591929, "psys": 76.838182, "uncore": 11.602021 }, "net_j": { "core": 8.451572439402389, "dram": 5.045162081942696, "package-0": 34.04864086852117, "psys": 54.774656647497316, "uncore": 11.600481525806272 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 13, "gross_j": { "core": 2.575311, "dram": 0.333373, "package-0": 3.406729, "psys": 5.199755, "uncore": 0.0 }, "logits": { "l2": 531.4957347674297, "sha256": "ea56709a94624ac035abbd9e592a608814486168ca89335794227ce270636dc5", "sum": -124549.59498099051, "top1": 546 }, "net_j": { "core": 2.2572617795109746, "dram": 0.2698306459886558, "package-0": 2.850528098691881, "psys": 3.913850967969597, "uncore": 0.0 }, "npu_busy_delta_us": 119722, "npu_utilization": 0.9246980347907574, "seconds": 0.129471455, "watts": { "core": 19.89095588676284, "dram": 2.574876446704024, "package-0": 26.312587589287535, "psys": 40.16140082769595, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 460.95611433507025, "decode_tokens_per_s": 10.022305897904792, "end_to_end_ms": 2221.469611, "handoff_estimate_ms": 32.22111100000001, "handoff_plus_first_decode_ms": 129.47145500000002, "output_tokens_per_s_e2e": 7.202439286485473, "prefill_tokens_per_s": 1390.7130327687628, "steady_tpot_ms": 97.250344, "steady_tpot_p95_ms": 99.3982875, "ttft_ms": 724.808049 }, "prefill": { "gpu_active_fraction": 0.9861111111111112, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 72, "gross_j": { "core": 4.160329, "dram": 1.5733, "package-0": 19.428844, "psys": 32.394814, "uncore": 11.602021 }, "logits": { "l2": 877.5494660304123, "sha256": "5e812874ce95b07d0da2ea5f5b3e9c6c0c235c65f52b97c6d2e966cef93ec2be", "sum": -271925.10265737865, "top1": 198 }, "net_j": { "core": 2.379823521100259, "dram": 1.2175767277170888, "package-0": 16.315116034795786, "psys": 25.196056616318863, "uncore": 11.600481525806272 }, "npu_busy_delta_us": 11733, "npu_utilization": 0.01618773414035307, "seconds": 0.724808049, "watts": { "core": 5.739904524708169, "dram": 2.170643665134022, "package-0": 26.805502542094427, "psys": 44.69433534119045, "uncore": 16.007025606306424 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 528.2898786352387, "sha256": "4e0d1a2576ffa7766379d7748d45a9bb599bb9599bebd7b0b95e95ac55cbc585", "sum": -93911.72267914936, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 136, "gross_j": { "core": 7.173017, "dram": 4.228748, "package-0": 20.756356, "psys": 39.243613, "uncore": 0.0 }, "net_j": { "core": 3.8144871387911548, "dram": 3.5577547082369514, "package-0": 14.8829967350335, "psys": 25.664749063208856, "uncore": 0.0 }, "npu_busy_delta_us": 1353782, "npu_utilization": 0.990192946151855, "seconds": 1.367190107, "token_latencies_s": [ 0.102315247, 0.097827617, 0.097505718, 0.09739017, 0.09711434, 0.097241242, 0.097330274, 0.097231508, 0.097003388, 0.097155065, 0.09736003, 0.097210354, 0.097219653, 0.097259446 ], "watts": { "core": 5.246539572861318, "dram": 3.0930212106925374, "package-0": 15.181762868036902, "psys": 28.703845060809822, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 33.750633016228676, "compile_target": "NPU", "context_length": 1024, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 395594, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 2.4565200143075967, "dram": 0.4907827289910671, "package-0": 4.295934585025856, "psys": 9.931950112327097, "uncore": 0.002123975024632864 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_3b_ov_fp16/openvino_model.xml", "key": "llama32_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU+GPU", "key": "npu_gpu_combined", "label": "NPU + iGPU (subgraph combination)", "mode": "combined", "prefill_device": "NPU+GPU" }, "prompt_length": 1008, "prompt_sha256": "2669029615323cd09063f0eb8d822f5daea575e1396736877003e47194c143f8", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU,GPU", "NPUW_SUBMODEL_DEVICE": "0:GPU,last:NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "NPU,GPU", "NPUW_SUBMODEL_DEVICE": "0:GPU,last:NPU" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 1024, "NPUW_LLM_MIN_RESPONSE_LEN": 1024, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.043780725449323654, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 107.71627699999999, "dram": 19.438302999999998, "package-0": 234.553965, "psys": 404.213076, "uncore": 65.215653 }, "net_j": { "core": 77.11498131454753, "dram": 13.32453746188281, "package-0": 181.03876312252137, "psys": 280.48905162800594, "uncore": 65.1891942749016 } }, "first_decode": { "gpu_active_fraction": 0.5337837837837838, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 148, "gross_j": { "core": 17.623185, "dram": 1.700923, "package-0": 26.469476, "psys": 43.057812, "uncore": 4.252552 }, "logits": { "l2": 504.4882724745259, "sha256": "c2c1db075c1b7545142fc4eaec7eab893a508b06b424cdfbd622a43353872895", "sum": -101006.92088515963, "top1": 287 }, "net_j": { "core": 13.932972281353123, "dram": 0.9636635198953609, "package-0": 20.01607348269718, "psys": 28.137921906694036, "uncore": 4.249361340203077 }, "npu_busy_delta_us": 14192, "npu_utilization": 0.0094474044455185, "seconds": 1.502211542, "watts": { "core": 11.731493539543047, "dram": 1.1322792778808246, "package-0": 17.62033858743977, "psys": 28.66294845709553, "uncore": 2.830860954734962 } }, "metrics": { "all_tokens_per_s_e2e": 82.20163356830707, "decode_tokens_per_s": 1.2894316514781647, "end_to_end_ms": 12457.173362000001, "handoff_estimate_ms": 780.9984665, "handoff_plus_first_decode_ms": 1502.211542, "output_tokens_per_s_e2e": 1.284400524504798, "prefill_tokens_per_s": 1223.0915298083162, "steady_tpot_ms": 721.2130755000001, "steady_tpot_p95_ms": 738.7687036, "ttft_ms": 824.141101 }, "prefill": { "gpu_active_fraction": 0.06097560975609756, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 82, "gross_j": { "core": 3.074211, "dram": 1.591731, "package-0": 18.341872, "psys": 28.927233, "uncore": 0.336242 }, "logits": { "l2": 868.0879947489402, "sha256": "8f622f28568639fbc823d20d44054c57cca137aa922702e247efb4f1b0ff4d7a", "sum": -268991.91000521276, "top1": 198 }, "net_j": { "core": 1.0496918907800015, "dram": 1.1872567813775174, "package-0": 14.80141574127281, "psys": 20.741904699349675, "uncore": 0.33449154488470256 }, "npu_busy_delta_us": 736047, "npu_utilization": 0.893107987342085, "seconds": 0.824141101, "watts": { "core": 3.730199836253525, "dram": 1.9313816506282944, "package-0": 22.25574234526619, "psys": 35.099854824495644, "uncore": 0.40799081564068235 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 1138.6761954498609, "sha256": "8f8ad56dcbeddedefeae83c67e079a5e7b4a1b66f1c7242fd23b39a60742d69c", "sum": -356217.6366803106, "top1": 90 }, "gpu_active_fraction": 0.993993993993994, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 999, "gross_j": { "core": 87.018881, "dram": 16.145649, "package-0": 189.742617, "psys": 332.228031, "uncore": 60.626859 }, "net_j": { "core": 62.132317142414415, "dram": 11.173617160609933, "package-0": 146.22127389855137, "psys": 231.60922502196223, "uncore": 60.605341389813816 }, "npu_busy_delta_us": 201749, "npu_utilization": 0.019914378666441784, "seconds": 10.130820719, "token_latencies_s": [ 0.73201316, 0.713476546, 0.712860235, 0.715058908, 0.721500303, 0.720925848, 0.724050688, 0.713012982, 0.719032783, 0.720298478, 0.725911012, 0.735033955, 0.738717584, 0.73886364 ], "watts": { "core": 8.589519389756756, "dram": 1.5937157953767158, "package-0": 18.729244378408982, "psys": 32.79379235059583, "uncore": 5.984397580572761 } } }, { "energy": { "gross_j": { "core": 97.409784, "dram": 20.292367, "package-0": 227.927028, "psys": 400.283826, "uncore": 67.829844 }, "net_j": { "core": 67.9407518651261, "dram": 14.404813905524023, "package-0": 176.39191589095978, "psys": 281.1376536433995, "uncore": 67.80436426122726 } }, "first_decode": { "gpu_active_fraction": 0.9868421052631579, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 76, "gross_j": { "core": 8.620278, "dram": 1.259274, "package-0": 16.349201, "psys": 27.453604, "uncore": 4.464588 }, "logits": { "l2": 1153.8101991094447, "sha256": "122341610e500cb6d0a918eba9b1179dddb1ce717cb641f1095dfb29faf2c7e7", "sum": -361862.7407342456, "top1": 90 }, "net_j": { "core": 6.741920964479739, "dram": 0.8840011805061825, "package-0": 13.064351332616944, "psys": 19.85922322505474, "uncore": 4.462963920648905 }, "npu_busy_delta_us": 11985, "npu_utilization": 0.01567401288185979, "seconds": 0.764641454, "watts": { "core": 11.27362106109408, "dram": 1.6468816769120653, "package-0": 21.381525830798076, "psys": 35.903891760490396, "uncore": 5.838799317830341 } }, "metrics": { "all_tokens_per_s_e2e": 85.3599970008565, "decode_tokens_per_s": 1.3420812457946254, "end_to_end_ms": 11996.251593, "handoff_estimate_ms": 19.12891849999998, "handoff_plus_first_decode_ms": 764.6414540000001, "output_tokens_per_s_e2e": 1.3337499531383827, "prefill_tokens_per_s": 1229.8967474227486, "steady_tpot_ms": 745.5125355000001, "steady_tpot_p95_ms": 751.2121039499999, "ttft_ms": 819.580995 }, "prefill": { "gpu_active_fraction": 0.04938271604938271, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 81, "gross_j": { "core": 3.345633, "dram": 1.58929, "package-0": 18.775525, "psys": 28.453907, "uncore": 0.449584 }, "logits": { "l2": 868.0879947489402, "sha256": "8f622f28568639fbc823d20d44054c57cca137aa922702e247efb4f1b0ff4d7a", "sum": -268991.91000521276, "top1": 198 }, "net_j": { "core": 1.3323158824363652, "dram": 1.187053802644686, "package-0": 15.254658658349594, "psys": 20.313869444648596, "uncore": 0.44784323043595625 }, "npu_busy_delta_us": 740369, "npu_utilization": 0.903350620032374, "seconds": 0.819580995, "watts": { "core": 4.082126135684734, "dram": 1.9391494064598216, "package-0": 22.90868762763343, "psys": 34.71762665750931, "uncore": 0.5485534715211399 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 1303.0924465167705, "sha256": "55d57314a17ef9ab9c3edbfcf30a0f6d7df14c83b86296dc6078fa81b9923749", "sum": -420117.63300922886, "top1": 90 }, "gpu_active_fraction": 0.9931573802541545, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 1023, "gross_j": { "core": 85.443873, "dram": 17.443803, "package-0": 192.802302, "psys": 344.376315, "uncore": 62.915672 }, "net_j": { "core": 59.86651501821, "dram": 12.333758922373153, "package-0": 148.07290589999323, "psys": 240.96456097369617, "uncore": 62.893557110142396 }, "npu_busy_delta_us": 201261, "npu_utilization": 0.01932966160740896, "seconds": 10.412029144, "token_latencies_s": [ 0.753083725, 0.733468279, 0.734218112, 0.733895279, 0.733983938, 0.744130347, 0.745302433, 0.742140772, 0.745722638, 0.748726863, 0.748456491, 0.749087426, 0.749552264, 0.750204308 ], "watts": { "core": 8.206265255148425, "dram": 1.6753509578920172, "package-0": 18.517264918635345, "psys": 33.07485123574103, "uncore": 6.042594688303919 } } }, { "energy": { "gross_j": { "core": 99.978931, "dram": 20.561775, "package-0": 231.591082, "psys": 414.558204, "uncore": 67.96882000000001 }, "net_j": { "core": 70.16604767755804, "dram": 14.605524633277684, "package-0": 179.4546467996874, "psys": 294.02180774084644, "uncore": 67.94304295799735 } }, "first_decode": { "gpu_active_fraction": 0.987012987012987, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 77, "gross_j": { "core": 8.253824, "dram": 1.301388, "package-0": 16.139302, "psys": 27.746633, "uncore": 4.528431 }, "logits": { "l2": 1309.6928914418934, "sha256": "57562c01e4413e6fbaea8390321d7b602247a4cea9321eea3fedec8e1740f1c0", "sum": -422736.6517739836, "top1": 90 }, "net_j": { "core": 6.329860808175593, "dram": 0.9170036268777255, "package-0": 12.774696799172899, "psys": 19.967862085104905, "uncore": 4.526767488307057 }, "npu_busy_delta_us": 11826, "npu_utilization": 0.015099460224940206, "seconds": 0.783206805, "watts": { "core": 10.538498832374165, "dram": 1.6616147761892852, "package-0": 20.606692762328592, "psys": 35.42695597492926, "uncore": 5.781909670715898 } }, "metrics": { "all_tokens_per_s_e2e": 84.37548517011194, "decode_tokens_per_s": 1.325303811963323, "end_to_end_ms": 12136.226511, "handoff_estimate_ms": 31.28212149999998, "handoff_plus_first_decode_ms": 783.2068049999999, "output_tokens_per_s_e2e": 1.3183669557829991, "prefill_tokens_per_s": 1232.1729781928193, "steady_tpot_ms": 751.9246835, "steady_tpot_p95_ms": 757.3541970499999, "ttft_ms": 818.066958 }, "prefill": { "gpu_active_fraction": 0.06172839506172839, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 81, "gross_j": { "core": 3.045341, "dram": 1.59222, "package-0": 18.398024, "psys": 28.837207, "uncore": 0.341307 }, "logits": { "l2": 868.0879947489402, "sha256": "8f622f28568639fbc823d20d44054c57cca137aa922702e247efb4f1b0ff4d7a", "sum": -268991.91000521276, "top1": 198 }, "net_j": { "core": 1.0357431446292682, "dram": 1.1907268658553394, "package-0": 14.883661862260904, "psys": 20.712206784600813, "uncore": 0.33956944621273066 }, "npu_busy_delta_us": 735297, "npu_utilization": 0.8988225142323863, "seconds": 0.818066958, "watts": { "core": 3.722606041252677, "dram": 1.9463199001370741, "package-0": 22.48963097712596, "psys": 35.25042384122303, "uncore": 0.41721157010719906 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 1378.3485702204835, "sha256": "daee1e6fdbbca988a1804f628833400f18d9971e52c6f7bf77318beff12d729e", "sum": -450508.6694248039, "top1": 90 }, "gpu_active_fraction": 0.9951969260326609, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 1041, "gross_j": { "core": 88.679766, "dram": 17.668167, "package-0": 197.053756, "psys": 357.974364, "uncore": 63.099082 }, "net_j": { "core": 62.80044372475318, "dram": 12.497794140544618, "package-0": 151.7962881382536, "psys": 253.3417388711407, "uncore": 63.07670602347756 }, "npu_busy_delta_us": 200000, "npu_utilization": 0.018984423070902607, "seconds": 10.534952748, "token_latencies_s": [ 0.76036756, 0.74992104, 0.747467182, 0.75232368, 0.749285046, 0.749908574, 0.755383249, 0.755731617, 0.751335529, 0.754336254, 0.754344248, 0.752925592, 0.75004468, 0.751525687 ], "watts": { "core": 8.417670977863223, "dram": 1.6770997860768004, "package-0": 18.704759358072064, "psys": 33.97968387356643, "uncore": 5.989498340367876 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 8.377430791035295, "compile_target": "AUTO:NPU,GPU", "context_length": 1024, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 395298, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 2.4565200143075967, "dram": 0.4907827289910671, "package-0": 4.295934585025856, "psys": 9.931950112327097, "uncore": 0.002123975024632864 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_3b_ov_fp16/openvino_model.xml", "key": "llama32_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": null, "key": "openvino_auto", "label": "OpenVINO AUTO", "mode": "stock", "prefill_device": null }, "prompt_length": 1008, "prompt_sha256": "2669029615323cd09063f0eb8d822f5daea575e1396736877003e47194c143f8", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.047323985025286674, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 16.195636999999998, "dram": 5.704088, "package-0": 50.023065, "psys": 83.195099, "uncore": 22.174504 }, "net_j": { "core": 11.512037060894261, "dram": 4.768361872702556, "package-0": 41.832438079135876, "psys": 64.2588474448064, "uncore": 22.170454430186567 } }, "first_decode": { "gpu_active_fraction": 0.9, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 10, "gross_j": { "core": 2.60357, "dram": 0.296753, "package-0": 3.974172, "psys": 5.712083, "uncore": 0.770018 }, "logits": { "l2": 534.553078595872, "sha256": "3565942ee06dde1770dddad80ce01950d2e4f39473f435d47f4fdea8aaa08c44", "sum": -125913.45951230347, "top1": 546 }, "net_j": { "core": 2.3612137247769156, "dram": 0.24833317298889318, "package-0": 3.550342070439115, "psys": 4.73221293140479, "uncore": 0.7698084520734051 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.098658376, "watts": { "core": 26.389751236124138, "dram": 3.0078845003489616, "package-0": 40.2821550600022, "psys": 57.89759807114603, "uncore": 7.804892308383425 } }, "metrics": { "all_tokens_per_s_e2e": 537.0818445973571, "decode_tokens_per_s": 11.343513880930132, "end_to_end_ms": 1906.5995440000002, "handoff_estimate_ms": 11.587197500000007, "handoff_plus_first_decode_ms": 98.658376, "output_tokens_per_s_e2e": 8.391903821833704, "prefill_tokens_per_s": 1725.2652967660842, "steady_tpot_ms": 87.0711785, "steady_tpot_p95_ms": 89.24730315000001, "ttft_ms": 584.25797 }, "prefill": { "gpu_active_fraction": 0.9137931034482759, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 58, "gross_j": { "core": 4.215443, "dram": 1.402157, "package-0": 17.850601, "psys": 28.261158, "uncore": 10.581516 }, "logits": { "l2": 878.279080814622, "sha256": "952e315a56fbaeb7978e914a4c86da9699720487343b981aaec4300879317855", "sum": -272259.80647715216, "top1": 198 }, "net_j": { "core": 2.7802016031762724, "dram": 1.115413279048619, "package-0": 15.340666980100002, "psys": 22.458336989230496, "uncore": 10.580275050663777 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.58425797, "watts": { "core": 7.215037220630468, "dram": 2.3998936634103596, "package-0": 30.55260161876782, "psys": 48.37102692839603, "uncore": 18.111034069419713 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 534.7418300276124, "sha256": "a801c56c0f265ea4dad1913c86c55ea9541f3f9ec2db193cbf0ca5420b0121b9", "sum": -97346.80173273385, "top1": 42 }, "gpu_active_fraction": 0.9918032786885246, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 122, "gross_j": { "core": 9.376624, "dram": 4.005178, "package-0": 28.198292, "psys": 49.221858, "uncore": 10.82297 }, "net_j": { "core": 6.370621732941074, "dram": 3.4046154206650434, "package-0": 22.941429028596758, "psys": 37.068297524171115, "uncore": 10.820370927449385 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.223683198, "token_latencies_s": [ 0.091057793, 0.088272424, 0.087096806, 0.087255071, 0.087049273, 0.086903285, 0.087185495, 0.087029006, 0.086999301, 0.086927765, 0.087093084, 0.086924588, 0.087160468, 0.086700812 ], "watts": { "core": 7.662623802733622, "dram": 3.2730513964285057, "package-0": 23.04378457274527, "psys": 40.2243473477847, "uncore": 8.844584952779583 } } }, { "energy": { "gross_j": { "core": 16.605548, "dram": 5.750108, "package-0": 50.596977, "psys": 84.290678, "uncore": 22.246525 }, "net_j": { "core": 11.852039633153733, "dram": 4.800415021732021, "package-0": 42.28409500660801, "psys": 65.07177985014967, "uncore": 22.24241498542989 } }, "first_decode": { "gpu_active_fraction": 0.9, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 10, "gross_j": { "core": 2.459283, "dram": 0.292297, "package-0": 3.797171, "psys": 5.481676, "uncore": 0.753355 }, "logits": { "l2": 534.553078595872, "sha256": "3565942ee06dde1770dddad80ce01950d2e4f39473f435d47f4fdea8aaa08c44", "sum": -125913.45951230347, "top1": 546 }, "net_j": { "core": 2.229860464782566, "dram": 0.24646117644869125, "package-0": 3.395959449560959, "psys": 4.554098304254901, "uncore": 0.7531566349420922 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.09339331, "watts": { "core": 26.332539236482788, "dram": 3.1297423766220516, "package-0": 40.65784797647712, "psys": 58.694525335915394, "uncore": 8.066477138458847 } }, "metrics": { "all_tokens_per_s_e2e": 529.183142328175, "decode_tokens_per_s": 11.418517217540597, "end_to_end_ms": 1935.057862, "handoff_estimate_ms": 6.424753999999991, "handoff_plus_first_decode_ms": 93.39331, "output_tokens_per_s_e2e": 8.268486598877734, "prefill_tokens_per_s": 1622.1378508130183, "steady_tpot_ms": 86.968556, "steady_tpot_p95_ms": 88.41024125, "ttft_ms": 621.4021819999999 }, "prefill": { "gpu_active_fraction": 0.8548387096774194, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 62, "gross_j": { "core": 4.44994, "dram": 1.450558, "package-0": 18.286452, "psys": 29.228624, "uncore": 10.660495 }, "logits": { "l2": 878.279080814622, "sha256": "952e315a56fbaeb7978e914a4c86da9699720487343b981aaec4300879317855", "sum": -272259.80647715216, "top1": 198 }, "net_j": { "core": 2.923453102982588, "dram": 1.1455845413170362, "package-0": 15.61694887513567, "psys": 23.056888528684798, "uncore": 10.659175157285178 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.621402182, "watts": { "core": 7.1611270911179385, "dram": 2.334330393452014, "package-0": 29.427724153051013, "psys": 47.0365647991883, "uncore": 17.15554806339576 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 534.7418300276124, "sha256": "a801c56c0f265ea4dad1913c86c55ea9541f3f9ec2db193cbf0ca5420b0121b9", "sum": -97346.80173273385, "top1": 42 }, "gpu_active_fraction": 0.9917355371900827, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 121, "gross_j": { "core": 9.696325, "dram": 4.007253, "package-0": 28.513354, "psys": 49.580378, "uncore": 10.832675 }, "net_j": { "core": 6.698726065388579, "dram": 3.408369303966293, "package-0": 23.27118668191138, "psys": 37.46079301720997, "uncore": 10.83008319320262 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.22026237, "token_latencies_s": [ 0.08960967, 0.087240858, 0.087345714, 0.087764395, 0.086776331, 0.087085376, 0.086990121, 0.086680053, 0.086946991, 0.086553458, 0.087400396, 0.086636014, 0.086724405, 0.086483082 ], "watts": { "core": 7.946098509945857, "dram": 3.283927373749959, "package-0": 23.366576484694846, "psys": 40.630916120112765, "uncore": 8.877332667400045 } } }, { "energy": { "gross_j": { "core": 16.382099999999998, "dram": 5.755051, "package-0": 50.564812, "psys": 84.393583, "uncore": 22.405399 }, "net_j": { "core": 11.63086274731996, "dram": 4.805811762634541, "package-0": 42.255901705530235, "psys": 65.18386718632726, "uncore": 22.401290949097902 } }, "first_decode": { "gpu_active_fraction": 0.9, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 10, "gross_j": { "core": 2.453241, "dram": 0.293945, "package-0": 3.746633, "psys": 5.442491, "uncore": 0.697752 }, "logits": { "l2": 534.553078595872, "sha256": "3565942ee06dde1770dddad80ce01950d2e4f39473f435d47f4fdea8aaa08c44", "sum": -125913.45951230347, "top1": 546 }, "net_j": { "core": 2.218598995990846, "dram": 0.24706639026231209, "package-0": 3.3362937012148364, "psys": 4.493810482641285, "uncore": 0.6975491220452744 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.095518051, "watts": { "core": 25.683532843441284, "dram": 3.07737644269982, "package-0": 39.224345144982074, "psys": 56.97866469239411, "uncore": 7.304922919752624 } }, "metrics": { "all_tokens_per_s_e2e": 529.4360944051087, "decode_tokens_per_s": 11.391117970778227, "end_to_end_ms": 1934.133337, "handoff_estimate_ms": 8.411922000000002, "handoff_plus_first_decode_ms": 95.518051, "output_tokens_per_s_e2e": 8.272438975079824, "prefill_tokens_per_s": 1632.8701980500973, "steady_tpot_ms": 87.10612900000001, "steady_tpot_p95_ms": 87.98141995, "ttft_ms": 617.317899 }, "prefill": { "gpu_active_fraction": 0.8524590163934426, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 61, "gross_j": { "core": 4.498158, "dram": 1.449581, "package-0": 18.488722, "psys": 29.511216, "uncore": 10.805026 }, "logits": { "l2": 878.279080814622, "sha256": "952e315a56fbaeb7978e914a4c86da9699720487343b981aaec4300879317855", "sum": -272259.80647715216, "top1": 198 }, "net_j": { "core": 2.9817042259161846, "dram": 1.146612036873748, "package-0": 15.836764687730401, "psys": 23.380045423685424, "uncore": 10.803714832200265 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.617317899, "watts": { "core": 7.286615222540307, "dram": 2.3481920779361687, "package-0": 29.950082493882135, "psys": 47.805540788312705, "uncore": 17.503179508488543 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 534.7418300276124, "sha256": "a801c56c0f265ea4dad1913c86c55ea9541f3f9ec2db193cbf0ca5420b0121b9", "sum": -97346.80173273385, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 121, "gross_j": { "core": 9.430701, "dram": 4.011525, "package-0": 28.329457, "psys": 49.439876, "uncore": 10.902621 }, "net_j": { "core": 6.430559525412929, "dram": 3.4121333354984804, "package-0": 23.082843316584995, "psys": 37.31001128000056, "uncore": 10.900026994852363 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.221297387, "token_latencies_s": [ 0.088140902, 0.087087482, 0.08706234, 0.087106793, 0.087015262, 0.087121043, 0.087105465, 0.087113481, 0.086876182, 0.08772084, 0.086807005, 0.087296786, 0.087895545, 0.086920479 ], "watts": { "core": 7.721871102308352, "dram": 3.284642252329653, "package-0": 23.196198814105873, "psys": 40.481439267993785, "uncore": 8.9270812465924 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 2.5578020960092545, "compile_target": "AUTO:GPU", "context_length": 2048, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 403449, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_3b_ov_fp16/openvino_model.xml", "key": "llama32_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "GPU", "key": "gpu_gpu", "label": "iGPU -> iGPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 2032, "prompt_sha256": "cd1a97358e68d07c27508cfacc4ba6882a7a2962b1281e2e2b0c7e5e320da4fa", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.04383872449398041, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 20.020274, "dram": 7.308209, "package-0": 70.552737, "psys": 117.574224, "uncore": 35.377962 }, "net_j": { "core": 10.146228027033347, "dram": 6.015469369054168, "package-0": 55.560811919046664, "psys": 90.98584646674911, "uncore": 35.37284906262151 } }, "first_decode": { "gpu_active_fraction": 0.9090909090909091, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 11, "gross_j": { "core": 2.416742, "dram": 0.305541, "package-0": 3.810903, "psys": 5.626634, "uncore": 0.768004 }, "logits": { "l2": 529.3319535900215, "sha256": "f984afc7983df3c72b7c030f6eb4d656d8481c00cf53a50b1f34deda53645068", "sum": -138429.60764685227, "top1": 319 }, "net_j": { "core": 2.0202993930298403, "dram": 0.25363754818005385, "package-0": 3.208977722472782, "psys": 4.559111554464755, "uncore": 0.7677987157397128 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.105108209, "watts": { "core": 22.992894874652468, "dram": 2.906918526220916, "package-0": 36.25694925502917, "psys": 53.531822619106755, "uncore": 7.30679370628416 } }, "metrics": { "all_tokens_per_s_e2e": 782.3093029721824, "decode_tokens_per_s": 11.196968461695255, "end_to_end_ms": 2617.8903819999996, "handoff_estimate_ms": 17.27638549999999, "handoff_plus_first_decode_ms": 105.10820899999999, "output_tokens_per_s_e2e": 6.111791429470175, "prefill_tokens_per_s": 1589.6832548369725, "steady_tpot_ms": 87.8318235, "steady_tpot_p95_ms": 89.73360699999999, "ttft_ms": 1278.2420610000001 }, "prefill": { "gpu_active_fraction": 0.905511811023622, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 127, "gross_j": { "core": 9.087379, "dram": 2.925225, "package-0": 39.002159, "psys": 62.381066, "uncore": 23.47339 }, "logits": { "l2": 813.1669789722381, "sha256": "d50d0ae7fdda5cdbe691764d1ca1d7d8554922da6b22bef84ee4b7e361ab62dd", "sum": -253097.8784024875, "top1": 319 }, "net_j": { "core": 4.266160754318068, "dram": 2.2940166875518053, "package-0": 31.68202468666213, "psys": 49.39871092110475, "uncore": 23.470893496839523 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.278242061, "watts": { "core": 7.109278654850961, "dram": 2.288474999572088, "package-0": 30.512342059443466, "psys": 48.80223230269685, "uncore": 18.36380660298112 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 553.0736012792826, "sha256": "7bd1634f8d377902a10467f3d09341ea554283d2d40c1594e26340bce4ce43e1", "sum": -121082.53280104871, "top1": 42 }, "gpu_active_fraction": 0.9917355371900827, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 121, "gross_j": { "core": 8.516153, "dram": 4.077443, "package-0": 27.739675, "psys": 49.566524, "uncore": 11.136568 }, "net_j": { "core": 3.859767879685439, "dram": 3.4678151333223086, "package-0": 20.66980950991175, "psys": 37.0280239911796, "uncore": 11.134156850042277 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.234540112, "token_latencies_s": [ 0.09163682, 0.08838888, 0.087822854, 0.087666229, 0.087725201, 0.087814205, 0.08784324, 0.0887088, 0.087757841, 0.087646695, 0.087840793, 0.087761656, 0.08791166, 0.087982589 ], "watts": { "core": 6.8982392043977585, "dram": 3.302803173721422, "package-0": 22.469642525475106, "psys": 40.14978818282415, "uncore": 9.020823132233716 } } }, { "energy": { "gross_j": { "core": 20.793465, "dram": 7.380598, "package-0": 71.720886, "psys": 119.79138, "uncore": 35.54245 }, "net_j": { "core": 10.707521335106975, "dram": 6.060116089464915, "package-0": 56.407233196602945, "psys": 92.63241409327102, "uncore": 35.53722733864085 } }, "first_decode": { "gpu_active_fraction": 0.9, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 10, "gross_j": { "core": 2.609307, "dram": 0.300232, "package-0": 3.907461, "psys": 5.690049, "uncore": 0.69055 }, "logits": { "l2": 529.3319535900215, "sha256": "f984afc7983df3c72b7c030f6eb4d656d8481c00cf53a50b1f34deda53645068", "sum": -138429.60764685227, "top1": 319 }, "net_j": { "core": 2.231430726554124, "dram": 0.2507593080948762, "package-0": 3.3337252895444887, "psys": 4.672521124725359, "uncore": 0.6903543296812438 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.100185746, "watts": { "core": 26.04469302449472, "dram": 2.9967536499653353, "package-0": 39.00216503852754, "psys": 56.79499556753313, "uncore": 6.892697090861608 } }, "metrics": { "all_tokens_per_s_e2e": 765.8736038269113, "decode_tokens_per_s": 11.225396339072269, "end_to_end_ms": 2674.0704859999996, "handoff_estimate_ms": 12.505427500000014, "handoff_plus_first_decode_ms": 100.18574600000001, "output_tokens_per_s_e2e": 5.983387529897745, "prefill_tokens_per_s": 1518.8948743346662, "steady_tpot_ms": 87.6803185, "steady_tpot_p95_ms": 90.99372635, "ttft_ms": 1337.814772 }, "prefill": { "gpu_active_fraction": 0.8796992481203008, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 133, "gross_j": { "core": 8.266031, "dram": 3.011101, "package-0": 38.60946, "psys": 62.960532, "uncore": 23.686951 }, "logits": { "l2": 813.1669789722381, "sha256": "d50d0ae7fdda5cdbe691764d1ca1d7d8554922da6b22bef84ee4b7e361ab62dd", "sum": -253097.8784024875, "top1": 319 }, "net_j": { "core": 3.2201189737103135, "dram": 2.350475105760719, "package-0": 30.948169454024544, "psys": 49.37313182768218, "uncore": 23.684338146841487 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.337814772, "watts": { "core": 6.178755963086346, "dram": 2.250760765257868, "package-0": 28.860093944305767, "psys": 47.06221916347624, "uncore": 17.70570298352185 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 553.0736012792826, "sha256": "7bd1634f8d377902a10467f3d09341ea554283d2d40c1594e26340bce4ce43e1", "sum": -121082.53280104871, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 122, "gross_j": { "core": 9.918127, "dram": 4.069265, "package-0": 29.203965, "psys": 51.140799, "uncore": 11.164949 }, "net_j": { "core": 5.255971634842537, "dram": 3.45888167560932, "package-0": 22.125338453033915, "psys": 38.58676114086349, "uncore": 11.162534862118115 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.236069968, "token_latencies_s": [ 0.096507482, 0.088024781, 0.087936399, 0.087922903, 0.087716069, 0.087548329, 0.087646186, 0.087683396, 0.087677241, 0.087499296, 0.087494305, 0.08732335, 0.087683446, 0.087384418 ], "watts": { "core": 8.023920374060896, "dram": 3.2920992381881087, "package-0": 23.626465941287233, "psys": 41.37370887082341, "uncore": 9.032618936665243 } } }, { "energy": { "gross_j": { "core": 19.931467, "dram": 7.370525, "package-0": 70.980775, "psys": 119.33660900000001, "uncore": 35.634918 }, "net_j": { "core": 9.861668672205102, "dram": 6.052156885286696, "package-0": 55.6916359253059, "psys": 92.22111851553885, "uncore": 35.629703698952085 } }, "first_decode": { "gpu_active_fraction": 0.9, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 10, "gross_j": { "core": 2.385187, "dram": 0.301207, "package-0": 3.681936, "psys": 5.470994, "uncore": 0.685301 }, "logits": { "l2": 529.3319535900215, "sha256": "f984afc7983df3c72b7c030f6eb4d656d8481c00cf53a50b1f34deda53645068", "sum": -138429.60764685227, "top1": 319 }, "net_j": { "core": 2.0087389814911294, "dram": 0.25192129950297937, "package-0": 3.1103688323151726, "psys": 4.457312064027524, "uncore": 0.6851060692542692 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.099807075, "watts": { "core": 23.897975168594012, "dram": 3.017892268659311, "package-0": 36.8905310570418, "psys": 54.815693176059916, "uncore": 6.866256725788228 } }, "metrics": { "all_tokens_per_s_e2e": 767.1015616375668, "decode_tokens_per_s": 11.229115187630274, "end_to_end_ms": 2669.789898, "handoff_estimate_ms": 12.054259999999983, "handoff_plus_first_decode_ms": 99.807075, "output_tokens_per_s_e2e": 5.99298095029349, "prefill_tokens_per_s": 1523.2649591568136, "steady_tpot_ms": 87.75281500000001, "steady_tpot_p95_ms": 90.78791505, "ttft_ms": 1333.976724 }, "prefill": { "gpu_active_fraction": 0.8863636363636364, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 132, "gross_j": { "core": 8.293618, "dram": 3.000542, "package-0": 38.77578, "psys": 63.534201, "uncore": 23.806884 }, "logits": { "l2": 813.1669789722381, "sha256": "d50d0ae7fdda5cdbe691764d1ca1d7d8554922da6b22bef84ee4b7e361ab62dd", "sum": -253097.8784024875, "top1": 319 }, "net_j": { "core": 3.262182157533374, "dram": 2.3418113710434776, "package-0": 31.136468880080244, "psys": 49.985781630666295, "uncore": 23.804278642838728 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.333976724, "watts": { "core": 6.217213427181208, "dram": 2.2493211058456217, "package-0": 29.067808532467314, "psys": 47.62766835203041, "uncore": 17.846551271609744 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 553.0736012792826, "sha256": "7bd1634f8d377902a10467f3d09341ea554283d2d40c1594e26340bce4ce43e1", "sum": -121082.53280104871, "top1": 42 }, "gpu_active_fraction": 0.991869918699187, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 123, "gross_j": { "core": 9.252662, "dram": 4.068776, "package-0": 28.523059, "psys": 50.331414, "uncore": 11.142733 }, "net_j": { "core": 4.590747533180599, "dram": 3.458424214740239, "package-0": 21.444798212910484, "psys": 37.77802482084503, "uncore": 11.14031898685909 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.236006099, "token_latencies_s": [ 0.09510276, 0.088289847, 0.087852981, 0.087528247, 0.08787454, 0.087843446, 0.087509492, 0.087503841, 0.087877606, 0.087510847, 0.087662184, 0.087578072, 0.087385574, 0.088464537 ], "watts": { "core": 7.4859355528147775, "dram": 3.291873723998509, "package-0": 23.076794704392476, "psys": 40.72100780143481, "uncore": 9.015111664105147 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 5.996397917158902, "compile_target": "NPU", "context_length": 2048, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 403518, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_3b_ov_fp16/openvino_model.xml", "key": "llama32_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "gpu_npu", "label": "iGPU -> NPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 2032, "prompt_sha256": "cd1a97358e68d07c27508cfacc4ba6882a7a2962b1281e2e2b0c7e5e320da4fa", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "GPU", "NPUW_ONLINE_PIPELINE": "NONE" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 2048, "NPUW_LLM_MIN_RESPONSE_LEN": 2048, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.043826090171933174, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 26.698051999999997, "dram": 9.004066, "package-0": 77.026047, "psys": 131.9111, "uncore": 25.475459 }, "net_j": { "core": 13.126488455076672, "dram": 7.227236309672879, "package-0": 56.420120670111004, "psys": 95.36621695642731, "uncore": 25.471903582146023 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 36, "gross_j": { "core": 4.900073, "dram": 0.531859, "package-0": 6.26512, "psys": 10.238316, "uncore": 0.0 }, "logits": { "l2": 887.927456298782, "sha256": "a9404b5df4f9d3f7570364ac78827e680855e76bd9d5afb0de3fd418bded0d8e", "sum": -221980.5972502306, "top1": 8 }, "net_j": { "core": 3.5355691043787756, "dram": 0.35321406890702123, "package-0": 4.193371475451029, "psys": 6.5640425743245885, "uncore": 0.0 }, "npu_busy_delta_us": 124142, "npu_utilization": 0.3431528777609907, "seconds": 0.361768786, "watts": { "core": 13.54476447285311, "dram": 1.4701627685479752, "package-0": 17.31802256704369, "psys": 28.300716911491637, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 569.172298907028, "decode_tokens_per_s": 8.437454835358947, "end_to_end_ms": 3598.2074390000002, "handoff_estimate_ms": 261.1783375, "handoff_plus_first_decode_ms": 361.76878600000003, "output_tokens_per_s_e2e": 4.446658585211156, "prefill_tokens_per_s": 1116.2258369756726, "steady_tpot_ms": 100.5904485, "steady_tpot_p95_ms": 103.65122000000001, "ttft_ms": 1820.420145 }, "prefill": { "gpu_active_fraction": 0.9832402234636871, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 179, "gross_j": { "core": 13.952357, "dram": 4.098744, "package-0": 48.870175, "psys": 82.584322, "uncore": 25.475459 }, "logits": { "l2": 600.6828465575093, "sha256": "c54e46c783110f50d983e9def0f6af3c07eb0aa1064735a83c5940836218a748", "sum": -153166.4828705974, "top1": 90 }, "net_j": { "core": 7.086174853705449, "dram": 3.1998028974257293, "package-0": 38.44513863585395, "psys": 64.09538204392076, "uncore": 25.471903582146023 }, "npu_busy_delta_us": 14319, "npu_utilization": 0.007865766614003275, "seconds": 1.820420145, "watts": { "core": 7.664360910486408, "dram": 2.251537377927665, "package-0": 26.84554724041466, "psys": 45.365528516495296, "uncore": 13.99427438219214 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 861.679016957534, "sha256": "ebe03c04ea266bd221813308a6fd828ac7e4973a369c68eb659d36c4bbad5510", "sum": -223266.26944584027, "top1": 735 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 140, "gross_j": { "core": 7.845622, "dram": 4.373463, "package-0": 21.890752, "psys": 39.088462, "uncore": 0.0 }, "net_j": { "core": 2.504744496992447, "dram": 3.6742193433401287, "package-0": 13.781610558806024, "psys": 24.706792338181955, "uncore": 0.0 }, "npu_busy_delta_us": 1405989, "npu_utilization": 0.9929171067021109, "seconds": 1.416018508, "token_latencies_s": [ 0.106603858, 0.101435403, 0.100607704, 0.100597683, 0.100571318, 0.102061338, 0.100583214, 0.100404997, 0.100654543, 0.100429012, 0.100630039, 0.100432121, 0.100522984, 0.100450332 ], "watts": { "core": 5.540621083463973, "dram": 3.0885634441156613, "package-0": 15.459368557914356, "psys": 27.604485237420356, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 23.818849, "dram": 8.891762, "package-0": 73.596186, "psys": 124.875839, "uncore": 25.280148 }, "net_j": { "core": 10.902868894360124, "dram": 7.2007632543204725, "package-0": 53.98564270083212, "psys": 90.09628093424675, "uncore": 25.276612393139285 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 20, "gross_j": { "core": 3.05175, "dram": 0.410643, "package-0": 4.0902, "psys": 6.37126, "uncore": 0.0 }, "logits": { "l2": 778.2465713227663, "sha256": "fe566b0d8145c772eb0c4796d094855e007986ea4936bca45001faf4234867f6", "sum": -167187.10537097976, "top1": 482 }, "net_j": { "core": 2.299653141681782, "dram": 0.31217622987155424, "package-0": 2.9482790706872297, "psys": 4.3460481197550695, "uncore": 0.0 }, "npu_busy_delta_us": 120506, "npu_utilization": 0.6043361643860966, "seconds": 0.199402265, "watts": { "core": 15.304490147090357, "dram": 2.0593697869981566, "package-0": 20.51230461198623, "psys": 31.951793526517868, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 598.062087386913, "decode_tokens_per_s": 9.29300683077449, "end_to_end_ms": 3424.3936259999996, "handoff_estimate_ms": 98.87701849999999, "handoff_plus_first_decode_ms": 199.402265, "output_tokens_per_s_e2e": 4.672360057710258, "prefill_tokens_per_s": 1122.4803622682073, "steady_tpot_ms": 100.5252465, "steady_tpot_p95_ms": 103.2223723, "ttft_ms": 1810.2766589999999 }, "prefill": { "gpu_active_fraction": 0.9776536312849162, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 179, "gross_j": { "core": 13.555874, "dram": 4.095449, "package-0": 48.221495, "psys": 80.659095, "uncore": 25.280148 }, "logits": { "l2": 719.4226608142055, "sha256": "0c501c54920f07eb1cd51683351d82fa8c33d783dc082c8bb6ae5bfd66f63919", "sum": -181298.67483598553, "top1": 264 }, "net_j": { "core": 6.727950616478253, "dram": 3.2015168490756176, "package-0": 37.85454753950602, "psys": 62.273176483292566, "uncore": 25.276612393139285 }, "npu_busy_delta_us": 11753, "npu_utilization": 0.00649237780400504, "seconds": 1.810276659, "watts": { "core": 7.488288562195951, "dram": 2.2623332072691773, "package-0": 26.637638374367395, "psys": 44.55622548022921, "uncore": 13.96479807344188 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 815.9284584349092, "sha256": "9eadb231f9593492a54cd405e74b46a6c85686c26e7ef1fd4d66d970c80b3e55", "sum": -190088.92014630325, "top1": 382 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 140, "gross_j": { "core": 7.211225, "dram": 4.38567, "package-0": 21.284491, "psys": 37.845484, "uncore": 0.0 }, "net_j": { "core": 1.8752651362000883, "dram": 3.687070175373301, "package-0": 13.18281609063887, "psys": 23.477056331199112, "uncore": 0.0 }, "npu_busy_delta_us": 1403104, "npu_utilization": 0.9917929021423288, "seconds": 1.414714702, "token_latencies_s": [ 0.107573939, 0.100879221, 0.100501066, 0.100397544, 0.100379416, 0.100445641, 0.100631787, 0.100549427, 0.100440995, 0.10065907, 0.10060175, 0.100477034, 0.100652609, 0.100497828 ], "watts": { "core": 5.097299822929245, "dram": 3.1000384698059076, "package-0": 15.045076558481966, "psys": 26.75131879699657, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 24.612486, "dram": 8.899635, "package-0": 74.324943, "psys": 128.319373, "uncore": 25.02966 }, "net_j": { "core": 11.663854247946592, "dram": 7.204361403114597, "package-0": 54.66482417341999, "psys": 93.45189208030197, "uncore": 25.026109441274645 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 20, "gross_j": { "core": 2.867302, "dram": 0.409362, "package-0": 3.889882, "psys": 6.263778, "uncore": 0.0 }, "logits": { "l2": 730.2819384422855, "sha256": "0079145e0fc0317bb3a1425172a56437e9c22be22e53eb09799308790eff75c3", "sum": -132213.13601575233, "top1": 66 }, "net_j": { "core": 2.1146502672167515, "dram": 0.3108225840470251, "package-0": 2.747118595735032, "psys": 4.237071979278087, "uncore": 0.0 }, "npu_busy_delta_us": 120547, "npu_utilization": 0.6040960949524984, "seconds": 0.199549378, "watts": { "core": 14.3688846777563, "dram": 2.0514321021837514, "package-0": 19.49333061814906, "psys": 31.389614253771317, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 596.5539966338031, "decode_tokens_per_s": 9.287245410181095, "end_to_end_ms": 3433.050506, "handoff_estimate_ms": 98.755864, "handoff_plus_first_decode_ms": 199.549378, "output_tokens_per_s_e2e": 4.660578098701587, "prefill_tokens_per_s": 1117.753451453235, "steady_tpot_ms": 100.793514, "steady_tpot_p95_ms": 102.57646704999999, "ttft_ms": 1817.932208 }, "prefill": { "gpu_active_fraction": 0.9832402234636871, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 179, "gross_j": { "core": 13.966029, "dram": 4.109547, "package-0": 48.485227, "psys": 81.597874, "uncore": 25.02966 }, "logits": { "l2": 758.2016488784138, "sha256": "492b5c2c1dc0d4d194f12400bb823c47d03bde1c1182589d4e78e784bf614b52", "sum": -183229.291040726, "top1": 264 }, "net_j": { "core": 7.109230747120069, "dram": 3.2118344648177453, "package-0": 38.07443835430946, "psys": 63.13420256115502, "uncore": 25.026109441274645 }, "npu_busy_delta_us": 11373, "npu_utilization": 0.00625600885992994, "seconds": 1.817932208, "watts": { "core": 7.682370628861206, "dram": 2.260561192499649, "package-0": 26.67053633058247, "psys": 44.88499276316249, "uncore": 13.768203176033945 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 746.5391437933649, "sha256": "089041255ec886dbd43cd10c181c994eadc76e524ffff1628e8a99374ddb8519", "sum": -144414.58760984242, "top1": 4194 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 140, "gross_j": { "core": 7.779155, "dram": 4.380726, "package-0": 21.949834, "psys": 40.457721, "uncore": 0.0 }, "net_j": { "core": 2.439973233609771, "dram": 3.6817043542498267, "package-0": 13.843267223375499, "psys": 26.08061753986886, "uncore": 0.0 }, "npu_busy_delta_us": 1402547, "npu_utilization": 0.9908009282939046, "seconds": 1.41556892, "token_latencies_s": [ 0.104712135, 0.101426492, 0.100711233, 0.100867738, 0.100934798, 0.101137329, 0.100764231, 0.100919526, 0.100822797, 0.100697364, 0.100594681, 0.100649415, 0.100584998, 0.100717427 ], "watts": { "core": 5.495426531404773, "dram": 3.094675178372806, "package-0": 15.506015772089713, "psys": 28.58053778123357, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 7.915970413945615, "compile_target": "AUTO:NPU,GPU", "context_length": 2048, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 403365, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_3b_ov_fp16/openvino_model.xml", "key": "llama32_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": null, "key": "openvino_auto", "label": "OpenVINO AUTO", "mode": "stock", "prefill_device": null }, "prompt_length": 2032, "prompt_sha256": "cd1a97358e68d07c27508cfacc4ba6882a7a2962b1281e2e2b0c7e5e320da4fa", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.047580779530107975, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 18.503737, "dram": 7.3125420000000005, "package-0": 69.08313600000001, "psys": 117.557134, "uncore": 35.352082 }, "net_j": { "core": 8.580078729246372, "dram": 6.013306978643074, "package-0": 54.01588375796674, "psys": 90.83516274969902, "uncore": 35.34694337258802 } }, "first_decode": { "gpu_active_fraction": 0.9090909090909091, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 11, "gross_j": { "core": 2.504877, "dram": 0.309081, "package-0": 3.919424, "psys": 5.84935, "uncore": 0.78076 }, "logits": { "l2": 529.3319535900215, "sha256": "f984afc7983df3c72b7c030f6eb4d656d8481c00cf53a50b1f34deda53645068", "sum": -138429.60764685227, "top1": 319 }, "net_j": { "core": 2.0911683236457157, "dram": 0.2549170227354055, "package-0": 3.291283367656931, "psys": 4.73533427576651, "uncore": 0.7805457750955458 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.109685935, "watts": { "core": 22.836811301284893, "dram": 2.817872683494014, "package-0": 35.73315028950612, "psys": 53.328168283381096, "uncore": 7.118141446303029 } }, "metrics": { "all_tokens_per_s_e2e": 778.398228946693, "decode_tokens_per_s": 11.11239944154533, "end_to_end_ms": 2631.0440129999997, "handoff_estimate_ms": 21.822345000000006, "handoff_plus_first_decode_ms": 109.685935, "output_tokens_per_s_e2e": 6.0812361636460395, "prefill_tokens_per_s": 1586.0124603273755, "steady_tpot_ms": 87.86358999999999, "steady_tpot_p95_ms": 91.53992875, "ttft_ms": 1281.200527 }, "prefill": { "gpu_active_fraction": 0.9133858267716536, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 127, "gross_j": { "core": 7.011152, "dram": 2.925469, "package-0": 36.942593, "psys": 61.15762, "uncore": 23.483094 }, "logits": { "l2": 813.1669789722381, "sha256": "d50d0ae7fdda5cdbe691764d1ca1d7d8554922da6b22bef84ee4b7e361ab62dd", "sum": -253097.8784024875, "top1": 319 }, "net_j": { "core": 2.1787751399259925, "dram": 2.2927997683814296, "package-0": 29.605516380394647, "psys": 48.14521751672597, "uncore": 23.480591718732278 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.281200527, "watts": { "core": 5.4723299376226375, "dram": 2.283381046408202, "package-0": 28.834356700198267, "psys": 47.734619765731644, "uncore": 18.328976225905034 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 553.0736012792826, "sha256": "7bd1634f8d377902a10467f3d09341ea554283d2d40c1594e26340bce4ce43e1", "sum": -121082.53280104871, "top1": 42 }, "gpu_active_fraction": 0.991869918699187, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 123, "gross_j": { "core": 8.987708, "dram": 4.077992, "package-0": 28.221119, "psys": 50.550164, "uncore": 11.088228 }, "net_j": { "core": 4.310135265674664, "dram": 3.4655901875262387, "package-0": 21.119084009915163, "psys": 37.95461095720654, "uncore": 11.085805878760198 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.240157551, "token_latencies_s": [ 0.096881593, 0.088663648, 0.088054924, 0.087911311, 0.087829574, 0.087763997, 0.087897606, 0.087750979, 0.087544915, 0.087744268, 0.087782994, 0.087769238, 0.088626289, 0.08790289 ], "watts": { "core": 7.247230799629263, "dram": 3.2882854252765825, "package-0": 22.75607561091244, "psys": 40.76108229896993, "uncore": 8.940983338011382 } } }, { "energy": { "gross_j": { "core": 21.156806, "dram": 7.409404, "package-0": 72.095153, "psys": 120.088499, "uncore": 35.469391 }, "net_j": { "core": 11.041868410243916, "dram": 6.085126118104263, "package-0": 56.73747824749112, "psys": 92.85145958393754, "uncore": 35.46415332512738 } }, "first_decode": { "gpu_active_fraction": 0.9090909090909091, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 11, "gross_j": { "core": 2.407525, "dram": 0.306945, "package-0": 3.78948, "psys": 5.642014, "uncore": 0.74951 }, "logits": { "l2": 529.3319535900215, "sha256": "f984afc7983df3c72b7c030f6eb4d656d8481c00cf53a50b1f34deda53645068", "sum": -138429.60764685227, "top1": 319 }, "net_j": { "core": 2.0063025474150904, "dram": 0.2544157564961882, "package-0": 3.18029740489992, "psys": 4.5616206057064055, "uncore": 0.7493022406599559 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.106375482, "watts": { "core": 22.632329882180937, "dram": 2.8854863379138442, "package-0": 35.62362236816939, "psys": 53.038669192587065, "uncore": 7.04589051826811 } }, "metrics": { "all_tokens_per_s_e2e": 763.6782683117967, "decode_tokens_per_s": 11.122532279262616, "end_to_end_ms": 2681.7576, "handoff_estimate_ms": 18.484246999999996, "handoff_plus_first_decode_ms": 106.37548199999999, "output_tokens_per_s_e2e": 5.966236471185912, "prefill_tokens_per_s": 1524.2166136629965, "steady_tpot_ms": 87.891235, "steady_tpot_p95_ms": 91.87286215000002, "ttft_ms": 1333.143847 }, "prefill": { "gpu_active_fraction": 0.8939393939393939, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 132, "gross_j": { "core": 8.291422, "dram": 3.017387, "package-0": 38.604699, "psys": 62.925742, "uncore": 23.638001 }, "logits": { "l2": 813.1669789722381, "sha256": "d50d0ae7fdda5cdbe691764d1ca1d7d8554922da6b22bef84ee4b7e361ab62dd", "sum": -253097.8784024875, "top1": 319 }, "net_j": { "core": 3.2631275670608213, "dram": 2.3590676537744146, "package-0": 30.970157533413136, "psys": 49.38578167397346, "uncore": 23.63539726951033 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.333143847, "watts": { "core": 6.219450375635271, "dram": 2.2633619071115882, "package-0": 28.95763955770633, "psys": 47.20101446036978, "uncore": 17.731020589558327 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 553.0736012792826, "sha256": "7bd1634f8d377902a10467f3d09341ea554283d2d40c1594e26340bce4ce43e1", "sum": -121082.53280104871, "top1": 42 }, "gpu_active_fraction": 0.991869918699187, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 123, "gross_j": { "core": 10.457859, "dram": 4.085072, "package-0": 29.700974, "psys": 51.520743, "uncore": 11.08188 }, "net_j": { "core": 5.772438295768004, "dram": 3.4716427078336607, "package-0": 22.58702330917806, "psys": 38.90405730425767, "uncore": 11.079453814957093 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.242238271, "token_latencies_s": [ 0.097536617, 0.088419048, 0.088368702, 0.088131056, 0.088823148, 0.08782937, 0.087889673, 0.087706404, 0.087763903, 0.087746802, 0.088616436, 0.087721156, 0.087755781, 0.087892797 ], "watts": { "core": 8.41856127293634, "dram": 3.288477013923845, "package-0": 23.909240838386633, "psys": 41.47412312335691, "uncore": 8.920897269634997 } } }, { "energy": { "gross_j": { "core": 20.825264, "dram": 7.384259, "package-0": 71.662109, "psys": 119.028869, "uncore": 35.399751 }, "net_j": { "core": 10.763769832330647, "dram": 6.066978090767145, "package-0": 56.385578267094395, "psys": 91.93573957655872, "uncore": 35.39454099897771 } }, "first_decode": { "gpu_active_fraction": 0.9, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 10, "gross_j": { "core": 2.447198, "dram": 0.302307, "package-0": 3.768972, "psys": 5.542344, "uncore": 0.707639 }, "logits": { "l2": 529.3319535900215, "sha256": "f984afc7983df3c72b7c030f6eb4d656d8481c00cf53a50b1f34deda53645068", "sum": -138429.60764685227, "top1": 319 }, "net_j": { "core": 2.065541434221394, "dram": 0.25233938091624436, "package-0": 3.1894966100259246, "psys": 4.514636727140632, "uncore": 0.707441372186006 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.101188009, "watts": { "core": 24.184664014883428, "dram": 2.987577312643833, "package-0": 37.24721967797588, "psys": 54.77273497890447, "uncore": 6.9933088613296075 } }, "metrics": { "all_tokens_per_s_e2e": 767.7346817382437, "decode_tokens_per_s": 11.219421973091185, "end_to_end_ms": 2667.5882289999995, "handoff_estimate_ms": 13.265295499999997, "handoff_plus_first_decode_ms": 101.188009, "output_tokens_per_s_e2e": 5.997927201080029, "prefill_tokens_per_s": 1527.1065669282907, "steady_tpot_ms": 87.9227135, "steady_tpot_p95_ms": 89.9502872, "ttft_ms": 1330.620956 }, "prefill": { "gpu_active_fraction": 0.8939393939393939, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 132, "gross_j": { "core": 8.981483, "dram": 3.007439, "package-0": 39.282798, "psys": 63.428182, "uncore": 23.619691 }, "logits": { "l2": 813.1669789722381, "sha256": "d50d0ae7fdda5cdbe691764d1ca1d7d8554922da6b22bef84ee4b7e361ab62dd", "sum": -253097.8784024875, "top1": 319 }, "net_j": { "core": 3.9627042984642706, "dram": 2.350365481790241, "package-0": 31.66270442363783, "psys": 49.91384519848941, "uncore": 23.61709219690675 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.330620956, "watts": { "core": 6.749843341562404, "dram": 2.2601770898308327, "package-0": 29.522154917872797, "psys": 47.66810691954862, "uncore": 17.75087856049067 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 553.0736012792826, "sha256": "7bd1634f8d377902a10467f3d09341ea554283d2d40c1594e26340bce4ce43e1", "sum": -121082.53280104871, "top1": 42 }, "gpu_active_fraction": 0.991869918699187, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 123, "gross_j": { "core": 9.396583, "dram": 4.074513, "package-0": 28.610339, "psys": 50.058343, "uncore": 11.072421 }, "net_j": { "core": 4.735524099644983, "dram": 3.4642732280606596, "package-0": 21.53337723343064, "psys": 37.50725765092868, "uncore": 11.070007429884955 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.235779264, "token_latencies_s": [ 0.092201726, 0.088244896, 0.088126783, 0.087921287, 0.087696855, 0.087731693, 0.087639581, 0.088042578, 0.087880844, 0.087783678, 0.08792414, 0.087771034, 0.088737974, 0.088048731 ], "watts": { "core": 7.603771380323145, "dram": 3.297120382819435, "package-0": 23.151658094175627, "psys": 40.507511703967225, "uncore": 8.959869551590081 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 2.5552231147885323, "compile_target": "AUTO:GPU", "context_length": 4096, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 403852, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_3b_ov_fp16/openvino_model.xml", "key": "llama32_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "GPU", "key": "gpu_gpu", "label": "iGPU -> iGPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 4080, "prompt_sha256": "66f8de7a9ef70b33481de3e8dae7cef6e4434d9723c6203a6a3dbe07ef228a16", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.04400048963725567, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 29.317245999999997, "dram": 11.250032, "package-0": 125.571579, "psys": 207.82448, "uncore": 71.998839 }, "net_j": { "core": 11.522534278419908, "dram": 8.920295119422017, "package-0": 98.55357789918703, "psys": 159.90769749320518, "uncore": 71.98962461645857 } }, "first_decode": { "gpu_active_fraction": 0.8333333333333334, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 12, "gross_j": { "core": 2.001277, "dram": 0.320922, "package-0": 3.435172, "psys": 5.311571, "uncore": 0.771116 }, "logits": { "l2": 612.9391833447571, "sha256": "2baff3d89206006a57e2184111b8b052e1406719a9a1ea9356400754881d45b5", "sum": -156671.62222059013, "top1": 437 }, "net_j": { "core": 1.5705418287129977, "dram": 0.2645288630944757, "package-0": 2.7811797626389487, "psys": 4.1517071121486255, "uncore": 0.7708929585068237 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.114200143, "watts": { "core": 17.52429504400883, "dram": 2.810171612482131, "package-0": 30.08027757023036, "psys": 46.51107135653937, "uncore": 6.752320791752424 } }, "metrics": { "all_tokens_per_s_e2e": 868.1858007577679, "decode_tokens_per_s": 10.565813551347448, "end_to_end_ms": 4717.8841170000005, "handoff_estimate_ms": 21.207544999999996, "handoff_plus_first_decode_ms": 114.200143, "output_tokens_per_s_e2e": 3.3913507842100308, "prefill_tokens_per_s": 1237.0342071169125, "steady_tpot_ms": 92.99259800000002, "steady_tpot_p95_ms": 94.43623765000001, "ttft_ms": 3298.211138 }, "prefill": { "gpu_active_fraction": 0.9296636085626911, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 327, "gross_j": { "core": 18.231337, "dram": 6.70146, "package-0": 92.669807, "psys": 150.688884, "uncore": 59.30124 }, "logits": { "l2": 664.5119473649122, "sha256": "8c551b400c21b7ca89d3bd6971b7ce08a428c386302f3d228b8083472325153d", "sum": -177372.0653137702, "top1": 72 }, "net_j": { "core": 5.791286557443822, "dram": 5.07277138926368, "package-0": 73.78187547236821, "psys": 117.19088747406408, "uncore": 59.29479834486976 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 3.298211138, "watts": { "core": 5.527643997665743, "dram": 2.031846876869045, "package-0": 28.096990496549587, "psys": 45.688064740262845, "uncore": 17.979819216776896 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 493.57484335745835, "sha256": "2d68b390a1647172ee9cabcff42617290735e761c3331785f10c4fbb6e5d57c9", "sum": -72888.27841649903, "top1": 42 }, "gpu_active_fraction": 0.9923076923076923, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 130, "gross_j": { "core": 9.084632, "dram": 4.22765, "package-0": 29.4666, "psys": 51.824025, "uncore": 11.926483 }, "net_j": { "core": 4.1607058922630875, "dram": 3.582994867063861, "package-0": 21.990522664179878, "psys": 38.565102906992465, "uncore": 11.923933313081985 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.305472836, "token_latencies_s": [ 0.095775809, 0.093471366, 0.092632614, 0.093668092, 0.092907505, 0.092847205, 0.09371493, 0.092613468, 0.09312234, 0.092566274, 0.092846817, 0.093077691, 0.092796978, 0.093400508 ], "watts": { "core": 6.9588824443375845, "dram": 3.2384051842500376, "package-0": 22.571591830502094, "psys": 39.697513093255964, "uncore": 9.135757306558004 } } }, { "energy": { "gross_j": { "core": 33.620459000000004, "dram": 11.410677, "package-0": 130.722444, "psys": 214.424928, "uncore": 72.420896 }, "net_j": { "core": 15.443009817651124, "dram": 9.030830986333312, "package-0": 103.12332636871619, "psys": 165.47752763595605, "uncore": 72.41148342894003 } }, "first_decode": { "gpu_active_fraction": 0.8181818181818182, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 11, "gross_j": { "core": 2.506646, "dram": 0.315673, "package-0": 3.916799, "psys": 5.729844, "uncore": 0.760862 }, "logits": { "l2": 612.9391833447571, "sha256": "2baff3d89206006a57e2184111b8b052e1406719a9a1ea9356400754881d45b5", "sum": -156671.62222059013, "top1": 437 }, "net_j": { "core": 2.0988071390976213, "dram": 0.26227751653032927, "package-0": 3.2975706042147643, "psys": 4.631634245245796, "uncore": 0.7606508145789461 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.10812968, "watts": { "core": 23.181849793692166, "dram": 2.9193927143777727, "package-0": 36.22316278009886, "psys": 52.990483278966515, "uncore": 7.036569422937347 } }, "metrics": { "all_tokens_per_s_e2e": 849.9056105328265, "decode_tokens_per_s": 10.614448028170655, "end_to_end_ms": 4819.3587019999995, "handoff_estimate_ms": 15.117160500000004, "handoff_plus_first_decode_ms": 108.12968000000001, "output_tokens_per_s_e2e": 3.3199437911438534, "prefill_tokens_per_s": 1197.8190740915454, "steady_tpot_ms": 93.0125195, "steady_tpot_p95_ms": 94.15325495, "ttft_ms": 3406.190541 }, "prefill": { "gpu_active_fraction": 0.9169139465875371, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 337, "gross_j": { "core": 20.938179, "dram": 6.861128, "package-0": 96.219481, "psys": 154.843232, "uncore": 59.71933 }, "logits": { "l2": 664.5119473649122, "sha256": "8c551b400c21b7ca89d3bd6971b7ce08a428c386302f3d228b8083472325153d", "sum": -177372.0653137702, "top1": 72 }, "net_j": { "core": 8.09085650484514, "dram": 5.17911811403239, "package-0": 76.71318167087709, "psys": 120.24855199467956, "uncore": 59.71267745295102 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 3.406190541, "watts": { "core": 6.147095633074275, "dram": 2.0143112716136216, "package-0": 28.248414127693394, "psys": 45.4593570547996, "uncore": 17.532586413227317 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 493.57484335745835, "sha256": "2d68b390a1647172ee9cabcff42617290735e761c3331785f10c4fbb6e5d57c9", "sum": -72888.27841649903, "top1": 42 }, "gpu_active_fraction": 0.9922480620155039, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 129, "gross_j": { "core": 10.175634, "dram": 4.233876, "package-0": 30.586164, "psys": 53.851852, "uncore": 11.940704 }, "net_j": { "core": 5.253346173708362, "dram": 3.589435355770593, "package-0": 23.112574093624332, "psys": 40.59734139603069, "uncore": 11.938155161410064 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.305038481, "token_latencies_s": [ 0.095537987, 0.093321664, 0.093286226, 0.093186799, 0.092806638, 0.093083915, 0.09297743, 0.092971573, 0.092974747, 0.09340763, 0.092640208, 0.092915735, 0.093047609, 0.092853653 ], "watts": { "core": 7.797190771112596, "dram": 3.2442537608207127, "package-0": 23.436982468565233, "psys": 41.26457019009541, "uncore": 9.149694950642608 } } }, { "energy": { "gross_j": { "core": 35.532989, "dram": 11.419953000000001, "package-0": 132.688686, "psys": 216.76445999999999, "uncore": 72.479001 }, "net_j": { "core": 17.363264051302657, "dram": 9.041118266105288, "package-0": 105.10129619867857, "psys": 167.83785909759968, "uncore": 72.46959242867054 } }, "first_decode": { "gpu_active_fraction": 0.9090909090909091, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 11, "gross_j": { "core": 2.61004, "dram": 0.316344, "package-0": 4.018239, "psys": 5.940049, "uncore": 0.762755 }, "logits": { "l2": 612.9391833447571, "sha256": "2baff3d89206006a57e2184111b8b052e1406719a9a1ea9356400754881d45b5", "sum": -156671.62222059013, "top1": 437 }, "net_j": { "core": 2.1984882480597467, "dram": 0.2624624137308097, "package-0": 3.39337326118173, "psys": 4.8318413427724725, "uncore": 0.7625418919851662 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.109114073, "watts": { "core": 23.920287532479886, "dram": 2.89920439501878, "package-0": 36.82603801253025, "psys": 54.438889839626825, "uncore": 6.990436513170945 } }, "metrics": { "all_tokens_per_s_e2e": 850.2669186723851, "decode_tokens_per_s": 10.594658249592095, "end_to_end_ms": 4817.310788000001, "handoff_estimate_ms": 16.117296000000003, "handoff_plus_first_decode_ms": 109.114073, "output_tokens_per_s_e2e": 3.3213551510640045, "prefill_tokens_per_s": 1199.4697750914913, "steady_tpot_ms": 92.99677700000001, "steady_tpot_p95_ms": 94.99643845, "ttft_ms": 3401.502968 }, "prefill": { "gpu_active_fraction": 0.9169139465875371, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 337, "gross_j": { "core": 22.149845, "dram": 6.859663, "package-0": 97.440119, "psys": 156.79214, "uncore": 59.743438 }, "logits": { "l2": 664.5119473649122, "sha256": "8c551b400c21b7ca89d3bd6971b7ce08a428c386302f3d228b8083472325153d", "sum": -177372.0653137702, "top1": 72 }, "net_j": { "core": 9.320202890403632, "dram": 5.17996788298976, "package-0": 77.96066408869943, "psys": 122.24506892494479, "uncore": 59.736794608134666 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 3.401502968, "watts": { "core": 6.511781764818968, "dram": 2.0166564793660355, "package-0": 28.64619549554366, "psys": 46.094959044586666, "uncore": 17.563835328689326 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 493.57484335745835, "sha256": "2d68b390a1647172ee9cabcff42617290735e761c3331785f10c4fbb6e5d57c9", "sum": -72888.27841649903, "top1": 42 }, "gpu_active_fraction": 0.9922480620155039, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 129, "gross_j": { "core": 10.773104, "dram": 4.243946, "package-0": 31.230328, "psys": 54.032271, "uncore": 11.972808 }, "net_j": { "core": 5.844572912839278, "dram": 3.598687969384719, "package-0": 23.7472588487974, "psys": 40.76094882988241, "uncore": 11.970255928550712 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.306693747, "token_latencies_s": [ 0.096694802, 0.093496106, 0.093174391, 0.094081935, 0.092715433, 0.093077501, 0.092916285, 0.092825469, 0.092931103, 0.092687669, 0.093062451, 0.092663412, 0.093395732, 0.092927428 ], "watts": { "core": 8.244551582751242, "dram": 3.247850546268819, "package-0": 23.900265897576077, "psys": 41.35037083023556, "uncore": 9.16267337123792 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 8.330430769361556, "compile_target": "NPU", "context_length": 4096, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 403921, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_3b_ov_fp16/openvino_model.xml", "key": "llama32_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "gpu_npu", "label": "iGPU -> NPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 4080, "prompt_sha256": "66f8de7a9ef70b33481de3e8dae7cef6e4434d9723c6203a6a3dbe07ef228a16", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "GPU", "NPUW_ONLINE_PIPELINE": "NONE" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 4096, "NPUW_LLM_MIN_RESPONSE_LEN": 4096, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.04439142346382141, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 42.042739, "dram": 14.471765, "package-0": 135.735371, "psys": 235.814582, "uncore": 55.869975000000004 }, "net_j": { "core": 19.336425457843905, "dram": 11.498986519021546, "package-0": 101.26000472961552, "psys": 174.67206367467514, "uncore": 55.861174704536296 } }, "first_decode": { "gpu_active_fraction": 0.046511627906976744, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 43, "gross_j": { "core": 6.138046, "dram": 0.646116, "package-0": 7.800151, "psys": 12.842374, "uncore": 0.009461 }, "logits": { "l2": 795.940332625534, "sha256": "0a1480a30cc1c194375e5006bdf6f9aa21a60cde87b2a188b89fab018ccd6bea", "sum": -176995.81349948607, "top1": 4194 }, "net_j": { "core": 4.482238075707148, "dram": 0.42933267421562504, "package-0": 5.286110816532128, "psys": 8.383690463972318, "uncore": 0.008603596434036525 }, "npu_busy_delta_us": 132967, "npu_utilization": 0.30288488496474575, "seconds": 0.439001768, "watts": { "core": 13.981825239482863, "dram": 1.4717845054327892, "package-0": 17.76792616470738, "psys": 29.25358150266037, "uncore": 0.021551166053618263 } }, "metrics": { "all_tokens_per_s_e2e": 680.3885631443926, "decode_tokens_per_s": 7.679604594484617, "end_to_end_ms": 6020.0894339999995, "handoff_estimate_ms": 331.29756, "handoff_plus_first_decode_ms": 439.001768, "output_tokens_per_s_e2e": 2.6577678247827836, "prefill_tokens_per_s": 1003.2300387751951, "steady_tpot_ms": 107.704208, "steady_tpot_p95_ms": 110.33944955, "ttft_ms": 4066.863872 }, "prefill": { "gpu_active_fraction": 0.9875311720698254, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 401, "gross_j": { "core": 28.478504, "dram": 9.136696, "package-0": 105.439611, "psys": 180.9313, "uncore": 55.860514 }, "logits": { "l2": 644.3204748970329, "sha256": "92de318a3b00199052669a6bdf346331eeba162cc1006f9acf0cb48fca979529", "sum": -117752.07326419652, "top1": 264 }, "net_j": { "core": 13.139282345025004, "dram": 7.128439223037589, "package-0": 82.1498205249381, "psys": 139.62654837687626, "uncore": 55.85257110810226 }, "npu_busy_delta_us": 14168, "npu_utilization": 0.003483765487589942, "seconds": 4.066863872, "watts": { "core": 7.002571243181262, "dram": 2.2466195790091104, "package-0": 25.9265159392087, "psys": 44.48914586143295, "uncore": 13.735525888779984 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 778.4076845265583, "sha256": "6400a338520784bec892978554b291b5e5482e8a25c932a63eee2c6e825f41eb", "sum": -170108.64136687852, "top1": 4194 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 150, "gross_j": { "core": 7.426189, "dram": 4.688953, "package-0": 22.495609, "psys": 42.040908, "uncore": 0.0 }, "net_j": { "core": 1.7149050371117527, "dram": 3.9412146217683333, "package-0": 13.824073388145296, "psys": 26.66182483382657, "uncore": 0.0 }, "npu_busy_delta_us": 1499090, "npu_utilization": 0.9900055764148162, "seconds": 1.514223794, "token_latencies_s": [ 0.114102581, 0.108313148, 0.10751789, 0.107497365, 0.107580084, 0.107286321, 0.107339312, 0.107522486, 0.107883348, 0.107679407, 0.10810087, 0.107893555, 0.107740762, 0.107729009 ], "watts": { "core": 4.904287615493645, "dram": 3.0966050187426917, "package-0": 14.856198330218552, "psys": 27.763999064460613, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 46.555178, "dram": 14.38894, "package-0": 139.743783, "psys": 240.19658900000002, "uncore": 55.433513 }, "net_j": { "core": 24.289525731451064, "dram": 11.473854210930684, "package-0": 105.93747993005869, "psys": 180.2406631150008, "uncore": 55.42547767506553 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 27, "gross_j": { "core": 4.367969, "dram": 0.51721, "package-0": 5.643906, "psys": 9.042274, "uncore": 0.0 }, "logits": { "l2": 769.6036712572533, "sha256": "3cd0382f5a114231fcf2dbc64be60b7477ce00ebf44cb96ae1c6b8dec4e0356c", "sum": -160694.08502659574, "top1": 4194 }, "net_j": { "core": 3.3274896555157207, "dram": 0.38098733226299286, "package-0": 4.064129237415097, "psys": 6.240519004890777, "uncore": 0.0 }, "npu_busy_delta_us": 129842, "npu_utilization": 0.4706796488776655, "seconds": 0.275860663, "watts": { "core": 15.833968324798814, "dram": 1.8748958056408351, "package-0": 20.459263523193954, "psys": 32.778410309265446, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 693.8541866602355, "decode_tokens_per_s": 8.384259272919909, "end_to_end_ms": 5903.257599, "handoff_estimate_ms": 168.1576455, "handoff_plus_first_decode_ms": 275.860663, "output_tokens_per_s_e2e": 2.710367916641545, "prefill_tokens_per_s": 991.6895472860117, "steady_tpot_ms": 107.7030175, "steady_tpot_p95_ms": 110.319164, "ttft_ms": 4114.190788 }, "prefill": { "gpu_active_fraction": 0.9778325123152709, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 406, "gross_j": { "core": 33.655127, "dram": 9.193031, "package-0": 110.413841, "psys": 187.468453, "uncore": 55.433513 }, "logits": { "l2": 542.5382000105269, "sha256": "cfd528695d632340b303a5014801f9770a9431866307b0e28a5e11d8e28e4813", "sum": 93871.5732476227, "top1": 14924 }, "net_j": { "core": 18.137399725555284, "dram": 7.161403733446306, "package-0": 86.85302253084672, "psys": 145.68302964258802, "uncore": 55.42547767506553 }, "npu_busy_delta_us": 14682, "npu_utilization": 0.0035686240032483394, "seconds": 4.114190788, "watts": { "core": 8.180254328059615, "dram": 2.234468811415753, "package-0": 26.837316665539138, "psys": 45.56630031519093, "uncore": 13.473734169471383 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 752.0471312612783, "sha256": "d32a92f79d95feacf0e26e8999682795635ded2922ddc5b9e3abee1402fb5063", "sum": -149188.8564177081, "top1": 4194 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 150, "gross_j": { "core": 8.532082, "dram": 4.678699, "package-0": 23.686036, "psys": 43.685862, "uncore": 0.0 }, "net_j": { "core": 2.824636350380061, "dram": 3.931463145221385, "package-0": 15.02032816179687, "psys": 28.317114467522025, "uncore": 0.0 }, "npu_busy_delta_us": 1499187, "npu_utilization": 0.9907354671942556, "seconds": 1.513206148, "token_latencies_s": [ 0.112835054, 0.107687895, 0.10771814, 0.108001443, 0.108964454, 0.107637014, 0.107849571, 0.107833572, 0.10758187, 0.107358476, 0.107729926, 0.107294444, 0.107378499, 0.10729835 ], "watts": { "core": 5.638413517733078, "dram": 3.0919111756080437, "package-0": 15.65288115654682, "psys": 28.86973599581225, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 39.387473, "dram": 14.329676000000001, "package-0": 132.323087, "psys": 232.33223, "uncore": 55.281902 }, "net_j": { "core": 17.269484427388246, "dram": 11.433922783648349, "package-0": 98.74098412851103, "psys": 172.7739261340072, "uncore": 55.27392373752541 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 0, "gpu_sample_count": 26, "gross_j": { "core": 3.72637, "dram": 0.511229, "package-0": 4.987414, "psys": 8.372538, "uncore": 0.0 }, "logits": { "l2": 752.0494720512006, "sha256": "c4a380935b3e73c9d50bf91e3f1398b1007b698509fefbb1e1e6e91942deba27", "sum": -156186.7489903532, "top1": 4194 }, "net_j": { "core": 2.72089169532343, "dram": 0.37958877307880623, "package-0": 3.460779887144425, "psys": 5.6650321974721765, "uncore": 0.0 }, "npu_busy_delta_us": 132146, "npu_utilization": 0.4957069466179144, "seconds": 0.266580892, "watts": { "core": 13.978383717014497, "dram": 1.91772559602659, "package-0": 18.70882028558896, "psys": 31.407119757105473, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 698.486482825295, "decode_tokens_per_s": 8.43106963214957, "end_to_end_ms": 5864.107754, "handoff_estimate_ms": 158.90403450000002, "handoff_plus_first_decode_ms": 266.580892, "output_tokens_per_s_e2e": 2.7284628235363084, "prefill_tokens_per_s": 998.782350409472, "steady_tpot_ms": 107.6768575, "steady_tpot_p95_ms": 109.48473709999999, "ttft_ms": 4084.974067 }, "prefill": { "gpu_active_fraction": 0.9826302729528535, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 403, "gross_j": { "core": 28.642078, "dram": 9.144142, "package-0": 105.18186, "psys": 181.811973, "uncore": 55.281902 }, "logits": { "l2": 803.7338034246765, "sha256": "5075f102276a647d3b8b6d63b673bd613e0ac84f61524d76ac961b7991a6528e", "sum": 218158.74183682352, "top1": 7998 }, "net_j": { "core": 13.234549093033506, "dram": 7.126942233307525, "package-0": 81.78835750990939, "psys": 140.32328674570468, "uncore": 55.27392373752541 }, "npu_busy_delta_us": 11712, "npu_utilization": 0.002867092864704837, "seconds": 4.084974067, "watts": { "core": 7.011569114081233, "dram": 2.2384822645191105, "package-0": 25.748476801774515, "psys": 44.50749748174594, "uncore": 13.532987258496592 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 750.6369392275984, "sha256": "d113d3583ce066650c8eb822292a21463debdce54ba13c47cac7304db50ab8e8", "sum": -154246.8292356953, "top1": 4194 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 150, "gross_j": { "core": 7.019025, "dram": 4.674305, "package-0": 22.153813, "psys": 42.147719, "uncore": 0.0 }, "net_j": { "core": 1.3140436390313095, "dram": 3.9273917772620184, "package-0": 13.49184673145722, "psys": 26.785607190830355, "uncore": 0.0 }, "npu_busy_delta_us": 1497198, "npu_utilization": 0.9898484237702262, "seconds": 1.512552795, "token_latencies_s": [ 0.112196164, 0.107883842, 0.107862353, 0.107684435, 0.107649603, 0.107772679, 0.107536952, 0.107580107, 0.107524802, 0.107533906, 0.107567178, 0.108010707, 0.108024738, 0.10766928 ], "watts": { "core": 4.640515705106346, "dram": 3.0903417159729623, "package-0": 14.646637838515911, "psys": 27.865287836118146, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 3.1715340623632073, "compile_target": "AUTO:NPU,GPU", "context_length": 4096, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 403768, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_3b_ov_fp16/openvino_model.xml", "key": "llama32_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": null, "key": "openvino_auto", "label": "OpenVINO AUTO", "mode": "stock", "prefill_device": null }, "prompt_length": 4080, "prompt_sha256": "66f8de7a9ef70b33481de3e8dae7cef6e4434d9723c6203a6a3dbe07ef228a16", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.04399902559816837, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 30.4326, "dram": 11.267245, "package-0": 126.691265, "psys": 209.48530300000002, "uncore": 71.962157 }, "net_j": { "core": 12.658991993471382, "dram": 8.940271080915403, "package-0": 99.70530601402503, "psys": 161.62534760791368, "uncore": 71.95295354429643 } }, "first_decode": { "gpu_active_fraction": 0.8333333333333334, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 12, "gross_j": { "core": 2.583794, "dram": 0.322448, "package-0": 4.039601, "psys": 6.001998, "uncore": 0.788511 }, "logits": { "l2": 612.9391833447571, "sha256": "2baff3d89206006a57e2184111b8b052e1406719a9a1ea9356400754881d45b5", "sum": -156671.62222059013, "top1": 437 }, "net_j": { "core": 2.152811284539373, "dram": 0.2660224538708879, "package-0": 3.385232912281824, "psys": 4.841467536566286, "uncore": 0.7882878303245303 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.114265774, "watts": { "core": 22.61214281014716, "dram": 2.821912360213829, "package-0": 35.352676996700694, "psys": 52.52664721809, "uncore": 6.900675262568124 } }, "metrics": { "all_tokens_per_s_e2e": 869.216651992036, "decode_tokens_per_s": 10.593454419911968, "end_to_end_ms": 4712.288922, "handoff_estimate_ms": 21.5509755, "handoff_plus_first_decode_ms": 114.26577400000001, "output_tokens_per_s_e2e": 3.3953775468438905, "prefill_tokens_per_s": 1237.7438291294693, "steady_tpot_ms": 92.7147985, "steady_tpot_p95_ms": 94.30396769999999, "ttft_ms": 3296.3202109999997 }, "prefill": { "gpu_active_fraction": 0.9325153374233128, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 326, "gross_j": { "core": 18.352187, "dram": 6.707747, "package-0": 92.737617, "psys": 150.897746, "uncore": 59.233064 }, "logits": { "l2": 664.5119473649122, "sha256": "8c551b400c21b7ca89d3bd6971b7ce08a428c386302f3d228b8083472325153d", "sum": -177372.0653137702, "top1": 72 }, "net_j": { "core": 5.919268674303224, "dram": 5.079992147334275, "package-0": 73.86051428174558, "psys": 117.41895451080265, "uncore": 59.226626037992816 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 3.296320211, "watts": { "core": 5.567477012323546, "dram": 2.0349197197577724, "package-0": 28.133679698510335, "psys": 45.77763576986423, "uncore": 17.96945084471346 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 493.57484335745835, "sha256": "2d68b390a1647172ee9cabcff42617290735e761c3331785f10c4fbb6e5d57c9", "sum": -72888.27841649903, "top1": 42 }, "gpu_active_fraction": 0.9921875, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 128, "gross_j": { "core": 9.496619, "dram": 4.23705, "package-0": 29.914047, "psys": 52.585559, "uncore": 11.940582 }, "net_j": { "core": 4.586912034628784, "dram": 3.5942564797102396, "package-0": 22.459558819997618, "psys": 39.36492556054475, "uncore": 11.938039675979088 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.301702937, "token_latencies_s": [ 0.095749842, 0.093016564, 0.092660296, 0.09352542, 0.092782285, 0.092539888, 0.092575364, 0.093426931, 0.092509679, 0.092391097, 0.092769301, 0.092327928, 0.092529806, 0.092874187 ], "watts": { "core": 7.295534741503008, "dram": 3.255005331527496, "package-0": 22.980701778965106, "psys": 40.397511218029926, "uncore": 9.173046830115586 } } }, { "energy": { "gross_j": { "core": 33.737644, "dram": 11.427095, "package-0": 131.091706, "psys": 215.956601, "uncore": 72.526487 }, "net_j": { "core": 15.584494342279042, "dram": 9.050430352774754, "package-0": 103.52948273336894, "psys": 167.07463327992537, "uncore": 72.51708701161871 } }, "first_decode": { "gpu_active_fraction": 0.9090909090909091, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 11, "gross_j": { "core": 1.923029, "dram": 0.313964, "package-0": 3.459403, "psys": 5.121142, "uncore": 0.75598 }, "logits": { "l2": 612.9391833447571, "sha256": "2baff3d89206006a57e2184111b8b052e1406719a9a1ea9356400754881d45b5", "sum": -156671.62222059013, "top1": 437 }, "net_j": { "core": 1.515362010513625, "dram": 0.26059101845016064, "package-0": 2.8404355593895056, "psys": 4.023395052701664, "uncore": 0.7557689035766871 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.108084112, "watts": { "core": 17.791967426257802, "dram": 2.9048117636383046, "package-0": 32.00658205898014, "psys": 47.3810803941286, "uncore": 6.994367497787279 } }, "metrics": { "all_tokens_per_s_e2e": 851.0432810034584, "decode_tokens_per_s": 10.610753560155404, "end_to_end_ms": 4812.916207, "handoff_estimate_ms": 15.01799999999999, "handoff_plus_first_decode_ms": 108.08411199999999, "output_tokens_per_s_e2e": 3.3243878164197596, "prefill_tokens_per_s": 1200.262643236685, "steady_tpot_ms": 93.066112, "steady_tpot_p95_ms": 94.6056683, "ttft_ms": 3399.256007 }, "prefill": { "gpu_active_fraction": 0.9169139465875371, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 337, "gross_j": { "core": 21.931828, "dram": 6.876203, "package-0": 97.316096, "psys": 157.912804, "uncore": 59.810699 }, "logits": { "l2": 664.5119473649122, "sha256": "8c551b400c21b7ca89d3bd6971b7ce08a428c386302f3d228b8083472325153d", "sum": -177372.0653137702, "top1": 72 }, "net_j": { "core": 9.110660880940292, "dram": 5.197617454098566, "package-0": 77.84950880508796, "psys": 123.38855399025039, "uncore": 59.80405999661898 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 3.399256007, "watts": { "core": 6.451949472130476, "dram": 2.022855291228437, "package-0": 28.62864573883211, "psys": 46.45510772793054, "uncore": 17.59523227342494 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 493.57484335745835, "sha256": "2d68b390a1647172ee9cabcff42617290735e761c3331785f10c4fbb6e5d57c9", "sum": -72888.27841649903, "top1": 42 }, "gpu_active_fraction": 0.9923076923076923, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 130, "gross_j": { "core": 9.882787, "dram": 4.236928, "package-0": 30.316207, "psys": 52.922655, "uncore": 11.959808 }, "net_j": { "core": 4.958471450825124, "dram": 3.592221880226027, "package-0": 22.839538368891464, "psys": 39.662684236973305, "uncore": 11.957258111423043 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.305576088, "token_latencies_s": [ 0.096251376, 0.093719518, 0.092846601, 0.092841851, 0.092646383, 0.093118375, 0.093553867, 0.093072083, 0.093112069, 0.093060141, 0.093137184, 0.0929697, 0.092669052, 0.092527111 ], "watts": { "core": 7.569675249750745, "dram": 3.2452555151270506, "package-0": 23.220559321395903, "psys": 40.53586419545392, "uncore": 9.160559932068855 } } }, { "energy": { "gross_j": { "core": 32.793555999999995, "dram": 11.423739, "package-0": 130.21152, "psys": 216.322261, "uncore": 72.718137 }, "net_j": { "core": 14.653619383592828, "dram": 9.04880424366228, "package-0": 102.66935830954654, "psys": 167.47587275076995, "uncore": 72.70874385354075 } }, "first_decode": { "gpu_active_fraction": 0.9090909090909091, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 11, "gross_j": { "core": 2.439202, "dram": 0.317321, "package-0": 3.854177, "psys": 5.788315, "uncore": 0.759825 }, "logits": { "l2": 612.9391833447571, "sha256": "2baff3d89206006a57e2184111b8b052e1406719a9a1ea9356400754881d45b5", "sum": -156671.62222059013, "top1": 437 }, "net_j": { "core": 2.031070990127524, "dram": 0.2638872675133794, "package-0": 3.234505029667495, "psys": 4.689318559915978, "uncore": 0.7596136632995827 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.108207137, "watts": { "core": 22.541969666936108, "dram": 2.9325329991865514, "package-0": 35.61851008034711, "psys": 53.49291331864737, "uncore": 7.021949023565793 } }, "metrics": { "all_tokens_per_s_e2e": 851.6631767764982, "decode_tokens_per_s": 10.629307023993828, "end_to_end_ms": 4809.413054000001, "handoff_estimate_ms": 15.408518499999996, "handoff_plus_first_decode_ms": 108.20713699999999, "output_tokens_per_s_e2e": 3.326809284283196, "prefill_tokens_per_s": 1200.6284233949712, "steady_tpot_ms": 92.7986185, "steady_tpot_p95_ms": 94.44744735, "ttft_ms": 3398.220399 }, "prefill": { "gpu_active_fraction": 0.9196428571428571, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 336, "gross_j": { "core": 20.965462, "dram": 6.869917, "package-0": 96.557798, "psys": 157.600976, "uncore": 60.018096 }, "logits": { "l2": 664.5119473649122, "sha256": "8c551b400c21b7ca89d3bd6971b7ce08a428c386302f3d228b8083472325153d", "sum": -177372.0653137702, "top1": 72 }, "net_j": { "core": 8.148200942505085, "dram": 5.1918428473612215, "package-0": 77.09714144208331, "psys": 123.0872440534473, "uncore": 60.011459019239524 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 3.398220399, "watts": { "core": 6.169541565393916, "dram": 2.021621964844194, "package-0": 28.414224700791696, "psys": 46.37750277950703, "uncore": 17.661625484227457 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 493.57484335745835, "sha256": "2d68b390a1647172ee9cabcff42617290735e761c3331785f10c4fbb6e5d57c9", "sum": -72888.27841649903, "top1": 42 }, "gpu_active_fraction": 0.9922480620155039, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 129, "gross_j": { "core": 9.388892, "dram": 4.236501, "package-0": 29.799545, "psys": 52.93297, "uncore": 11.940216 }, "net_j": { "core": 4.474347450960219, "dram": 3.59307412878768, "package-0": 22.337711837795737, "psys": 39.699310137406684, "uncore": 11.937671171001652 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.302985518, "token_latencies_s": [ 0.096126099, 0.093173871, 0.092844728, 0.092642769, 0.093070971, 0.092752509, 0.092687359, 0.093051435, 0.092973292, 0.092493251, 0.093543558, 0.092651087, 0.09274077, 0.092204888 ], "watts": { "core": 7.205676402613709, "dram": 3.2513799589290597, "package-0": 22.87020430260837, "psys": 40.62437323267318, "uncore": 9.163736538167724 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 2.562057756818831, "compile_target": "AUTO:GPU", "context_length": 8192, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 404334, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_3b_ov_fp16/openvino_model.xml", "key": "llama32_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "GPU", "key": "gpu_gpu", "label": "iGPU -> iGPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 8176, "prompt_sha256": "998ace73fce48442f97bcb84c31be293d0d94180b17f70a4d127bba1092ad744", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.04385981895029545, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 59.225496, "dram": 21.396856, "package-0": 266.189321, "psys": 440.650849, "uncore": 159.666278 }, "net_j": { "core": 22.4822087513465, "dram": 16.586314938351506, "package-0": 210.40138855894233, "psys": 341.71021252184516, "uncore": 159.64725174396509 } }, "first_decode": { "gpu_active_fraction": 0.8461538461538461, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 13, "gross_j": { "core": 2.197016, "dram": 0.348571, "package-0": 3.797109, "psys": 5.899887, "uncore": 0.878477 }, "logits": { "l2": 550.3755398990971, "sha256": "27bdcad5725470bbc36ac628654ced5364aca34207ec7bb0e1c27625c40cf33c", "sum": -138718.30373605376, "top1": 546 }, "net_j": { "core": 1.7118527744143335, "dram": 0.2850519794374833, "package-0": 3.060477759885503, "psys": 4.593461747126099, "uncore": 0.8782257747983395 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.128630568, "watts": { "core": 17.08004585659608, "dram": 2.7098613138363814, "package-0": 29.51949182094881, "psys": 45.8669124433937, "uncore": 6.829457520548304 } }, "metrics": { "all_tokens_per_s_e2e": 840.9218228464197, "decode_tokens_per_s": 9.820576869039325, "end_to_end_ms": 9741.690341999998, "handoff_estimate_ms": 29.180919499999998, "handoff_plus_first_decode_ms": 128.630568, "output_tokens_per_s_e2e": 1.6424254352469134, "prefill_tokens_per_s": 995.3391975985407, "steady_tpot_ms": 99.4496485, "steady_tpot_p95_ms": 101.72606005, "ttft_ms": 8214.28516 }, "prefill": { "gpu_active_fraction": 0.9446494464944649, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 813, "gross_j": { "core": 46.080693, "dram": 16.525409, "package-0": 229.107324, "psys": 377.018688, "uncore": 145.504876 }, "logits": { "l2": 774.371456062169, "sha256": "f744ae7d8c64e3ef134b46cd4f17547d69d31eb2f2cfcb5f70b3df06e0cd1022", "sum": -244942.2377996092, "top1": 27 }, "net_j": { "core": 15.098406704676524, "dram": 12.469115413669758, "package-0": 182.06641328844265, "psys": 293.59101031664153, "uncore": 145.48883288149477 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 8.21428516, "watts": { "core": 5.609823874193332, "dram": 2.0117890574911574, "package-0": 27.89132828205833, "psys": 45.897930331895125, "uncore": 17.713638273546376 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 467.6968416061623, "sha256": "26b6c86fb5c84fff5365d961a08fcfa0b1bf1e3e9f2d6b350bbbe31be223463e", "sum": -62206.398901291075, "top1": 42 }, "gpu_active_fraction": 0.9928057553956835, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 139, "gross_j": { "core": 10.947787, "dram": 4.522876, "package-0": 33.284888, "psys": 57.732274, "uncore": 13.282925 }, "net_j": { "core": 5.671949272255644, "dram": 3.8321475452442657, "package-0": 25.27449751061418, "psys": 43.52574045807751, "uncore": 13.280193087671979 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.398774614, "token_latencies_s": [ 0.103799705, 0.099957361, 0.100360113, 0.099441587, 0.099367391, 0.100609482, 0.09924028, 0.099167948, 0.100404216, 0.099230005, 0.099052791, 0.09945771, 0.099501575, 0.099136618 ], "watts": { "core": 7.826698376154545, "dram": 3.233455879690422, "package-0": 23.795747840187786, "psys": 41.273464232315554, "uncore": 9.496115290522424 } } }, { "energy": { "gross_j": { "core": 67.83863099999999, "dram": 21.560370000000002, "package-0": 275.498013, "psys": 444.63826600000004, "uncore": 159.881426 }, "net_j": { "core": 30.44214537873348, "dram": 16.664310253765876, "package-0": 218.7183188138557, "psys": 343.93872694202236, "uncore": 159.86206150750317 } }, "first_decode": { "gpu_active_fraction": 0.8461538461538461, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 13, "gross_j": { "core": 2.763603, "dram": 0.346374, "package-0": 4.34087, "psys": 6.394942, "uncore": 0.866575 }, "logits": { "l2": 550.3755398990971, "sha256": "27bdcad5725470bbc36ac628654ced5364aca34207ec7bb0e1c27625c40cf33c", "sum": -138718.30373605376, "top1": 546 }, "net_j": { "core": 2.2975947622579693, "dram": 0.28536280773639433, "package-0": 3.6333220904132912, "psys": 5.140096418704074, "uncore": 0.8663336935543994 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.123552036, "watts": { "core": 22.367927631722715, "dram": 2.803466549106483, "package-0": 35.13394145928926, "psys": 51.759098490291166, "uncore": 7.013846376436888 } }, "metrics": { "all_tokens_per_s_e2e": 826.2335772251334, "decode_tokens_per_s": 9.861020473541428, "end_to_end_ms": 9914.871806000001, "handoff_estimate_ms": 24.143331000000003, "handoff_plus_first_decode_ms": 123.552036, "output_tokens_per_s_e2e": 1.6137374555178388, "prefill_tokens_per_s": 974.0602763729432, "steady_tpot_ms": 99.408705, "steady_tpot_p95_ms": 101.59664469999998, "ttft_ms": 8393.731064000001 }, "prefill": { "gpu_active_fraction": 0.9314079422382672, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 831, "gross_j": { "core": 53.764266, "dram": 16.680866, "package-0": 237.476321, "psys": 380.822328, "uncore": 145.706231 }, "logits": { "l2": 774.371456062169, "sha256": "f744ae7d8c64e3ef134b46cd4f17547d69d31eb2f2cfcb5f70b3df06e0cd1022", "sum": -244942.2377996092, "top1": 27 }, "net_j": { "core": 22.10515338760097, "dram": 12.535960287926558, "package-0": 189.4077738498242, "psys": 295.5721234930767, "uncore": 145.68983741010902 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 8.393731064, "watts": { "core": 6.405288135879212, "dram": 1.9873005071061687, "package-0": 28.292105047124487, "psys": 45.36985103481748, "uncore": 17.35893488712328 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 467.6968416061623, "sha256": "26b6c86fb5c84fff5365d961a08fcfa0b1bf1e3e9f2d6b350bbbe31be223463e", "sum": -62206.398901291075, "top1": 42 }, "gpu_active_fraction": 0.9927536231884058, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 138, "gross_j": { "core": 11.310762, "dram": 4.53313, "package-0": 33.680822, "psys": 57.420996, "uncore": 13.30862 }, "net_j": { "core": 6.039397228874539, "dram": 3.8429871581029236, "package-0": 25.6772228736182, "psys": 43.226507030241606, "uncore": 13.305890403839753 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.397588706, "token_latencies_s": [ 0.10406896, 0.100265398, 0.099652442, 0.099870934, 0.09952613, 0.099416567, 0.099140365, 0.099119543, 0.099182989, 0.099400843, 0.099159644, 0.100129151, 0.099225483, 0.0994006 ], "watts": { "core": 8.093054810361354, "dram": 3.2435365143827943, "package-0": 24.099237390374274, "psys": 41.0857613212567, "uncore": 9.52255834843588 } } }, { "energy": { "gross_j": { "core": 63.324484, "dram": 21.633184, "package-0": 271.834753, "psys": 448.484143, "uncore": 160.576127 }, "net_j": { "core": 25.868534451965292, "dram": 16.72933905862344, "package-0": 214.96477376377908, "psys": 347.6244822108849, "uncore": 160.55673171614018 } }, "first_decode": { "gpu_active_fraction": 0.8461538461538461, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 13, "gross_j": { "core": 2.728081, "dram": 0.347045, "package-0": 4.302235, "psys": 6.456588, "uncore": 0.861021 }, "logits": { "l2": 550.3755398990971, "sha256": "27bdcad5725470bbc36ac628654ced5364aca34207ec7bb0e1c27625c40cf33c", "sum": -138718.30373605376, "top1": 546 }, "net_j": { "core": 2.2568610052231572, "dram": 0.2853514689176154, "package-0": 3.5867739946853643, "psys": 5.187708438709191, "uncore": 0.8607769948240691 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.124933821, "watts": { "core": 21.836208787690886, "dram": 2.7778306724485757, "package-0": 34.43611157942571, "psys": 51.68006508021555, "uncore": 6.8918167483246995 } }, "metrics": { "all_tokens_per_s_e2e": 824.9218739170508, "decode_tokens_per_s": 9.825767383491202, "end_to_end_ms": 9930.637384000001, "handoff_estimate_ms": 25.2402075, "handoff_plus_first_decode_ms": 124.933821, "output_tokens_per_s_e2e": 1.6111755349942398, "prefill_tokens_per_s": 972.865539972189, "steady_tpot_ms": 99.6936135, "steady_tpot_p95_ms": 102.18888539999999, "ttft_ms": 8404.039062 }, "prefill": { "gpu_active_fraction": 0.9326923076923077, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 832, "gross_j": { "core": 50.071894, "dram": 16.750445, "package-0": 234.59144, "psys": 384.07232, "uncore": 146.354117 }, "logits": { "l2": 774.371456062169, "sha256": "f744ae7d8c64e3ef134b46cd4f17547d69d31eb2f2cfcb5f70b3df06e0cd1022", "sum": -244942.2377996092, "top1": 27 }, "net_j": { "core": 18.373902125565756, "dram": 12.60044909865739, "package-0": 186.4638618320195, "psys": 298.71742320178726, "uncore": 146.3377032778114 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 8.404039062, "watts": { "core": 5.958074876925173, "dram": 1.993142211313534, "package-0": 27.914130130681677, "psys": 45.70092037489865, "uncore": 17.414735452832428 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 467.6968416061623, "sha256": "26b6c86fb5c84fff5365d961a08fcfa0b1bf1e3e9f2d6b350bbbe31be223463e", "sum": -62206.398901291075, "top1": 42 }, "gpu_active_fraction": 0.9927536231884058, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 138, "gross_j": { "core": 10.524509, "dram": 4.535694, "package-0": 32.941078, "psys": 57.955235, "uncore": 13.360989 }, "net_j": { "core": 5.237771321176379, "dram": 3.8435384910484363, "package-0": 24.91413793707423, "psys": 43.71935057038842, "uncore": 13.358251443504688 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.401664501, "token_latencies_s": [ 0.104194957, 0.101108693, 0.100045986, 0.099687146, 0.099022089, 0.099154508, 0.099700081, 0.100846795, 0.099643308, 0.099451213, 0.099772853, 0.099615578, 0.099607513, 0.09978594 ], "watts": { "core": 7.508579258796539, "dram": 3.23593413171559, "package-0": 23.501399925944188, "psys": 41.34743724953623, "uncore": 9.532230423519872 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 11.38440308533609, "compile_target": "NPU", "context_length": 8192, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 404406, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_3b_ov_fp16/openvino_model.xml", "key": "llama32_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "gpu_npu", "label": "iGPU -> NPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 8176, "prompt_sha256": "998ace73fce48442f97bcb84c31be293d0d94180b17f70a4d127bba1092ad744", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "GPU", "NPUW_ONLINE_PIPELINE": "NONE" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 8192, "NPUW_LLM_MIN_RESPONSE_LEN": 8192, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.044112238101661205, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 86.191186, "dram": 28.397510999999998, "package-0": 291.347886, "psys": 500.813536, "uncore": 133.409998 }, "net_j": { "core": 39.7499536501519, "dram": 22.31728596023605, "package-0": 220.83540530442116, "psys": 375.7587183287195, "uncore": 133.3904416045126 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 56, "gross_j": { "core": 8.534097, "dram": 0.848326, "package-0": 10.710055, "psys": 17.219194, "uncore": 0.0 }, "logits": { "l2": 809.4602388694859, "sha256": "02515d7fe1a284c67bb7f8751a7f64870233f7377c943f6424cceca3b74dcbf9", "sum": -186452.78511033393, "top1": 4194 }, "net_j": { "core": 6.381426452547956, "dram": 0.5664919335311953, "package-0": 7.441620190223687, "psys": 11.422581610603938, "uncore": 0.0 }, "npu_busy_delta_us": 149585, "npu_utilization": 0.26209224101856354, "seconds": 0.570734179, "watts": { "core": 14.95284024333857, "dram": 1.4863767253020954, "package-0": 18.765399715092233, "psys": 30.170251990462972, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 665.3189531609914, "decode_tokens_per_s": 6.522411948421407, "end_to_end_ms": 12312.891375, "handoff_estimate_ms": 447.624303, "handoff_plus_first_decode_ms": 570.734179, "output_tokens_per_s_e2e": 1.2994510803925614, "prefill_tokens_per_s": 816.5280105048917, "steady_tpot_ms": 123.10987600000001, "steady_tpot_p95_ms": 125.08208115, "ttft_ms": 10013.128631000001 }, "prefill": { "gpu_active_fraction": 0.9888438133874239, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 986, "gross_j": { "core": 68.037058, "dram": 22.254216, "package-0": 253.695578, "psys": 436.433208, "uncore": 133.409998 }, "logits": { "l2": 821.0241594431237, "sha256": "53efb3fb82e8f96410679387a5bb401865f2ce272df9e8e28dcbcf94fe2e1c8a", "sum": 229391.01477363706, "top1": 7998 }, "net_j": { "core": 30.26997152627665, "dram": 17.309636081736606, "package-0": 196.3531944937754, "psys": 334.7357319862653, "uncore": 133.3904416045126 }, "npu_busy_delta_us": 14349, "npu_utilization": 0.0014330186427023838, "seconds": 10.013128631, "watts": { "core": 6.794785177268337, "dram": 2.222503756828049, "package-0": 25.336294713579814, "psys": 43.58609821997401, "uncore": 13.323507858170448 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 798.1518897193432, "sha256": "7353c5ffa2362e60360074d60553031a1ea2eb55d106d91bd58ec13397d04511", "sum": -180498.95023651794, "top1": 4194 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 171, "gross_j": { "core": 9.620031, "dram": 5.294969, "package-0": 26.942253, "psys": 47.161134, "uncore": 0.0 }, "net_j": { "core": 3.098555671327297, "dram": 4.441157944968246, "package-0": 17.040590620422087, "psys": 29.60040473185029, "uncore": 0.0 }, "npu_busy_delta_us": 1718527, "npu_utilization": 0.9939263207025154, "seconds": 1.729028565, "token_latencies_s": [ 0.127413598, 0.123744848, 0.12322284, 0.123032849, 0.123826649, 0.123082879, 0.123060194, 0.122900025, 0.123136873, 0.123061888, 0.123009146, 0.122924988, 0.123158256, 0.123419675 ], "watts": { "core": 5.563835783129472, "dram": 3.0623953283270366, "package-0": 15.582306472767847, "psys": 27.276087251918824, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 88.253742, "dram": 28.280263, "package-0": 292.576887, "psys": 505.99620300000004, "uncore": 132.704189 }, "net_j": { "core": 42.37595633406719, "dram": 22.273806085300127, "package-0": 222.9198965963943, "psys": 382.45860868303794, "uncore": 132.6845922394805 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 38, "gross_j": { "core": 5.941391, "dram": 0.711058, "package-0": 7.685588, "psys": 12.708647, "uncore": 0.0 }, "logits": { "l2": 750.7986950958871, "sha256": "ba2622eda8d54300ed0e5f25cf093e0b4e612efaabeb35cd823240152f32e4dd", "sum": -158572.60981203616, "top1": 4194 }, "net_j": { "core": 4.444766939104243, "dram": 0.5151155017126701, "package-0": 5.413239273173071, "psys": 8.678606460854347, "uncore": 0.0 }, "npu_busy_delta_us": 150219, "npu_utilization": 0.37857839961375367, "seconds": 0.396797599, "watts": { "core": 14.973354211248642, "dram": 1.7919916899497166, "package-0": 19.369038571223815, "psys": 32.02803402043771, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 673.4900484408342, "decode_tokens_per_s": 7.043213831919347, "end_to_end_ms": 12163.505636, "handoff_estimate_ms": 273.27189699999997, "handoff_plus_first_decode_ms": 396.797599, "output_tokens_per_s_e2e": 1.3154102508610044, "prefill_tokens_per_s": 814.8461417421682, "steady_tpot_ms": 123.525702, "steady_tpot_p95_ms": 124.9987826, "ttft_ms": 10033.796052 }, "prefill": { "gpu_active_fraction": 0.9858156028368794, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 987, "gross_j": { "core": 74.010553, "dram": 22.264409, "package-0": 259.182075, "psys": 445.078634, "uncore": 132.704189 }, "logits": { "l2": 945.9286264695784, "sha256": "8ea9865ceac26fc4449fd72654c4156b870e7e63017d5e7597987edcf6438156", "sum": 266038.023974292, "top1": 7998 }, "net_j": { "core": 36.16551403961037, "dram": 17.309623309035103, "package-0": 201.72133496159248, "psys": 343.1712511101162, "uncore": 132.6845922394805 }, "npu_busy_delta_us": 11663, "npu_utilization": 0.0011623716427518235, "seconds": 10.033796052, "watts": { "core": 7.376126903162214, "dram": 2.218941752913357, "package-0": 25.830909224862925, "psys": 44.3579510380106, "uncore": 13.2257211839131 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 746.970184871683, "sha256": "7f28dff8e147b239d2d5b734afb59f3655057aa1de98d1b34e87ed48de2fcc0b", "sum": -156238.34350851923, "top1": 4194 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 172, "gross_j": { "core": 8.301798, "dram": 5.304796, "package-0": 25.709224, "psys": 48.208922, "uncore": 0.0 }, "net_j": { "core": 1.765675355352574, "dram": 4.4490672745523545, "package-0": 15.78532236162875, "psys": 30.60875111206738, "uncore": 0.0 }, "npu_busy_delta_us": 1717374, "npu_utilization": 0.9910335982816807, "seconds": 1.732911985, "token_latencies_s": [ 0.127564174, 0.123385, 0.123617418, 0.123434516, 0.123427281, 0.1235311, 0.123520304, 0.123541957, 0.123600584, 0.123533853, 0.123555496, 0.123261997, 0.123441636, 0.12345531 ], "watts": { "core": 4.790663387327198, "dram": 3.061203365155328, "package-0": 14.835850996783313, "psys": 27.81960215942531, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 84.735501, "dram": 28.418934, "package-0": 289.050957, "psys": 504.788808, "uncore": 132.352994 }, "net_j": { "core": 38.52525309884055, "dram": 22.368950135106967, "package-0": 218.88918376102873, "psys": 380.35597464425325, "uncore": 132.33324516043228 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 40, "gross_j": { "core": 5.955795, "dram": 0.723326, "package-0": 7.741252, "psys": 12.8309, "uncore": 0.0 }, "logits": { "l2": 719.7956473364802, "sha256": "de7c0de46ead8a45b790320cf84e4976d58ab67f8c2d4bc531c013b20fe8fa29", "sum": -141453.3275534436, "top1": 67692 }, "net_j": { "core": 4.411194505066401, "dram": 0.5211022834762981, "package-0": 5.396059870710108, "psys": 8.671670722581343, "uncore": 0.0 }, "npu_busy_delta_us": 150080, "npu_utilization": 0.3664800506359781, "seconds": 0.409517516, "watts": { "core": 14.543443851129435, "dram": 1.7662883069450928, "package-0": 18.903347714191547, "psys": 31.331748945263673, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 668.6445862960204, "decode_tokens_per_s": 7.009384025014176, "end_to_end_ms": 12251.650828999998, "handoff_estimate_ms": 286.1944635, "handoff_plus_first_decode_ms": 409.517516, "output_tokens_per_s_e2e": 1.3059464576094149, "prefill_tokens_per_s": 808.5712907438741, "steady_tpot_ms": 123.3230525, "steady_tpot_p95_ms": 124.93267545, "ttft_ms": 10111.662500999999 }, "prefill": { "gpu_active_fraction": 0.9819639278557114, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 998, "gross_j": { "core": 70.39545, "dram": 22.395695, "package-0": 255.50606, "psys": 443.372339, "uncore": 132.352994 }, "logits": { "l2": 979.6350729049458, "sha256": "df38e347e9e126abe4db0af89e15a3d83431e738275de280974d22de94ea3543", "sum": 273512.2029115502, "top1": 7998 }, "net_j": { "core": 32.25671772744415, "dram": 17.402458102202516, "package-0": 197.59940061456618, "psys": 340.6741122468968, "uncore": 132.33324516043228 }, "npu_busy_delta_us": 11673, "npu_utilization": 0.0011544095739791148, "seconds": 10.111662501, "watts": { "core": 6.9618077139182795, "dram": 2.2148380642436556, "package-0": 25.268452143723305, "psys": 43.84762040427599, "uncore": 13.089142758365487 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 716.9232841831466, "sha256": "5e3c25a5a0af9c851d52716cf0605f2af73f44412a6d337bfd2ac01c749cd5bb", "sum": -139669.67636439204, "top1": 67692 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 171, "gross_j": { "core": 8.384256, "dram": 5.299913, "package-0": 25.803645, "psys": 48.585569, "uncore": 0.0 }, "net_j": { "core": 1.8573408663300013, "dram": 4.445389749428153, "package-0": 15.89372327575246, "psys": 31.010191674775072, "uncore": 0.0 }, "npu_busy_delta_us": 1717137, "npu_utilization": 0.9922946911860423, "seconds": 1.730470812, "token_latencies_s": [ 0.127591557, 0.123425006, 0.123381172, 0.123413131, 0.123231025, 0.123300249, 0.12350097, 0.123387958, 0.123338739, 0.123296494, 0.123307366, 0.123056069, 0.12307591, 0.123127331 ], "watts": { "core": 4.8450721860543, "dram": 3.0627000254772283, "package-0": 14.911343676567022, "psys": 28.076503032054607, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 3.1997664403170347, "compile_target": "AUTO:NPU,GPU", "context_length": 8192, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 404249, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_3b_ov_fp16/openvino_model.xml", "key": "llama32_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": null, "key": "openvino_auto", "label": "OpenVINO AUTO", "mode": "stock", "prefill_device": null }, "prompt_length": 8176, "prompt_sha256": "998ace73fce48442f97bcb84c31be293d0d94180b17f70a4d127bba1092ad744", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.04428545292466879, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 60.950953999999996, "dram": 21.32807, "package-0": 267.101367, "psys": 432.939077, "uncore": 159.02138300000001 }, "net_j": { "core": 24.204821571830056, "dram": 16.5171564389396, "package-0": 211.3091146764677, "psys": 333.9907791531073, "uncore": 159.00235527068608 } }, "first_decode": { "gpu_active_fraction": 0.8461538461538461, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 13, "gross_j": { "core": 2.205866, "dram": 0.349792, "package-0": 3.802603, "psys": 5.842942, "uncore": 0.873167 }, "logits": { "l2": 550.3755398990971, "sha256": "27bdcad5725470bbc36ac628654ced5364aca34207ec7bb0e1c27625c40cf33c", "sum": -138718.30373605376, "top1": 546 }, "net_j": { "core": 1.7176740536662036, "dram": 0.2858764502585738, "package-0": 3.0613732037645978, "psys": 4.528361146988676, "uncore": 0.8729142064787708 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.129433568, "watts": { "core": 17.042456868684944, "dram": 2.7024828675046644, "package-0": 29.378800714201127, "psys": 45.14240077195431, "uncore": 6.746062968765568 } }, "metrics": { "all_tokens_per_s_e2e": 840.856711952097, "decode_tokens_per_s": 9.80120117157312, "end_to_end_ms": 9742.44468, "handoff_estimate_ms": 29.815155999999995, "handoff_plus_first_decode_ms": 129.433568, "output_tokens_per_s_e2e": 1.6422982655314395, "prefill_tokens_per_s": 995.6137446175929, "steady_tpot_ms": 99.618412, "steady_tpot_p95_ms": 101.63124915, "ttft_ms": 8212.020017 }, "prefill": { "gpu_active_fraction": 0.9446494464944649, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 813, "gross_j": { "core": 48.994564, "dram": 16.444782, "package-0": 231.186969, "psys": 370.090544, "uncore": 144.856624 }, "logits": { "l2": 774.371456062169, "sha256": "f744ae7d8c64e3ef134b46cd4f17547d69d31eb2f2cfcb5f70b3df06e0cd1022", "sum": -244942.2377996092, "top1": 27 }, "net_j": { "core": 18.02082127329617, "dram": 12.3896069632263, "package-0": 184.15903012805333, "psys": 286.6858720458596, "uncore": 144.84058530548992 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 8.212020017, "watts": { "core": 5.966201238985605, "dram": 2.0025258055821906, "package-0": 28.152265644921894, "psys": 45.066931550807496, "uncore": 17.639584864640742 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 467.6968416061623, "sha256": "26b6c86fb5c84fff5365d961a08fcfa0b1bf1e3e9f2d6b350bbbe31be223463e", "sum": -62206.398901291075, "top1": 42 }, "gpu_active_fraction": 0.9928057553956835, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 139, "gross_j": { "core": 9.750524, "dram": 4.533496, "package-0": 32.111795, "psys": 57.005591, "uncore": 13.291592 }, "net_j": { "core": 4.46632624486768, "dram": 3.841673025454729, "package-0": 24.088711344649766, "psys": 42.776545960259085, "uncore": 13.2888557587174 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.400991095, "token_latencies_s": [ 0.104067884, 0.100310782, 0.100319215, 0.09950612, 0.099805906, 0.100233306, 0.099607027, 0.099629797, 0.100236208, 0.099498137, 0.099591541, 0.09948711, 0.099322784, 0.099344216 ], "watts": { "core": 6.959733030994034, "dram": 3.235920639452744, "package-0": 22.92077024229765, "psys": 40.689474189698544, "uncore": 9.487278004432998 } } }, { "energy": { "gross_j": { "core": 62.875570999999994, "dram": 21.659246, "package-0": 270.850441, "psys": 446.837173, "uncore": 159.990069 }, "net_j": { "core": 25.37821603315127, "dram": 16.749980137353344, "package-0": 213.91759524092666, "psys": 345.86601760063456, "uncore": 159.97065227575857 } }, "first_decode": { "gpu_active_fraction": 0.7692307692307693, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 13, "gross_j": { "core": 2.696282, "dram": 0.345031, "package-0": 4.268483, "psys": 6.349837, "uncore": 0.849973 }, "logits": { "l2": 550.3755398990971, "sha256": "27bdcad5725470bbc36ac628654ced5364aca34207ec7bb0e1c27625c40cf33c", "sum": -138718.30373605376, "top1": 546 }, "net_j": { "core": 2.2263685078491418, "dram": 0.2835085201490122, "package-0": 3.5550056789360203, "psys": 5.08447552897968, "uncore": 0.8497296713518196 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.12458743, "watts": { "core": 21.641685682094895, "dram": 2.769388533016533, "package-0": 34.260944302326486, "psys": 50.966915362167754, "uncore": 6.822301415158816 } }, "metrics": { "all_tokens_per_s_e2e": 824.0109767162062, "decode_tokens_per_s": 9.885514147172476, "end_to_end_ms": 9941.615138, "handoff_estimate_ms": 25.80042099999999, "handoff_plus_first_decode_ms": 124.58743, "output_tokens_per_s_e2e": 1.6093964388988402, "prefill_tokens_per_s": 970.5322642211062, "steady_tpot_ms": 98.78700900000001, "steady_tpot_p95_ms": 102.62800239999999, "ttft_ms": 8424.243378 }, "prefill": { "gpu_active_fraction": 0.9303721488595438, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 833, "gross_j": { "core": 49.195308, "dram": 16.786639, "package-0": 233.241773, "psys": 382.295163, "uncore": 145.848076 }, "logits": { "l2": 774.371456062169, "sha256": "f744ae7d8c64e3ef134b46cd4f17547d69d31eb2f2cfcb5f70b3df06e0cd1022", "sum": -244942.2377996092, "top1": 27 }, "net_j": { "core": 17.421110358354095, "dram": 12.626666011691132, "package-0": 184.99849037247964, "psys": 296.7350628115679, "uncore": 145.83162281725828 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 8.424243378, "watts": { "core": 5.839730144605515, "dram": 1.9926583607304704, "package-0": 27.68696991935363, "psys": 45.380355937765025, "uncore": 17.31289914781947 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 467.6968416061623, "sha256": "26b6c86fb5c84fff5365d961a08fcfa0b1bf1e3e9f2d6b350bbbe31be223463e", "sum": -62206.398901291075, "top1": 42 }, "gpu_active_fraction": 0.9927536231884058, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 138, "gross_j": { "core": 10.983981, "dram": 4.527576, "package-0": 33.340185, "psys": 58.192173, "uncore": 13.29202 }, "net_j": { "core": 5.730737166948033, "dram": 3.8398056055132006, "package-0": 25.364099189511013, "psys": 44.04647926008698, "uncore": 13.289299787148465 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.39278433, "token_latencies_s": [ 0.108252234, 0.09959957, 0.098821045, 0.098817327, 0.098861228, 0.098751283, 0.098757316, 0.098673786, 0.098694255, 0.098816977, 0.098816702, 0.098500365, 0.098690247, 0.09869122 ], "watts": { "core": 7.886347342807913, "dram": 3.250737319826107, "package-0": 23.937794446610408, "psys": 41.7811801486882, "uncore": 9.543487612328322 } } }, { "energy": { "gross_j": { "core": 60.341703, "dram": 21.691596, "package-0": 267.946579, "psys": 441.157135, "uncore": 159.68068200000002 }, "net_j": { "core": 22.860696204017362, "dram": 16.78447048877868, "package-0": 211.03855493544845, "psys": 340.2300012043576, "uncore": 159.66127374110036 } }, "first_decode": { "gpu_active_fraction": 0.8461538461538461, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 13, "gross_j": { "core": 2.650994, "dram": 0.343933, "package-0": 4.21465, "psys": 6.277328, "uncore": 0.859007 }, "logits": { "l2": 550.3755398990971, "sha256": "27bdcad5725470bbc36ac628654ced5364aca34207ec7bb0e1c27625c40cf33c", "sum": -138718.30373605376, "top1": 546 }, "net_j": { "core": 2.186928635699177, "dram": 0.28317617453823396, "package-0": 3.510051986854133, "psys": 5.027714099080647, "uncore": 0.8587666996050363 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.123036925, "watts": { "core": 21.546328470091396, "dram": 2.795364074646696, "package-0": 34.255163642947025, "psys": 51.01987066077927, "uncore": 6.981700818676994 } }, "metrics": { "all_tokens_per_s_e2e": 824.3703873456008, "decode_tokens_per_s": 9.83424115114686, "end_to_end_ms": 9937.280772999999, "handoff_estimate_ms": 23.322992, "handoff_plus_first_decode_ms": 123.03692500000001, "output_tokens_per_s_e2e": 1.6100984127843765, "prefill_tokens_per_s": 971.9450880806507, "steady_tpot_ms": 99.71393300000001, "steady_tpot_p95_ms": 102.0876229, "ttft_ms": 8411.997859 }, "prefill": { "gpu_active_fraction": 0.9326923076923077, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 832, "gross_j": { "core": 46.987917, "dram": 16.807818, "package-0": 230.63802, "psys": 377.466868, "uncore": 145.514764 }, "logits": { "l2": 774.371456062169, "sha256": "f744ae7d8c64e3ef134b46cd4f17547d69d31eb2f2cfcb5f70b3df06e0cd1022", "sum": -244942.2377996092, "top1": 27 }, "net_j": { "core": 15.259906478487245, "dram": 12.653891967599463, "package-0": 182.46486403045114, "psys": 292.03113836753204, "uncore": 145.49833473368057 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 8.411997859, "watts": { "core": 5.585821321831129, "dram": 1.998076828088741, "package-0": 27.4177459226574, "psys": 44.87243985638299, "uncore": 17.298478487404 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 467.6968416061623, "sha256": "26b6c86fb5c84fff5365d961a08fcfa0b1bf1e3e9f2d6b350bbbe31be223463e", "sum": -62206.398901291075, "top1": 42 }, "gpu_active_fraction": 0.9928057553956835, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 139, "gross_j": { "core": 10.702792, "dram": 4.539845, "package-0": 33.093909, "psys": 57.412939, "uncore": 13.306911 }, "net_j": { "core": 5.413861089830939, "dram": 3.8474023466409815, "package-0": 25.063638918143177, "psys": 43.17114873774494, "uncore": 13.304172307814763 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.402245989, "token_latencies_s": [ 0.103922517, 0.101099603, 0.100158725, 0.100695298, 0.100046673, 0.099575185, 0.099419181, 0.09966279, 0.099392544, 0.099364709, 0.099648997, 0.099564595, 0.099765076, 0.09990528 ], "watts": { "core": 7.6326066067998575, "dram": 3.237552494792695, "package-0": 23.600644437286384, "psys": 40.94355730048731, "uncore": 9.48971229326868 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 0.9332987340167165, "compile_target": "AUTO:GPU", "context_length": 1024, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 392871, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 2.4565200143075967, "dram": 0.4907827289910671, "package-0": 4.295934585025856, "psys": 9.931950112327097, "uncore": 0.002123975024632864 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_0_5b_ov_fp16/openvino_model.xml", "key": "qwen25_0_5b_fp16", "size_b": 0.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "GPU", "key": "gpu_gpu", "label": "iGPU -> iGPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 1008, "prompt_sha256": "2669029615323cd09063f0eb8d822f5daea575e1396736877003e47194c143f8", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.038815573789179325, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 6.166489, "dram": 0.9864470000000001, "package-0": 11.843659, "psys": 18.605971, "uncore": 3.52825 }, "net_j": { "core": 5.104705244065135, "dram": 0.7743155594658269, "package-0": 9.986823504106466, "psys": 14.313075671403713, "uncore": 3.5273319524506084 } }, "first_decode": { "gpu_active_fraction": 0.75, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 4, "gross_j": { "core": 1.332578, "dram": 0.065063, "package-0": 1.628414, "psys": 2.174006, "uncore": 0.145324 }, "logits": { "l2": 712.6588257239783, "sha256": "990e8b5de951b6913e147a016df0e6896d435e87a38f418db97e9b4ce13bdab3", "sum": 24349.479562914465, "top1": 78 }, "net_j": { "core": 1.2409058243885105, "dram": 0.04674801772543606, "package-0": 1.46809872986166, "psys": 1.8033664491073382, "uncore": 0.14524473770605634 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.037317903, "watts": { "core": 35.708812469982576, "dram": 1.7434795304548596, "package-0": 43.63626755769208, "psys": 58.25638166217432, "uncore": 3.894216671285094 } }, "metrics": { "all_tokens_per_s_e2e": 2369.104330887212, "decode_tokens_per_s": 59.777697502971996, "end_to_end_ms": 432.230859, "handoff_estimate_ms": 22.103882, "handoff_plus_first_decode_ms": 37.317903, "output_tokens_per_s_e2e": 37.01725517011269, "prefill_tokens_per_s": 5559.81017087183, "steady_tpot_ms": 15.214021, "steady_tpot_p95_ms": 15.5608924, "ttft_ms": 181.301154 }, "prefill": { "gpu_active_fraction": 0.6111111111111112, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 18, "gross_j": { "core": 1.311703, "dram": 0.286376, "package-0": 3.858328, "psys": 6.459456, "uncore": 1.851985 }, "logits": { "l2": 838.9345161742121, "sha256": "e1f6e0509b78af0035fc75a4b5671053a3706868fabf97ca68685fc192833e98", "sum": 177352.29018324683, "top1": 291 }, "net_j": { "core": 0.8663330865819363, "dram": 0.1973965248706503, "package-0": 3.0794701022263014, "psys": 4.658781983164668, "uncore": 1.851599920876967 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.181301154, "watts": { "core": 7.234940159288783, "dram": 1.5795597197357059, "package-0": 21.281320691428142, "psys": 35.62832258640781, "uncore": 10.214965316767923 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 720.1119452755735, "sha256": "9d31e7729784e25aa0cb77989028372b1f68cbebfb778541584dec15714a8c90", "sum": 72014.2167466206, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 22, "gross_j": { "core": 3.522208, "dram": 0.635008, "package-0": 6.356917, "psys": 9.972509, "uncore": 1.530941 }, "net_j": { "core": 2.9974663330946885, "dram": 0.5301710168697406, "package-0": 5.439254672018505, "psys": 7.850927239131707, "uncore": 1.5304872938675853 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.213611802, "token_latencies_s": [ 0.015784664, 0.015339931, 0.015102557, 0.015052076, 0.01507876, 0.015395917, 0.01517187, 0.015128386, 0.015362293, 0.0154404, 0.015105483, 0.015256172, 0.015288043, 0.015078236 ], "watts": { "core": 16.48882677371918, "dram": 2.972719644020418, "package-0": 29.759203098712685, "psys": 46.685196728971, "uncore": 7.166930785968465 } } }, { "energy": { "gross_j": { "core": 6.070908, "dram": 1.028623, "package-0": 11.842437, "psys": 18.839551, "uncore": 3.545096 }, "net_j": { "core": 4.964701560495243, "dram": 0.8076164492511504, "package-0": 9.907915614475579, "psys": 14.367050427195585, "uncore": 3.5441395433719607 } }, "first_decode": { "gpu_active_fraction": 0.6666666666666666, "gpu_freq_mhz_after": 2100, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 1900, "gpu_sample_count": 3, "gross_j": { "core": 0.819212, "dram": 0.053833, "package-0": 1.002744, "psys": 1.362789, "uncore": 0.065002 }, "logits": { "l2": 712.6588257239783, "sha256": "990e8b5de951b6913e147a016df0e6896d435e87a38f418db97e9b4ce13bdab3", "sum": 24349.479562914465, "top1": 78 }, "net_j": { "core": 0.7597050110648914, "dram": 0.04194422960386476, "package-0": 0.8986788467229406, "psys": 1.122196437352027, "uncore": 0.06495054861448182 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.024224101, "watts": { "core": 33.818055827954154, "dram": 2.222290932489094, "package-0": 41.39447734303948, "psys": 56.257567618298815, "uncore": 2.683360674561256 } }, "metrics": { "all_tokens_per_s_e2e": 2273.966598655079, "decode_tokens_per_s": 61.96977063360149, "end_to_end_ms": 450.31444200000004, "handoff_estimate_ms": 8.991238500000001, "handoff_plus_first_decode_ms": 24.224101, "output_tokens_per_s_e2e": 35.53072810398561, "prefill_tokens_per_s": 4840.081870368971, "steady_tpot_ms": 15.2328625, "steady_tpot_p95_ms": 16.608904, "ttft_ms": 208.26094 }, "prefill": { "gpu_active_fraction": 0.5714285714285714, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 21, "gross_j": { "core": 1.810421, "dram": 0.334838, "package-0": 4.549976, "psys": 7.603801, "uncore": 1.94012 }, "logits": { "l2": 838.9345161742121, "sha256": "e1f6e0509b78af0035fc75a4b5671053a3706868fabf97ca68685fc192833e98", "sum": 177352.29018324683, "top1": 291 }, "net_j": { "core": 1.2988238326914865, "dram": 0.23262712752455514, "package-0": 3.6553006251440054, "psys": 5.535363733573653, "uncore": 1.9396776589648335 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.20826094, "watts": { "core": 8.693041527614348, "dram": 1.6077810846335372, "package-0": 21.847476535926514, "psys": 36.51093191070779, "uncore": 9.315813133274055 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 720.1119452755735, "sha256": "9d31e7729784e25aa0cb77989028372b1f68cbebfb778541584dec15714a8c90", "sum": 72014.2167466206, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 2100, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 22, "gross_j": { "core": 3.441275, "dram": 0.639952, "package-0": 6.289717, "psys": 9.872961, "uncore": 1.539974 }, "net_j": { "core": 2.906172716738865, "dram": 0.5330450921227304, "package-0": 5.353936142608633, "psys": 7.7094902562699055, "uncore": 1.5395113357926453 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.217829401, "token_latencies_s": [ 0.017152317, 0.016282604, 0.016316297, 0.016097107, 0.016078418, 0.015614612, 0.015297622, 0.015066855, 0.014909969, 0.014972133, 0.015168103, 0.014985535, 0.014934098, 0.014917102 ], "watts": { "core": 15.798028109162363, "dram": 2.9378586961270665, "package-0": 28.874509001656758, "psys": 45.324281087289954, "uncore": 7.06963335954819 } } }, { "energy": { "gross_j": { "core": 6.166732, "dram": 1.03192, "package-0": 11.956938999999998, "psys": 18.926952999999997, "uncore": 3.551992 }, "net_j": { "core": 5.054146837800091, "dram": 0.8096390582779157, "package-0": 10.01126257593537, "psys": 14.428662628936411, "uncore": 3.551030028152213 } }, "first_decode": { "gpu_active_fraction": 0.6666666666666666, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2100, "gpu_sample_count": 3, "gross_j": { "core": 0.806272, "dram": 0.052979, "package-0": 0.982724, "psys": 1.34924, "uncore": 0.065185 }, "logits": { "l2": 712.6588257239783, "sha256": "990e8b5de951b6913e147a016df0e6896d435e87a38f418db97e9b4ce13bdab3", "sum": 24349.479562914465, "top1": 78 }, "net_j": { "core": 0.749675326477962, "dram": 0.041671675117165075, "package-0": 0.8837483735997929, "psys": 1.1204141265405545, "uncore": 0.06513606495353673 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.02303937, "watts": { "core": 34.99540134995011, "dram": 2.299498640804848, "package-0": 42.65411771242009, "psys": 58.56236520356242, "uncore": 2.8292874327726847 } }, "metrics": { "all_tokens_per_s_e2e": 2260.9293922967113, "decode_tokens_per_s": 61.171514515309156, "end_to_end_ms": 452.911092, "handoff_estimate_ms": 7.680473, "handoff_plus_first_decode_ms": 23.03937, "output_tokens_per_s_e2e": 35.327021754636114, "prefill_tokens_per_s": 4853.1788458658275, "steady_tpot_ms": 15.358897, "steady_tpot_p95_ms": 17.65771065, "ttft_ms": 207.69891900000002 }, "prefill": { "gpu_active_fraction": 0.5714285714285714, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 21, "gross_j": { "core": 1.851802, "dram": 0.336242, "package-0": 4.592457, "psys": 7.615214, "uncore": 1.932734 }, "logits": { "l2": 838.9345161742121, "sha256": "e1f6e0509b78af0035fc75a4b5671053a3706868fabf97ca68685fc192833e98", "sum": 177352.29018324683, "top1": 291 }, "net_j": { "core": 1.3415854485264476, "dram": 0.23430695772468538, "package-0": 3.7001960305954156, "psys": 5.552358698107733, "uncore": 1.9322928526834007 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.207698919, "watts": { "core": 8.91579989398019, "dram": 1.6188914300512078, "package-0": 22.11112615371869, "psys": 36.66467806700525, "uncore": 9.305460082823059 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 720.1119452755735, "sha256": "9d31e7729784e25aa0cb77989028372b1f68cbebfb778541584dec15714a8c90", "sum": 72014.2167466206, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 22, "gross_j": { "core": 3.508658, "dram": 0.642699, "package-0": 6.381758, "psys": 9.962499, "uncore": 1.554073 }, "net_j": { "core": 2.9628860627956812, "dram": 0.5336604254360653, "package-0": 5.427318171740163, "psys": 7.755889804288123, "uncore": 1.5536011105152754 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.222172803, "token_latencies_s": [ 0.018899561, 0.016467577, 0.016190065, 0.016147038, 0.016989022, 0.015474793, 0.015108714, 0.01518469, 0.015155258, 0.015127353, 0.015147554, 0.015243001, 0.015802612, 0.015202327 ], "watts": { "core": 15.792473032804109, "dram": 2.892788817180292, "package-0": 28.7242988962965, "psys": 44.84121758143367, "uncore": 6.9948840677857405 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 2.1501153903082013, "compile_target": "NPU", "context_length": 1024, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 392939, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 2.4565200143075967, "dram": 0.4907827289910671, "package-0": 4.295934585025856, "psys": 9.931950112327097, "uncore": 0.002123975024632864 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_0_5b_ov_fp16/openvino_model.xml", "key": "qwen25_0_5b_fp16", "size_b": 0.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "gpu_npu", "label": "iGPU -> NPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 1008, "prompt_sha256": "2669029615323cd09063f0eb8d822f5daea575e1396736877003e47194c143f8", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "GPU", "NPUW_ONLINE_PIPELINE": "NONE" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 1024, "NPUW_LLM_MIN_RESPONSE_LEN": 1024, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.039530511014163494, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 7.110149, "dram": 1.022093, "package-0": 11.441499, "psys": 18.053055, "uncore": 1.5514480000000002 }, "net_j": { "core": 5.822267380230705, "dram": 0.7647899646954137, "package-0": 9.189266101606341, "psys": 12.846023946464287, "uncore": 1.5508980303238797 } }, "first_decode": { "gpu_active_fraction": 0.07142857142857142, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 14, "gross_j": { "core": 3.039298, "dram": 0.125183, "package-0": 3.423453, "psys": 4.859484, "uncore": 0.006652 }, "logits": { "l2": 714.5644081016375, "sha256": "f7f64affdcb433a14d202661deba5ad255d97611c0d28dc97208fa573f8d89b8", "sum": 24380.198635727167, "top1": 78 }, "net_j": { "core": 2.6901048271610217, "dram": 0.05541846426945424, "package-0": 2.8127879049400257, "psys": 3.447661929753406, "uncore": 0.0063500779421447745 }, "npu_busy_delta_us": 27100, "npu_utilization": 0.19064431256344683, "seconds": 0.142149533, "watts": { "core": 21.380991803891472, "dram": 0.8806430619789655, "package-0": 24.083462870046855, "psys": 34.18571906247487, "uncore": 0.04679579214657005 } }, "metrics": { "all_tokens_per_s_e2e": 1953.1892186656023, "decode_tokens_per_s": 36.8110678885641, "end_to_end_ms": 524.270762, "handoff_estimate_ms": 124.520972, "handoff_plus_first_decode_ms": 142.149533, "output_tokens_per_s_e2e": 30.518581541650036, "prefill_tokens_per_s": 8631.273461939822, "steady_tpot_ms": 17.628561, "steady_tpot_p95_ms": 23.970691000000002, "ttft_ms": 116.784621 }, "prefill": { "gpu_active_fraction": 0.9166666666666666, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 12, "gross_j": { "core": 0.723204, "dram": 0.216858, "package-0": 2.811882, "psys": 4.684071, "uncore": 1.544796 }, "logits": { "l2": 840.9680755108035, "sha256": "8fb49fad0852bc3258a6ca67a848f5fb785395020a78359d88430d551db8ad9a", "sum": 178291.6503057778, "top1": 291 }, "net_j": { "core": 0.4363202411501727, "dram": 0.1595421250014325, "package-0": 2.310182907646963, "psys": 3.524171970340973, "uncore": 1.544547952381735 }, "npu_busy_delta_us": 5719, "npu_utilization": 0.04897048901670024, "seconds": 0.116784621, "watts": { "core": 6.192630449175324, "dram": 1.8569054567553034, "package-0": 24.07750246498638, "psys": 40.108628686648736, "uncore": 13.227734840189274 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 723.4091300558687, "sha256": "1c462e5b54f0c0f9e4e380bd8290810c4daf6d05e264abf4482ad7bacb911518", "sum": 73007.79722458497, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 27, "gross_j": { "core": 3.347647, "dram": 0.680052, "package-0": 5.206164, "psys": 8.5095, "uncore": 0.0 }, "net_j": { "core": 2.6958423119195105, "dram": 0.5498293754245269, "package-0": 4.066295289019352, "psys": 5.874190046369908, "uncore": 0.0 }, "npu_busy_delta_us": 248723, "npu_utilization": 0.9373866722529295, "seconds": 0.265336608, "token_latencies_s": [ 0.026586083, 0.022562403, 0.022048623, 0.019552663, 0.018827059, 0.01810178, 0.017711709, 0.017545413, 0.017389837, 0.017190402, 0.017081727, 0.017024312, 0.016983586, 0.016690714 ], "watts": { "core": 12.616604339797696, "dram": 2.5629784187186115, "package-0": 19.62097894912413, "psys": 32.07058409369581, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 4.605396, "dram": 0.955686, "package-0": 8.691201999999999, "psys": 14.287378, "uncore": 1.554684 }, "net_j": { "core": 3.609645936461058, "dram": 0.7567472855867913, "package-0": 6.949845483023134, "psys": 10.261463332985626, "uncore": 1.5543844131486093 } }, "first_decode": { "gpu_active_fraction": 0.3333333333333333, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 3, "gross_j": { "core": 0.82379, "dram": 0.055908, "package-0": 0.979185, "psys": 1.360592, "uncore": 0.016113 }, "logits": { "l2": 714.5644081016375, "sha256": "f7f64affdcb433a14d202661deba5ad255d97611c0d28dc97208fa573f8d89b8", "sum": 24380.198635727167, "top1": 78 }, "net_j": { "core": 0.7620824579795547, "dram": 0.04357958594445183, "package-0": 0.8712715442257398, "psys": 1.1111023865094543, "uncore": 0.016059645955530773 }, "npu_busy_delta_us": 21826, "npu_utilization": 0.868872816462421, "seconds": 0.025119902, "watts": { "core": 32.79431583769714, "dram": 2.2256456255283164, "package-0": 38.980446659385855, "psys": 54.16390557574628, "uncore": 0.6414435852496558 } }, "metrics": { "all_tokens_per_s_e2e": 2526.212738275766, "decode_tokens_per_s": 51.827849389430625, "end_to_end_ms": 405.34986799999996, "handoff_estimate_ms": 7.256595499999998, "handoff_plus_first_decode_ms": 25.119902, "output_tokens_per_s_e2e": 39.47207403555884, "prefill_tokens_per_s": 8694.888595877277, "steady_tpot_ms": 17.863306500000004, "steady_tpot_p95_ms": 23.0763647, "ttft_ms": 115.93018 }, "prefill": { "gpu_active_fraction": 0.9166666666666666, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 12, "gross_j": { "core": 0.625914, "dram": 0.218139, "package-0": 2.71148, "psys": 4.606617, "uncore": 1.538571 }, "logits": { "l2": 840.9680755108035, "sha256": "8fb49fad0852bc3258a6ca67a848f5fb785395020a78359d88430d551db8ad9a", "sum": 178291.6503057778, "top1": 291 }, "net_j": { "core": 0.34112919256771773, "dram": 0.1612424698871744, "package-0": 2.2134515302897273, "psys": 3.4552042357268995, "uncore": 1.5383247671930786 }, "npu_busy_delta_us": 5583, "npu_utilization": 0.048158296657522665, "seconds": 0.11593018, "watts": { "core": 5.399060020436439, "dram": 1.8816411740238823, "package-0": 23.388905287648136, "psys": 39.73613255840714, "uncore": 13.271531192309025 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 723.4091300558687, "sha256": "1c462e5b54f0c0f9e4e380bd8290810c4daf6d05e264abf4482ad7bacb911518", "sum": 73007.79722458497, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 26, "gross_j": { "core": 3.155692, "dram": 0.681639, "package-0": 5.000537, "psys": 8.320169, "uncore": 0.0 }, "net_j": { "core": 2.5064342859137856, "dram": 0.551925229755165, "package-0": 3.865122408507667, "psys": 5.695156710749272, "uncore": 0.0 }, "npu_busy_delta_us": 251347, "npu_utilization": 0.9509920677726164, "seconds": 0.264299786, "token_latencies_s": [ 0.023258002, 0.02297856, 0.022379059, 0.020607237, 0.019030997, 0.018320396, 0.018006654, 0.017719959, 0.017524268, 0.017384537, 0.016986401, 0.016817675, 0.016751431, 0.016496031 ], "watts": { "core": 11.939820488541752, "dram": 2.5790372754974538, "package-0": 18.919943431206562, "psys": 31.4800444068464, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 4.676867, "dram": 0.952085, "package-0": 8.757728, "psys": 14.378259, "uncore": 1.553341 }, "net_j": { "core": 3.679534780354562, "dram": 0.7528301900992262, "package-0": 7.013604626210564, "psys": 10.34594752146848, "uncore": 1.5530400064675625 } }, "first_decode": { "gpu_active_fraction": 0.3333333333333333, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 3, "gross_j": { "core": 0.861509, "dram": 0.054931, "package-0": 1.030881, "psys": 1.423092, "uncore": 0.022461 }, "logits": { "l2": 714.5644081016375, "sha256": "f7f64affdcb433a14d202661deba5ad255d97611c0d28dc97208fa573f8d89b8", "sum": 24380.198635727167, "top1": 78 }, "net_j": { "core": 0.7961867649306213, "dram": 0.04188041437300608, "package-0": 0.9166462096578464, "psys": 1.1589878298096186, "uncore": 0.022404520592125252 }, "npu_busy_delta_us": 21625, "npu_utilization": 0.8132337366132795, "seconds": 0.026591371, "watts": { "core": 32.39806627495814, "dram": 2.0657453126429624, "package-0": 38.76750093103511, "psys": 53.517060101940594, "uncore": 0.844672506731601 } }, "metrics": { "all_tokens_per_s_e2e": 2522.2051840912736, "decode_tokens_per_s": 51.568909467527156, "end_to_end_ms": 405.99393200000003, "handoff_estimate_ms": 8.573653999999996, "handoff_plus_first_decode_ms": 26.591371, "output_tokens_per_s_e2e": 39.40945600142615, "prefill_tokens_per_s": 8756.004703850813, "steady_tpot_ms": 18.017717, "steady_tpot_p95_ms": 23.08701755, "ttft_ms": 115.120998 }, "prefill": { "gpu_active_fraction": 0.9166666666666666, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 12, "gross_j": { "core": 0.494505, "dram": 0.215698, "package-0": 2.565728, "psys": 4.437183, "uncore": 1.53088 }, "logits": { "l2": 840.9680755108035, "sha256": "8fb49fad0852bc3258a6ca67a848f5fb785395020a78359d88430d551db8ad9a", "sum": 178291.6503057778, "top1": 291 }, "net_j": { "core": 0.21170796434593514, "dram": 0.15919860243738482, "package-0": 2.0711757232291075, "psys": 3.2938069909826924, "uncore": 1.5306354858754372 }, "npu_busy_delta_us": 4145, "npu_utilization": 0.03600559473954526, "seconds": 0.115120998, "watts": { "core": 4.295523914759668, "dram": 1.8736633954476316, "package-0": 22.287228607938232, "psys": 38.54364605143538, "uncore": 13.29800841372136 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 723.4091300558687, "sha256": "1c462e5b54f0c0f9e4e380bd8290810c4daf6d05e264abf4482ad7bacb911518", "sum": 73007.79722458497, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 26, "gross_j": { "core": 3.320853, "dram": 0.681456, "package-0": 5.161119, "psys": 8.517984, "uncore": 0.0 }, "net_j": { "core": 2.671640051078006, "dram": 0.5517511732888354, "package-0": 4.0257826933236105, "psys": 5.893152700676169, "uncore": 0.0 }, "npu_busy_delta_us": 254059, "npu_utilization": 0.9613194243141356, "seconds": 0.264281563, "token_latencies_s": [ 0.022965451, 0.023312784, 0.021587818, 0.020058527, 0.019119158, 0.01823842, 0.017797014, 0.017611399, 0.01730654, 0.019252841, 0.01700627, 0.016841375, 0.01667363, 0.016487893 ], "watts": { "core": 12.565587104538201, "dram": 2.578522664481139, "package-0": 19.528865129346915, "psys": 32.23071599587899, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 3.8849987629801035, "compile_target": "NPU", "context_length": 1024, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 393081, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 2.4565200143075967, "dram": 0.4907827289910671, "package-0": 4.295934585025856, "psys": 9.931950112327097, "uncore": 0.002123975024632864 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_0_5b_ov_fp16/openvino_model.xml", "key": "qwen25_0_5b_fp16", "size_b": 0.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU+GPU", "key": "npu_gpu_combined", "label": "NPU + iGPU (subgraph combination)", "mode": "combined", "prefill_device": "NPU+GPU" }, "prompt_length": 1008, "prompt_sha256": "2669029615323cd09063f0eb8d822f5daea575e1396736877003e47194c143f8", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU,GPU", "NPUW_SUBMODEL_DEVICE": "0:GPU,last:NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "NPU,GPU", "NPUW_SUBMODEL_DEVICE": "0:GPU,last:NPU" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 1024, "NPUW_LLM_MIN_RESPONSE_LEN": 1024, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.03892374038696289, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 24.098327, "dram": 2.623834, "package-0": 38.396387, "psys": 63.814412, "uncore": 6.060897 }, "net_j": { "core": 19.08077772474327, "dram": 1.621388869565455, "package-0": 29.621753234305892, "psys": 43.52797069370386, "uncore": 6.056558688370768 } }, "first_decode": { "gpu_active_fraction": 0.17391304347826086, "gpu_freq_mhz_after": 2050, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 69, "gross_j": { "core": 10.890964, "dram": 0.583129, "package-0": 12.970182, "psys": 19.720348, "uncore": 0.358275 }, "logits": { "l2": 958.485918292106, "sha256": "0de2e58c6569eebaed9a2446c17498b4a5555fdcc3ad76d584b7a237d8eacab0", "sum": 107240.04813884199, "top1": 96630 }, "net_j": { "core": 9.169017531592287, "dram": 0.23910510358575832, "package-0": 9.958861374028679, "psys": 12.758350409622643, "uncore": 0.3567861575434554 }, "npu_busy_delta_us": 5730, "npu_utilization": 0.008174388658550167, "seconds": 0.700969851, "watts": { "core": 15.536993473346973, "dram": 0.8318888453877312, "package-0": 18.503195225153842, "psys": 28.13294747536867, "uncore": 0.5111132803912847 } }, "metrics": { "all_tokens_per_s_e2e": 501.33568334957124, "decode_tokens_per_s": 8.23159884768899, "end_to_end_ms": 2042.5436170000003, "handoff_estimate_ms": 622.4121865, "handoff_plus_first_decode_ms": 700.969851, "output_tokens_per_s_e2e": 7.833370052337051, "prefill_tokens_per_s": 4575.63311135657, "steady_tpot_ms": 78.5576645, "steady_tpot_p95_ms": 86.10422714999999, "ttft_ms": 220.29738300000002 }, "prefill": { "gpu_active_fraction": 0.13636363636363635, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 800, "gpu_sample_count": 22, "gross_j": { "core": 0.807554, "dram": 0.256103, "package-0": 2.566644, "psys": 4.804187, "uncore": 0.042908 }, "logits": { "l2": 839.9427313198568, "sha256": "a824a46e6afb71eec854c6d4700ff61ce067e09421e5019d2f0eb31fb7898035", "sum": 177091.2838851288, "top1": 291 }, "net_j": { "core": 0.2663890695609139, "dram": 0.1479848491816697, "package-0": 1.620260853379613, "psys": 2.616204382167784, "uncore": 0.04244009386051602 }, "npu_busy_delta_us": 180194, "npu_utilization": 0.8179579691148668, "seconds": 0.220297383, "watts": { "core": 3.6657448627067892, "dram": 1.1625331019025316, "package-0": 11.650814753437176, "psys": 21.80773522852062, "uncore": 0.19477308089492829 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 1318.474203054553, "sha256": "3478025624cc65876a42b2a858ed37b3779732cb44a02c80b3e2714ea2d2dced", "sum": 371578.8326460272, "top1": 81826 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 2050, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 111, "gross_j": { "core": 12.399809, "dram": 1.784602, "package-0": 22.859561, "psys": 39.289877, "uncore": 5.659714 }, "net_j": { "core": 9.645371123590069, "dram": 1.234298916798027, "package-0": 18.0426310068976, "psys": 28.153415901913426, "uncore": 5.657332436966797 }, "npu_busy_delta_us": 73315, "npu_utilization": 0.06538530652348538, "seconds": 1.121276383, "token_latencies_s": [ 0.097667681, 0.079877752, 0.07968476, 0.079762935, 0.078480599, 0.078007595, 0.078859713, 0.078330556, 0.078517422, 0.078050959, 0.078337537, 0.078597907, 0.078286445, 0.078760557 ], "watts": { "core": 11.058655286062507, "dram": 1.5915808332868453, "package-0": 20.38708863093926, "psys": 35.040314409261924, "uncore": 5.047563728094681 } } }, { "energy": { "gross_j": { "core": 15.588095, "dram": 2.195795, "package-0": 28.654346, "psys": 48.830381, "uncore": 6.131087 }, "net_j": { "core": 12.018391142543287, "dram": 1.4826117201371836, "package-0": 22.41168802439587, "psys": 34.397719991115494, "uncore": 6.128000535491503 } }, "first_decode": { "gpu_active_fraction": 0.9, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 10, "gross_j": { "core": 2.668267, "dram": 0.144348, "package-0": 3.418204, "psys": 4.933154, "uncore": 0.355284 }, "logits": { "l2": 1318.5428560315058, "sha256": "ecc3520c68a3ff34e89690bcb65dd6268467d08c74ede766137410ed2d52fabb", "sum": 371439.9113267623, "top1": 81826 }, "net_j": { "core": 2.4174995634001264, "dram": 0.09424772556308686, "package-0": 2.9796647259809674, "psys": 3.9192767852409314, "uncore": 0.3550671795510612 }, "npu_busy_delta_us": 4350, "npu_utilization": 0.04261263825609257, "seconds": 0.102082391, "watts": { "core": 26.138366998084912, "dram": 1.4140342774690693, "package-0": 33.484756445408884, "psys": 48.325219968642784, "uncore": 3.4803651885465734 } }, "metrics": { "all_tokens_per_s_e2e": 704.6737194729557, "decode_tokens_per_s": 12.094934056625442, "end_to_end_ms": 1453.154803, "handoff_estimate_ms": 21.9319895, "handoff_plus_first_decode_ms": 102.082391, "output_tokens_per_s_e2e": 11.010526866764932, "prefill_tokens_per_s": 4733.146603186907, "steady_tpot_ms": 80.1504015, "steady_tpot_p95_ms": 86.93882674999999, "ttft_ms": 212.966148 }, "prefill": { "gpu_active_fraction": 0.09090909090909091, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 1750, "gpu_sample_count": 22, "gross_j": { "core": 0.678465, "dram": 0.248718, "package-0": 2.406855, "psys": 4.640552, "uncore": 0.058045 }, "logits": { "l2": 839.9427313198568, "sha256": "a824a46e6afb71eec854c6d4700ff61ce067e09421e5019d2f0eb31fb7898035", "sum": 177091.2838851288, "top1": 291 }, "net_j": { "core": 0.15530939506800623, "dram": 0.14419789270184452, "package-0": 1.4919663593670651, "psys": 2.52538284244953, "uncore": 0.057592665220555735 }, "npu_busy_delta_us": 184692, "npu_utilization": 0.8672364210672581, "seconds": 0.212966148, "watts": { "core": 3.1857880060825443, "dram": 1.1678757508446835, "package-0": 11.30158488850538, "psys": 21.790092198127187, "uncore": 0.27255505414879366 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 1316.4160478001859, "sha256": "ca8d3e330d15bc8bb96b83f7f7b690e76f28ebabb35d42beaeffef5da51f3929", "sum": 369103.7374538556, "top1": 89372 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 113, "gross_j": { "core": 12.241363, "dram": 1.802729, "package-0": 22.829287, "psys": 39.256675, "uncore": 5.717758 }, "net_j": { "core": 9.445582184075155, "dram": 1.2441661018722523, "package-0": 17.940056939047835, "psys": 27.95306036342503, "uncore": 5.715340690719886 }, "npu_busy_delta_us": 62586, "npu_utilization": 0.05499135008714793, "seconds": 1.138106264, "token_latencies_s": [ 0.097083055, 0.08147655, 0.079121465, 0.079371177, 0.078459493, 0.080402221, 0.079898582, 0.079365039, 0.080583146, 0.081307318, 0.080788845, 0.079793174, 0.079416239, 0.080987266 ], "watts": { "core": 10.755905126975033, "dram": 1.583972478689389, "package-0": 20.05901181824969, "psys": 34.4929785923751, "uncore": 5.023922792502968 } } }, { "energy": { "gross_j": { "core": 15.326438, "dram": 2.207942, "package-0": 28.527821, "psys": 48.314634999999996, "uncore": 6.230819 }, "net_j": { "core": 11.694591133219905, "dram": 1.482343305027267, "package-0": 22.176488028297715, "psys": 33.63072374082853, "uncore": 6.2276788049299805 } }, "first_decode": { "gpu_active_fraction": 0.9230769230769231, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 13, "gross_j": { "core": 2.770745, "dram": 0.163818, "package-0": 3.732412, "psys": 5.484788, "uncore": 0.495909 }, "logits": { "l2": 1315.6363375493609, "sha256": "4a59eb917d5b1c11e3987b53e9dab89eeae07096734a9e6ce2de62e2cd3c22a2", "sum": 368620.2270870283, "top1": 89372 }, "net_j": { "core": 2.4429652531692607, "dram": 0.09833160553928387, "package-0": 3.159194463615266, "psys": 4.159542500515758, "uncore": 0.4956255925814591 }, "npu_busy_delta_us": 4474, "npu_utilization": 0.03353004769293299, "seconds": 0.133432557, "watts": { "core": 20.76513455408038, "dram": 1.2277213573895611, "package-0": 27.972273663315917, "psys": 41.105320345468606, "uncore": 3.7165517258280523 } }, "metrics": { "all_tokens_per_s_e2e": 692.6163428473894, "decode_tokens_per_s": 11.788574398479327, "end_to_end_ms": 1478.451975, "handoff_estimate_ms": 53.18247600000001, "handoff_plus_first_decode_ms": 133.432557, "output_tokens_per_s_e2e": 10.82213035699046, "prefill_tokens_per_s": 4892.408558360183, "steady_tpot_ms": 80.250081, "steady_tpot_p95_ms": 86.65143395, "ttft_ms": 206.03348799999998 }, "prefill": { "gpu_active_fraction": 0.047619047619047616, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2400, "gpu_sample_count": 21, "gross_j": { "core": 0.540587, "dram": 0.243347, "package-0": 2.286555, "psys": 4.354787, "uncore": 0.078552 }, "logits": { "l2": 839.9427313198568, "sha256": "a824a46e6afb71eec854c6d4700ff61ce067e09421e5019d2f0eb31fb7898035", "sum": 177091.2838851288, "top1": 291 }, "net_j": { "core": 0.03446161311039597, "dram": 0.14222932249581172, "package-0": 1.40144861322729, "psys": 2.3084726757152563, "uncore": 0.07811439001725 }, "npu_busy_delta_us": 183843, "npu_utilization": 0.8922966930502095, "seconds": 0.206033488, "watts": { "core": 2.623782207676842, "dram": 1.1811041125508686, "package-0": 11.097977431707607, "psys": 21.136306734757603, "uncore": 0.3812584097979257 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 1313.8573886044735, "sha256": "cde0be62f1d20f004a2b7e7db6ca788a3e41569493213ae305108322f77e0c23", "sum": 367252.4857843369, "top1": 89372 }, "gpu_active_fraction": 0.9910714285714286, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 112, "gross_j": { "core": 12.015106, "dram": 1.800777, "package-0": 22.508854, "psys": 38.47506, "uncore": 5.656358 }, "net_j": { "core": 9.217164266940248, "dram": 1.2417823769921714, "package-0": 17.61584495145516, "psys": 27.162708564597516, "uncore": 5.653938822331272 }, "npu_busy_delta_us": 74318, "npu_utilization": 0.06524926958491928, "seconds": 1.13898593, "token_latencies_s": [ 0.096948186, 0.080219025, 0.080689727, 0.080281137, 0.080410467, 0.079389672, 0.079738538, 0.079822903, 0.079175359, 0.079584611, 0.079686347, 0.081107029, 0.080941774, 0.080919268 ], "watts": { "core": 10.548950328122139, "dram": 1.5810353337727359, "package-0": 19.762187931505, "psys": 33.780101216878066, "uncore": 4.966135095277252 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 1.9877989795058966, "compile_target": "AUTO:NPU,GPU", "context_length": 1024, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 392786, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 2.4565200143075967, "dram": 0.4907827289910671, "package-0": 4.295934585025856, "psys": 9.931950112327097, "uncore": 0.002123975024632864 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_0_5b_ov_fp16/openvino_model.xml", "key": "qwen25_0_5b_fp16", "size_b": 0.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": null, "key": "openvino_auto", "label": "OpenVINO AUTO", "mode": "stock", "prefill_device": null }, "prompt_length": 1008, "prompt_sha256": "2669029615323cd09063f0eb8d822f5daea575e1396736877003e47194c143f8", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.04277300927788019, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 5.998336, "dram": 0.998837, "package-0": 11.695039999999999, "psys": 18.551467, "uncore": 3.541006 }, "net_j": { "core": 4.895510088230332, "dram": 0.7785058374497378, "package-0": 9.766430443375091, "psys": 14.092634230682913, "uncore": 3.5400524662695707 } }, "first_decode": { "gpu_active_fraction": 0.8, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 5, "gross_j": { "core": 1.713984, "dram": 0.072143, "package-0": 2.084529, "psys": 2.777031, "uncore": 0.187622 }, "logits": { "l2": 712.6588257239783, "sha256": "990e8b5de951b6913e147a016df0e6896d435e87a38f418db97e9b4ce13bdab3", "sum": 24349.479562914465, "top1": 78 }, "net_j": { "core": 1.585145629701802, "dram": 0.04640266545950007, "package-0": 1.8592179064128376, "psys": 2.2561248975044728, "uncore": 0.18751060278803594 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.052447515, "watts": { "core": 32.67998493350924, "dram": 1.3755275154599793, "package-0": 39.745047977964255, "psys": 52.948762205416216, "uncore": 3.5773286875460166 } }, "metrics": { "all_tokens_per_s_e2e": 2280.9370616024767, "decode_tokens_per_s": 56.051410832320784, "end_to_end_ms": 448.938297, "handoff_estimate_ms": 37.251965999999996, "handoff_plus_first_decode_ms": 52.447515, "output_tokens_per_s_e2e": 35.6396415875387, "prefill_tokens_per_s": 5559.022744758106, "steady_tpot_ms": 15.195549, "steady_tpot_p95_ms": 16.1355517, "ttft_ms": 181.326835 }, "prefill": { "gpu_active_fraction": 0.6111111111111112, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 18, "gross_j": { "core": 1.033628, "dram": 0.286864, "package-0": 3.569755, "psys": 6.132004, "uncore": 1.851008 }, "logits": { "l2": 838.9345161742121, "sha256": "e1f6e0509b78af0035fc75a4b5671053a3706868fabf97ca68685fc192833e98", "sum": 177352.29018324683, "top1": 291 }, "net_j": { "core": 0.5881950006914488, "dram": 0.19787192107938706, "package-0": 2.790786778330223, "psys": 4.331074920753833, "uncore": 1.8506228663311644 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.181326835, "watts": { "core": 5.700358692082173, "dram": 1.582027282393144, "package-0": 19.686854402990047, "psys": 33.81741042355921, "uncore": 10.2081305285012 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 720.1119452755735, "sha256": "9d31e7729784e25aa0cb77989028372b1f68cbebfb778541584dec15714a8c90", "sum": 72014.2167466206, "top1": 42 }, "gpu_active_fraction": 0.9545454545454546, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 22, "gross_j": { "core": 3.250724, "dram": 0.63983, "package-0": 6.040756, "psys": 9.642432, "uncore": 1.502376 }, "net_j": { "core": 2.722169457837081, "dram": 0.5342312509108507, "package-0": 5.11642575863203, "psys": 7.5054344124246075, "uncore": 1.5019189971503706 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.215163947, "token_latencies_s": [ 0.01732816, 0.015319979, 0.015055705, 0.015043114, 0.015227711, 0.015446988, 0.015493378, 0.015282708, 0.015270521, 0.015133025, 0.015146154, 0.015163387, 0.015136484, 0.015087242 ], "watts": { "core": 15.108125898062282, "dram": 2.9736859214615543, "package-0": 28.07513100696187, "psys": 44.81434800970629, "uncore": 6.982470906243415 } } }, { "energy": { "gross_j": { "core": 6.037643, "dram": 1.026122, "package-0": 11.773589, "psys": 18.811109, "uncore": 3.5179970000000003 }, "net_j": { "core": 4.927513753510475, "dram": 0.8043317202743555, "package-0": 9.832207453720489, "psys": 14.32274813578901, "uncore": 3.51703715160474 } }, "first_decode": { "gpu_active_fraction": 0.6666666666666666, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 1950, "gpu_sample_count": 3, "gross_j": { "core": 0.88513, "dram": 0.055054, "package-0": 1.073178, "psys": 1.45898, "uncore": 0.066528 }, "logits": { "l2": 712.6588257239783, "sha256": "990e8b5de951b6913e147a016df0e6896d435e87a38f418db97e9b4ce13bdab3", "sum": 24349.479562914465, "top1": 78 }, "net_j": { "core": 0.8200700819638291, "dram": 0.04205582211593866, "package-0": 0.9594019467365738, "psys": 1.1959364031704132, "uncore": 0.06647174739867429 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.026484587, "watts": { "core": 33.420570235813, "dram": 2.0787184636860676, "package-0": 40.52085086318318, "psys": 55.087889420363624, "uncore": 2.511951573947519 } }, "metrics": { "all_tokens_per_s_e2e": 2265.9311990972883, "decode_tokens_per_s": 61.55369402594316, "end_to_end_ms": 451.911338, "handoff_estimate_ms": 11.253072999999999, "handoff_plus_first_decode_ms": 26.484587, "output_tokens_per_s_e2e": 35.40517498589513, "prefill_tokens_per_s": 4840.9949746918255, "steady_tpot_ms": 15.231514, "steady_tpot_p95_ms": 16.6062088, "ttft_ms": 208.221658 }, "prefill": { "gpu_active_fraction": 0.5714285714285714, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 21, "gross_j": { "core": 1.734492, "dram": 0.334961, "package-0": 4.465076, "psys": 7.540264, "uncore": 1.931453 }, "logits": { "l2": 838.9345161742121, "sha256": "e1f6e0509b78af0035fc75a4b5671053a3706868fabf97ca68685fc192833e98", "sum": 177352.29018324683, "top1": 291 }, "net_j": { "core": 1.2229913297106885, "dram": 0.23276940645171534, "package-0": 3.570569378046374, "psys": 5.472216880437966, "uncore": 1.9310107423988205 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.208221658, "watts": { "core": 8.330026840915847, "dram": 1.6086751167834807, "package-0": 21.443859600810594, "psys": 36.21267870223183, "uncore": 9.275946693306995 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 720.1119452755735, "sha256": "9d31e7729784e25aa0cb77989028372b1f68cbebfb778541584dec15714a8c90", "sum": 72014.2167466206, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 22, "gross_j": { "core": 3.418021, "dram": 0.636107, "package-0": 6.235335, "psys": 9.811865, "uncore": 1.520016 }, "net_j": { "core": 2.8844523418359573, "dram": 0.5295064917067015, "package-0": 5.302236128937542, "psys": 7.654594852180631, "uncore": 1.519554661807245 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.217205093, "token_latencies_s": [ 0.018152538, 0.01577357, 0.015673576, 0.015386031, 0.015580588, 0.01542957, 0.01522996, 0.015233068, 0.015164214, 0.015094769, 0.015152342, 0.015167017, 0.015092245, 0.015043933 ], "watts": { "core": 15.736375942160805, "dram": 2.9286007579941966, "package-0": 28.707130730125193, "psys": 45.17327316998041, "uncore": 6.998067950460076 } } }, { "energy": { "gross_j": { "core": 5.887314, "dram": 1.042111, "package-0": 11.709137, "psys": 18.89546, "uncore": 3.5533360000000003 }, "net_j": { "core": 4.736853507198252, "dram": 0.8122630293076882, "package-0": 9.697224623570218, "psys": 14.24403597163487, "uncore": 3.5523412800957024 } }, "first_decode": { "gpu_active_fraction": 0.6666666666666666, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 1900, "gpu_sample_count": 3, "gross_j": { "core": 0.804808, "dram": 0.053283, "package-0": 0.978452, "psys": 1.334348, "uncore": 0.062561 }, "logits": { "l2": 712.6588257239783, "sha256": "990e8b5de951b6913e147a016df0e6896d435e87a38f418db97e9b4ce13bdab3", "sum": 24349.479562914465, "top1": 78 }, "net_j": { "core": 0.746232909127959, "dram": 0.04158041146758662, "package-0": 0.8760165395793122, "psys": 1.0975231921507893, "uncore": 0.06251035435927524 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.023844744, "watts": { "core": 33.752008409064906, "dram": 2.234580501262668, "package-0": 41.03428411728807, "psys": 55.95983752226486, "uncore": 2.6236809252387028 } }, "metrics": { "all_tokens_per_s_e2e": 2186.495329817863, "decode_tokens_per_s": 61.19568611046857, "end_to_end_ms": 468.329379, "handoff_estimate_ms": 8.4692085, "handoff_plus_first_decode_ms": 23.844744000000002, "output_tokens_per_s_e2e": 34.16398952840411, "prefill_tokens_per_s": 4515.844525960018, "steady_tpot_ms": 15.3755355, "steady_tpot_p95_ms": 17.291089500000002, "ttft_ms": 223.21406199999998 }, "prefill": { "gpu_active_fraction": 0.5454545454545454, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 22, "gross_j": { "core": 1.633907, "dram": 0.345092, "package-0": 4.422962, "psys": 7.600811, "uncore": 1.944514 }, "logits": { "l2": 838.9345161742121, "sha256": "e1f6e0509b78af0035fc75a4b5671053a3706868fabf97ca68685fc192833e98", "sum": 177352.29018324683, "top1": 291 }, "net_j": { "core": 1.0855771892221031, "dram": 0.23554239350245876, "package-0": 3.4640489911900945, "psys": 5.3838600718461125, "uncore": 1.9440398989071652 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.223214062, "watts": { "core": 7.319910696307296, "dram": 1.5460137094767803, "package-0": 19.81488961927497, "psys": 34.05166740794314, "uncore": 8.711431450945058 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 720.1119452755735, "sha256": "9d31e7729784e25aa0cb77989028372b1f68cbebfb778541584dec15714a8c90", "sum": 72014.2167466206, "top1": 42 }, "gpu_active_fraction": 0.9545454545454546, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 22, "gross_j": { "core": 3.448599, "dram": 0.643736, "package-0": 6.307723, "psys": 9.960301, "uncore": 1.546261 }, "net_j": { "core": 2.90504340884819, "dram": 0.5351402243376429, "package-0": 5.357159092800812, "psys": 7.762652707637969, "uncore": 1.5457910268292618 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.221270573, "token_latencies_s": [ 0.019060071, 0.016338561, 0.016201103, 0.016299304, 0.016259874, 0.01551903, 0.015116966, 0.01501455, 0.015250364, 0.015247183, 0.015074194, 0.015232958, 0.015500707, 0.015121262 ], "watts": { "core": 15.585438918712432, "dram": 2.909270723495618, "package-0": 28.506831769265588, "psys": 45.014123952216636, "uncore": 6.988100491790203 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 0.9142003450542688, "compile_target": "AUTO:GPU", "context_length": 2048, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 395961, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_0_5b_ov_fp16/openvino_model.xml", "key": "qwen25_0_5b_fp16", "size_b": 0.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "GPU", "key": "gpu_gpu", "label": "iGPU -> iGPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 2032, "prompt_sha256": "cd1a97358e68d07c27508cfacc4ba6882a7a2962b1281e2e2b0c7e5e320da4fa", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.03955871798098087, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 7.614482, "dram": 1.349363, "package-0": 16.965653, "psys": 27.241691, "uncore": 6.301742 }, "net_j": { "core": 5.008566568409632, "dram": 1.0081887576951474, "package-0": 13.00904908862839, "psys": 20.224601601738687, "uncore": 6.300392615720058 } }, "first_decode": { "gpu_active_fraction": 0.8, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 5, "gross_j": { "core": 1.621517, "dram": 0.070496, "package-0": 1.858332, "psys": 2.546014, "uncore": 0.09967 }, "logits": { "l2": 752.85195897198, "sha256": "4970ddf30ee86b82fca779e34e85f3a0321c66b97a1a267f9f290f6c9784648a", "sum": -146160.83817428164, "top1": 522 }, "net_j": { "core": 1.4451380894002102, "dram": 0.0474039454947186, "package-0": 1.5905330226798906, "psys": 2.0710689696882034, "uncore": 0.0995786682068069 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.046763065, "watts": { "core": 34.67516511161106, "dram": 1.5075145309658382, "package-0": 39.7393113560884, "psys": 54.444977034760235, "uncore": 2.131382962173245 } }, "metrics": { "all_tokens_per_s_e2e": 2964.23971744647, "decode_tokens_per_s": 49.79433908873345, "end_to_end_ms": 690.902287, "handoff_estimate_ms": 30.324757, "handoff_plus_first_decode_ms": 46.763065, "output_tokens_per_s_e2e": 23.158122792550547, "prefill_tokens_per_s": 5214.759488786759, "steady_tpot_ms": 16.438308, "steady_tpot_p95_ms": 26.155030949999997, "ttft_ms": 389.663225 }, "prefill": { "gpu_active_fraction": 0.6666666666666666, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 39, "gross_j": { "core": 2.511834, "dram": 0.614562, "package-0": 8.523781, "psys": 14.199976, "uncore": 4.516529 }, "logits": { "l2": 658.700794245309, "sha256": "d8b9f589d3ed262681597fb65c434b4174e3993033527e14721527676e3f0747", "sum": -95245.78444925009, "top1": 220 }, "net_j": { "core": 1.042119063087699, "dram": 0.42214252470675884, "package-0": 6.292288835099008, "psys": 10.242395114123134, "uncore": 4.515767958330027 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.389663225, "watts": { "core": 6.446166429998622, "dram": 1.5771619197577602, "package-0": 21.874738115201914, "psys": 36.44166318235445, "uncore": 11.59085258815481 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 726.9945112251366, "sha256": "82a7d15255d9e580be3390c3a34d6d3e2feb60a349b88eded778069599535d54", "sum": -57824.15603592619, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 25, "gross_j": { "core": 3.481131, "dram": 0.664305, "package-0": 6.58354, "psys": 10.495701, "uncore": 1.685543 }, "net_j": { "core": 2.5213094159217233, "dram": 0.5386422874936699, "package-0": 5.126227230849491, "psys": 7.91113751792735, "uncore": 1.6850459891832237 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.254475997, "token_latencies_s": [ 0.026521564, 0.025957667, 0.021161474, 0.016808544, 0.016487786, 0.016380679, 0.016314393, 0.016352132, 0.016308652, 0.016494606, 0.016401723, 0.016397908, 0.016474893, 0.016375879 ], "watts": { "core": 13.679604524744233, "dram": 2.6104819622732434, "package-0": 25.870966525774136, "psys": 41.244365377218664, "uncore": 6.623583441545569 } } }, { "energy": { "gross_j": { "core": 7.980022, "dram": 1.427792, "package-0": 17.574966, "psys": 28.345996, "uncore": 6.388472999999999 }, "net_j": { "core": 5.31761053903866, "dram": 1.0792211285566065, "package-0": 13.532583245586757, "psys": 21.176776688412616, "uncore": 6.3870943611813304 } }, "first_decode": { "gpu_active_fraction": 0.6666666666666666, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 1900, "gpu_sample_count": 3, "gross_j": { "core": 0.971921, "dram": 0.057922, "package-0": 1.173642, "psys": 1.590877, "uncore": 0.07434 }, "logits": { "l2": 752.85195897198, "sha256": "4970ddf30ee86b82fca779e34e85f3a0321c66b97a1a267f9f290f6c9784648a", "sum": -146160.83817428164, "top1": 522 }, "net_j": { "core": 0.8622817690720452, "dram": 0.043567703968258424, "package-0": 1.0071749670830579, "psys": 1.295645514107709, "uncore": 0.07428322706988659 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.029068478, "watts": { "core": 33.43556549469154, "dram": 1.9926051855896962, "package-0": 40.375075709158224, "psys": 54.72859638540415, "uncore": 2.557409438498982 } }, "metrics": { "all_tokens_per_s_e2e": 2901.3389312250233, "decode_tokens_per_s": 56.753306339796076, "end_to_end_ms": 705.8809909999999, "handoff_estimate_ms": 12.520836999999997, "handoff_plus_first_decode_ms": 29.068478, "output_tokens_per_s_e2e": 22.666710400195495, "prefill_tokens_per_s": 4601.665926712596, "steady_tpot_ms": 16.547641000000002, "steady_tpot_p95_ms": 18.1297297, "ttft_ms": 441.579209 }, "prefill": { "gpu_active_fraction": 0.6136363636363636, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 44, "gross_j": { "core": 3.536735, "dram": 0.712462, "package-0": 9.944432, "psys": 16.56496, "uncore": 4.677235 }, "logits": { "l2": 658.700794245309, "sha256": "d8b9f589d3ed262681597fb65c434b4174e3993033527e14721527676e3f0747", "sum": -95245.78444925009, "top1": 220 }, "net_j": { "core": 1.8712055949676967, "dram": 0.49440590888135655, "package-0": 7.415631533734093, "psys": 12.080098907093362, "uncore": 4.676372562497352 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.441579209, "watts": { "core": 8.009287864818836, "dram": 1.613440998758617, "package-0": 22.520154475841732, "psys": 37.512998036100925, "uncore": 10.592063450161215 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 726.9945112251366, "sha256": "82a7d15255d9e580be3390c3a34d6d3e2feb60a349b88eded778069599535d54", "sum": -57824.15603592619, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 23, "gross_j": { "core": 3.471366, "dram": 0.657408, "package-0": 6.456892, "psys": 10.190159, "uncore": 1.636898 }, "net_j": { "core": 2.5841231749989184, "dram": 0.5412475157069916, "package-0": 5.109776744769607, "psys": 7.801032267211546, "uncore": 1.6364385716140921 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.235233304, "token_latencies_s": [ 0.020142235, 0.017046073, 0.016604627, 0.016623763, 0.016571168, 0.016388021, 0.016310621, 0.016478148, 0.016615487, 0.016391979, 0.016413113, 0.016524114, 0.016422428, 0.016665236 ], "watts": { "core": 14.757119595616444, "dram": 2.7947063142045567, "package-0": 27.44888538401858, "psys": 43.31937198824534, "uncore": 6.958615009718182 } } }, { "energy": { "gross_j": { "core": 8.231913, "dram": 1.4384130000000002, "package-0": 17.964798000000002, "psys": 28.874437, "uncore": 6.444440999999999 }, "net_j": { "core": 5.523284780440212, "dram": 1.0837912922917516, "package-0": 13.852243587049882, "psys": 21.580767323350216, "uncore": 6.443038429411801 } }, "first_decode": { "gpu_active_fraction": 0.75, "gpu_freq_mhz_after": 1900, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 1900, "gpu_sample_count": 4, "gross_j": { "core": 1.338681, "dram": 0.0625, "package-0": 1.614803, "psys": 2.114984, "uncore": 0.094055 }, "logits": { "l2": 752.85195897198, "sha256": "4970ddf30ee86b82fca779e34e85f3a0321c66b97a1a267f9f290f6c9784648a", "sum": -146160.83817428164, "top1": 522 }, "net_j": { "core": 1.197080614412732, "dram": 0.043961254008029055, "package-0": 1.3998088245656908, "psys": 1.7336889846480503, "uncore": 0.0939816770778403 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.037542289, "watts": { "core": 35.657948293989215, "dram": 1.6647892727052418, "package-0": 43.012907390915885, "psys": 56.33604280229158, "uncore": 2.5053080807086645 } }, "metrics": { "all_tokens_per_s_e2e": 2851.833989930976, "decode_tokens_per_s": 54.51941527970728, "end_to_end_ms": 718.1343680000001, "handoff_estimate_ms": 20.8518715, "handoff_plus_first_decode_ms": 37.542289, "output_tokens_per_s_e2e": 22.27995304633575, "prefill_tokens_per_s": 4586.876138344373, "steady_tpot_ms": 16.6904175, "steady_tpot_p95_ms": 18.34303435, "ttft_ms": 443.00302400000004 }, "prefill": { "gpu_active_fraction": 0.6363636363636364, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 44, "gross_j": { "core": 3.416251, "dram": 0.715453, "package-0": 9.854345, "psys": 16.48769, "uncore": 4.696338 }, "logits": { "l2": 658.700794245309, "sha256": "d8b9f589d3ed262681597fb65c434b4174e3993033527e14721527676e3f0747", "sum": -95245.78444925009, "top1": 220 }, "net_j": { "core": 1.745351310992605, "dram": 0.49669381524142, "package-0": 7.317390743966612, "psys": 11.988368053034002, "uncore": 4.695472781679261 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.443003024, "watts": { "core": 7.711574898865701, "dram": 1.615007034353788, "package-0": 22.24441926157145, "psys": 37.21800779400549, "uncore": 10.601142081594459 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 726.9945112251366, "sha256": "82a7d15255d9e580be3390c3a34d6d3e2feb60a349b88eded778069599535d54", "sum": -57824.15603592619, "top1": 42 }, "gpu_active_fraction": 0.9583333333333334, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 24, "gross_j": { "core": 3.476981, "dram": 0.66046, "package-0": 6.49565, "psys": 10.271763, "uncore": 1.654048 }, "net_j": { "core": 2.580852855034875, "dram": 0.5431362230423027, "package-0": 5.135044018517579, "psys": 7.858710285668163, "uncore": 1.6535839706546993 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.237589055, "token_latencies_s": [ 0.020231909, 0.017325948, 0.017130183, 0.017158318, 0.01676068, 0.016601071, 0.016842128, 0.01662665, 0.016534849, 0.016754185, 0.016387876, 0.01641993, 0.016365636, 0.016426358 ], "watts": { "core": 14.63443254993375, "dram": 2.7798418576141906, "package-0": 27.339853681391176, "psys": 43.23331729233066, "uncore": 6.961802175609479 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 3.1983783105388284, "compile_target": "NPU", "context_length": 2048, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 396029, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_0_5b_ov_fp16/openvino_model.xml", "key": "qwen25_0_5b_fp16", "size_b": 0.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "gpu_npu", "label": "iGPU -> NPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 2032, "prompt_sha256": "cd1a97358e68d07c27508cfacc4ba6882a7a2962b1281e2e2b0c7e5e320da4fa", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "GPU", "NPUW_ONLINE_PIPELINE": "NONE" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 2048, "NPUW_LLM_MIN_RESPONSE_LEN": 2048, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.03935159929096699, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 8.949988000000001, "dram": 1.463071, "package-0": 16.844744, "psys": 27.576833999999998, "uncore": 4.024343 }, "net_j": { "core": 6.162054949215223, "dram": 1.0980664665959126, "package-0": 12.611779768062025, "psys": 20.069615915879385, "uncore": 4.023731913936043 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 16, "gross_j": { "core": 2.839958, "dram": 0.134277, "package-0": 3.244681, "psys": 4.804675, "uncore": 0.0 }, "logits": { "l2": 1443.2633782892876, "sha256": "8dfbc68a4d27bc8c6dd2a02d6083dc30ff3beb0b60761b166fbe665205c7e7e4", "sum": -388910.4892301038, "top1": 8 }, "net_j": { "core": 2.246151140195466, "dram": 0.05653402990461359, "package-0": 2.34309435769363, "psys": 3.2056991583295424, "uncore": 0.0 }, "npu_busy_delta_us": 24839, "npu_utilization": 0.15777296401243726, "seconds": 0.157435085, "watts": { "core": 18.038914261074652, "dram": 0.8529039127460057, "package-0": 20.609643650905387, "psys": 30.518451462074033, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 2770.711448918199, "decode_tokens_per_s": 35.188442366426685, "end_to_end_ms": 739.160334, "handoff_estimate_ms": 139.6647425, "handoff_plus_first_decode_ms": 157.43508500000002, "output_tokens_per_s_e2e": 21.64618319467343, "prefill_tokens_per_s": 6494.419532585504, "steady_tpot_ms": 17.7703425, "steady_tpot_p95_ms": 24.969379049999997, "ttft_ms": 312.884006 }, "prefill": { "gpu_active_fraction": 0.967741935483871, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 31, "gross_j": { "core": 2.344964, "dram": 0.628844, "package-0": 7.914835, "psys": 13.506679, "uncore": 4.024343 }, "logits": { "l2": 1214.4736652113506, "sha256": "6309e0e955846c137342cf9ddc36ea672b062c49bbf09e8acb3ff6ad93269bdf", "sum": 281703.18898873776, "top1": 89372 }, "net_j": { "core": 1.1648416080571682, "dram": 0.4743388468330124, "package-0": 6.123035920392642, "psys": 10.328899619761636, "uncore": 4.023731913936043 }, "npu_busy_delta_us": 5856, "npu_utilization": 0.018716201172647988, "seconds": 0.312884006, "watts": { "core": 7.494675199217438, "dram": 2.009831080978936, "package-0": 25.29638731357844, "psys": 43.16832673128073, "uncore": 12.862092413889638 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 1281.626957611578, "sha256": "20ba5eabdad4a4b070d6d6d3912744bc0c1fa18ee32a52ccd4fb0399db15ebad", "sum": -343781.3722482808, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 27, "gross_j": { "core": 3.765066, "dram": 0.69995, "package-0": 5.685228, "psys": 9.26548, "uncore": 0.0 }, "net_j": { "core": 2.751062200962589, "dram": 0.5671935898582865, "package-0": 4.145649489975752, "psys": 6.535017137788207, "uncore": 0.0 }, "npu_busy_delta_us": 256774, "npu_utilization": 0.9551138699354994, "seconds": 0.268841243, "token_latencies_s": [ 0.02553388, 0.024665417, 0.022123729, 0.019989949, 0.0201231, 0.018341175, 0.017858462, 0.017682223, 0.017409616, 0.017279776, 0.017121826, 0.016946542, 0.017021695, 0.016718193 ], "watts": { "core": 14.004793155937016, "dram": 2.6035811774609297, "package-0": 21.147157097469602, "psys": 34.464503647604396, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 7.15196, "dram": 1.392758, "package-0": 14.817345000000001, "psys": 24.471799, "uncore": 3.953237 }, "net_j": { "core": 4.77961377566322, "dram": 1.0821633368881125, "package-0": 11.215373026331571, "psys": 18.083654036431717, "uncore": 3.95262917805632 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 0, "gpu_sample_count": 5, "gross_j": { "core": 1.615658, "dram": 0.070984, "package-0": 1.806148, "psys": 2.437432, "uncore": 0.0 }, "logits": { "l2": 1215.1658719840748, "sha256": "67f1824ed2e475f93e13a8b3a0737153bdb4a3bba006f63c1e57e63dd98c2618", "sum": -177402.685341008, "top1": 8 }, "net_j": { "core": 1.4294050486398115, "dram": 0.046599206546217756, "package-0": 1.5233571055132642, "psys": 1.9358986061911159, "uncore": 0.0 }, "npu_busy_delta_us": 23715, "npu_utilization": 0.48024588589357753, "seconds": 0.049380954, "watts": { "core": 32.71824193594964, "dram": 1.4374772913459712, "package-0": 36.5758020794819, "psys": 49.359759230248976, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 3256.083763568839, "decode_tokens_per_s": 47.20488633261632, "end_to_end_ms": 628.976448, "handoff_estimate_ms": 31.236676999999997, "handoff_plus_first_decode_ms": 49.380953999999996, "output_tokens_per_s_e2e": 25.438154402881555, "prefill_tokens_per_s": 6529.295809598845, "steady_tpot_ms": 18.144277, "steady_tpot_p95_ms": 23.3607074, "ttft_ms": 311.212734 }, "prefill": { "gpu_active_fraction": 0.967741935483871, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 31, "gross_j": { "core": 2.538934, "dram": 0.623411, "package-0": 8.038187, "psys": 13.643459, "uncore": 3.953237 }, "logits": { "l2": 1294.8704681928841, "sha256": "f67be6ad2198c7836d5609b7d08a700e9459eafcb8c610546c38ffc62933f614", "sum": 344172.9917854555, "top1": 89372 }, "net_j": { "core": 1.3651152396663435, "dram": 0.46973113713674153, "package-0": 6.255958827099116, "psys": 10.482653749466333, "uncore": 3.95262917805632 }, "npu_busy_delta_us": 4171, "npu_utilization": 0.013402407884762197, "seconds": 0.311212734, "watts": { "core": 8.158194452287418, "dram": 2.0031667470264893, "package-0": 25.82859286214169, "psys": 43.839655352920104, "uncore": 12.702683946088145 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 1228.9703970899452, "sha256": "063abfdbf40b7782053c5f3c313384a0d3cb22f1fd987cddfffb471ee7aa5ff4", "sum": -187181.752090808, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 27, "gross_j": { "core": 2.997368, "dram": 0.698363, "package-0": 4.97301, "psys": 8.390908, "uncore": 0.0 }, "net_j": { "core": 1.9850934873570654, "dram": 0.5658329932051532, "package-0": 3.4360570937191905, "psys": 5.665101680774267, "uncore": 0.0 }, "npu_busy_delta_us": 257207, "npu_utilization": 0.9583588752124019, "seconds": 0.26838276, "token_latencies_s": [ 0.023516723, 0.023276699, 0.022803482, 0.020529341, 0.019389887, 0.018663185, 0.018351903, 0.017936651, 0.01785845, 0.017481147, 0.017311715, 0.017215874, 0.017055887, 0.016967341 ], "watts": { "core": 11.168258348636105, "dram": 2.6021157245718762, "package-0": 18.52954340286239, "psys": 31.264705676325857, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 6.682905, "dram": 1.384519, "package-0": 14.293115, "psys": 23.554443, "uncore": 3.963246 }, "net_j": { "core": 4.369907773228524, "dram": 1.0816944851555632, "package-0": 10.781253577524536, "psys": 17.32611029148973, "uncore": 3.962576104690427 } }, "first_decode": { "gpu_active_fraction": 0.25, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 4, "gross_j": { "core": 1.210813, "dram": 0.063293, "package-0": 1.403256, "psys": 1.87884, "uncore": 0.016784 }, "logits": { "l2": 1332.6647311881006, "sha256": "ce35f508c161565adce06026e2f517cd92518ab10af6c84aa72fcae548a07a08", "sum": -237999.51964376122, "top1": 151643 }, "net_j": { "core": 1.0691443493855584, "dram": 0.04474531654649845, "package-0": 1.188158176661292, "psys": 1.4973611637179292, "uncore": 0.016710641729128135 }, "npu_busy_delta_us": 22856, "npu_utilization": 0.6085134157826059, "seconds": 0.037560388, "watts": { "core": 32.23643483129088, "dram": 1.6850997385863, "package-0": 37.359997452635476, "psys": 50.0218474846426, "uncore": 0.4468537438963623 } }, "metrics": { "all_tokens_per_s_e2e": 3339.631337737878, "decode_tokens_per_s": 48.73189457829246, "end_to_end_ms": 613.2413409999999, "handoff_estimate_ms": 19.081272, "handoff_plus_first_decode_ms": 37.560388, "output_tokens_per_s_e2e": 26.09086982607717, "prefill_tokens_per_s": 6652.812838734416, "steady_tpot_ms": 18.479116, "steady_tpot_p95_ms": 23.452067549999995, "ttft_ms": 305.434716 }, "prefill": { "gpu_active_fraction": 0.967741935483871, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 31, "gross_j": { "core": 2.372369, "dram": 0.624938, "package-0": 7.854838, "psys": 13.233853, "uncore": 3.946462 }, "logits": { "l2": 1317.0179500179788, "sha256": "f133d0b67d8a6a6da60f1d83f451f8880173f1bcb958e32ccb1b50ebc8687a23", "sum": 359899.42827031016, "top1": 89372 }, "net_j": { "core": 1.2203435186198506, "dram": 0.47411137839858986, "package-0": 6.105698918058769, "psys": 10.131731690182198, "uncore": 3.9458654629612986 }, "npu_busy_delta_us": 4197, "npu_utilization": 0.01374107061228757, "seconds": 0.305434716, "watts": { "core": 7.767188455421027, "dram": 2.0460608020733306, "package-0": 25.7169129392613, "psys": 43.327926744253915, "uncore": 12.92080367183932 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 1286.3926898566062, "sha256": "ec3d7969f0d2cefd54957f9226c67c9e51f1f296cab99825f211f20021ce44f2", "sum": -189694.47188457847, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 27, "gross_j": { "core": 3.099723, "dram": 0.696288, "package-0": 5.035021, "psys": 8.44175, "uncore": 0.0 }, "net_j": { "core": 2.080419905223115, "dram": 0.562837790210475, "package-0": 3.487396482804475, "psys": 5.6970174375896026, "uncore": 0.0 }, "npu_busy_delta_us": 258158, "npu_utilization": 0.9552695455293241, "seconds": 0.270246237, "token_latencies_s": [ 0.023975885, 0.023170012, 0.022205197, 0.020709119, 0.019696798, 0.018954403, 0.018650373, 0.018307859, 0.018001305, 0.017715389, 0.017528287, 0.017228937, 0.017063598, 0.017014403 ], "watts": { "core": 11.46999504751661, "dram": 2.5764947098967377, "package-0": 18.631234447123866, "psys": 31.237252713346756, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 2.040728172287345, "compile_target": "AUTO:NPU,GPU", "context_length": 2048, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 395874, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_0_5b_ov_fp16/openvino_model.xml", "key": "qwen25_0_5b_fp16", "size_b": 0.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": null, "key": "openvino_auto", "label": "OpenVINO AUTO", "mode": "stock", "prefill_device": null }, "prompt_length": 2032, "prompt_sha256": "cd1a97358e68d07c27508cfacc4ba6882a7a2962b1281e2e2b0c7e5e320da4fa", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.0431777061894536, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 7.671978, "dram": 1.331906, "package-0": 17.13582, "psys": 26.943474, "uncore": 6.3591750000000005 }, "net_j": { "core": 5.215483435339661, "dram": 1.0102943844946957, "package-0": 13.406084228464792, "psys": 20.328738250097036, "uncore": 6.3579029882113085 } }, "first_decode": { "gpu_active_fraction": 0.75, "gpu_freq_mhz_after": 2100, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2100, "gpu_sample_count": 4, "gross_j": { "core": 1.408565, "dram": 0.067627, "package-0": 1.680172, "psys": 2.233698, "uncore": 0.109131 }, "logits": { "l2": 752.85195897198, "sha256": "4970ddf30ee86b82fca779e34e85f3a0321c66b97a1a267f9f290f6c9784648a", "sum": -146160.83817428164, "top1": 522 }, "net_j": { "core": 1.2604030021787365, "dram": 0.048229188109307444, "package-0": 1.455215221667988, "psys": 1.8347341765645226, "uncore": 0.10905427937111035 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.039281959, "watts": { "core": 35.85780943358757, "dram": 1.721579109636564, "package-0": 42.77210309190537, "psys": 56.863202774586675, "uncore": 2.7781455604085328 } }, "metrics": { "all_tokens_per_s_e2e": 3144.545130998449, "decode_tokens_per_s": 55.71604541802646, "end_to_end_ms": 651.286566, "handoff_estimate_ms": 22.8349735, "handoff_plus_first_decode_ms": 39.281959, "output_tokens_per_s_e2e": 24.566758835925384, "prefill_tokens_per_s": 5318.476497280693, "steady_tpot_ms": 16.446985499999997, "steady_tpot_p95_ms": 16.7151854, "ttft_ms": 382.0643 }, "prefill": { "gpu_active_fraction": 0.6578947368421053, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 38, "gross_j": { "core": 2.738824, "dram": 0.60858, "package-0": 8.897682, "psys": 14.468713, "uncore": 4.599902 }, "logits": { "l2": 658.700794245309, "sha256": "d8b9f589d3ed262681597fb65c434b4174e3993033527e14721527676e3f0747", "sum": -95245.78444925009, "top1": 220 }, "net_j": { "core": 1.2977703604861806, "dram": 0.4199129474848967, "package-0": 6.70970675050505, "psys": 10.58830993949583, "uncore": 4.599155799603724 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.3820643, "watts": { "core": 7.16848970186432, "dram": 1.5928732414936437, "package-0": 23.28844123881765, "psys": 37.869837616338394, "uncore": 12.039601711020893 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 726.9945112251366, "sha256": "82a7d15255d9e580be3390c3a34d6d3e2feb60a349b88eded778069599535d54", "sum": -57824.15603592619, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 2100, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 23, "gross_j": { "core": 3.524589, "dram": 0.655699, "package-0": 6.557966, "psys": 10.241063, "uncore": 1.650142 }, "net_j": { "core": 2.657310072674744, "dram": 0.5421522489004915, "package-0": 5.241162256291754, "psys": 7.905694134036684, "uncore": 1.6496929092364736 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.229940307, "token_latencies_s": [ 0.016858227, 0.016458946, 0.016226336, 0.016165594, 0.016198213, 0.016528615, 0.016466194, 0.016565852, 0.016362389, 0.016476792, 0.016638163, 0.016320655, 0.016435025, 0.016206259 ], "watts": { "core": 15.328278221355946, "dram": 2.851605308155042, "package-0": 28.520297661427406, "psys": 44.53792000895258, "uncore": 7.176392958368973 } } }, { "energy": { "gross_j": { "core": 8.254679, "dram": 1.460263, "package-0": 18.109451, "psys": 29.291917, "uncore": 6.407698 }, "net_j": { "core": 5.413217183586314, "dram": 1.0882503205012364, "package-0": 13.795213169966411, "psys": 21.640559114087203, "uncore": 6.406226646071685 } }, "first_decode": { "gpu_active_fraction": 0.75, "gpu_freq_mhz_after": 1900, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 1900, "gpu_sample_count": 4, "gross_j": { "core": 1.186887, "dram": 0.062805, "package-0": 1.419674, "psys": 2.024836, "uncore": 0.083374 }, "logits": { "l2": 752.85195897198, "sha256": "4970ddf30ee86b82fca779e34e85f3a0321c66b97a1a267f9f290f6c9784648a", "sum": -146160.83817428164, "top1": 522 }, "net_j": { "core": 1.0605367438820092, "dram": 0.046262846569575175, "package-0": 1.227834344921803, "psys": 1.6846058329250604, "uncore": 0.08330857383639335 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.033499046, "watts": { "core": 35.430471661790016, "dram": 1.8748295100702272, "package-0": 42.37953522616734, "psys": 60.44458698913397, "uncore": 2.4888469958219113 } }, "metrics": { "all_tokens_per_s_e2e": 2718.5155112787256, "decode_tokens_per_s": 50.75007511434034, "end_to_end_ms": 753.3523319999999, "handoff_estimate_ms": 17.1116635, "handoff_plus_first_decode_ms": 33.499046, "output_tokens_per_s_e2e": 21.238402431865044, "prefill_tokens_per_s": 4438.752637374331, "steady_tpot_ms": 16.387382499999998, "steady_tpot_p95_ms": 28.636520799999996, "ttft_ms": 457.786267 }, "prefill": { "gpu_active_fraction": 0.6086956521739131, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 46, "gross_j": { "core": 3.780874, "dram": 0.726195, "package-0": 10.315464, "psys": 17.005572, "uncore": 4.68706 }, "logits": { "l2": 658.700794245309, "sha256": "d8b9f589d3ed262681597fb65c434b4174e3993033527e14721527676e3f0747", "sum": -95245.78444925009, "top1": 220 }, "net_j": { "core": 2.054215512986701, "dram": 0.5001357066381704, "package-0": 7.693850251333384, "psys": 12.356105322470647, "uncore": 4.686165908890611 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.457786267, "watts": { "core": 8.2590376176575, "dram": 1.5863188836112465, "package-0": 22.53336271443896, "psys": 37.147405297765296, "uncore": 10.238533433332545 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 726.9945112251366, "sha256": "82a7d15255d9e580be3390c3a34d6d3e2feb60a349b88eded778069599535d54", "sum": -57824.15603592619, "top1": 42 }, "gpu_active_fraction": 0.9615384615384616, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 26, "gross_j": { "core": 3.286918, "dram": 0.671263, "package-0": 6.374313, "psys": 10.261509, "uncore": 1.637264 }, "net_j": { "core": 2.298464926717604, "dram": 0.5418517672934908, "package-0": 4.873528573711224, "psys": 7.599847958691497, "uncore": 1.6367521633446815 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.262067019, "token_latencies_s": [ 0.029648017, 0.028091869, 0.024304728, 0.016569629, 0.016349441, 0.01634223, 0.016422857, 0.01629344, 0.01620432, 0.016351908, 0.016222598, 0.016349102, 0.016442159, 0.016441698 ], "watts": { "core": 12.542280263049811, "dram": 2.561417314400787, "package-0": 24.32321710806349, "psys": 39.15604885786868, "uncore": 6.247501140156824 } } }, { "energy": { "gross_j": { "core": 8.148967, "dram": 1.430722, "package-0": 17.74196, "psys": 28.552111999999997, "uncore": 6.377242 }, "net_j": { "core": 5.470221093595741, "dram": 1.0800125741047237, "package-0": 13.674776390598883, "psys": 21.338908043878686, "uncore": 6.375854902946795 } }, "first_decode": { "gpu_active_fraction": 0.6666666666666666, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 1750, "gpu_sample_count": 3, "gross_j": { "core": 0.940122, "dram": 0.057373, "package-0": 1.134763, "psys": 1.555477, "uncore": 0.067444 }, "logits": { "l2": 752.85195897198, "sha256": "4970ddf30ee86b82fca779e34e85f3a0321c66b97a1a267f9f290f6c9784648a", "sum": -146160.83817428164, "top1": 522 }, "net_j": { "core": 0.8369853042785467, "dram": 0.043870035325847725, "package-0": 0.9781688723661635, "psys": 1.2777552422846954, "uncore": 0.06739059418559612 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.027344471, "watts": { "core": 34.38069802118315, "dram": 2.098157247218277, "package-0": 41.498809759384265, "psys": 56.88451606908029, "uncore": 2.4664583929965223 } }, "metrics": { "all_tokens_per_s_e2e": 2883.6471589780886, "decode_tokens_per_s": 56.11062744373706, "end_to_end_ms": 710.2117169999999, "handoff_estimate_ms": 10.788283499999999, "handoff_plus_first_decode_ms": 27.344471, "output_tokens_per_s_e2e": 22.528493429516317, "prefill_tokens_per_s": 4588.122524908854, "steady_tpot_ms": 16.5561875, "steady_tpot_p95_ms": 20.14588655, "ttft_ms": 442.88268 }, "prefill": { "gpu_active_fraction": 0.6363636363636364, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 44, "gross_j": { "core": 3.656485, "dram": 0.713377, "package-0": 10.073888, "psys": 16.679584, "uncore": 4.68883 }, "logits": { "l2": 658.700794245309, "sha256": "d8b9f589d3ed262681597fb65c434b4174e3993033527e14721527676e3f0747", "sum": -95245.78444925009, "top1": 220 }, "net_j": { "core": 1.9860392192985992, "dram": 0.4946772422744296, "package-0": 7.537622920352433, "psys": 12.18148431647477, "uncore": 4.687965016720171 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.44288268, "watts": { "core": 8.256102948076453, "dram": 1.6107584067184566, "package-0": 22.74617738494538, "psys": 37.661405047494746, "uncore": 10.587070146884047 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 726.9945112251366, "sha256": "82a7d15255d9e580be3390c3a34d6d3e2feb60a349b88eded778069599535d54", "sum": -57824.15603592619, "top1": 42 }, "gpu_active_fraction": 0.9583333333333334, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 24, "gross_j": { "core": 3.55236, "dram": 0.659972, "package-0": 6.533309, "psys": 10.317051, "uncore": 1.620968 }, "net_j": { "core": 2.6471965700185955, "dram": 0.5414652965044463, "package-0": 5.158984597880285, "psys": 7.87966848511922, "uncore": 1.6204992920410275 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.239984566, "token_latencies_s": [ 0.02277963, 0.016815926, 0.018727717, 0.01678692, 0.016704349, 0.016471615, 0.01664076, 0.016647662, 0.016404419, 0.016269311, 0.016459391, 0.016440634, 0.016351194, 0.016463857 ], "watts": { "core": 14.802451921012286, "dram": 2.7500601851204047, "package-0": 27.223871555139922, "psys": 42.990477145934456, "uncore": 6.7544677018937955 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 0.9078262252733111, "compile_target": "AUTO:GPU", "context_length": 4096, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 396367, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_0_5b_ov_fp16/openvino_model.xml", "key": "qwen25_0_5b_fp16", "size_b": 0.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "GPU", "key": "gpu_gpu", "label": "iGPU -> iGPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 4080, "prompt_sha256": "66f8de7a9ef70b33481de3e8dae7cef6e4434d9723c6203a6a3dbe07ef228a16", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.03983466699719429, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 12.173551999999999, "dram": 2.246699, "package-0": 31.7335, "psys": 51.562001, "uncore": 14.243920999999999 }, "net_j": { "core": 7.083099794976163, "dram": 1.5802417723461608, "package-0": 24.004583374046508, "psys": 37.854665079881904, "uncore": 14.241285083231562 } }, "first_decode": { "gpu_active_fraction": 0.6666666666666666, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 1900, "gpu_sample_count": 6, "gross_j": { "core": 1.791621, "dram": 0.078308, "package-0": 2.086481, "psys": 2.804558, "uncore": 0.107849 }, "logits": { "l2": 731.5050659415324, "sha256": "c9bf661180d16881f80683aef6a956fd8906c6d1b49f9d814a4006bfb3fa18cd", "sum": -52036.174412652384, "top1": 78 }, "net_j": { "core": 1.5983440114728615, "dram": 0.05300359859221261, "package-0": 1.79302539529609, "psys": 2.2841106014848633, "uncore": 0.10774891811444397 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.051243226, "watts": { "core": 34.96307980297727, "dram": 1.5281629614809964, "package-0": 40.71720621180251, "psys": 54.730316939842936, "uncore": 2.1046489149609746 } }, "metrics": { "all_tokens_per_s_e2e": 3034.9201648542553, "decode_tokens_per_s": 43.86502438393833, "end_to_end_ms": 1349.6236399999998, "handoff_estimate_ms": 31.199350500000005, "handoff_plus_first_decode_ms": 51.243226, "output_tokens_per_s_e2e": 11.855156893961935, "prefill_tokens_per_s": 4048.9624797126016, "steady_tpot_ms": 20.0438755, "steady_tpot_p95_ms": 23.857719749999998, "ttft_ms": 1007.6655489999999 }, "prefill": { "gpu_active_fraction": 0.73, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 100, "gross_j": { "core": 6.547285, "dram": 1.46069, "package-0": 22.34705, "psys": 37.008938, "uncore": 12.173675 }, "logits": { "l2": 651.1399731232168, "sha256": "7cccf7c80f58f7fe8b63aa7d53f62f7cd36a27b775f8f747cddefd8226dc491f", "sum": -4677.085442299489, "top1": 62 }, "net_j": { "core": 2.7466155660875904, "dram": 0.9630949903358066, "package-0": 16.576431596289574, "psys": 26.774669893023734, "uncore": 12.171706953176527 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.007665549, "watts": { "core": 6.497478262006157, "dram": 1.4495781873753433, "package-0": 22.177050730946444, "psys": 36.72740229804165, "uncore": 12.081066989023359 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 713.6949237157127, "sha256": "c6991e9de21ae0c23772f8b7e984041fd8586adc9216fbb0440d1c7c586214f4", "sum": 28851.84398850752, "top1": 42 }, "gpu_active_fraction": 0.9655172413793104, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 29, "gross_j": { "core": 3.834646, "dram": 0.707701, "package-0": 7.299969, "psys": 11.748505, "uncore": 1.962397 }, "net_j": { "core": 2.738140217415711, "dram": 0.5641431834181415, "package-0": 5.635126382460843, "psys": 8.79588458537331, "uncore": 1.9618292119405916 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.290714865, "token_latencies_s": [ 0.02685148, 0.020833562, 0.020735698, 0.022245695, 0.020263753, 0.019823998, 0.019666624, 0.021071469, 0.019740248, 0.021032095, 0.019667173, 0.019524074, 0.019672873, 0.01955748 ], "watts": { "core": 13.19040221765062, "dram": 2.434347483400961, "package-0": 25.110408440930602, "psys": 40.41246738449374, "uncore": 6.750246500123067 } } }, { "energy": { "gross_j": { "core": 13.123991, "dram": 2.449518, "package-0": 33.405189, "psys": 55.373577000000004, "uncore": 14.503931 }, "net_j": { "core": 7.641205094654028, "dram": 1.731695270873853, "package-0": 25.080585716513614, "psys": 40.60978290946809, "uncore": 14.50109192662821 } }, "first_decode": { "gpu_active_fraction": 0.75, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 1900, "gpu_sample_count": 4, "gross_j": { "core": 1.342343, "dram": 0.068054, "package-0": 1.592892, "psys": 2.169367, "uncore": 0.09198 }, "logits": { "l2": 731.5050659415324, "sha256": "c9bf661180d16881f80683aef6a956fd8906c6d1b49f9d814a4006bfb3fa18cd", "sum": -52036.174412652384, "top1": 78 }, "net_j": { "core": 1.1885890935430734, "dram": 0.04792407870595432, "package-0": 1.359444935562888, "psys": 1.755345541797223, "uncore": 0.09190038379226065 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.040764533, "watts": { "core": 32.929188714120684, "dram": 1.6694414235041035, "package-0": 39.075438445474155, "psys": 53.21702078618195, "uncore": 2.2563732056000743 } }, "metrics": { "all_tokens_per_s_e2e": 2817.7492814720413, "decode_tokens_per_s": 46.14865166161512, "end_to_end_ms": 1453.6424610000001, "handoff_estimate_ms": 21.256463499999995, "handoff_plus_first_decode_ms": 40.764533, "output_tokens_per_s_e2e": 11.006833130750161, "prefill_tokens_per_s": 3615.079520598528, "steady_tpot_ms": 19.5080695, "steady_tpot_p95_ms": 23.44630155, "ttft_ms": 1128.605879 }, "prefill": { "gpu_active_fraction": 0.7053571428571429, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 112, "gross_j": { "core": 8.214579, "dram": 1.676143, "package-0": 24.804624, "psys": 41.804459, "uncore": 12.45999 }, "logits": { "l2": 651.1399731232168, "sha256": "7cccf7c80f58f7fe8b63aa7d53f62f7cd36a27b775f8f747cddefd8226dc491f", "sum": -4677.085442299489, "top1": 62 }, "net_j": { "core": 3.957752048333375, "dram": 1.118826483795405, "package-0": 18.341414195560034, "psys": 30.341870878119476, "uncore": 12.45778574758974 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.128605879, "watts": { "core": 7.27851870422518, "dram": 1.4851446649251416, "package-0": 21.978109862388905, "psys": 37.040795000147256, "uncore": 11.040160459770208 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 713.6949237157127, "sha256": "c6991e9de21ae0c23772f8b7e984041fd8586adc9216fbb0440d1c7c586214f4", "sum": 28851.84398850752, "top1": 42 }, "gpu_active_fraction": 0.9655172413793104, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 29, "gross_j": { "core": 3.567069, "dram": 0.705321, "package-0": 7.007673, "psys": 11.399751, "uncore": 1.951961 }, "net_j": { "core": 2.49486395277758, "dram": 0.5649447083724938, "package-0": 5.37972658539069, "psys": 8.51256648955139, "uncore": 1.951405795246209 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.284272049, "token_latencies_s": [ 0.026313824, 0.020670523, 0.021902251, 0.02035076, 0.019545107, 0.019262625, 0.019415048, 0.019098347, 0.019386232, 0.019517558, 0.019344895, 0.020663806, 0.019271766, 0.019498581 ], "watts": { "core": 12.548082066274478, "dram": 2.4811479091284134, "package-0": 24.65129098921716, "psys": 40.10155426853098, "uncore": 6.866524538260179 } } }, { "energy": { "gross_j": { "core": 13.166531, "dram": 2.441583, "package-0": 33.326574, "psys": 54.908367999999996, "uncore": 14.399315999999999 }, "net_j": { "core": 7.737410665675533, "dram": 1.7307863279396978, "package-0": 25.08345202846858, "psys": 40.289082094538706, "uncore": 14.396504715510778 } }, "first_decode": { "gpu_active_fraction": 0.6, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2000, "gpu_sample_count": 5, "gross_j": { "core": 1.592586, "dram": 0.070861, "package-0": 1.877009, "psys": 2.525018, "uncore": 0.111755 }, "logits": { "l2": 731.5050659415324, "sha256": "c9bf661180d16881f80683aef6a956fd8906c6d1b49f9d814a4006bfb3fa18cd", "sum": -52036.174412652384, "top1": 78 }, "net_j": { "core": 1.4165011921403539, "dram": 0.04780745030508654, "package-0": 1.6096565636680942, "psys": 2.0508649160402777, "uncore": 0.1116638204978634 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.04668509, "watts": { "core": 34.11337538387524, "dram": 1.517850774197929, "package-0": 40.20574877332356, "psys": 54.086176121755365, "uncore": 2.393804959999006 } }, "metrics": { "all_tokens_per_s_e2e": 2845.602066983094, "decode_tokens_per_s": 44.477808863252505, "end_to_end_ms": 1439.414192, "handoff_estimate_ms": 26.82702, "handoff_plus_first_decode_ms": 46.685089999999995, "output_tokens_per_s_e2e": 11.115633074152711, "prefill_tokens_per_s": 3701.797147118028, "steady_tpot_ms": 19.858069999999998, "steady_tpot_p95_ms": 23.73208415, "ttft_ms": 1102.167363 }, "prefill": { "gpu_active_fraction": 0.6972477064220184, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 109, "gross_j": { "core": 7.796001, "dram": 1.65881, "package-0": 24.176146, "psys": 40.703326, "uncore": 12.315093 }, "logits": { "l2": 651.1399731232168, "sha256": "7cccf7c80f58f7fe8b63aa7d53f62f7cd36a27b775f8f747cddefd8226dc491f", "sum": -4677.085442299489, "top1": 62 }, "net_j": { "core": 3.6388937020805168, "dram": 1.1145490791142907, "package-0": 17.86434217262038, "psys": 29.509258382133034, "uncore": 12.312940384005694 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.102167363, "watts": { "core": 7.073336828610121, "dram": 1.505043658237955, "package-0": 21.93509516938944, "psys": 36.93025883946447, "uncore": 11.173523562228725 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 713.6949237157127, "sha256": "c6991e9de21ae0c23772f8b7e984041fd8586adc9216fbb0440d1c7c586214f4", "sum": 28851.84398850752, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 29, "gross_j": { "core": 3.777944, "dram": 0.711912, "package-0": 7.273419, "psys": 11.680024, "uncore": 1.972468 }, "net_j": { "core": 2.6820157714546626, "dram": 0.5684297985203205, "package-0": 5.609453292180108, "psys": 8.728958796365394, "uncore": 1.9719005110072196 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.290561739, "token_latencies_s": [ 0.028723609, 0.020800673, 0.020679015, 0.020717207, 0.019930645, 0.019642804, 0.019785495, 0.019738085, 0.019739745, 0.019631231, 0.019536176, 0.02104434, 0.019687741, 0.020872349 ], "watts": { "core": 13.002207424150916, "dram": 2.4501230012255677, "package-0": 25.032266894575546, "psys": 40.198079899294655, "uncore": 6.788464327025522 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 3.790025919675827, "compile_target": "NPU", "context_length": 4096, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 396436, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_0_5b_ov_fp16/openvino_model.xml", "key": "qwen25_0_5b_fp16", "size_b": 0.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "gpu_npu", "label": "iGPU -> NPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 4080, "prompt_sha256": "66f8de7a9ef70b33481de3e8dae7cef6e4434d9723c6203a6a3dbe07ef228a16", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "GPU", "NPUW_ONLINE_PIPELINE": "NONE" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 4096, "NPUW_LLM_MIN_RESPONSE_LEN": 4096, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.03980210516601801, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 11.947664, "dram": 2.564935, "package-0": 28.757006, "psys": 48.551816, "uncore": 10.149876 }, "net_j": { "core": 7.202749779567352, "dram": 1.9437166387542306, "package-0": 21.552725332983442, "psys": 35.77492889605537, "uncore": 10.14836493559327 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 18, "gross_j": { "core": 2.852715, "dram": 0.15979, "package-0": 3.322623, "psys": 5.06798, "uncore": 0.0 }, "logits": { "l2": 1194.5389349351794, "sha256": "e1f0572991cfbd0bb62604abe32d92b58e7c7dae72ac6196e0b2ea2540eacc99", "sum": -145173.30570931733, "top1": 151643 }, "net_j": { "core": 2.1830452256811927, "dram": 0.07211483023892871, "package-0": 2.3058524569973446, "psys": 3.2647239861082875, "uncore": 0.0 }, "npu_busy_delta_us": 27636, "npu_utilization": 0.1556532431664968, "seconds": 0.177548501, "watts": { "core": 16.067243507733135, "dram": 0.8999794371679882, "package-0": 18.713889338891125, "psys": 28.544200437941182, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 3255.93157800964, "decode_tokens_per_s": 30.970805863321115, "end_to_end_ms": 1258.0116939999998, "handoff_estimate_ms": 157.0572, "handoff_plus_first_decode_ms": 177.548501, "output_tokens_per_s_e2e": 12.718482726600156, "prefill_tokens_per_s": 5273.466644476229, "steady_tpot_ms": 20.491301, "steady_tpot_p95_ms": 28.679078849999996, "ttft_ms": 773.684613 }, "prefill": { "gpu_active_fraction": 0.987012987012987, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 77, "gross_j": { "core": 5.708116, "dram": 1.624813, "package-0": 19.831736, "psys": 34.060459, "uncore": 10.149876 }, "logits": { "l2": 1316.4831161793027, "sha256": "e44a9a14f0e2b695504bf9bdfb6472c0d1fa984ec295e707921b018c2f670cae", "sum": 359625.1051243767, "top1": 89372 }, "net_j": { "core": 2.7899657635097266, "dram": 1.2427600431783385, "package-0": 15.401060890707981, "psys": 26.202598058398856, "uncore": 10.14836493559327 }, "npu_busy_delta_us": 5404, "npu_utilization": 0.006984758271262143, "seconds": 0.773684613, "watts": { "core": 7.377833168823793, "dram": 2.10009734289494, "package-0": 25.63284272011236, "psys": 44.023699615698575, "uncore": 13.118880522443582 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 1245.6294987734661, "sha256": "5a20b8596177229ad661bfa977ae7e167f5368166f38b44599912f24e0db964d", "sum": -187297.29206892848, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 30, "gross_j": { "core": 3.386833, "dram": 0.780332, "package-0": 5.602647, "psys": 9.423377, "uncore": 0.0 }, "net_j": { "core": 2.229738790376433, "dram": 0.6288417653369635, "package-0": 3.8458119852781154, "psys": 6.307606851548226, "uncore": 0.0 }, "npu_busy_delta_us": 287678, "npu_utilization": 0.9377382214886057, "seconds": 0.30677858, "token_latencies_s": [ 0.033055289, 0.026322658, 0.023812932, 0.022443461, 0.021143438, 0.02091615, 0.020546206, 0.020436396, 0.020137225, 0.019855986, 0.01985136, 0.019418918, 0.019410153, 0.01938896 ], "watts": { "core": 11.039991775175437, "dram": 2.543632609551814, "package-0": 18.262836342745963, "psys": 30.717193488541472, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 10.995272, "dram": 2.47503, "package-0": 27.288567, "psys": 45.607854, "uncore": 9.983251000000001 }, "net_j": { "core": 6.740093321162535, "dram": 1.9179292806039978, "package-0": 20.82785979508564, "psys": 34.14970427149738, "uncore": 9.981628450628 } }, "first_decode": { "gpu_active_fraction": 0.16666666666666666, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 6, "gross_j": { "core": 1.945124, "dram": 0.086181, "package-0": 2.185236, "psys": 2.944329, "uncore": 0.010804 }, "logits": { "l2": 1076.3716215015206, "sha256": "938aa50fdd336de86a3bb8faeb3e45e76c6d3a3148e88756f8b550ab3d963cd8", "sum": -83517.90235760808, "top1": 151643 }, "net_j": { "core": 1.7148955245361954, "dram": 0.056038799404707546, "package-0": 1.8356763498722581, "psys": 2.3243803344483394, "uncore": 0.010684784049313386 }, "npu_busy_delta_us": 25329, "npu_utilization": 0.414956616399615, "seconds": 0.061040116, "watts": { "core": 31.86632214132752, "dram": 1.4118747742877815, "package-0": 35.799997496728224, "psys": 48.235966655109245, "uncore": 0.17699835301754668 } }, "metrics": { "all_tokens_per_s_e2e": 3630.6621205092188, "decode_tokens_per_s": 41.8477013338037, "end_to_end_ms": 1128.168875, "handoff_estimate_ms": 40.7916245, "handoff_plus_first_decode_ms": 61.040116, "output_tokens_per_s_e2e": 14.182273908239136, "prefill_tokens_per_s": 5300.585773969865, "steady_tpot_ms": 20.2484915, "steady_tpot_p95_ms": 26.01549035, "ttft_ms": 769.726248 }, "prefill": { "gpu_active_fraction": 0.9868421052631579, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 76, "gross_j": { "core": 5.739915, "dram": 1.618404, "package-0": 19.639599, "psys": 33.500768, "uncore": 9.972447 }, "logits": { "l2": 1334.0000819701768, "sha256": "4ec570d789e7f405abaa32f13c5b5ba60aa64d12f6cb3a84109463b050062787", "sum": 371782.7636419125, "top1": 89372 }, "net_j": { "core": 2.836694753801345, "dram": 1.238305722161245, "package-0": 15.231592338528067, "psys": 25.683109850600566, "uncore": 9.970943666578686 }, "npu_busy_delta_us": 5632, "npu_utilization": 0.007316887029166244, "seconds": 0.769726248, "watts": { "core": 7.457086223724567, "dram": 2.102570886994099, "package-0": 25.515043888694308, "psys": 43.52296428379041, "uncore": 12.955835955850112 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 1276.7456685137324, "sha256": "c285e1961626b95265953e042fd1dcac6f2895b14eba95347b13315d1aba89d0", "sum": -156027.74945994467, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 30, "gross_j": { "core": 3.310233, "dram": 0.770445, "package-0": 5.463732, "psys": 9.162757, "uncore": 0.0 }, "net_j": { "core": 2.188503042824994, "dram": 0.6235847590380453, "package-0": 3.760591106685316, "psys": 6.142214086448472, "uncore": 0.0 }, "npu_busy_delta_us": 287886, "npu_utilization": 0.9680012419262997, "seconds": 0.297402511, "token_latencies_s": [ 0.025895454, 0.026238415, 0.024245749, 0.022298555, 0.021324219, 0.020759729, 0.020372359, 0.020124624, 0.019862358, 0.019490558, 0.019343102, 0.019181057, 0.019005239, 0.019225364 ], "watts": { "core": 11.130481006597824, "dram": 2.590580010267634, "package-0": 18.371505948717427, "psys": 30.809279212844306, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 10.950778, "dram": 2.4689870000000003, "package-0": 27.393546, "psys": 45.691716, "uncore": 10.064854 }, "net_j": { "core": 6.695186313787788, "dram": 1.9118322084432375, "package-0": 20.932211719250972, "psys": 34.23245414420591, "uncore": 10.062650387229755 } }, "first_decode": { "gpu_active_fraction": 0.3333333333333333, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 6, "gross_j": { "core": 1.853084, "dram": 0.083801, "package-0": 2.117304, "psys": 2.851555, "uncore": 0.038147 }, "logits": { "l2": 1132.7733093910065, "sha256": "ba6f530cba6ce38dd32d9851f295a6bd9ea606a03641d5c0ab35981cde51d82e", "sum": -121606.2538428083, "top1": 51186 }, "net_j": { "core": 1.6358691832395387, "dram": 0.05536258652454415, "package-0": 1.7875032001797981, "psys": 2.266648916956126, "uncore": 0.038034522727885205 }, "npu_busy_delta_us": 26224, "npu_utilization": 0.4553582164904999, "seconds": 0.057589825, "watts": { "core": 32.17728131662147, "dram": 1.4551355209014787, "package-0": 36.76524455491921, "psys": 49.514909968905094, "uncore": 0.6623913165216252 } }, "metrics": { "all_tokens_per_s_e2e": 3630.309762872128, "decode_tokens_per_s": 41.678565086582346, "end_to_end_ms": 1128.278375, "handoff_estimate_ms": 37.2021285, "handoff_plus_first_decode_ms": 57.589825, "output_tokens_per_s_e2e": 14.18089751121925, "prefill_tokens_per_s": 5309.864786011121, "steady_tpot_ms": 20.3876965, "steady_tpot_p95_ms": 26.38795515, "ttft_ms": 768.381148 }, "prefill": { "gpu_active_fraction": 0.9868421052631579, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 76, "gross_j": { "core": 5.631028, "dram": 1.610591, "package-0": 19.576, "psys": 33.383825, "uncore": 9.989476 }, "logits": { "l2": 1341.2597838450145, "sha256": "55370d341ddc06cd6b806e3e24b2ff4f1e4aa07f6b532c5ea1c0e77cc32101c3", "sum": 376821.0504594818, "top1": 89372 }, "net_j": { "core": 2.732881143936225, "dram": 1.2311569455716451, "package-0": 15.17569634953458, "psys": 25.579828242561774, "uncore": 9.987975293660442 }, "npu_busy_delta_us": 4538, "npu_utilization": 0.005905923136989822, "seconds": 0.768381148, "watts": { "core": 7.3284307074124095, "dram": 2.0960834400898136, "package-0": 25.47693947327297, "psys": 43.44695999751415, "uncore": 13.00067814781942 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 1295.5273517640333, "sha256": "d5f59dc8ca74197855cb3401a475dc3f2f32742d13458fbb24ee3aea7556bd73", "sum": -187058.67358023673, "top1": 42 }, "gpu_active_fraction": 0.06666666666666667, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 30, "gross_j": { "core": 3.466666, "dram": 0.774595, "package-0": 5.700242, "psys": 9.456336, "uncore": 0.037231 }, "net_j": { "core": 2.326435986612025, "dram": 0.6253126763470483, "package-0": 3.969012169536594, "psys": 6.385976984688012, "uncore": 0.036640570841427786 }, "npu_busy_delta_us": 290036, "npu_utilization": 0.9594075371002659, "seconds": 0.302307402, "token_latencies_s": [ 0.026277177, 0.026593686, 0.024474226, 0.022381668, 0.021525552, 0.020739282, 0.023106132, 0.020036111, 0.020015511, 0.020003811, 0.01951297, 0.019395297, 0.019172401, 0.019045849 ], "watts": { "core": 11.467354014705865, "dram": 2.5622759974630065, "package-0": 18.85578044827364, "psys": 31.280530802219655, "uncore": 0.12315609791122481 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 1.350780539214611, "compile_target": "AUTO:NPU,GPU", "context_length": 4096, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 396280, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_0_5b_ov_fp16/openvino_model.xml", "key": "qwen25_0_5b_fp16", "size_b": 0.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": null, "key": "openvino_auto", "label": "OpenVINO AUTO", "mode": "stock", "prefill_device": null }, "prompt_length": 4080, "prompt_sha256": "66f8de7a9ef70b33481de3e8dae7cef6e4434d9723c6203a6a3dbe07ef228a16", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.040753516368567944, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 11.879669, "dram": 2.247492, "package-0": 31.45158, "psys": 51.033440000000006, "uncore": 14.252771 }, "net_j": { "core": 6.801667613264014, "dram": 1.5826648707154083, "package-0": 23.741567654569312, "psys": 37.359631071567144, "uncore": 14.250141530462455 } }, "first_decode": { "gpu_active_fraction": 0.8, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 1950, "gpu_sample_count": 5, "gross_j": { "core": 1.674007, "dram": 0.075989, "package-0": 1.973384, "psys": 2.661859, "uncore": 0.117066 }, "logits": { "l2": 731.5050659415324, "sha256": "c9bf661180d16881f80683aef6a956fd8906c6d1b49f9d814a4006bfb3fa18cd", "sum": -52036.174412652384, "top1": 78 }, "net_j": { "core": 1.483971391415538, "dram": 0.05110896974891198, "package-0": 1.6848498353412904, "psys": 2.15013984053505, "uncore": 0.11696759655231151 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.050383844, "watts": { "core": 33.22507508557704, "dram": 1.508201716407347, "package-0": 39.1669996437747, "psys": 52.831598160712, "uncore": 2.323482900590118 } }, "metrics": { "all_tokens_per_s_e2e": 3042.3615254630645, "decode_tokens_per_s": 44.8805488036593, "end_to_end_ms": 1346.3225740000003, "handoff_estimate_ms": 30.657608499999995, "handoff_plus_first_decode_ms": 50.383843999999996, "output_tokens_per_s_e2e": 11.884224708840096, "prefill_tokens_per_s": 4031.2139982578155, "steady_tpot_ms": 19.7262355, "steady_tpot_p95_ms": 22.792098749999997, "ttft_ms": 1012.102062 }, "prefill": { "gpu_active_fraction": 0.74, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 100, "gross_j": { "core": 6.873701, "dram": 1.468319, "package-0": 22.724368, "psys": 37.347683, "uncore": 12.197234 }, "logits": { "l2": 651.1399731232168, "sha256": "7cccf7c80f58f7fe8b63aa7d53f62f7cd36a27b775f8f747cddefd8226dc491f", "sum": -4677.085442299489, "top1": 62 }, "net_j": { "core": 3.0562981178030983, "dram": 0.968533197238532, "package-0": 16.928342928778207, "psys": 27.06835583196211, "uncore": 12.195257288332002 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.012102062, "watts": { "core": 6.791509728195771, "dram": 1.4507617908597836, "package-0": 22.452644701755382, "psys": 36.90110355688614, "uncore": 12.05138736294759 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 713.6949237157127, "sha256": "c6991e9de21ae0c23772f8b7e984041fd8586adc9216fbb0440d1c7c586214f4", "sum": 28851.84398850752, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 28, "gross_j": { "core": 3.331961, "dram": 0.703184, "package-0": 6.753828, "psys": 11.023898, "uncore": 1.938471 }, "net_j": { "core": 2.2613981040453774, "dram": 0.5630227037279644, "package-0": 5.128374890449814, "psys": 8.141135399069984, "uncore": 1.9379166455781418 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.283836668, "token_latencies_s": [ 0.02613058, 0.020524535, 0.020036894, 0.020026238, 0.019896589, 0.020994455, 0.019629611, 0.019503504, 0.019547554, 0.019274896, 0.01982286, 0.019396263, 0.019531048, 0.01949757 ], "watts": { "core": 11.73900829472815, "dram": 2.4774247984055395, "package-0": 23.79476918042175, "psys": 38.83887898514931, "uncore": 6.829529861871124 } } }, { "energy": { "gross_j": { "core": 13.175198, "dram": 2.480158, "package-0": 33.601415, "psys": 55.254863, "uncore": 14.557091 }, "net_j": { "core": 7.552141487204246, "dram": 1.7439706234128591, "package-0": 25.063836564918784, "psys": 40.113354659615084, "uncore": 14.554179292286229 } }, "first_decode": { "gpu_active_fraction": 0.5, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2000, "gpu_sample_count": 6, "gross_j": { "core": 1.860774, "dram": 0.075928, "package-0": 2.137995, "psys": 2.824761, "uncore": 0.089965 }, "logits": { "l2": 731.5050659415324, "sha256": "c9bf661180d16881f80683aef6a956fd8906c6d1b49f9d814a4006bfb3fa18cd", "sum": -52036.174412652384, "top1": 78 }, "net_j": { "core": 1.664254997875035, "dram": 0.05019914447548576, "package-0": 1.8396169931619377, "psys": 2.2955836561571883, "uncore": 0.08986323934845977 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.052102776, "watts": { "core": 35.71352896820699, "dram": 1.4572736009305913, "package-0": 41.034185971204295, "psys": 54.21517271939599, "uncore": 1.726683430456757 } }, "metrics": { "all_tokens_per_s_e2e": 2747.4587904457026, "decode_tokens_per_s": 42.67533289256163, "end_to_end_ms": 1490.8321879999999, "handoff_estimate_ms": 32.602312500000004, "handoff_plus_first_decode_ms": 52.102776000000006, "output_tokens_per_s_e2e": 10.732260900178526, "prefill_tokens_per_s": 3581.0171932174894, "steady_tpot_ms": 19.500463500000002, "steady_tpot_p95_ms": 32.233098700000006, "ttft_ms": 1139.341081 }, "prefill": { "gpu_active_fraction": 0.6902654867256637, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 113, "gross_j": { "core": 7.796245, "dram": 1.692867, "package-0": 24.510069, "psys": 41.102189, "uncore": 12.54904 }, "logits": { "l2": 651.1399731232168, "sha256": "7cccf7c80f58f7fe8b63aa7d53f62f7cd36a27b775f8f747cddefd8226dc491f", "sum": -4677.085442299489, "top1": 62 }, "net_j": { "core": 3.4989274766860987, "dram": 1.130249337052581, "package-0": 17.98538170008525, "psys": 29.530569726305686, "uncore": 12.546814780930498 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.139341081, "watts": { "core": 6.842766516552913, "dram": 1.4858298609878704, "package-0": 21.512494729398774, "psys": 36.07540330585166, "uncore": 11.014296078032844 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 713.6949237157127, "sha256": "c6991e9de21ae0c23772f8b7e984041fd8586adc9216fbb0440d1c7c586214f4", "sum": 28851.84398850752, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 30, "gross_j": { "core": 3.518179, "dram": 0.711363, "package-0": 6.953351, "psys": 11.327913, "uncore": 1.918086 }, "net_j": { "core": 2.388959012643112, "dram": 0.5635221418847923, "package-0": 5.238837871671596, "psys": 8.28720127715221, "uncore": 1.9175012720072706 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.299388331, "token_latencies_s": [ 0.033592562, 0.03150108, 0.019775257, 0.019514018, 0.019912938, 0.019486909, 0.019429031, 0.019411923, 0.019301678, 0.019409982, 0.019374999, 0.019550814, 0.019701982, 0.019399599 ], "watts": { "core": 11.751222862456853, "dram": 2.3760545296603426, "package-0": 23.225190430017125, "psys": 37.83685543843057, "uncore": 6.406682563723567 } } }, { "energy": { "gross_j": { "core": 13.194912, "dram": 2.443354, "package-0": 33.474402000000005, "psys": 54.61656, "uncore": 14.518579 }, "net_j": { "core": 7.731456499365104, "dram": 1.7280620652737015, "package-0": 25.1791483853329, "psys": 39.90481793553073, "uncore": 14.515749936217901 } }, "first_decode": { "gpu_active_fraction": 0.5, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 1950, "gpu_sample_count": 6, "gross_j": { "core": 1.600154, "dram": 0.075684, "package-0": 1.877559, "psys": 2.553644, "uncore": 0.096192 }, "logits": { "l2": 731.5050659415324, "sha256": "c9bf661180d16881f80683aef6a956fd8906c6d1b49f9d814a4006bfb3fa18cd", "sum": -52036.174412652384, "top1": 78 }, "net_j": { "core": 1.3913281928071148, "dram": 0.04834390074771877, "package-0": 1.5604953702603133, "psys": 1.991327456656816, "uncore": 0.0960838666898947 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.05536566, "watts": { "core": 28.901561003697964, "dram": 1.3669845171176502, "package-0": 33.911977207532615, "psys": 46.12324679232578, "uncore": 1.7373946233098279 } }, "metrics": { "all_tokens_per_s_e2e": 2827.7188390128445, "decode_tokens_per_s": 42.97830372995891, "end_to_end_ms": 1448.517421, "handoff_estimate_ms": 35.464899499999994, "handoff_plus_first_decode_ms": 55.36566, "output_tokens_per_s_e2e": 11.045776714893924, "prefill_tokens_per_s": 3710.76374837605, "steady_tpot_ms": 19.9007605, "steady_tpot_p95_ms": 24.891375349999993, "ttft_ms": 1099.504112 }, "prefill": { "gpu_active_fraction": 0.6972477064220184, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 109, "gross_j": { "core": 8.526407, "dram": 1.658993, "package-0": 25.030515, "psys": 41.232133, "uncore": 12.435454 }, "logits": { "l2": 651.1399731232168, "sha256": "7cccf7c80f58f7fe8b63aa7d53f62f7cd36a27b775f8f747cddefd8226dc491f", "sum": -4677.085442299489, "top1": 62 }, "net_j": { "core": 4.379344837275009, "dram": 1.1160472182578274, "package-0": 18.73396286525744, "psys": 30.065114460864784, "uncore": 12.433306585535782 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.099504112, "watts": { "core": 7.7547749998774, "dram": 1.5088556576494185, "package-0": 22.76527638852523, "psys": 37.50066284426956, "uncore": 11.310056837695575 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 713.6949237157127, "sha256": "c6991e9de21ae0c23772f8b7e984041fd8586adc9216fbb0440d1c7c586214f4", "sum": 28851.84398850752, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 29, "gross_j": { "core": 3.068351, "dram": 0.708677, "package-0": 6.566328, "psys": 10.830783, "uncore": 1.986933 }, "net_j": { "core": 1.9607834692829802, "dram": 0.5636709462681554, "package-0": 4.884690149815148, "psys": 7.848376018009131, "uncore": 1.9863594839922254 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.293647649, "token_latencies_s": [ 0.031669173, 0.020945545, 0.020480787, 0.020005821, 0.019894239, 0.019690039, 0.019767622, 0.019636265, 0.019509863, 0.019850594, 0.021241792, 0.019907282, 0.021150328, 0.019873929 ], "watts": { "core": 10.449090978419513, "dram": 2.4133583306842685, "package-0": 22.361248327242695, "psys": 36.883601952488306, "uncore": 6.766384838313485 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 0.9227810930460691, "compile_target": "AUTO:GPU", "context_length": 8192, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 396852, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_0_5b_ov_fp16/openvino_model.xml", "key": "qwen25_0_5b_fp16", "size_b": 0.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "GPU", "key": "gpu_gpu", "label": "iGPU -> iGPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 8176, "prompt_sha256": "998ace73fce48442f97bcb84c31be293d0d94180b17f70a4d127bba1092ad744", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.039894359186291695, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 22.641299, "dram": 4.593249999999999, "package-0": 71.594605, "psys": 115.939156, "uncore": 37.757960000000004 }, "net_j": { "core": 11.152216496772057, "dram": 3.0890649561190755, "package-0": 54.15054360826083, "psys": 85.00188226815493, "uncore": 37.75201077112907 } }, "first_decode": { "gpu_active_fraction": 0.7142857142857143, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 7, "gross_j": { "core": 2.093134, "dram": 0.095886, "package-0": 2.497369, "psys": 3.405509, "uncore": 0.168029 }, "logits": { "l2": 751.8547055410498, "sha256": "87ebcfb2f1baeab184e419908d200db26fa2774137672858533c8384b10828b9", "sum": 20114.015098012285, "top1": 78 }, "net_j": { "core": 1.8299457595340742, "dram": 0.06142860839166613, "package-0": 2.097766000980581, "psys": 2.696807792918463, "uncore": 0.1678927169610168 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.069778687, "watts": { "core": 29.996752446775044, "dram": 1.3741445149290354, "package-0": 35.789853712781955, "psys": 48.80442935247549, "uncore": 2.408027539985096 } }, "metrics": { "all_tokens_per_s_e2e": 2689.3559239239694, "decode_tokens_per_s": 37.56383805239734, "end_to_end_ms": 3046.082494, "handoff_estimate_ms": 46.99103600000001, "handoff_plus_first_decode_ms": 69.778687, "output_tokens_per_s_e2e": 5.252648288914003, "prefill_tokens_per_s": 3089.0571753033405, "steady_tpot_ms": 22.787651, "steady_tpot_p95_ms": 26.263037849999996, "ttft_ms": 2646.762276 }, "prefill": { "gpu_active_fraction": 0.7977099236641222, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 262, "gross_j": { "core": 17.217058, "dram": 3.733999, "package-0": 61.81869, "psys": 100.485582, "uncore": 35.272798 }, "logits": { "l2": 655.8555450667305, "sha256": "e30c9d66db32ee71e32369110e91308f4656be42fddef82b606d730823ce7e00", "sum": 5070.977332231589, "top1": 8 }, "net_j": { "core": 7.234114263153401, "dram": 2.4270021481059962, "package-0": 46.66142366764463, "psys": 73.60396956927202, "uncore": 35.26762867363478 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.646762276, "watts": { "core": 6.504950654661666, "dram": 1.4107798927991066, "package-0": 23.356343922743743, "psys": 37.965473103183975, "uncore": 13.326772230299085 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 724.3605232286193, "sha256": "546555705e83653d0439120b2a2a1aa452842d59c8f62055b758ef9c1470d280", "sum": 84226.24423345178, "top1": 42 }, "gpu_active_fraction": 0.9696969696969697, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 33, "gross_j": { "core": 3.331107, "dram": 0.763365, "package-0": 7.278546, "psys": 12.048065, "uncore": 2.317133 }, "net_j": { "core": 2.0881564740845815, "dram": 0.6006341996214131, "package-0": 5.391353939635625, "psys": 8.701104905964442, "uncore": 2.3164893805332727 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.329541531, "token_latencies_s": [ 0.029823589, 0.024345818, 0.024109697, 0.023475631, 0.023261902, 0.022716271, 0.022857281, 0.022666854, 0.022666097, 0.022701305, 0.022651963, 0.022655375, 0.022814111, 0.022761191 ], "watts": { "core": 10.108307107427986, "dram": 2.3164455104749755, "package-0": 22.086885309760852, "psys": 36.560080799041984, "uncore": 7.031383853102267 } } }, { "energy": { "gross_j": { "core": 24.198791, "dram": 4.986986, "package-0": 74.670951, "psys": 122.276848, "uncore": 38.38759700000001 }, "net_j": { "core": 11.920416480937828, "dram": 3.3794644847220856, "package-0": 56.02849466450851, "psys": 89.21420497592791, "uncore": 38.38123906322338 } }, "first_decode": { "gpu_active_fraction": 0.5, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 1950, "gpu_sample_count": 6, "gross_j": { "core": 1.63702, "dram": 0.081848, "package-0": 1.943293, "psys": 2.64721, "uncore": 0.115356 }, "logits": { "l2": 751.8547055410498, "sha256": "87ebcfb2f1baeab184e419908d200db26fa2774137672858533c8384b10828b9", "sum": 20114.015098012285, "top1": 78 }, "net_j": { "core": 1.4296665622122018, "dram": 0.0547006677539247, "package-0": 1.628464892747548, "psys": 2.088858185372689, "uncore": 0.11524862910609032 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.054975293, "watts": { "core": 29.77737653894814, "dram": 1.4888142569790397, "package-0": 35.34847917954707, "psys": 48.152721987311644, "uncore": 2.0983244236642813 } }, "metrics": { "all_tokens_per_s_e2e": 2516.4757796349845, "decode_tokens_per_s": 39.09940484893042, "end_to_end_ms": 3255.3462529999997, "handoff_estimate_ms": 32.187742500000006, "handoff_plus_first_decode_ms": 54.975293, "output_tokens_per_s_e2e": 4.914991757099579, "prefill_tokens_per_s": 2847.0854303571946, "steady_tpot_ms": 22.7875505, "steady_tpot_p95_ms": 26.0898805, "ttft_ms": 2871.7087 }, "prefill": { "gpu_active_fraction": 0.7640845070422535, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 284, "gross_j": { "core": 19.335217, "dram": 4.145497, "package-0": 65.556351, "psys": 107.772978, "uncore": 35.956878 }, "logits": { "l2": 655.8555450667305, "sha256": "e30c9d66db32ee71e32369110e91308f4656be42fddef82b606d730823ce7e00", "sum": 5070.977332231589, "top1": 8 }, "net_j": { "core": 8.503830048226439, "dram": 2.7274194248091774, "package-0": 49.11087949334702, "psys": 78.6067166446301, "uncore": 35.951269336301436 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.8717087, "watts": { "core": 6.733000808891235, "dram": 1.4435645927457752, "package-0": 22.82834293046506, "psys": 37.52921666462897, "uncore": 12.521074299771422 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 724.3605232286193, "sha256": "546555705e83653d0439120b2a2a1aa452842d59c8f62055b758ef9c1470d280", "sum": 84226.24423345178, "top1": 42 }, "gpu_active_fraction": 0.9696969696969697, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 33, "gross_j": { "core": 3.226554, "dram": 0.759641, "package-0": 7.171307, "psys": 11.85666, "uncore": 2.315363 }, "net_j": { "core": 1.986919870499188, "dram": 0.5973443921589834, "package-0": 5.289150278413937, "psys": 8.518630145925131, "uncore": 2.3147210978158577 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.32866226, "token_latencies_s": [ 0.029580556, 0.02420987, 0.024210286, 0.023369866, 0.022887726, 0.022622746, 0.022806878, 0.022566795, 0.022561561, 0.022768223, 0.023077653, 0.022547966, 0.022693308, 0.022734013 ], "watts": { "core": 9.817233046471475, "dram": 2.3113119224580276, "package-0": 21.81968504689282, "psys": 36.07551411591949, "uncore": 7.044809464889581 } } }, { "energy": { "gross_j": { "core": 26.123528999999998, "dram": 4.96721, "package-0": 76.570728, "psys": 124.765123, "uncore": 38.406639999999996 }, "net_j": { "core": 13.912156539102966, "dram": 3.368460594572713, "package-0": 58.030001982458366, "psys": 91.88290004110453, "uncore": 38.4003167579502 } }, "first_decode": { "gpu_active_fraction": 0.5, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 1950, "gpu_sample_count": 6, "gross_j": { "core": 1.868587, "dram": 0.082092, "package-0": 2.178766, "psys": 2.918755, "uncore": 0.112304 }, "logits": { "l2": 751.8547055410498, "sha256": "87ebcfb2f1baeab184e419908d200db26fa2774137672858533c8384b10828b9", "sum": 20114.015098012285, "top1": 78 }, "net_j": { "core": 1.6629554785937994, "dram": 0.05517010619666239, "package-0": 1.8665523064802017, "psys": 2.3650398827265344, "uncore": 0.11219752074166243 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.054518764, "watts": { "core": 34.274199613182724, "dram": 1.50575680695916, "package-0": 39.96359858781832, "psys": 53.53670527086785, "uncore": 2.059914637830014 } }, "metrics": { "all_tokens_per_s_e2e": 2530.28332314397, "decode_tokens_per_s": 39.39493332412788, "end_to_end_ms": 3237.5821020000003, "handoff_estimate_ms": 31.659954999999997, "handoff_plus_first_decode_ms": 54.518764, "output_tokens_per_s_e2e": 4.941959615515566, "prefill_tokens_per_s": 2861.920914919376, "steady_tpot_ms": 22.858809, "steady_tpot_p95_ms": 25.109816749999997, "ttft_ms": 2856.822478 }, "prefill": { "gpu_active_fraction": 0.7632508833922261, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 283, "gross_j": { "core": 20.284677, "dram": 4.12505, "package-0": 66.48152, "psys": 109.147548, "uncore": 35.976532 }, "logits": { "l2": 655.8555450667305, "sha256": "e30c9d66db32ee71e32369110e91308f4656be42fddef82b606d730823ce7e00", "sum": 5070.977332231589, "top1": 8 }, "net_j": { "core": 9.509437257980688, "dram": 2.7143233854402165, "package-0": 50.121297717874114, "psys": 80.13247727251171, "uncore": 35.97095241021584 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.856822478, "watts": { "core": 7.10043314073924, "dram": 1.4439294117035435, "package-0": 23.271141455923534, "psys": 38.20592593363094, "uncore": 12.59319830932806 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 724.3605232286193, "sha256": "546555705e83653d0439120b2a2a1aa452842d59c8f62055b758ef9c1470d280", "sum": 84226.24423345178, "top1": 42 }, "gpu_active_fraction": 0.9696969696969697, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 33, "gross_j": { "core": 3.970265, "dram": 0.760068, "package-0": 7.910442, "psys": 12.69882, "uncore": 2.317804 }, "net_j": { "core": 2.7397638025284787, "dram": 0.5989671029358344, "package-0": 6.042151958104049, "psys": 9.3853828858663, "uncore": 2.317166826992699 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.32624086, "token_latencies_s": [ 0.028226011, 0.023431866, 0.023351737, 0.023308951, 0.022954834, 0.022869557, 0.022782074, 0.022603696, 0.022873462, 0.022848061, 0.022744495, 0.022622921, 0.022841177, 0.022753609 ], "watts": { "core": 12.169735575120786, "dram": 2.3297756142501584, "package-0": 24.24724481170139, "psys": 38.92467669439076, "uncore": 7.1045791137259755 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 5.3119816817343235, "compile_target": "NPU", "context_length": 8192, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 396920, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_0_5b_ov_fp16/openvino_model.xml", "key": "qwen25_0_5b_fp16", "size_b": 0.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "gpu_npu", "label": "iGPU -> NPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 8176, "prompt_sha256": "998ace73fce48442f97bcb84c31be293d0d94180b17f70a4d127bba1092ad744", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "GPU", "NPUW_ONLINE_PIPELINE": "NONE" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 8192, "NPUW_LLM_MIN_RESPONSE_LEN": 8192, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.03929801285266876, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 24.575803, "dram": 5.850815, "package-0": 67.833628, "psys": 117.651248, "uncore": 28.270435 }, "net_j": { "core": 13.908018087245983, "dram": 4.454156693072982, "package-0": 51.63655613811141, "psys": 88.92552670192482, "uncore": 28.26562753730757 } }, "first_decode": { "gpu_active_fraction": 0.047619047619047616, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 21, "gross_j": { "core": 3.50463, "dram": 0.196289, "package-0": 4.085927, "psys": 6.237105, "uncore": 0.017395 }, "logits": { "l2": 1201.221305411654, "sha256": "2158e56b7a1abccc4fecc5a5e247427368c958ccb0da6d510335fdeba75d67e2", "sum": -118090.05621039867, "top1": 151643 }, "net_j": { "core": 2.712959902789019, "dram": 0.0926411830765907, "package-0": 2.8839213764985185, "psys": 4.105332119454381, "uncore": 0.016985060686111747 }, "npu_busy_delta_us": 31923, "npu_utilization": 0.15209086998550356, "seconds": 0.209894256, "watts": { "core": 16.697122002233353, "dram": 0.9351804272337971, "package-0": 19.46659750422136, "psys": 29.715463009144948, "uncore": 0.08287506448008754 } }, "metrics": { "all_tokens_per_s_e2e": 2896.405612149763, "decode_tokens_per_s": 26.00807155299033, "end_to_end_ms": 2828.333147, "handoff_estimate_ms": 186.513786, "handoff_plus_first_decode_ms": 209.894256, "output_tokens_per_s_e2e": 5.657042211230006, "prefill_tokens_per_s": 3631.2131455376903, "steady_tpot_ms": 23.38047, "steady_tpot_p95_ms": 40.0217218, "ttft_ms": 2251.589117 }, "prefill": { "gpu_active_fraction": 0.9955156950672646, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 223, "gross_j": { "core": 17.377213, "dram": 4.779101, "package-0": 57.429541, "psys": 100.405321, "uncore": 28.25304 }, "logits": { "l2": 1341.7647947586395, "sha256": "3d80fac3fd6fb24da3914052cfc080db8cd8781f8c36dad3eb072ab6b4a68430", "sum": 377329.91663436964, "top1": 89372 }, "net_j": { "core": 8.88476633135664, "dram": 3.667244482184873, "package-0": 44.53532068151385, "psys": 77.53725062387568, "uncore": 28.24864247662146 }, "npu_busy_delta_us": 5500, "npu_utilization": 0.0024427192148308826, "seconds": 2.251589117, "watts": { "core": 7.717754926419819, "dram": 2.12254578951227, "package-0": 25.506226054475995, "psys": 44.593092159629585, "uncore": 12.548044306433729 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 1096.0236381099567, "sha256": "c4c493829be4b9bbf7279ba05873d05092edb29b9da7fe7529d2f6f62c3529ec", "sum": -78989.97184104472, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 37, "gross_j": { "core": 3.69396, "dram": 0.875425, "package-0": 6.31816, "psys": 11.008822, "uncore": 0.0 }, "net_j": { "core": 2.310291853100323, "dram": 0.6942710278115183, "package-0": 4.21731408009904, "psys": 7.282943958594764, "uncore": 0.0 }, "npu_busy_delta_us": 333312, "npu_utilization": 0.9085789977888877, "seconds": 0.366849774, "token_latencies_s": [ 0.045486589, 0.037079101, 0.02737205, 0.025429651, 0.024865887, 0.024243291, 0.023558889, 0.023202051, 0.022905424, 0.022743629, 0.022416924, 0.022510618, 0.022520066, 0.022486937 ], "watts": { "core": 10.069407866120152, "dram": 2.386331032604098, "package-0": 17.22274469767017, "psys": 30.009073959522187, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 22.110480000000003, "dram": 5.710678000000001, "package-0": 64.55726800000001, "psys": 111.63399199999999, "uncore": 27.856497 }, "net_j": { "core": 12.046308446961056, "dram": 4.393046559466799, "package-0": 49.27667214921024, "psys": 84.53365303619366, "uncore": 27.851962883760265 } }, "first_decode": { "gpu_active_fraction": 0.1, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 10, "gross_j": { "core": 2.47674, "dram": 0.130676, "package-0": 2.85247, "psys": 4.055532, "uncore": 0.027161 }, "logits": { "l2": 1219.6496415972515, "sha256": "ae511b6f669e4df1ef974305fd17a19af9ab3f992052f05a1989fdf62e185803", "sum": -153737.0175356716, "top1": 151643 }, "net_j": { "core": 2.0855684835716124, "dram": 0.0794626551190238, "package-0": 2.2585479052322626, "psys": 3.002203305002304, "uncore": 0.02695844519387279 }, "npu_busy_delta_us": 29903, "npu_utilization": 0.2883309245653194, "seconds": 0.103710693, "watts": { "core": 23.881240481152698, "dram": 1.2600050797076439, "package-0": 27.504107025878223, "psys": 39.10428021149179, "uncore": 0.2618919921786657 } }, "metrics": { "all_tokens_per_s_e2e": 3070.121760909111, "decode_tokens_per_s": 33.29764299570672, "end_to_end_ms": 2668.29808, "handoff_estimate_ms": 80.27002750000001, "handoff_plus_first_decode_ms": 103.710693, "output_tokens_per_s_e2e": 5.996331564275607, "prefill_tokens_per_s": 3686.510036786897, "steady_tpot_ms": 23.440665499999998, "steady_tpot_p95_ms": 31.256373099999998, "ttft_ms": 2217.815744 }, "prefill": { "gpu_active_fraction": 0.9954337899543378, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 219, "gross_j": { "core": 16.232686, "dram": 4.714221, "package-0": 55.720072, "psys": 97.193721, "uncore": 27.829336 }, "logits": { "l2": 1345.2523121521842, "sha256": "1fb23ce8ab84b58eaf4dcab2813fae764591cfe95c15809970ac11972f3a51f4", "sum": 380191.34651395306, "top1": 89372 }, "net_j": { "core": 7.8676242822151945, "dram": 3.619042100945389, "package-0": 43.01926233049613, "psys": 74.66866696942994, "uncore": 27.825004438566392 }, "npu_busy_delta_us": 4158, "npu_utilization": 0.0018748176043248434, "seconds": 2.217815744, "watts": { "core": 7.319222096747817, "dram": 2.12561436302979, "package-0": 25.123850865764258, "psys": 43.824073872207116, "uncore": 12.548082984480788 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 1049.9828109634605, "sha256": "241d2b084d70ef6175e411c1167bb1dd4753085dbbef69255281655f2b57032e", "sum": -35860.37525684759, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 35, "gross_j": { "core": 3.401054, "dram": 0.865781, "package-0": 5.984726, "psys": 10.384739, "uncore": 0.0 }, "net_j": { "core": 2.09311568117425, "dram": 0.6945418034023864, "package-0": 3.998861913481844, "psys": 6.862782761761412, "uncore": 0.0 }, "npu_busy_delta_us": 331642, "npu_utilization": 0.9563700109123398, "seconds": 0.346771643, "token_latencies_s": [ 0.030863468, 0.031986054, 0.027159793, 0.025537363, 0.02465389, 0.024103283, 0.023645351, 0.02323598, 0.022970673, 0.02266734, 0.022343926, 0.022241234, 0.022530057, 0.02280368 ], "watts": { "core": 9.807762741430388, "dram": 2.49668915402059, "package-0": 17.258406564691334, "psys": 29.9469094709108, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 20.748542999999998, "dram": 5.682359, "package-0": 63.035422000000004, "psys": 109.313625, "uncore": 27.794362 }, "net_j": { "core": 10.781093783607577, "dram": 4.377390737066231, "package-0": 47.90168125081999, "psys": 82.4737355023696, "uncore": 27.789879570387832 } }, "first_decode": { "gpu_active_fraction": 0.1111111111111111, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 9, "gross_j": { "core": 2.413629, "dram": 0.122437, "package-0": 2.76574, "psys": 3.859488, "uncore": 0.025818 }, "logits": { "l2": 1307.4508778747368, "sha256": "70535f068ff0b5d9b427a82892e9a89bfc10f7877cc29c0f02f467344c6b60c9", "sum": -241774.7024954781, "top1": 151643 }, "net_j": { "core": 2.0789245420782434, "dram": 0.07861649130535636, "package-0": 2.2575527613379487, "psys": 2.958211206915354, "uncore": 0.025644684731027274 }, "npu_busy_delta_us": 29919, "npu_utilization": 0.33715473586953837, "seconds": 0.08873967, "watts": { "core": 27.198985526991475, "dram": 1.3797324240669364, "package-0": 31.166895256653532, "psys": 43.49225098538229, "uncore": 0.29094090613589163 } }, "metrics": { "all_tokens_per_s_e2e": 3099.9136709612935, "decode_tokens_per_s": 34.377516319602876, "end_to_end_ms": 2642.654238, "handoff_estimate_ms": 65.39738550000001, "handoff_plus_first_decode_ms": 88.73967, "output_tokens_per_s_e2e": 6.054518888596276, "prefill_tokens_per_s": 3705.7138493398033, "steady_tpot_ms": 23.342284499999998, "steady_tpot_p95_ms": 31.46177735, "ttft_ms": 2206.322542 }, "prefill": { "gpu_active_fraction": 0.9908256880733946, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 218, "gross_j": { "core": 14.744774, "dram": 4.695605, "package-0": 54.085433, "psys": 94.918214, "uncore": 27.768544 }, "logits": { "l2": 1350.4952051518917, "sha256": "c479574faa13fa43ac899e354ac4ef981a21beba7736c1d8f95eba41660cb26f", "sum": 383524.78882252425, "top1": 89372 }, "net_j": { "core": 6.423061845552484, "dram": 3.6061015554311924, "package-0": 41.450441679694016, "psys": 72.50988968276823, "uncore": 27.764234885656805 }, "npu_busy_delta_us": 4810, "npu_utilization": 0.0021800982895455543, "seconds": 2.206322542, "watts": { "core": 6.682963945350471, "dram": 2.128249569414044, "package-0": 24.513837832147754, "psys": 43.02100540293532, "uncore": 12.585895068101967 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 997.4966629813678, "sha256": "3397ea1d16a2a94fab376b2782534dfa279ee052f3d3ce5772911a05e328d0af", "sum": -29625.88963482529, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 35, "gross_j": { "core": 3.59014, "dram": 0.864317, "package-0": 6.184249, "psys": 10.535923, "uncore": 0.0 }, "net_j": { "core": 2.2791073959768493, "dram": 0.6926726903296824, "package-0": 4.1936868097880255, "psys": 7.00563461268602, "uncore": 0.0 }, "npu_busy_delta_us": 332979, "npu_utilization": 0.9579592599745083, "seconds": 0.347592026, "token_latencies_s": [ 0.03278367, 0.030749989, 0.027886746, 0.025532029, 0.024503872, 0.02396962, 0.023495316, 0.023189253, 0.022995276, 0.022671885, 0.022561967, 0.022376598, 0.022445836, 0.022410346 ], "watts": { "core": 10.328602877673609, "dram": 2.4865846606043833, "package-0": 17.791688351331743, "psys": 30.311175780539916, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 1.3113034395501018, "compile_target": "AUTO:NPU,GPU", "context_length": 8192, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 396764, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_0_5b_ov_fp16/openvino_model.xml", "key": "qwen25_0_5b_fp16", "size_b": 0.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": null, "key": "openvino_auto", "label": "OpenVINO AUTO", "mode": "stock", "prefill_device": null }, "prompt_length": 8176, "prompt_sha256": "998ace73fce48442f97bcb84c31be293d0d94180b17f70a4d127bba1092ad744", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.03969423286616802, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 22.445866, "dram": 4.602283, "package-0": 71.283997, "psys": 113.860608, "uncore": 37.622889 }, "net_j": { "core": 10.937785561111768, "dram": 3.095610689561734, "package-0": 53.811090732625104, "psys": 82.87217749948371, "uncore": 37.61692993369712 } }, "first_decode": { "gpu_active_fraction": 0.5714285714285714, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2050, "gpu_sample_count": 7, "gross_j": { "core": 2.186091, "dram": 0.090271, "package-0": 2.560357, "psys": 3.409293, "uncore": 0.156372 }, "logits": { "l2": 751.8547055410498, "sha256": "87ebcfb2f1baeab184e419908d200db26fa2774137672858533c8384b10828b9", "sum": 20114.015098012285, "top1": 78 }, "net_j": { "core": 1.947039034982144, "dram": 0.058973601762222695, "package-0": 2.1974005017403915, "psys": 2.7655848469934394, "uncore": 0.15624821510668613 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.063379474, "watts": { "core": 34.49209755196137, "dram": 1.424293928346581, "package-0": 40.39725858248681, "psys": 53.791752831523965, "uncore": 2.4672341080015907 } }, "metrics": { "all_tokens_per_s_e2e": 2684.9162425121453, "decode_tokens_per_s": 37.92086452533005, "end_to_end_ms": 3051.1193869999997, "handoff_estimate_ms": 40.368918, "handoff_plus_first_decode_ms": 63.379474, "output_tokens_per_s_e2e": 5.243977036156534, "prefill_tokens_per_s": 3078.8247076970656, "steady_tpot_ms": 23.010556, "steady_tpot_p95_ms": 26.6206027, "ttft_ms": 2655.558785 }, "prefill": { "gpu_active_fraction": 0.8015267175572519, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 262, "gross_j": { "core": 16.263447, "dram": 3.74999, "package-0": 60.771939, "psys": 97.721734, "uncore": 35.144258 }, "logits": { "l2": 655.8555450667305, "sha256": "e30c9d66db32ee71e32369110e91308f4656be42fddef82b606d730823ce7e00", "sum": 5070.977332231589, "top1": 8 }, "net_j": { "core": 6.247324970103076, "dram": 2.438649346747291, "package-0": 45.56429752405242, "psys": 70.7507805854498, "uncore": 35.1390714933892 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.655558785, "watts": { "core": 6.1243031379551995, "dram": 1.4121284082212473, "package-0": 22.884802755364348, "psys": 36.79893457903625, "uncore": 13.234223319970678 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 724.3605232286193, "sha256": "546555705e83653d0439120b2a2a1aa452842d59c8f62055b758ef9c1470d280", "sum": 84226.24423345178, "top1": 42 }, "gpu_active_fraction": 0.9696969696969697, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 33, "gross_j": { "core": 3.996328, "dram": 0.762022, "package-0": 7.951701, "psys": 12.729581, "uncore": 2.322259 }, "net_j": { "core": 2.743421556026549, "dram": 0.5979877410522202, "package-0": 6.049392706832295, "psys": 9.355812067040477, "uncore": 2.321610225201232 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.332181128, "token_latencies_s": [ 0.030880964, 0.024326562, 0.024064383, 0.023416013, 0.023134239, 0.02300785, 0.022789099, 0.022861265, 0.023013262, 0.02287143, 0.022856356, 0.022959546, 0.023098143, 0.022873856 ], "watts": { "core": 12.030569057493237, "dram": 2.2939954614158573, "package-0": 23.937846944754792, "psys": 38.3212046892682, "uncore": 6.990942001979112 } } }, { "energy": { "gross_j": { "core": 26.724359, "dram": 4.9849120000000005, "package-0": 77.11448899999999, "psys": 124.629258, "uncore": 38.264795 }, "net_j": { "core": 14.443491455271161, "dram": 3.377064090345446, "package-0": 58.46824746296883, "psys": 91.55990187090423, "uncore": 38.25843577229521 } }, "first_decode": { "gpu_active_fraction": 0.5714285714285714, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2000, "gpu_sample_count": 7, "gross_j": { "core": 2.099421, "dram": 0.085266, "package-0": 2.435175, "psys": 3.251029, "uncore": 0.127258 }, "logits": { "l2": 751.8547055410498, "sha256": "87ebcfb2f1baeab184e419908d200db26fa2774137672858533c8384b10828b9", "sum": 20114.015098012285, "top1": 78 }, "net_j": { "core": 1.874450511994681, "dram": 0.05581219084882378, "package-0": 2.093598638006628, "psys": 2.6452388016049704, "uncore": 0.1271415067184893 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.059646074, "watts": { "core": 35.19797463953788, "dram": 1.4295324785332895, "package-0": 40.82707941515145, "psys": 54.50533089570992, "uncore": 2.1335519920389063 } }, "metrics": { "all_tokens_per_s_e2e": 2515.964933093783, "decode_tokens_per_s": 38.37590018029254, "end_to_end_ms": 3256.007225, "handoff_estimate_ms": 36.61781, "handoff_plus_first_decode_ms": 59.646074, "output_tokens_per_s_e2e": 4.913994009948795, "prefill_tokens_per_s": 2853.615810537976, "steady_tpot_ms": 23.028264, "steady_tpot_p95_ms": 26.27539825, "ttft_ms": 2865.136915 }, "prefill": { "gpu_active_fraction": 0.7667844522968198, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 283, "gross_j": { "core": 19.898448, "dram": 4.136586, "package-0": 65.998184, "psys": 107.858977, "uncore": 35.821686 }, "logits": { "l2": 655.8555450667305, "sha256": "e30c9d66db32ee71e32369110e91308f4656be42fddef82b606d730823ce7e00", "sum": 5070.977332231589, "top1": 8 }, "net_j": { "core": 9.091848223297106, "dram": 2.721753635905093, "package-0": 49.590347265622256, "psys": 78.75946141131644, "uncore": 35.81609017149324 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.865136915, "watts": { "core": 6.945025173430499, "dram": 1.443765559105925, "package-0": 23.03491454613435, "psys": 37.6453133654173, "uncore": 12.502608797667179 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 724.3605232286193, "sha256": "546555705e83653d0439120b2a2a1aa452842d59c8f62055b758ef9c1470d280", "sum": 84226.24423345178, "top1": 42 }, "gpu_active_fraction": 0.9696969696969697, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 33, "gross_j": { "core": 4.72649, "dram": 0.76306, "package-0": 8.68113, "psys": 13.519252, "uncore": 2.315851 }, "net_j": { "core": 3.4771927199793744, "dram": 0.599498263591529, "package-0": 6.784301559339948, "psys": 10.155201657982818, "uncore": 2.3152040940834846 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.331224236, "token_latencies_s": [ 0.030248507, 0.024136032, 0.023949366, 0.023264652, 0.022959251, 0.023174113, 0.023070902, 0.022985626, 0.022950056, 0.022904245, 0.022951011, 0.022750485, 0.023188016, 0.022661465 ], "watts": { "core": 14.269758931529395, "dram": 2.303756540327562, "package-0": 26.20922340960581, "psys": 40.81601081872523, "uncore": 6.991792110285069 } } }, { "energy": { "gross_j": { "core": 24.925228999999998, "dram": 4.999315, "package-0": 75.394217, "psys": 124.082691, "uncore": 38.331445 }, "net_j": { "core": 12.623916824554648, "dram": 3.3887904181444397, "package-0": 56.716934046628566, "psys": 90.95828250822466, "uncore": 38.325075185741646 } }, "first_decode": { "gpu_active_fraction": 0.6666666666666666, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 1950, "gpu_sample_count": 6, "gross_j": { "core": 1.947139, "dram": 0.082031, "package-0": 2.24853, "psys": 3.00219, "uncore": 0.1109 }, "logits": { "l2": 751.8547055410498, "sha256": "87ebcfb2f1baeab184e419908d200db26fa2774137672858533c8384b10828b9", "sum": 20114.015098012285, "top1": 78 }, "net_j": { "core": 1.7449360244264938, "dram": 0.055557981671475125, "package-0": 1.9415219235495462, "psys": 2.457707113489821, "uncore": 0.11079529609699193 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.053609759, "watts": { "core": 36.32060722377058, "dram": 1.5301505086042264, "package-0": 41.94255004951617, "psys": 56.00081134481504, "uncore": 2.068653209203944 } }, "metrics": { "all_tokens_per_s_e2e": 2511.783430078562, "decode_tokens_per_s": 38.28302305627987, "end_to_end_ms": 3261.427678, "handoff_estimate_ms": 30.879824, "handoff_plus_first_decode_ms": 53.609759, "output_tokens_per_s_e2e": 4.905827011872192, "prefill_tokens_per_s": 2849.1685594538003, "steady_tpot_ms": 22.729935, "steady_tpot_p95_ms": 31.340761199999996, "ttft_ms": 2869.6090910000003 }, "prefill": { "gpu_active_fraction": 0.7640845070422535, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 284, "gross_j": { "core": 19.043957, "dram": 4.153492, "package-0": 65.269303, "psys": 107.971525, "uncore": 35.932159 }, "logits": { "l2": 655.8555450667305, "sha256": "e30c9d66db32ee71e32369110e91308f4656be42fddef82b606d730823ce7e00", "sum": 5070.977332231589, "top1": 8 }, "net_j": { "core": 8.220489262848181, "dram": 2.7364512320729397, "package-0": 48.83585536609726, "psys": 78.82658814207586, "uncore": 35.92655443699618 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.869609091, "watts": { "core": 6.636429003423449, "dram": 1.447406900482251, "package-0": 22.74501541157823, "psys": 37.62586525761742, "uncore": 12.521621538172077 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 724.3605232286193, "sha256": "546555705e83653d0439120b2a2a1aa452842d59c8f62055b758ef9c1470d280", "sum": 84226.24423345178, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 34, "gross_j": { "core": 3.934133, "dram": 0.763792, "package-0": 7.876384, "psys": 13.108976, "uncore": 2.288386 }, "net_j": { "core": 2.6584915372799727, "dram": 0.596781204400025, "package-0": 5.939556756981758, "psys": 9.673987252658968, "uncore": 2.2877254526484743 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.338208828, "token_latencies_s": [ 0.036353374, 0.028641662, 0.02324467, 0.022730129, 0.022715643, 0.022739338, 0.022724185, 0.02267181, 0.022729741, 0.022777227, 0.02277903, 0.022720517, 0.022663035, 0.022695453 ], "watts": { "core": 11.632259936159915, "dram": 2.2583443623180646, "package-0": 23.288522793970355, "psys": 38.760005401160015, "uncore": 6.76619239519082 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 1.6709308847784996, "compile_target": "AUTO:GPU", "context_length": 1024, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 393335, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 2.4565200143075967, "dram": 0.4907827289910671, "package-0": 4.295934585025856, "psys": 9.931950112327097, "uncore": 0.002123975024632864 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_1_5b_ov_fp16/openvino_model.xml", "key": "qwen25_1_5b_fp16", "size_b": 1.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "GPU", "key": "gpu_gpu", "label": "iGPU -> iGPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 1008, "prompt_sha256": "2669029615323cd09063f0eb8d822f5daea575e1396736877003e47194c143f8", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.0455717658624053, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 10.434726999999999, "dram": 2.773919, "package-0": 26.957389, "psys": 44.033579, "uncore": 10.756931 }, "net_j": { "core": 7.956433139067665, "dram": 2.2787861220971055, "package-0": 22.623376556776588, "psys": 34.01359510119007, "uncore": 10.754788198706438 } }, "first_decode": { "gpu_active_fraction": 0.8333333333333334, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 6, "gross_j": { "core": 2.050776, "dram": 0.152954, "package-0": 2.744622, "psys": 3.765798, "uncore": 0.375732 }, "logits": { "l2": 990.1133752307505, "sha256": "9ad38a6a770523e4f4bdb5ee760a99cce289de34be1c5c32497a309a9efb2956", "sum": 291550.6807441355, "top1": 78 }, "net_j": { "core": 1.9041113370444558, "dram": 0.12365218966967964, "package-0": 2.4881364862153816, "psys": 3.172818456893646, "uncore": 0.37560518968325135 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.059704241, "watts": { "core": 34.348916687509686, "dram": 2.5618615602198176, "package-0": 45.97030217669127, "psys": 63.07421276823535, "uncore": 6.293221280545213 } }, "metrics": { "all_tokens_per_s_e2e": 1015.0033191401506, "decode_tokens_per_s": 22.221603605698732, "end_to_end_ms": 1008.8636960000001, "handoff_estimate_ms": 16.121927, "handoff_plus_first_decode_ms": 59.704240999999996, "output_tokens_per_s_e2e": 15.859426861564852, "prefill_tokens_per_s": 3019.3661335044185, "steady_tpot_ms": 43.582314, "steady_tpot_p95_ms": 45.64459405, "ttft_ms": 333.84490500000004 }, "prefill": { "gpu_active_fraction": 0.8181818181818182, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 33, "gross_j": { "core": 1.807735, "dram": 0.684874, "package-0": 8.468484, "psys": 13.987269, "uncore": 5.107287 }, "logits": { "l2": 1146.1615279083735, "sha256": "9992a7f0f85fa0507bed1eb81bdf57b5726e3f270c5327fd51897bcdd6e10001", "sum": 371186.4616530363, "top1": 291 }, "net_j": { "core": 0.9876383091928818, "dram": 0.5210286864643364, "package-0": 7.034308126575828, "psys": 10.671538058285421, "uncore": 5.10657792175968 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.333844905, "watts": { "core": 5.41489467991132, "dram": 2.0514735727358184, "package-0": 25.36652161877384, "psys": 41.89750626866688, "uncore": 15.298382343142244 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 946.9425538034262, "sha256": "d666fa852f546efdb063aa00cdee49c7db96fa7b0654c593aeeadfd6ccfa0e85", "sum": 270664.63174004806, "top1": 42 }, "gpu_active_fraction": 0.9836065573770492, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 61, "gross_j": { "core": 6.576216, "dram": 1.936091, "package-0": 15.744283, "psys": 26.280512, "uncore": 5.273912 }, "net_j": { "core": 5.064683492830327, "dram": 1.6341052459630896, "package-0": 13.100931943985378, "psys": 20.169238586011005, "uncore": 5.272605087263507 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.61531455, "token_latencies_s": [ 0.047299509, 0.044753486, 0.044034376, 0.043584331, 0.044182667, 0.043532346, 0.043443539, 0.043244585, 0.043237435, 0.043233129, 0.043337202, 0.043677565, 0.043580297, 0.044111402 ], "watts": { "core": 10.687567846396611, "dram": 3.146506124387925, "package-0": 25.587373157355046, "psys": 42.710694879553884, "uncore": 8.571082871354172 } } }, { "energy": { "gross_j": { "core": 9.975317, "dram": 2.820732, "package-0": 26.794425999999998, "psys": 44.762031, "uncore": 10.913119 }, "net_j": { "core": 7.410699591157208, "dram": 2.308352730383246, "package-0": 22.30945190663298, "psys": 34.39303257398403, "uncore": 10.91090156098367 } }, "first_decode": { "gpu_active_fraction": 0.8333333333333334, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 6, "gross_j": { "core": 1.512021, "dram": 0.145202, "package-0": 2.110224, "psys": 3.011894, "uncore": 0.301147 }, "logits": { "l2": 990.1133752307505, "sha256": "9ad38a6a770523e4f4bdb5ee760a99cce289de34be1c5c32497a309a9efb2956", "sum": 291550.6807441355, "top1": 78 }, "net_j": { "core": 1.3820457006522604, "dram": 0.11923452090608569, "package-0": 1.8829246576722616, "psys": 2.4863911824215355, "uncore": 0.30103461976779106 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.052910336, "watts": { "core": 28.577044001383776, "dram": 2.7443031168806034, "package-0": 39.88302020988867, "psys": 56.92449203119783, "uncore": 5.691647847407357 } }, "metrics": { "all_tokens_per_s_e2e": 980.8388908137426, "decode_tokens_per_s": 22.553990554533097, "end_to_end_ms": 1044.00428, "handoff_estimate_ms": 9.532815, "handoff_plus_first_decode_ms": 52.910336, "output_tokens_per_s_e2e": 15.325607668964729, "prefill_tokens_per_s": 2660.0973240940975, "steady_tpot_ms": 43.377521, "steady_tpot_p95_ms": 45.06358405, "ttft_ms": 378.93350399999997 }, "prefill": { "gpu_active_fraction": 0.7105263157894737, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 38, "gross_j": { "core": 2.321222, "dram": 0.744322, "package-0": 9.391822, "psys": 15.725607, "uncore": 5.351488 }, "logits": { "l2": 1146.1615279083735, "sha256": "9992a7f0f85fa0507bed1eb81bdf57b5726e3f270c5327fd51897bcdd6e10001", "sum": 371186.4616530363, "top1": 291 }, "net_j": { "core": 1.3903642633322923, "dram": 0.5583479808007326, "package-0": 7.7639484547413655, "psys": 11.9620583423827, "uncore": 5.350683154701507 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.378933504, "watts": { "core": 6.125671062329713, "dram": 1.9642549210956022, "package-0": 24.78488151842071, "psys": 41.499647917118466, "uncore": 14.122498917382613 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 946.9425538034262, "sha256": "d666fa852f546efdb063aa00cdee49c7db96fa7b0654c593aeeadfd6ccfa0e85", "sum": 270664.63174004806, "top1": 42 }, "gpu_active_fraction": 0.9836065573770492, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 61, "gross_j": { "core": 6.142074, "dram": 1.931208, "package-0": 15.29238, "psys": 26.02453, "uncore": 5.260484 }, "net_j": { "core": 4.6382896271726555, "dram": 1.6307702286764276, "package-0": 12.662578794219353, "psys": 19.944583049179794, "uncore": 5.259183786514372 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.61216044, "token_latencies_s": [ 0.04690371, 0.044072747, 0.043788318, 0.043391651, 0.043325283, 0.043351694, 0.04308949, 0.043349156, 0.043239634, 0.043385532, 0.04336951, 0.043548176, 0.043253087, 0.044067795 ], "watts": { "core": 10.033438292745608, "dram": 3.1547415902928977, "package-0": 24.981000078998896, "psys": 42.51259686104512, "uncore": 8.5933092964975 } } }, { "energy": { "gross_j": { "core": 9.011085000000001, "dram": 2.8106, "package-0": 25.564327, "psys": 43.079784000000004, "uncore": 10.739536000000001 }, "net_j": { "core": 6.486228325435229, "dram": 2.3061644400058015, "package-0": 21.148886031751488, "psys": 32.87154210231628, "uncore": 10.737352939212252 } }, "first_decode": { "gpu_active_fraction": 0.8333333333333334, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 6, "gross_j": { "core": 1.789241, "dram": 0.147949, "package-0": 2.376154, "psys": 3.312919, "uncore": 0.277831 }, "logits": { "l2": 990.1133752307505, "sha256": "9ad38a6a770523e4f4bdb5ee760a99cce289de34be1c5c32497a309a9efb2956", "sum": 291550.6807441355, "top1": 78 }, "net_j": { "core": 1.6528156578115332, "dram": 0.12069288103788024, "package-0": 2.137574895263247, "psys": 2.76133802985475, "uncore": 0.27771304288267645 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.055536019, "watts": { "core": 32.21766760055308, "dram": 2.6640188235314453, "package-0": 42.785817975177515, "psys": 59.65351963740865, "uncore": 5.002717245541133 } }, "metrics": { "all_tokens_per_s_e2e": 996.2848663814116, "decode_tokens_per_s": 22.477790538086463, "end_to_end_ms": 1027.8184829999998, "handoff_estimate_ms": 12.014416, "handoff_plus_first_decode_ms": 55.536018999999996, "output_tokens_per_s_e2e": 15.566951037209556, "prefill_tokens_per_s": 2796.169961729199, "steady_tpot_ms": 43.521603, "steady_tpot_p95_ms": 44.418626749999994, "ttft_ms": 360.493108 }, "prefill": { "gpu_active_fraction": 0.75, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 36, "gross_j": { "core": 2.164362, "dram": 0.730161, "package-0": 9.003578, "psys": 15.070762, "uncore": 5.185656 }, "logits": { "l2": 1146.1615279083735, "sha256": "9992a7f0f85fa0507bed1eb81bdf57b5726e3f270c5327fd51897bcdd6e10001", "sum": 371186.4616530363, "top1": 291 }, "net_j": { "core": 1.2788034651780502, "dram": 0.5532372086732884, "package-0": 7.454923189679338, "psys": 11.490362435506256, "uncore": 5.184890321642055 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.360493108, "watts": { "core": 6.003892867766004, "dram": 2.0254506502243586, "package-0": 24.97572852349787, "psys": 41.8059642904463, "uncore": 14.384896368115863 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 946.9425538034262, "sha256": "d666fa852f546efdb063aa00cdee49c7db96fa7b0654c593aeeadfd6ccfa0e85", "sum": 270664.63174004806, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 61, "gross_j": { "core": 5.057482, "dram": 1.93249, "package-0": 14.184595, "psys": 24.696103, "uncore": 5.276049 }, "net_j": { "core": 3.5546092024456453, "dram": 1.6322343502946326, "package-0": 11.556387946808904, "psys": 18.61984163695528, "uncore": 5.274749574687521 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.611789356, "token_latencies_s": [ 0.04513961, 0.044030405, 0.043728651, 0.043572165, 0.043488254, 0.043425616, 0.043927817, 0.043484576, 0.043170101, 0.043554952, 0.043998679, 0.043355741, 0.043409205, 0.043478174 ], "watts": { "core": 8.266704790463862, "dram": 3.1587506076192673, "package-0": 23.18542299058894, "psys": 40.367003377548144, "uncore": 8.623963376048016 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 3.4886896423995495, "compile_target": "NPU", "context_length": 1024, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 393404, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 2.4565200143075967, "dram": 0.4907827289910671, "package-0": 4.295934585025856, "psys": 9.931950112327097, "uncore": 0.002123975024632864 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_1_5b_ov_fp16/openvino_model.xml", "key": "qwen25_1_5b_fp16", "size_b": 1.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "gpu_npu", "label": "iGPU -> NPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 1008, "prompt_sha256": "2669029615323cd09063f0eb8d822f5daea575e1396736877003e47194c143f8", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "GPU", "NPUW_ONLINE_PIPELINE": "NONE" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 1024, "NPUW_LLM_MIN_RESPONSE_LEN": 1024, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.04515166487544775, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 11.611725, "dram": 3.028618, "package-0": 25.76147, "psys": 43.123241, "uncore": 5.447313 }, "net_j": { "core": 8.524323082877697, "dram": 2.411792763668828, "package-0": 20.36225623565407, "psys": 30.640573796409228, "uncore": 5.446590373366908 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 0, "gpu_sample_count": 23, "gross_j": { "core": 3.662466, "dram": 0.273559, "package-0": 4.408558, "psys": 6.869672, "uncore": 0.0 }, "logits": { "l2": 991.8733295788431, "sha256": "8ff682a14bc39705e7bd3c784b3399bc49fa56c100079d57fa5ecae59ced5fc7", "sum": 291802.32683754154, "top1": 78 }, "net_j": { "core": 3.087492529951442, "dram": 0.15868631543633147, "package-0": 3.40305085938338, "psys": 4.544998157664751, "uncore": 0.0 }, "npu_busy_delta_us": 65246, "npu_utilization": 0.27875739178014153, "seconds": 0.234060161, "watts": { "core": 15.647541146483277, "dram": 1.1687550706247698, "package-0": 18.835148968388516, "psys": 29.350026807851336, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 814.7551119601545, "decode_tokens_per_s": 16.364903911732, "end_to_end_ms": 1256.8193620000002, "handoff_estimate_ms": 186.644851, "handoff_plus_first_decode_ms": 234.060161, "output_tokens_per_s_e2e": 12.730548624377414, "prefill_tokens_per_s": 2962.756597647449, "steady_tpot_ms": 47.415310000000005, "steady_tpot_p95_ms": 54.75503085, "ttft_ms": 340.223696 }, "prefill": { "gpu_active_fraction": 0.9705882352941176, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 34, "gross_j": { "core": 2.118464, "dram": 0.71826, "package-0": 9.260231, "psys": 15.358969, "uncore": 5.447313 }, "logits": { "l2": 1145.1161856425492, "sha256": "2ae057c3aedd79f7ab7ca960943e32321cda2cc25585ad5dfc0e9014c0ed5485", "sum": 370676.0797875095, "top1": 291 }, "net_j": { "core": 1.2826976814342963, "dram": 0.5512840860096928, "package-0": 7.798652257708277, "psys": 11.979884224296459, "uncore": 5.446590373366908 }, "npu_busy_delta_us": 8568, "npu_utilization": 0.025183431080003312, "seconds": 0.340223696, "watts": { "core": 6.226679754839886, "dram": 2.111140430383191, "package-0": 27.218065963283166, "psys": 45.1437368430681, "uncore": 16.01097473234198 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 954.2613549585964, "sha256": "1873f2d4527206e819430b61339692e2b6ffc74aac45fa6cf5359f3882dc15ad", "sum": 273881.8022980206, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 67, "gross_j": { "core": 5.830795, "dram": 2.036799, "package-0": 12.092681, "psys": 20.8946, "uncore": 0.0 }, "net_j": { "core": 4.154132871491957, "dram": 1.7018223622228037, "package-0": 9.160553118562412, "psys": 14.115691414448019, "uncore": 0.0 }, "npu_busy_delta_us": 669584, "npu_utilization": 0.9810244230444831, "seconds": 0.682535505, "token_latencies_s": [ 0.05903209, 0.052451999, 0.049565249, 0.04833845, 0.04773908, 0.047415147, 0.0474718, 0.047261906, 0.047311857, 0.047415473, 0.047000139, 0.047188932, 0.046964112, 0.047347732 ], "watts": { "core": 8.54284496159654, "dram": 2.9841656369217007, "package-0": 17.717292230826878, "psys": 30.613205975270105, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 8.833534, "dram": 2.916252, "package-0": 22.609560000000002, "psys": 38.069848, "uncore": 5.403734 }, "net_j": { "core": 6.1429467238449185, "dram": 2.3787054877432166, "package-0": 17.904291218820102, "psys": 27.19154110895417, "uncore": 5.402855637825849 } }, "first_decode": { "gpu_active_fraction": 0.14285714285714285, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 7, "gross_j": { "core": 2.207514, "dram": 0.164245, "package-0": 2.647088, "psys": 3.674551, "uncore": 0.023133 }, "logits": { "l2": 991.8733295788431, "sha256": "8ff682a14bc39705e7bd3c784b3399bc49fa56c100079d57fa5ecae59ced5fc7", "sum": 291802.32683754154, "top1": 78 }, "net_j": { "core": 2.0388429292134433, "dram": 0.13054657786611612, "package-0": 2.352117927594727, "psys": 2.992597402761729, "uncore": 0.022987162335481846 }, "npu_busy_delta_us": 61351, "npu_utilization": 0.8935139777970567, "seconds": 0.068662608, "watts": { "core": 32.15016242901814, "dram": 2.3920588626636494, "package-0": 38.55210393406554, "psys": 53.516041802548486, "uncore": 0.3369082630825791 } }, "metrics": { "all_tokens_per_s_e2e": 934.9172639534885, "decode_tokens_per_s": 19.98933057486968, "end_to_end_ms": 1095.2840849999998, "handoff_estimate_ms": 21.257724000000007, "handoff_plus_first_decode_ms": 68.662608, "output_tokens_per_s_e2e": 14.608082249273258, "prefill_tokens_per_s": 2922.72380879346, "steady_tpot_ms": 47.404883999999996, "steady_tpot_p95_ms": 54.15565405, "ttft_ms": 344.883768 }, "prefill": { "gpu_active_fraction": 0.9714285714285714, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 35, "gross_j": { "core": 1.550472, "dram": 0.718504, "package-0": 8.638405, "psys": 14.633446, "uncore": 5.380601 }, "logits": { "l2": 1145.1161856425492, "sha256": "2ae057c3aedd79f7ab7ca960943e32321cda2cc25585ad5dfc0e9014c0ed5485", "sum": 370676.0797875095, "top1": 291 }, "net_j": { "core": 0.7032581212981822, "dram": 0.549241003156238, "package-0": 7.1568068932347675, "psys": 11.208077621672608, "uncore": 5.379868475490367 }, "npu_busy_delta_us": 6651, "npu_utilization": 0.019284757988378278, "seconds": 0.344883768, "watts": { "core": 4.495636338559141, "dram": 2.0833221701521194, "package-0": 25.0472936145838, "psys": 42.43008038580697, "uncore": 15.601201039997918 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 954.2613549585964, "sha256": "1873f2d4527206e819430b61339692e2b6ffc74aac45fa6cf5359f3882dc15ad", "sum": 273881.8022980206, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 68, "gross_j": { "core": 5.075548, "dram": 2.033503, "package-0": 11.324067, "psys": 19.761851, "uncore": 0.0 }, "net_j": { "core": 3.4008456733332926, "dram": 1.6989179067208622, "package-0": 8.395366397990607, "psys": 12.990866084519833, "uncore": 0.0 }, "npu_busy_delta_us": 667292, "npu_utilization": 0.9788104591996393, "seconds": 0.681737709, "token_latencies_s": [ 0.058657777, 0.051731434, 0.049256804, 0.048283387, 0.0475006, 0.047398644, 0.047336664, 0.047613775, 0.04734079, 0.047295518, 0.047350557, 0.047411124, 0.047344449, 0.04718353 ], "watts": { "core": 7.445015778054901, "dram": 2.982823119734455, "package-0": 16.61059209503108, "psys": 28.987469431590444, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 8.460244, "dram": 2.9150310000000004, "package-0": 22.19733, "psys": 37.127041, "uncore": 5.402574 }, "net_j": { "core": 5.779125186174984, "dram": 2.3793761710002768, "package-0": 17.50861955929925, "psys": 26.287016026038074, "uncore": 5.401700764583309 } }, "first_decode": { "gpu_active_fraction": 0.14285714285714285, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 7, "gross_j": { "core": 2.097712, "dram": 0.160644, "package-0": 2.520501, "psys": 3.504813, "uncore": 0.020081 }, "logits": { "l2": 991.8733295788431, "sha256": "8ff682a14bc39705e7bd3c784b3399bc49fa56c100079d57fa5ecae59ced5fc7", "sum": 291802.32683754154, "top1": 78 }, "net_j": { "core": 1.9349193963468012, "dram": 0.12812002388128813, "package-0": 2.2358111248323014, "psys": 2.846626618972228, "uncore": 0.019940245020459643 }, "npu_busy_delta_us": 62070, "npu_utilization": 0.9366285314343672, "seconds": 0.066269602, "watts": { "core": 31.654211534271777, "dram": 2.4240978540960607, "package-0": 38.03404462878772, "psys": 52.887189514130476, "uncore": 0.30301977669942853 } }, "metrics": { "all_tokens_per_s_e2e": 938.2189560865737, "decode_tokens_per_s": 20.091977160201182, "end_to_end_ms": 1091.429664, "handoff_estimate_ms": 18.838401499999996, "handoff_plus_first_decode_ms": 66.26960199999999, "output_tokens_per_s_e2e": 14.659671188852714, "prefill_tokens_per_s": 2922.8996656289773, "steady_tpot_ms": 47.4312005, "steady_tpot_p95_ms": 53.66817335, "ttft_ms": 344.863018 }, "prefill": { "gpu_active_fraction": 0.9714285714285714, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 35, "gross_j": { "core": 1.836543, "dram": 0.719053, "package-0": 8.928688, "psys": 14.844322, "uncore": 5.382493 }, "logits": { "l2": 1145.1161856425492, "sha256": "2ae057c3aedd79f7ab7ca960943e32321cda2cc25585ad5dfc0e9014c0ed5485", "sum": 370676.0797875095, "top1": 291 }, "net_j": { "core": 0.989380094088479, "dram": 0.5498001868978646, "package-0": 7.447179033877405, "psys": 11.419159709637437, "uncore": 5.381760519562849 }, "npu_busy_delta_us": 6709, "npu_utilization": 0.01945410104831826, "seconds": 0.344863018, "watts": { "core": 5.325427500608372, "dram": 2.085039457608644, "package-0": 25.890534890580813, "psys": 43.044110922905624, "uncore": 15.607625982093563 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 954.2613549585964, "sha256": "1873f2d4527206e819430b61339692e2b6ffc74aac45fa6cf5359f3882dc15ad", "sum": 273881.8022980206, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 68, "gross_j": { "core": 4.525989, "dram": 2.035334, "package-0": 10.748141, "psys": 18.777906, "uncore": 0.0 }, "net_j": { "core": 2.854825695739704, "dram": 1.701455960221124, "package-0": 7.825629400589544, "psys": 12.02122969742841, "uncore": 0.0 }, "npu_busy_delta_us": 667254, "npu_utilization": 0.9808274280844883, "seconds": 0.680297044, "token_latencies_s": [ 0.058306184, 0.051170783, 0.049181799, 0.048039222, 0.047425901, 0.047389814, 0.047449361, 0.047272791, 0.047186459, 0.047390032, 0.047141211, 0.047494376, 0.0474365, 0.047380614 ], "watts": { "core": 6.6529599678813245, "dram": 2.991831315380521, "package-0": 15.799188155813889, "psys": 27.6025100588266, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 8.697680806741118, "compile_target": "NPU", "context_length": 1024, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 393554, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 2.4565200143075967, "dram": 0.4907827289910671, "package-0": 4.295934585025856, "psys": 9.931950112327097, "uncore": 0.002123975024632864 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_1_5b_ov_fp16/openvino_model.xml", "key": "qwen25_1_5b_fp16", "size_b": 1.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU+GPU", "key": "npu_gpu_combined", "label": "NPU + iGPU (subgraph combination)", "mode": "combined", "prefill_device": "NPU+GPU" }, "prompt_length": 1008, "prompt_sha256": "2669029615323cd09063f0eb8d822f5daea575e1396736877003e47194c143f8", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU,GPU", "NPUW_SUBMODEL_DEVICE": "0:GPU,last:NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "NPU,GPU", "NPUW_SUBMODEL_DEVICE": "0:GPU,last:NPU" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 1024, "NPUW_LLM_MIN_RESPONSE_LEN": 1024, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.048760765232145786, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 57.393957, "dram": 9.769201, "package-0": 121.404231, "psys": 215.112425, "uncore": 33.586767 }, "net_j": { "core": 41.64916887460104, "dram": 6.623584421033316, "package-0": 93.869922527911, "psys": 151.45471031425197, "uncore": 33.57315362150045 } }, "first_decode": { "gpu_active_fraction": 0.422680412371134, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 97, "gross_j": { "core": 13.39169, "dram": 1.015806, "package-0": 18.551955, "psys": 29.517502, "uncore": 2.178339 }, "logits": { "l2": 1882.7353047579231, "sha256": "1902aae974c485506c3338d22a40ea0aea7c444494aded21d1f363e35a2038ca", "sum": 661811.1864299811, "top1": 69754 }, "net_j": { "core": 10.9813444259671, "dram": 0.5342483608415665, "package-0": 14.336769757037285, "psys": 19.77223965516588, "uncore": 2.176254948671225 }, "npu_busy_delta_us": 8736, "npu_utilization": 0.008903353559002255, "seconds": 0.981203312, "watts": { "core": 13.648231550200883, "dram": 1.0352655637998907, "package-0": 18.90735056956269, "psys": 30.08296205180359, "uncore": 2.2200689432650424 } }, "metrics": { "all_tokens_per_s_e2e": 159.76566179338394, "decode_tokens_per_s": 2.512044668460185, "end_to_end_ms": 6409.387277, "handoff_estimate_ms": 626.926127, "handoff_plus_first_decode_ms": 981.203312, "output_tokens_per_s_e2e": 2.496338465521624, "prefill_tokens_per_s": 2300.551129431613, "steady_tpot_ms": 354.277185, "steady_tpot_p95_ms": 366.59889565000003, "ttft_ms": 438.155878 }, "prefill": { "gpu_active_fraction": 0.06818181818181818, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 1900, "gpu_sample_count": 44, "gross_j": { "core": 1.441524, "dram": 0.645018, "package-0": 7.825969, "psys": 13.256497, "uncore": 0.161193 }, "logits": { "l2": 1145.1872934436176, "sha256": "241c43a634ad9ad84cb70858b42518580d814415f305684145ca096fb8068099", "sum": 370344.48207614943, "top1": 291 }, "net_j": { "core": 0.36518531630648243, "dram": 0.4299786624716829, "package-0": 5.94368001006743, "psys": 8.904754678281122, "uncore": 0.16026236785823192 }, "npu_busy_delta_us": 393312, "npu_utilization": 0.8976531406934588, "seconds": 0.438155878, "watts": { "core": 3.289979827681326, "dram": 1.4721199289719444, "package-0": 17.861152601038484, "psys": 30.255207485770622, "uncore": 0.36788962123657737 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 3230.8548509007255, "sha256": "3dc7ccdc989a4822f462f6221f23dadd86c14033eb527cada2f58c1e598e55d5", "sum": 1207703.5942871496, "top1": 283 }, "gpu_active_fraction": 0.9979716024340771, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 493, "gross_j": { "core": 42.560743, "dram": 8.108377, "package-0": 95.026307, "psys": 172.338426, "uncore": 31.247235 }, "net_j": { "core": 30.302639132327453, "dram": 5.659357397720067, "package-0": 73.58947276080629, "psys": 122.77771598080497, "uncore": 31.236636304970997 }, "npu_busy_delta_us": 124876, "npu_utilization": 0.025025109643235562, "seconds": 4.990028087, "token_latencies_s": [ 0.369897229, 0.353039063, 0.351452315, 0.352740454, 0.351920798, 0.353892769, 0.354661601, 0.352234731, 0.352330242, 0.357071229, 0.357833043, 0.355913636, 0.362166688, 0.36482287 ], "watts": { "core": 8.529159006314828, "dram": 1.6249161044050855, "package-0": 19.043240908315152, "psys": 34.536564322949474, "uncore": 6.261935695593611 } } }, { "energy": { "gross_j": { "core": 47.142763, "dram": 10.038181999999999, "package-0": 111.652301, "psys": 202.13907, "uncore": 34.269138 }, "net_j": { "core": 32.31522683642132, "dram": 7.027489271528754, "package-0": 85.2990121552312, "psys": 141.21180378549022, "uncore": 34.256108535464314 } }, "first_decode": { "gpu_active_fraction": 0.975609756097561, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 41, "gross_j": { "core": 5.00249, "dram": 0.647276, "package-0": 8.948097, "psys": 15.133933, "uncore": 2.271478 }, "logits": { "l2": 3233.3883385391673, "sha256": "d7d4781f681a7ddbc3a5ad43821a5de1a0bc4a68985453853a781c9d9377abee", "sum": 1208728.0883883238, "top1": 283 }, "net_j": { "core": 3.9943947589986752, "dram": 0.4458708697425975, "package-0": 7.185151375460387, "psys": 11.058105575926831, "uncore": 2.2706063730228667 }, "npu_busy_delta_us": 11230, "npu_utilization": 0.027365191936897616, "seconds": 0.410375342, "watts": { "core": 12.190035530935969, "dram": 1.5772780032188192, "package-0": 21.80466534950826, "psys": 36.87827081969267, "uncore": 5.535123014286761 } }, "metrics": { "all_tokens_per_s_e2e": 166.9255416649712, "decode_tokens_per_s": 2.64201365027035, "end_to_end_ms": 6134.471632000001, "handoff_estimate_ms": 37.32217599999993, "handoff_plus_first_decode_ms": 410.375342, "output_tokens_per_s_e2e": 2.608211588515175, "prefill_tokens_per_s": 2205.766827255172, "steady_tpot_ms": 373.05316600000003, "steady_tpot_p95_ms": 389.02324304999996, "ttft_ms": 456.98393300000004 }, "prefill": { "gpu_active_fraction": 0.06521739130434782, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 1950, "gpu_sample_count": 46, "gross_j": { "core": 0.880674, "dram": 0.635862, "package-0": 7.240643, "psys": 12.296905, "uncore": 0.160461 }, "logits": { "l2": 1145.1872934436176, "sha256": "241c43a634ad9ad84cb70858b42518580d814415f305684145ca096fb8068099", "sum": 370344.48207614943, "top1": 291 }, "net_j": { "core": 0.0, "dram": 0.41158217825718907, "package-0": 5.2774699174241615, "psys": 7.758163375308972, "uncore": 0.1594903775396495 }, "npu_busy_delta_us": 399276, "npu_utilization": 0.8737199957531111, "seconds": 0.456983933, "watts": { "core": 1.9271443401052788, "dram": 1.3914318515001272, "package-0": 15.844414818846596, "psys": 26.908834451297878, "uncore": 0.35113050681368263 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 3244.1967264885366, "sha256": "1c7b735d072ce209290e8ce01671cb97d65a146d735d72abae9c93f26e69c07c", "sum": 1213023.6846235134, "top1": 283 }, "gpu_active_fraction": 0.9980769230769231, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 520, "gross_j": { "core": 41.259599, "dram": 8.755044, "package-0": 95.463561, "psys": 174.708232, "uncore": 31.837199 }, "net_j": { "core": 28.320832077422644, "dram": 6.170036223528967, "package-0": 72.83639086234665, "psys": 122.39553483425442, "uncore": 31.826011784901794 }, "npu_busy_delta_us": 127020, "npu_utilization": 0.024115680735610323, "seconds": 5.267112357, "token_latencies_s": [ 0.41296307, 0.372307188, 0.371511816, 0.376132567, 0.372543283, 0.374152864, 0.375724969, 0.372111054, 0.373419747, 0.372686585, 0.371189293, 0.372352259, 0.374926239, 0.375027675 ], "watts": { "core": 7.833438173227107, "dram": 1.6622094625273245, "package-0": 18.124458817197773, "psys": 33.16964214135522, "uncore": 6.044526268304931 } } }, { "energy": { "gross_j": { "core": 50.119991, "dram": 10.028783, "package-0": 115.11945599999999, "psys": 207.57100200000002, "uncore": 34.613558999999995 }, "net_j": { "core": 35.25964192780879, "dram": 7.013005840000079, "package-0": 88.72166195168798, "psys": 146.5408423515746, "uncore": 34.60050753141864 } }, "first_decode": { "gpu_active_fraction": 0.9767441860465116, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 43, "gross_j": { "core": 5.127551, "dram": 0.669066, "package-0": 9.258339, "psys": 15.566061, "uncore": 2.369745 }, "logits": { "l2": 3244.8463463137473, "sha256": "988d0c153e14ceb46d50b742f501554610cb77a788fc4aae049ed148b7abb1c7", "sum": 1213281.8226122651, "top1": 283 }, "net_j": { "core": 4.053848076303634, "dram": 0.4545532551624949, "package-0": 7.380659397519452, "psys": 11.224975313133395, "uncore": 2.368816646809094 }, "npu_busy_delta_us": 7291, "npu_utilization": 0.016681045593745274, "seconds": 0.437082913, "watts": { "core": 11.731300509567163, "dram": 1.5307530450177997, "package-0": 21.18211150477987, "psys": 35.61351985406942, "uncore": 5.421728760190632 } }, "metrics": { "all_tokens_per_s_e2e": 166.64411454288808, "decode_tokens_per_s": 2.641463475621028, "end_to_end_ms": 6144.8314740000005, "handoff_estimate_ms": 65.30061900000005, "handoff_plus_first_decode_ms": 437.082913, "output_tokens_per_s_e2e": 2.603814289732626, "prefill_tokens_per_s": 2162.3419166902218, "steady_tpot_ms": 371.782294, "steady_tpot_p95_ms": 390.4651399, "ttft_ms": 466.161245 }, "prefill": { "gpu_active_fraction": 0.06521739130434782, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 1900, "gpu_sample_count": 46, "gross_j": { "core": 0.910582, "dram": 0.657713, "package-0": 7.353313, "psys": 12.34909, "uncore": 0.1604 }, "logits": { "l2": 1145.1872934436176, "sha256": "241c43a634ad9ad84cb70858b42518580d814415f305684145ca096fb8068099", "sum": 370344.48207614943, "top1": 291 }, "net_j": { "core": 0.0, "dram": 0.42892911202902656, "package-0": 5.350714785405788, "psys": 7.719199770359711, "uncore": 0.15940988515816823 }, "npu_busy_delta_us": 399904, "npu_utilization": 0.8578662518373873, "seconds": 0.466161245, "watts": { "core": 1.9533627253805708, "dram": 1.4109130843770592, "package-0": 15.77418345877294, "psys": 26.491026726170684, "uncore": 0.3440869478542773 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 3249.8824613058905, "sha256": "2cbd4ab13a8143a7e97bc1e3d8bae02c5d101b7bab2ad7c8989726a89a5ff0ea", "sum": 1215262.375621643, "top1": 283 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 515, "gross_j": { "core": 44.081858, "dram": 8.702004, "package-0": 98.507804, "psys": 179.655851, "uncore": 32.083414 }, "net_j": { "core": 31.205793851505156, "dram": 6.1295234728085575, "package-0": 75.99028776876274, "psys": 127.59666726808152, "uncore": 32.07228099945138 }, "npu_busy_delta_us": 126205, "npu_utilization": 0.024077629998599456, "seconds": 5.241587316, "token_latencies_s": [ 0.414582814, 0.3774787, 0.366906066, 0.368730497, 0.370806839, 0.372245928, 0.371846289, 0.371718299, 0.371105392, 0.372687579, 0.370395877, 0.37225142, 0.373026945, 0.367747079 ], "watts": { "core": 8.410020732734846, "dram": 1.6601848782404218, "package-0": 18.793506253211483, "psys": 34.2750850399074, "uncore": 6.120934759984831 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 4.743245515041053, "compile_target": "AUTO:NPU,GPU", "context_length": 1024, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 393251, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 2.4565200143075967, "dram": 0.4907827289910671, "package-0": 4.295934585025856, "psys": 9.931950112327097, "uncore": 0.002123975024632864 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_1_5b_ov_fp16/openvino_model.xml", "key": "qwen25_1_5b_fp16", "size_b": 1.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": null, "key": "openvino_auto", "label": "OpenVINO AUTO", "mode": "stock", "prefill_device": null }, "prompt_length": 1008, "prompt_sha256": "2669029615323cd09063f0eb8d822f5daea575e1396736877003e47194c143f8", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.05435913521796465, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 9.007484, "dram": 2.775261, "package-0": 25.480098, "psys": 42.233107000000004, "uncore": 10.7233 }, "net_j": { "core": 6.520063238649946, "dram": 2.2783046787430323, "package-0": 21.130124535929944, "psys": 32.176222152245636, "uncore": 10.72114930733636 } }, "first_decode": { "gpu_active_fraction": 0.7142857142857143, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 7, "gross_j": { "core": 2.052119, "dram": 0.154296, "package-0": 2.766167, "psys": 3.78179, "uncore": 0.391722 }, "logits": { "l2": 990.1133752307505, "sha256": "9ad38a6a770523e4f4bdb5ee760a99cce289de34be1c5c32497a309a9efb2956", "sum": 291550.6807441355, "top1": 78 }, "net_j": { "core": 1.900753122736674, "dram": 0.1240549444064184, "package-0": 2.5014600554845963, "psys": 3.1698029887271777, "uncore": 0.39159112488356845 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.061618011, "watts": { "core": 33.30388252876257, "dram": 2.5040730379953353, "package-0": 44.89218257953831, "psys": 61.37474966532107, "uncore": 6.357264599144559 } }, "metrics": { "all_tokens_per_s_e2e": 1011.2790460451487, "decode_tokens_per_s": 22.150324892843745, "end_to_end_ms": 1012.579074, "handoff_estimate_ms": 18.002156000000006, "handoff_plus_first_decode_ms": 61.618011, "output_tokens_per_s_e2e": 15.801235094455448, "prefill_tokens_per_s": 3005.473324718756, "steady_tpot_ms": 43.615854999999996, "steady_tpot_p95_ms": 45.577434, "ttft_ms": 335.388104 }, "prefill": { "gpu_active_fraction": 0.7941176470588235, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 34, "gross_j": { "core": 1.733577, "dram": 0.684752, "package-0": 8.376077, "psys": 13.872828, "uncore": 5.088305 }, "logits": { "l2": 1146.1615279083735, "sha256": "9992a7f0f85fa0507bed1eb81bdf57b5726e3f270c5327fd51897bcdd6e10001", "sum": 371186.4616530363, "top1": 291 }, "net_j": { "core": 0.9096894099633221, "dram": 0.5201493110477402, "package-0": 6.935271644620151, "psys": 10.541770082804028, "uncore": 5.0875926440435455 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.335388104, "watts": { "core": 5.168868481990047, "dram": 2.0416705059998193, "package-0": 24.97428173540705, "psys": 41.36350644088438, "uncore": 15.171393795171698 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 946.9425538034262, "sha256": "d666fa852f546efdb063aa00cdee49c7db96fa7b0654c593aeeadfd6ccfa0e85", "sum": 270664.63174004806, "top1": 42 }, "gpu_active_fraction": 0.9836065573770492, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 61, "gross_j": { "core": 5.221788, "dram": 1.936213, "package-0": 14.337854, "psys": 24.578489, "uncore": 5.243273 }, "net_j": { "core": 3.70962070594995, "dram": 1.6341004232888738, "package-0": 11.693392835825197, "psys": 18.464649080714427, "uncore": 5.241965538409245 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.615572959, "token_latencies_s": [ 0.047086916, 0.04462669, 0.044764636, 0.043614718, 0.043616992, 0.043427905, 0.043407817, 0.043649949, 0.043628233, 0.043494906, 0.043435934, 0.043486358, 0.043712771, 0.043584855 ], "watts": { "core": 8.482809265180864, "dram": 3.1453834540513013, "package-0": 23.291884073809683, "psys": 39.92782437995299, "uncore": 8.517711707995932 } } }, { "energy": { "gross_j": { "core": 10.167272, "dram": 2.813286, "package-0": 26.940544, "psys": 44.178109, "uncore": 10.910433 }, "net_j": { "core": 7.652518446906631, "dram": 2.3108689204163757, "package-0": 22.542771257002222, "psys": 34.01071500823536, "uncore": 10.908258674650007 } }, "first_decode": { "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 5, "gross_j": { "core": 1.81512, "dram": 0.145935, "package-0": 2.419123, "psys": 3.332755, "uncore": 0.30072 }, "logits": { "l2": 990.1133752307505, "sha256": "9ad38a6a770523e4f4bdb5ee760a99cce289de34be1c5c32497a309a9efb2956", "sum": 291550.6807441355, "top1": 78 }, "net_j": { "core": 1.6869454944095887, "dram": 0.120327297528879, "package-0": 2.1949728655507026, "psys": 2.8145329672945287, "uncore": 0.30060917678379045 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.052177269, "watts": { "core": 34.78756237701901, "dram": 2.7969075959111622, "package-0": 46.36354194773973, "psys": 63.87369565087817, "uncore": 5.76342928182002 } }, "metrics": { "all_tokens_per_s_e2e": 1000.2874800819831, "decode_tokens_per_s": 22.428145358317238, "end_to_end_ms": 1023.705705, "handoff_estimate_ms": 8.667740499999994, "handoff_plus_first_decode_ms": 52.177268999999995, "output_tokens_per_s_e2e": 15.629491876280985, "prefill_tokens_per_s": 2840.211149299021, "steady_tpot_ms": 43.5095285, "steady_tpot_p95_ms": 46.3062299, "ttft_ms": 354.903191 }, "prefill": { "gpu_active_fraction": 0.7714285714285715, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 35, "gross_j": { "core": 2.71441, "dram": 0.727293, "package-0": 9.717443, "psys": 15.68569, "uncore": 5.350938 }, "logits": { "l2": 1146.1615279083735, "sha256": "9992a7f0f85fa0507bed1eb81bdf57b5726e3f270c5327fd51897bcdd6e10001", "sum": 371186.4616530363, "top1": 291 }, "net_j": { "core": 1.8425832081668683, "dram": 0.5531126433933821, "package-0": 8.192802107447061, "psys": 12.160809212282304, "uncore": 5.350184194486154 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.354903191, "watts": { "core": 7.6483110573102735, "dram": 2.049271515284854, "package-0": 27.380545586585043, "psys": 44.19709486353984, "uncore": 15.077176355960125 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 946.9425538034262, "sha256": "d666fa852f546efdb063aa00cdee49c7db96fa7b0654c593aeeadfd6ccfa0e85", "sum": 270664.63174004806, "top1": 42 }, "gpu_active_fraction": 0.9838709677419355, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 62, "gross_j": { "core": 5.637742, "dram": 1.940058, "package-0": 14.803978, "psys": 25.159664, "uncore": 5.258775 }, "net_j": { "core": 4.122989744330175, "dram": 1.6374289794941146, "package-0": 12.154996284004458, "psys": 19.035372828658524, "uncore": 5.257465303380062 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.616625245, "token_latencies_s": [ 0.050392568, 0.043657391, 0.043738194, 0.043583658, 0.043444801, 0.043682359, 0.043491157, 0.044105894, 0.043384604, 0.043478434, 0.043150542, 0.043482167, 0.0435279, 0.043483073 ], "watts": { "core": 9.14289845528462, "dram": 3.14625133455248, "package-0": 24.00806343892067, "psys": 40.80219582965663, "uncore": 8.528316092540129 } } }, { "energy": { "gross_j": { "core": 10.22562, "dram": 2.819756, "package-0": 26.826286, "psys": 44.152597, "uncore": 10.736239999999999 }, "net_j": { "core": 7.691431214705074, "dram": 2.3134559981348963, "package-0": 22.394525141417418, "psys": 33.90662446805531, "uncore": 10.734048870411671 } }, "first_decode": { "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 6, "gross_j": { "core": 2.008113, "dram": 0.150024, "package-0": 2.60357, "psys": 3.597098, "uncore": 0.28723 }, "logits": { "l2": 990.1133752307505, "sha256": "9ad38a6a770523e4f4bdb5ee760a99cce289de34be1c5c32497a309a9efb2956", "sum": 291550.6807441355, "top1": 78 }, "net_j": { "core": 1.8640829965206407, "dram": 0.12124856248982895, "package-0": 2.3516919490887096, "psys": 3.0147706422322686, "uncore": 0.28710546768257283 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.058631724, "watts": { "core": 34.2495983914783, "dram": 2.558751299893552, "package-0": 44.40548260187607, "psys": 61.350711775079304, "uncore": 4.8988837510559975 } }, "metrics": { "all_tokens_per_s_e2e": 992.6160628787688, "decode_tokens_per_s": 22.417209535138554, "end_to_end_ms": 1031.617398, "handoff_estimate_ms": 15.273837000000006, "handoff_plus_first_decode_ms": 58.631724000000006, "output_tokens_per_s_e2e": 15.509625982480763, "prefill_tokens_per_s": 2780.776944719597, "steady_tpot_ms": 43.357887, "steady_tpot_p95_ms": 44.75563305, "ttft_ms": 362.488621 }, "prefill": { "gpu_active_fraction": 0.7777777777777778, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 36, "gross_j": { "core": 2.569817, "dram": 0.734373, "package-0": 9.442664, "psys": 15.448264, "uncore": 5.197131 }, "logits": { "l2": 1146.1615279083735, "sha256": "9992a7f0f85fa0507bed1eb81bdf57b5726e3f270c5327fd51897bcdd6e10001", "sum": 371186.4616530363, "top1": 291 }, "net_j": { "core": 1.679356447554739, "dram": 0.5564698453574114, "package-0": 7.885436596367771, "psys": 11.848045099941755, "uncore": 5.196361083222282 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.362488621, "watts": { "core": 7.089372882686985, "dram": 2.025920146056116, "package-0": 26.049545980092986, "psys": 42.617238459466016, "uncore": 14.337363158221729 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 946.9425538034262, "sha256": "d666fa852f546efdb063aa00cdee49c7db96fa7b0654c593aeeadfd6ccfa0e85", "sum": 270664.63174004806, "top1": 42 }, "gpu_active_fraction": 0.9836065573770492, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 61, "gross_j": { "core": 5.64769, "dram": 1.935359, "package-0": 14.780052, "psys": 25.107235, "uncore": 5.251879 }, "net_j": { "core": 4.1479917706296945, "dram": 1.6357375902876559, "package-0": 12.157396595960936, "psys": 19.043808725881288, "uncore": 5.250582319506816 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.610497053, "token_latencies_s": [ 0.046123105, 0.043322607, 0.043596273, 0.043320962, 0.043528307, 0.043277703, 0.043357916, 0.044019302, 0.043260678, 0.043393456, 0.043330948, 0.043120893, 0.043357858, 0.043454986 ], "watts": { "core": 9.25097012712361, "dram": 3.1701365149751184, "package-0": 24.20986625139368, "psys": 41.12589057821381, "uncore": 8.602627931113043 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 1.6549215903505683, "compile_target": "AUTO:GPU", "context_length": 2048, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 397440, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_1_5b_ov_fp16/openvino_model.xml", "key": "qwen25_1_5b_fp16", "size_b": 1.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "GPU", "key": "gpu_gpu", "label": "iGPU -> iGPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 2032, "prompt_sha256": "cd1a97358e68d07c27508cfacc4ba6882a7a2962b1281e2e2b0c7e5e320da4fa", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.04699633829295635, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 13.147122000000001, "dram": 3.630912, "package-0": 38.231469, "psys": 62.412621, "uncore": 17.326982 }, "net_j": { "core": 7.6239568539536045, "dram": 2.9078026999115694, "package-0": 29.845557256631626, "psys": 47.54009554451513, "uncore": 17.324122017617896 } }, "first_decode": { "gpu_active_fraction": 0.8571428571428571, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 7, "gross_j": { "core": 2.388971, "dram": 0.162353, "package-0": 3.072624, "psys": 4.226369, "uncore": 0.364379 }, "logits": { "l2": 721.436499143929, "sha256": "33527b68f47cd3c749c20c0f2bfed74af94e75dcffa9e888e816a2d29c486e6e", "sum": 152204.33597268048, "top1": 220 }, "net_j": { "core": 2.124563591937882, "dram": 0.127735991323579, "package-0": 2.671169918930398, "psys": 3.5143848744053616, "uncore": 0.3642420856567277 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.070101923, "watts": { "core": 34.078537331993026, "dram": 2.315956439597242, "package-0": 43.830809035010354, "psys": 60.28891675339634, "uncore": 5.1978459991746595 } }, "metrics": { "all_tokens_per_s_e2e": 1398.5745163090473, "decode_tokens_per_s": 20.278340337242266, "end_to_end_ms": 1464.3481459999998, "handoff_estimate_ms": 23.565725499999996, "handoff_plus_first_decode_ms": 70.101923, "output_tokens_per_s_e2e": 10.926363408664432, "prefill_tokens_per_s": 2804.140810703604, "steady_tpot_ms": 46.5361975, "steady_tpot_p95_ms": 53.8247413, "ttft_ms": 724.64264 }, "prefill": { "gpu_active_fraction": 0.8194444444444444, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 72, "gross_j": { "core": 4.563831, "dram": 1.478451, "package-0": 19.339733, "psys": 31.46043, "uncore": 11.421479 }, "logits": { "l2": 834.8475507956192, "sha256": "765152f2d994661cc75a99376a54eb9c804048cab9a622294153aa34f8ebd025", "sum": 210672.0778277011, "top1": 220 }, "net_j": { "core": 1.8306551609621668, "dram": 1.1206154445305068, "package-0": 15.1899075359389, "psys": 24.100659620962176, "uncore": 11.420063718265679 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.72464264, "watts": { "core": 6.298043681227481, "dram": 2.0402484181720246, "package-0": 26.688648904237816, "psys": 43.41509630181299, "uncore": 15.761533160676274 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 677.6366610405684, "sha256": "ad393a0a123b7a19df0f9f33da818df6a7780c377cdd4707a8b74388cd4bd9a9", "sum": 116901.59847021359, "top1": 42 }, "gpu_active_fraction": 0.9848484848484849, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 66, "gross_j": { "core": 6.19432, "dram": 1.990108, "package-0": 15.819112, "psys": 26.725822, "uncore": 5.541124 }, "net_j": { "core": 3.668738101053556, "dram": 1.6594512640574837, "package-0": 11.984479801762328, "psys": 19.925051049147598, "uncore": 5.53981621369549 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.669603583, "token_latencies_s": [ 0.052774091, 0.055775949, 0.047196928, 0.046725191, 0.046533828, 0.046494774, 0.046538567, 0.046446849, 0.046454437, 0.046180024, 0.046253406, 0.046389088, 0.04854073, 0.047275331 ], "watts": { "core": 9.25072708280296, "dram": 2.972068923352819, "package-0": 23.62459282121255, "psys": 39.91290172053933, "uncore": 8.275230510527301 } } }, { "energy": { "gross_j": { "core": 13.10867, "dram": 3.71929, "package-0": 38.512414, "psys": 63.272422000000006, "uncore": 17.464005 }, "net_j": { "core": 7.454434210195799, "dram": 2.97902053921319, "package-0": 29.92749555999388, "psys": 48.04695553438617, "uncore": 17.461077147162815 } }, "first_decode": { "gpu_active_fraction": 0.7142857142857143, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 7, "gross_j": { "core": 1.861568, "dram": 0.153137, "package-0": 2.503961, "psys": 3.4887, "uncore": 0.320983 }, "logits": { "l2": 721.436499143929, "sha256": "33527b68f47cd3c749c20c0f2bfed74af94e75dcffa9e888e816a2d29c486e6e", "sum": 152204.33597268048, "top1": 220 }, "net_j": { "core": 1.631492235427416, "dram": 0.12301479277132733, "package-0": 2.154633213309647, "psys": 2.8691625473230142, "uncore": 0.32086386312543136 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.060999628, "watts": { "core": 30.51769430462756, "dram": 2.5104579326287038, "package-0": 41.04879131394047, "psys": 57.19215205705845, "uncore": 5.262048483312062 } }, "metrics": { "all_tokens_per_s_e2e": 1366.154208948247, "decode_tokens_per_s": 20.844968010130874, "end_to_end_ms": 1499.0987010000001, "handoff_estimate_ms": 14.474234500000001, "handoff_plus_first_decode_ms": 60.999628, "output_tokens_per_s_e2e": 10.67307975740818, "prefill_tokens_per_s": 2606.797320742437, "steady_tpot_ms": 46.5253935, "steady_tpot_p95_ms": 49.796110549999995, "ttft_ms": 779.500571 }, "prefill": { "gpu_active_fraction": 0.7922077922077922, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 77, "gross_j": { "core": 5.351366, "dram": 1.581722, "package-0": 20.582589, "psys": 33.612402, "uncore": 11.631989 }, "logits": { "l2": 834.8475507956192, "sha256": "765152f2d994661cc75a99376a54eb9c804048cab9a622294153aa34f8ebd025", "sum": 210672.0778277011, "top1": 220 }, "net_j": { "core": 2.411279384101444, "dram": 1.1967970667769536, "package-0": 16.118607527993372, "psys": 25.69547178335022, "uncore": 11.630466576588608 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.779500571, "watts": { "core": 6.865121334208746, "dram": 2.0291479683855163, "package-0": 26.40484146611356, "psys": 43.12043281364062, "uncore": 14.922361102414126 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 677.6366610405684, "sha256": "ad393a0a123b7a19df0f9f33da818df6a7780c377cdd4707a8b74388cd4bd9a9", "sum": 116901.59847021359, "top1": 42 }, "gpu_active_fraction": 0.9848484848484849, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 66, "gross_j": { "core": 5.895736, "dram": 1.984431, "package-0": 15.425864, "psys": 26.17132, "uncore": 5.511033 }, "net_j": { "core": 3.411662590666939, "dram": 1.659208679664909, "package-0": 11.654254818690859, "psys": 19.482321203712935, "uncore": 5.509746707448775 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.658598502, "token_latencies_s": [ 0.052068207, 0.048572674, 0.046855646, 0.046603811, 0.046443309, 0.046609289, 0.046416766, 0.046500834, 0.046319965, 0.046558583, 0.046549953, 0.046453821, 0.046215314, 0.046396657 ], "watts": { "core": 8.951942620725852, "dram": 3.013111924751994, "package-0": 23.42225795102097, "psys": 39.73789785510323, "uncore": 8.367818911316018 } } }, { "energy": { "gross_j": { "core": 14.061609, "dram": 3.7170330000000003, "package-0": 39.644611999999995, "psys": 64.618244, "uncore": 17.620744 }, "net_j": { "core": 8.384942202594605, "dram": 2.9738268046094114, "package-0": 31.025636195224855, "psys": 49.33237634725531, "uncore": 17.61780453203184 } }, "first_decode": { "gpu_active_fraction": 0.7142857142857143, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 7, "gross_j": { "core": 2.066157, "dram": 0.155701, "package-0": 2.720513, "psys": 3.73046, "uncore": 0.306029 }, "logits": { "l2": 721.436499143929, "sha256": "33527b68f47cd3c749c20c0f2bfed74af94e75dcffa9e888e816a2d29c486e6e", "sum": 152204.33597268048, "top1": 220 }, "net_j": { "core": 1.8192842632141077, "dram": 0.12337968293129936, "package-0": 2.345682096380861, "psys": 3.06569243194319, "uncore": 0.3059011653872083 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.065452983, "watts": { "core": 31.56704103157529, "dram": 2.378822062242755, "package-0": 41.56438523206803, "psys": 56.99449939508486, "uncore": 4.675554664941703 } }, "metrics": { "all_tokens_per_s_e2e": 1360.7559327169686, "decode_tokens_per_s": 20.416328324438073, "end_to_end_ms": 1505.045799, "handoff_estimate_ms": 18.99911950000001, "handoff_plus_first_decode_ms": 65.452983, "output_tokens_per_s_e2e": 10.630905724351317, "prefill_tokens_per_s": 2637.797167485111, "steady_tpot_ms": 46.4538635, "steady_tpot_p95_ms": 54.70955785, "ttft_ms": 770.339746 }, "prefill": { "gpu_active_fraction": 0.7792207792207793, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 77, "gross_j": { "core": 5.527451, "dram": 1.574337, "package-0": 20.859626, "psys": 33.895238, "uncore": 11.754304 }, "logits": { "l2": 834.8475507956192, "sha256": "765152f2d994661cc75a99376a54eb9c804048cab9a622294153aa34f8ebd025", "sum": 210672.0778277011, "top1": 220 }, "net_j": { "core": 2.621916788519233, "dram": 1.1939357709057004, "package-0": 16.448106007291372, "psys": 26.071348911157088, "uncore": 11.752799468370815 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.770339746, "watts": { "core": 7.1753418263842255, "dram": 2.043691771292819, "package-0": 27.078475579526955, "psys": 44.00037538761501, "uncore": 15.258597341023085 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 677.6366610405684, "sha256": "ad393a0a123b7a19df0f9f33da818df6a7780c377cdd4707a8b74388cd4bd9a9", "sum": 116901.59847021359, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 67, "gross_j": { "core": 6.468001, "dram": 1.986995, "package-0": 16.064473, "psys": 26.992546, "uncore": 5.560411 }, "net_j": { "core": 3.9437411508612636, "dram": 1.6565113507724116, "package-0": 12.231848091552623, "psys": 20.195335004155034, "uncore": 5.5591038982738175 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.66925307, "token_latencies_s": [ 0.057846764, 0.053020293, 0.046569169, 0.046005558, 0.046458924, 0.046382506, 0.048290515, 0.046751342, 0.046448803, 0.047179591, 0.045928717, 0.046342415, 0.046035256, 0.045965908 ], "watts": { "core": 9.664507030203088, "dram": 2.968974053417491, "package-0": 24.003585071339305, "psys": 40.33234543100415, "uncore": 8.308383254782829 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 4.149856457486749, "compile_target": "NPU", "context_length": 2048, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 397509, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_1_5b_ov_fp16/openvino_model.xml", "key": "qwen25_1_5b_fp16", "size_b": 1.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "gpu_npu", "label": "iGPU -> NPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 2032, "prompt_sha256": "cd1a97358e68d07c27508cfacc4ba6882a7a2962b1281e2e2b0c7e5e320da4fa", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "GPU", "NPUW_ONLINE_PIPELINE": "NONE" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 2048, "NPUW_LLM_MIN_RESPONSE_LEN": 2048, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.04626445472240448, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 14.136012000000001, "dram": 4.244863, "package-0": 37.968593, "psys": 65.43330399999999, "uncore": 12.25943 }, "net_j": { "core": 7.400698304893958, "dram": 3.3630555864682017, "package-0": 27.74225634021526, "psys": 47.29676059918477, "uncore": 12.257339588971764 } }, "first_decode": { "gpu_active_fraction": 0.04, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 25, "gross_j": { "core": 3.720999, "dram": 0.290161, "package-0": 4.528492, "psys": 7.148724, "uncore": 0.011536 }, "logits": { "l2": 1042.713244693195, "sha256": "38e7cb73dd1e63942c1fc408d3f6c090b083c85b5b9625f370be95e69b4098fc", "sum": 237734.52709483728, "top1": 13 }, "net_j": { "core": 2.7860254105715496, "dram": 0.16775146127079268, "package-0": 3.1089063464839537, "psys": 4.631070056783312, "uncore": 0.011051855868367108 }, "npu_busy_delta_us": 68627, "npu_utilization": 0.2768467046179593, "seconds": 0.247888087, "watts": { "core": 15.010802031805586, "dram": 1.1705322490951329, "package-0": 18.268292174928114, "psys": 28.838513728172824, "uncore": 0.04653712947488275 } }, "metrics": { "all_tokens_per_s_e2e": 1146.8742767303595, "decode_tokens_per_s": 15.57157535236046, "end_to_end_ms": 1785.7231969999998, "handoff_estimate_ms": 198.79146350000002, "handoff_plus_first_decode_ms": 247.888087, "output_tokens_per_s_e2e": 8.959955286955934, "prefill_tokens_per_s": 2470.728418948724, "steady_tpot_ms": 49.0966235, "steady_tpot_p95_ms": 59.35882599999999, "ttft_ms": 822.429525 }, "prefill": { "gpu_active_fraction": 0.9878048780487805, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 82, "gross_j": { "core": 6.038193, "dram": 1.849239, "package-0": 22.580265, "psys": 38.103784, "uncore": 12.247894 }, "logits": { "l2": 3172.3464720715606, "sha256": "f004143b6702c8233fb4ece6ff62551d0fe6c4f0eacaff333650b159de35221c", "sum": 1184720.837936934, "top1": 283 }, "net_j": { "core": 2.9361888079180076, "dram": 1.443115333425692, "package-0": 17.870441430632518, "psys": 29.750849570222105, "uncore": 12.246287733103397 }, "npu_busy_delta_us": 9040, "npu_utilization": 0.010991823281149834, "seconds": 0.822429525, "watts": { "core": 7.341897167419908, "dram": 2.2485075544922832, "package-0": 27.455562225833273, "psys": 46.33075885742307, "uncore": 14.892332567948603 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 911.2120236742918, "sha256": "d6f9dc82df9060085096789dc10c8db0740cc84a5a4516f697733fbdfebb4735", "sum": 196145.01117703877, "top1": 271 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 71, "gross_j": { "core": 4.37682, "dram": 2.105463, "package-0": 10.859836, "psys": 20.180796, "uncore": 0.0 }, "net_j": { "core": 1.678484086404401, "dram": 1.752188791771717, "package-0": 6.762908563098788, "psys": 12.914840972179356, "uncore": 0.0 }, "npu_busy_delta_us": 695949, "npu_utilization": 0.9728034203143663, "seconds": 0.715405585, "token_latencies_s": [ 0.069827999, 0.053721579, 0.051255127, 0.0500679, 0.049292558, 0.048827563, 0.048992746, 0.049012211, 0.049140611, 0.049001625, 0.049052636, 0.049228054, 0.048985064, 0.048972996 ], "watts": { "core": 6.117956152103566, "dram": 2.9430340552904686, "package-0": 15.17997095312025, "psys": 28.208887969472592, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 13.448879999999999, "dram": 4.13683, "package-0": 37.079495, "psys": 62.856467, "uncore": 12.373565000000001 }, "net_j": { "core": 7.373549958693754, "dram": 3.341429620360823, "package-0": 27.855222288419426, "psys": 46.497097272575225, "uncore": 12.371799312282544 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 0, "gpu_sample_count": 9, "gross_j": { "core": 2.361994, "dram": 0.184082, "package-0": 2.832817, "psys": 4.046986, "uncore": 0.006104 }, "logits": { "l2": 1135.1989390721888, "sha256": "d7d9a69882fb9e20540d9d9278969b3150f2fc54597af1b7836d34e8fe5c7737", "sum": 279393.15176730603, "top1": 819 }, "net_j": { "core": 2.027999367858062, "dram": 0.14035442381945956, "package-0": 2.3257075013991866, "psys": 3.1476205931767027, "uncore": 0.005931052290057462 }, "npu_busy_delta_us": 63352, "npu_utilization": 0.7154256888436924, "seconds": 0.088551475, "watts": { "core": 26.673683301153368, "dram": 2.078813481085436, "package-0": 31.990624662096252, "psys": 45.70207328562286, "uncore": 0.06893165811184963 } }, "metrics": { "all_tokens_per_s_e2e": 1271.4631090175278, "decode_tokens_per_s": 18.862249719637806, "end_to_end_ms": 1610.742762, "handoff_estimate_ms": 39.3654405, "handoff_plus_first_decode_ms": 88.55147500000001, "output_tokens_per_s_e2e": 9.933305539199436, "prefill_tokens_per_s": 2491.7119248165363, "steady_tpot_ms": 49.186034500000005, "steady_tpot_p95_ms": 56.62088194999999, "ttft_ms": 815.5035819999999 }, "prefill": { "gpu_active_fraction": 0.9876543209876543, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 81, "gross_j": { "core": 5.98143, "dram": 1.856501, "package-0": 22.647769, "psys": 37.993555, "uncore": 12.367461 }, "logits": { "l2": 3189.622647004578, "sha256": "382d46debf88546ab06200f51b873649d7eb5555857d5baa3b57d12ba7070aec", "sum": 1191507.7713742703, "top1": 283 }, "net_j": { "core": 2.905548780850091, "dram": 1.453797431172845, "package-0": 17.977608366592285, "psys": 29.710963312043518, "uncore": 12.365868259992487 }, "npu_busy_delta_us": 8549, "npu_utilization": 0.010483093132508154, "seconds": 0.815503582, "watts": { "core": 7.334645894909141, "dram": 2.27650870085326, "package-0": 27.77151382273144, "psys": 46.589071879760304, "uncore": 15.165428175887524 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 1211.5266628791376, "sha256": "f06f3f53b14c831c609644c43edf15f5e077fe256d545be2836d7ea42cae549c", "sum": 348738.0987180695, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 70, "gross_j": { "core": 5.105456, "dram": 2.096247, "package-0": 11.598909, "psys": 20.815926, "uncore": 0.0 }, "net_j": { "core": 2.440001809985601, "dram": 1.7472777653685183, "package-0": 7.551906420427954, "psys": 13.638513367355007, "uncore": 0.0 }, "npu_busy_delta_us": 691596, "npu_utilization": 0.9786444494601756, "seconds": 0.706687705, "token_latencies_s": [ 0.060445064, 0.054561707, 0.050538984, 0.049638074, 0.049234393, 0.049137676, 0.049290505, 0.049115283, 0.049128557, 0.049092839, 0.049116978, 0.049324404, 0.049071654, 0.048969642 ], "watts": { "core": 7.2244868049600495, "dram": 2.966298953793175, "package-0": 16.413061834718068, "psys": 29.455622126608247, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 13.824611, "dram": 4.131642, "package-0": 37.298122, "psys": 62.981955000000006, "uncore": 12.24124 }, "net_j": { "core": 7.7169512128683255, "dram": 3.332008913342711, "package-0": 28.024762507921537, "psys": 46.5355292177509, "uncore": 12.239458946692075 } }, "first_decode": { "gpu_active_fraction": 0.1111111111111111, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 9, "gross_j": { "core": 2.377924, "dram": 0.182983, "package-0": 2.842461, "psys": 4.053334, "uncore": 0.002746 }, "logits": { "l2": 1352.6577453890102, "sha256": "e62f0e0966003b386b36bde47a4495306738b10dbb4a67af95c88e4a6757f69a", "sum": 376909.0747744143, "top1": 13 }, "net_j": { "core": 2.0424688568565017, "dram": 0.13906420935091054, "package-0": 2.333133983719479, "psys": 3.1500357962252536, "uncore": 0.002572296014330975 }, "npu_busy_delta_us": 63344, "npu_utilization": 0.7122209052351992, "seconds": 0.088938698, "watts": { "core": 26.73666304402163, "dram": 2.0574058774730433, "package-0": 31.959777508773517, "psys": 45.574469731949534, "uncore": 0.030875199005049526 } }, "metrics": { "all_tokens_per_s_e2e": 1264.732858713221, "decode_tokens_per_s": 18.836398888604243, "end_to_end_ms": 1619.3142970000001, "handoff_estimate_ms": 39.74441449999999, "handoff_plus_first_decode_ms": 88.938698, "output_tokens_per_s_e2e": 9.88072545869704, "prefill_tokens_per_s": 2469.0645741444846, "steady_tpot_ms": 49.194283500000004, "steady_tpot_p95_ms": 56.315508099999995, "ttft_ms": 822.983741 }, "prefill": { "gpu_active_fraction": 0.9878048780487805, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 82, "gross_j": { "core": 6.502852, "dram": 1.85052, "package-0": 23.020082, "psys": 38.300378, "uncore": 12.238494 }, "logits": { "l2": 3199.9982517626586, "sha256": "4900c9dd8e9065590b01c6d5567d6284f9f760d8766220320e786ff3db59567a", "sum": 1195583.9069854915, "top1": 283 }, "net_j": { "core": 3.3987574399252987, "dram": 1.4441226561960299, "package-0": 18.307084590807982, "psys": 29.941814723290896, "uncore": 12.236886650677745 }, "npu_busy_delta_us": 8781, "npu_utilization": 0.010669712611005277, "seconds": 0.822983741, "watts": { "core": 7.901555858319198, "dram": 2.2485498896387064, "package-0": 27.971490629971022, "psys": 46.538438236290744, "uncore": 14.870881878090467 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 1248.264823861332, "sha256": "6193653dd623285c9088f54e32bb5cecc2517a2fd6a7aaa09c41215c455b3de1", "sum": 317309.1419824492, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 70, "gross_j": { "core": 4.943835, "dram": 2.098139, "package-0": 11.435579, "psys": 20.628243, "uncore": 0.0 }, "net_j": { "core": 2.275724916086525, "dram": 1.7488220477957706, "package-0": 7.384543933394076, "psys": 13.443678698234748, "uncore": 0.0 }, "npu_busy_delta_us": 691105, "npu_utilization": 0.9769761867968799, "seconds": 0.707391858, "token_latencies_s": [ 0.063301686, 0.05255372, 0.050674762, 0.049584771, 0.049100925, 0.049055602, 0.049253311, 0.048966893, 0.049053842, 0.048887522, 0.049193885, 0.049140436, 0.049194682, 0.049404227 ], "watts": { "core": 6.98882089762475, "dram": 2.9660208500731713, "package-0": 16.16583350610179, "psys": 29.16098449072056, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 8.34754111431539, "compile_target": "AUTO:NPU,GPU", "context_length": 2048, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 397355, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_1_5b_ov_fp16/openvino_model.xml", "key": "qwen25_1_5b_fp16", "size_b": 1.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": null, "key": "openvino_auto", "label": "OpenVINO AUTO", "mode": "stock", "prefill_device": null }, "prompt_length": 2032, "prompt_sha256": "cd1a97358e68d07c27508cfacc4ba6882a7a2962b1281e2e2b0c7e5e320da4fa", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.06940924748778343, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 13.624232, "dram": 3.6068029999999998, "package-0": 39.672506, "psys": 67.034192, "uncore": 17.298112 }, "net_j": { "core": 8.14529282271186, "dram": 2.889483896016317, "package-0": 31.3537432664774, "psys": 52.28075620212233, "uncore": 17.295274918524917 } }, "first_decode": { "gpu_active_fraction": 0.8571428571428571, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 7, "gross_j": { "core": 2.009577, "dram": 0.15741, "package-0": 2.757867, "psys": 3.983877, "uncore": 0.360167 }, "logits": { "l2": 721.436499143929, "sha256": "33527b68f47cd3c749c20c0f2bfed74af94e75dcffa9e888e816a2d29c486e6e", "sum": 152204.33597268048, "top1": 220 }, "net_j": { "core": 1.7566192162294858, "dram": 0.12429201038037897, "package-0": 2.3737970702757267, "psys": 3.3027238968770525, "uncore": 0.3600360144535118 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.067066302, "watts": { "core": 29.96403469509919, "dram": 2.3470803563912024, "package-0": 41.12150092903587, "psys": 59.40206752416438, "uncore": 5.370312500605745 } }, "metrics": { "all_tokens_per_s_e2e": 1409.8638025856274, "decode_tokens_per_s": 20.57018616979797, "end_to_end_ms": 1452.622584, "handoff_estimate_ms": 20.334507499999994, "handoff_plus_first_decode_ms": 67.066302, "output_tokens_per_s_e2e": 11.014560957700214, "prefill_tokens_per_s": 2808.9115821912337, "steady_tpot_ms": 46.7317945, "steady_tpot_p95_ms": 49.735279850000005, "ttft_ms": 723.411877 }, "prefill": { "gpu_active_fraction": 0.8194444444444444, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 72, "gross_j": { "core": 5.103136, "dram": 1.472653, "package-0": 20.405221, "psys": 34.211216, "uncore": 11.43094 }, "logits": { "l2": 834.8475507956192, "sha256": "765152f2d994661cc75a99376a54eb9c804048cab9a622294153aa34f8ebd025", "sum": 210672.0778277011, "top1": 220 }, "net_j": { "core": 2.3746022997344336, "dram": 1.115425207222689, "package-0": 16.262443770966343, "psys": 26.863945759059234, "uncore": 11.429527122038651 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.723411877, "watts": { "core": 7.054260736169804, "dram": 2.035704757996391, "package-0": 28.20692008074399, "psys": 47.29147680277857, "uncore": 15.801427047900127 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 677.6366610405684, "sha256": "ad393a0a123b7a19df0f9f33da818df6a7780c377cdd4707a8b74388cd4bd9a9", "sum": 116901.59847021359, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 66, "gross_j": { "core": 6.511519, "dram": 1.97674, "package-0": 16.509418, "psys": 28.839099, "uncore": 5.507005 }, "net_j": { "core": 4.014071306747941, "dram": 1.6497666784132492, "package-0": 12.717502425235333, "psys": 22.114086546186044, "uncore": 5.505711782032756 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.662144405, "token_latencies_s": [ 0.051925592, 0.047355167, 0.047225469, 0.046555757, 0.048555881, 0.047104171, 0.046852493, 0.046531202, 0.046263863, 0.046176471, 0.046450101, 0.046191371, 0.046611096, 0.04831796 ], "watts": { "core": 9.833986288836797, "dram": 2.9853608745663265, "package-0": 24.933259082661884, "psys": 43.55409300785377, "uncore": 8.316924463025554 } } }, { "energy": { "gross_j": { "core": 14.77218, "dram": 3.732962, "package-0": 40.64516, "psys": 65.94660900000001, "uncore": 17.642106 }, "net_j": { "core": 9.03250984034242, "dram": 2.981507215987359, "package-0": 31.930525162512343, "psys": 50.49108878679413, "uncore": 17.639133907893093 } }, "first_decode": { "gpu_active_fraction": 0.7142857142857143, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 7, "gross_j": { "core": 2.026118, "dram": 0.155761, "package-0": 2.665338, "psys": 3.70098, "uncore": 0.312316 }, "logits": { "l2": 721.436499143929, "sha256": "33527b68f47cd3c749c20c0f2bfed74af94e75dcffa9e888e816a2d29c486e6e", "sum": 152204.33597268048, "top1": 220 }, "net_j": { "core": 1.785999250134327, "dram": 0.12432393509931253, "package-0": 2.3007617849029613, "psys": 3.0543992577470283, "uncore": 0.3121916627085974 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.063662309, "watts": { "core": 31.82602126479578, "dram": 2.446675316159205, "package-0": 41.86681321910583, "psys": 58.1345549373649, "uncore": 4.905822690157216 } }, "metrics": { "all_tokens_per_s_e2e": 1345.8191512328403, "decode_tokens_per_s": 20.532744196156006, "end_to_end_ms": 1521.749782, "handoff_estimate_ms": 16.6687645, "handoff_plus_first_decode_ms": 63.662309, "output_tokens_per_s_e2e": 10.514212119006565, "prefill_tokens_per_s": 2568.2204376300456, "steady_tpot_ms": 46.9935445, "steady_tpot_p95_ms": 50.317366, "ttft_ms": 791.209341 }, "prefill": { "gpu_active_fraction": 0.759493670886076, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 79, "gross_j": { "core": 6.144089, "dram": 1.587582, "package-0": 21.795537, "psys": 35.291535, "uncore": 11.809785 }, "logits": { "l2": 834.8475507956192, "sha256": "765152f2d994661cc75a99376a54eb9c804048cab9a622294153aa34f8ebd025", "sum": 210672.0778277011, "top1": 220 }, "net_j": { "core": 3.1598397506841636, "dram": 1.1968751627244754, "package-0": 17.264502681229025, "psys": 27.25568567224055, "uncore": 11.808239708477627 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.791209341, "watts": { "core": 7.765440423434031, "dram": 2.006525855715346, "package-0": 27.547117899863117, "psys": 44.604547963748075, "uncore": 14.926245669791708 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 677.6366610405684, "sha256": "ad393a0a123b7a19df0f9f33da818df6a7780c377cdd4707a8b74388cd4bd9a9", "sum": 116901.59847021359, "top1": 42 }, "gpu_active_fraction": 0.9848484848484849, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 66, "gross_j": { "core": 6.601973, "dram": 1.989619, "package-0": 16.184285, "psys": 26.954094, "uncore": 5.520005 }, "net_j": { "core": 4.08667083952393, "dram": 1.660308118163571, "package-0": 12.365260696380357, "psys": 20.181003856806555, "uncore": 5.5187025367068685 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.666878132, "token_latencies_s": [ 0.053377475, 0.048602041, 0.047395822, 0.047230073, 0.046868216, 0.046658153, 0.046766056, 0.046636233, 0.04684158, 0.048669615, 0.047226088, 0.047110818, 0.046876271, 0.046578755 ], "watts": { "core": 9.899819297118592, "dram": 2.983482145430434, "package-0": 24.268729507537667, "psys": 40.41832038960906, "uncore": 8.277381931006248 } } }, { "energy": { "gross_j": { "core": 13.257594, "dram": 3.71868, "package-0": 38.737999, "psys": 63.272544, "uncore": 17.533768 }, "net_j": { "core": 7.538064301499767, "dram": 2.969862065399941, "package-0": 30.053943753248852, "psys": 47.87125709565368, "uncore": 17.53080633694284 } }, "first_decode": { "gpu_active_fraction": 0.7142857142857143, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 7, "gross_j": { "core": 2.093073, "dram": 0.156494, "package-0": 2.715142, "psys": 3.769705, "uncore": 0.320373 }, "logits": { "l2": 721.436499143929, "sha256": "33527b68f47cd3c749c20c0f2bfed74af94e75dcffa9e888e816a2d29c486e6e", "sum": 152204.33597268048, "top1": 220 }, "net_j": { "core": 1.8379431897010392, "dram": 0.12309164217098682, "package-0": 2.3277742469642435, "psys": 3.0827031635909528, "uncore": 0.3202408897437773 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.067642168, "watts": { "core": 30.943316305296424, "dram": 2.313556833364655, "package-0": 40.13978381059578, "psys": 55.73010315104034, "uncore": 4.736291125382025 } }, "metrics": { "all_tokens_per_s_e2e": 1350.5582503841788, "decode_tokens_per_s": 20.358070335927813, "end_to_end_ms": 1516.409973, "handoff_estimate_ms": 21.047647, "handoff_plus_first_decode_ms": 67.642168, "output_tokens_per_s_e2e": 10.551236331126397, "prefill_tokens_per_s": 2606.460049405989, "steady_tpot_ms": 46.594521, "steady_tpot_p95_ms": 53.8099788, "ttft_ms": 779.6014369999999 }, "prefill": { "gpu_active_fraction": 0.7922077922077922, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 77, "gross_j": { "core": 5.331956, "dram": 1.576229, "package-0": 20.611947, "psys": 33.352271, "uncore": 11.681244 }, "logits": { "l2": 834.8475507956192, "sha256": "765152f2d994661cc75a99376a54eb9c804048cab9a622294153aa34f8ebd025", "sum": 210672.0778277011, "top1": 220 }, "net_j": { "core": 2.3914889420753167, "dram": 1.1912542581690335, "package-0": 16.147387896658767, "psys": 25.434316346533954, "uncore": 11.6797213795897 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.779601437, "watts": { "core": 6.839335777160708, "dram": 2.0218395261885598, "package-0": 26.439082872034266, "psys": 42.781182046487174, "uncore": 14.983610144371758 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 677.6366610405684, "sha256": "ad393a0a123b7a19df0f9f33da818df6a7780c377cdd4707a8b74388cd4bd9a9", "sum": 116901.59847021359, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 66, "gross_j": { "core": 5.832565, "dram": 1.985957, "package-0": 15.41091, "psys": 26.150568, "uncore": 5.532151 }, "net_j": { "core": 3.3086321697234107, "dram": 1.6555161650599208, "package-0": 11.578781609625842, "psys": 19.354237585528775, "uncore": 5.530844067609363 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.669166368, "token_latencies_s": [ 0.057696464, 0.051717256, 0.047029774, 0.046733732, 0.048788985, 0.046665355, 0.046676283, 0.046523687, 0.046197304, 0.046466676, 0.046125517, 0.046097711, 0.046198812, 0.046221872 ], "watts": { "core": 8.716165783155438, "dram": 2.967807551260556, "package-0": 23.03001276955987, "psys": 39.079322049849345, "uncore": 8.267228098349378 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 1.671638049185276, "compile_target": "AUTO:GPU", "context_length": 4096, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 397842, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_1_5b_ov_fp16/openvino_model.xml", "key": "qwen25_1_5b_fp16", "size_b": 1.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "GPU", "key": "gpu_gpu", "label": "iGPU -> iGPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 4080, "prompt_sha256": "66f8de7a9ef70b33481de3e8dae7cef6e4434d9723c6203a6a3dbe07ef228a16", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.04640396311879158, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 13.511133999999998, "dram": 5.485217, "package-0": 59.912627, "psys": 99.782277, "uncore": 34.612826 }, "net_j": { "core": 4.442110618724019, "dram": 4.297873334490673, "package-0": 46.14298085274251, "psys": 75.36162733122013, "uncore": 34.608129916068506 } }, "first_decode": { "gpu_active_fraction": 0.75, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 8, "gross_j": { "core": 2.309564, "dram": 0.167969, "package-0": 3.033867, "psys": 4.224049, "uncore": 0.359802 }, "logits": { "l2": 979.727719530536, "sha256": "8eca8a515a86885daba895c0e4490a5dfd1090aa41c110d1f8b0c9ea55c06524", "sum": 284299.187461914, "top1": 78 }, "net_j": { "core": 2.0220074682353184, "dram": 0.13032123881200441, "package-0": 2.5972652267696854, "psys": 3.4497299772160224, "uncore": 0.3596530986738656 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.076239414, "watts": { "core": 30.29356967512893, "dram": 2.2031780044899087, "package-0": 39.79394437633007, "psys": 55.405055972754454, "uncore": 4.719369957381886 } }, "metrics": { "all_tokens_per_s_e2e": 1703.5038278928787, "decode_tokens_per_s": 21.21861532151427, "end_to_end_ms": 2404.456, "handoff_estimate_ms": 31.544564000000005, "handoff_plus_first_decode_ms": 76.23941400000001, "output_tokens_per_s_e2e": 6.654311827706557, "prefill_tokens_per_s": 2403.4927429296517, "steady_tpot_ms": 44.69485, "steady_tpot_p95_ms": 46.5722248, "ttft_ms": 1697.529569 }, "prefill": { "gpu_active_fraction": 0.844311377245509, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 167, "gross_j": { "core": 6.689741, "dram": 3.32714, "package-0": 42.845044, "psys": 71.014589, "uncore": 28.685656 }, "logits": { "l2": 1150.3575819756547, "sha256": "2c23050136374b58161b79874f63495a8c895d72d93c7555b2aaf74e81ff149c", "sum": 369774.01646265644, "top1": 220 }, "net_j": { "core": 0.2870722251634721, "dram": 2.4888834543335676, "package-0": 33.12376754926166, "psys": 53.773776563863144, "uncore": 28.682340596710354 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.697529569, "watts": { "core": 3.940868614112488, "dram": 1.959989422723275, "package-0": 25.23964517757393, "psys": 41.83408071167448, "uncore": 16.89847206425893 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 811.0208018948907, "sha256": "b95537717dfcf6809ff402f9c640cbc97a728f4a0d1eb86875f847d4d31b12aa", "sum": 202420.8223354963, "top1": 42 }, "gpu_active_fraction": 0.9841269841269841, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 63, "gross_j": { "core": 4.511829, "dram": 1.990108, "package-0": 14.033716, "psys": 24.543639, "uncore": 5.567368 }, "net_j": { "core": 2.1330309253252286, "dram": 1.6786686413451013, "package-0": 10.42194807671117, "psys": 18.138120790140967, "uncore": 5.56613622068429 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.630687017, "token_latencies_s": [ 0.049076706, 0.044911002, 0.044890823, 0.044562977, 0.044629914, 0.044604457, 0.044795915, 0.044648014, 0.044590616, 0.044683985, 0.045223658, 0.044705715, 0.044758398, 0.044575731 ], "watts": { "core": 7.153832056764853, "dram": 3.1554605475571416, "package-0": 22.251474379089686, "psys": 38.915719427279726, "uncore": 8.82746568413981 } } }, { "energy": { "gross_j": { "core": 19.837414, "dram": 5.683273, "package-0": 67.079846, "psys": 108.62521100000001, "uncore": 34.791049 }, "net_j": { "core": 10.272295710268732, "dram": 4.430979105311428, "package-0": 52.55697086019312, "psys": 82.86869974474936, "uncore": 34.78609603026833 } }, "first_decode": { "gpu_active_fraction": 0.625, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 8, "gross_j": { "core": 1.954402, "dram": 0.163085, "package-0": 2.62481, "psys": 3.706778, "uncore": 0.321899 }, "logits": { "l2": 979.727719530536, "sha256": "8eca8a515a86885daba895c0e4490a5dfd1090aa41c110d1f8b0c9ea55c06524", "sum": 284299.187461914, "top1": 78 }, "net_j": { "core": 1.6805868119871508, "dram": 0.12723629663481364, "package-0": 2.2090719331574333, "psys": 2.969461036603652, "uncore": 0.3217572141592797 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.072596193, "watts": { "core": 26.921549453702067, "dram": 2.2464676625673747, "package-0": 36.15630367834853, "psys": 51.060225706325944, "uncore": 4.434103038984427 } }, "metrics": { "all_tokens_per_s_e2e": 1615.1515932468096, "decode_tokens_per_s": 21.282415866569966, "end_to_end_ms": 2535.984868, "handoff_estimate_ms": 27.9302605, "handoff_plus_first_decode_ms": 72.596193, "output_tokens_per_s_e2e": 6.30918591112035, "prefill_tokens_per_s": 2228.0743578204356, "steady_tpot_ms": 44.665932500000004, "steady_tpot_p95_ms": 47.106267450000004, "ttft_ms": 1831.177665 }, "prefill": { "gpu_active_fraction": 0.8066298342541437, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 181, "gross_j": { "core": 12.734587, "dram": 3.538016, "package-0": 49.718928, "psys": 79.821329, "uncore": 28.878832 }, "logits": { "l2": 1150.3575819756547, "sha256": "2c23050136374b58161b79874f63495a8c895d72d93c7555b2aaf74e81ff149c", "sum": 369774.01646265644, "top1": 220 }, "net_j": { "core": 5.8278301039888625, "dram": 2.6337627298383044, "package-0": 39.23228633144328, "psys": 61.22313122126779, "uncore": 28.875255571898045 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.831177665, "watts": { "core": 6.95431538042487, "dram": 1.9320987076368692, "package-0": 27.151340337039333, "psys": 43.59016087060018, "uncore": 15.77063359387359 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 811.0208018948907, "sha256": "b95537717dfcf6809ff402f9c640cbc97a728f4a0d1eb86875f847d4d31b12aa", "sum": 202420.8223354963, "top1": 42 }, "gpu_active_fraction": 0.9841269841269841, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 63, "gross_j": { "core": 5.148425, "dram": 1.982172, "package-0": 14.736108, "psys": 25.097104, "uncore": 5.590318 }, "net_j": { "core": 2.76387879429272, "dram": 1.6699800788383095, "package-0": 11.115612595592406, "psys": 18.676107486877914, "uncore": 5.589083244211008 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.63221101, "token_latencies_s": [ 0.049636774, 0.045743687, 0.04519147, 0.044705648, 0.045360017, 0.044527445, 0.044626217, 0.044621231, 0.044313822, 0.04521814, 0.044709359, 0.044605963, 0.044476419, 0.044447921 ], "watts": { "core": 8.143523156928254, "dram": 3.135301297584172, "package-0": 23.308844305005064, "psys": 39.69735357819852, "uncore": 8.842487573887711 } } }, { "energy": { "gross_j": { "core": 18.014907, "dram": 5.691758, "package-0": 65.374405, "psys": 106.951265, "uncore": 35.025301 }, "net_j": { "core": 8.56128840175079, "dram": 4.454061978403341, "package-0": 51.02082166071997, "psys": 81.4949949933806, "uncore": 35.020405766574385 } }, "first_decode": { "gpu_active_fraction": 0.7142857142857143, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 7, "gross_j": { "core": 2.043757, "dram": 0.15979, "package-0": 2.702141, "psys": 3.737112, "uncore": 0.319336 }, "logits": { "l2": 979.727719530536, "sha256": "8eca8a515a86885daba895c0e4490a5dfd1090aa41c110d1f8b0c9ea55c06524", "sum": 284299.187461914, "top1": 78 }, "net_j": { "core": 1.7975150970902796, "dram": 0.12755127358912122, "package-0": 2.3282679017461585, "psys": 3.0740431124340803, "uncore": 0.3192084920429798 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.065285731, "watts": { "core": 31.304803801614778, "dram": 2.4475486075203783, "package-0": 41.38945767490909, "psys": 57.24240109986668, "uncore": 4.891359798054494 } }, "metrics": { "all_tokens_per_s_e2e": 1634.2013256294047, "decode_tokens_per_s": 21.51216734208275, "end_to_end_ms": 2506.423129, "handoff_estimate_ms": 20.579845, "handoff_plus_first_decode_ms": 65.285731, "output_tokens_per_s_e2e": 6.383598928239862, "prefill_tokens_per_s": 2255.211050258657, "steady_tpot_ms": 44.705886, "steady_tpot_p95_ms": 47.207606299999995, "ttft_ms": 1809.143317 }, "prefill": { "gpu_active_fraction": 0.8146067415730337, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 178, "gross_j": { "core": 10.683444, "dram": 3.541617, "package-0": 47.787048, "psys": 77.50145, "uncore": 29.090379 }, "logits": { "l2": 1150.3575819756547, "sha256": "2c23050136374b58161b79874f63495a8c895d72d93c7555b2aaf74e81ff149c", "sum": 369774.01646265644, "top1": 220 }, "net_j": { "core": 3.8597953069267303, "dram": 2.6482445043342517, "package-0": 37.426590871749084, "psys": 59.12704217341829, "uncore": 29.086845606641695 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.809143317, "watts": { "core": 5.905250236181261, "dram": 1.9576210279862534, "package-0": 26.414185958049227, "psys": 42.838756483105094, "uncore": 16.079643180640286 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 811.0208018948907, "sha256": "b95537717dfcf6809ff402f9c640cbc97a728f4a0d1eb86875f847d4d31b12aa", "sum": 202420.8223354963, "top1": 42 }, "gpu_active_fraction": 0.9838709677419355, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 62, "gross_j": { "core": 5.287706, "dram": 1.990351, "package-0": 14.885216, "psys": 25.712703, "uncore": 5.615586 }, "net_j": { "core": 2.9039779977337803, "dram": 1.6782662004799678, "package-0": 11.265962887224724, "psys": 19.293909707528236, "uncore": 5.614351667889709 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.631994081, "token_latencies_s": [ 0.048192979, 0.046677021, 0.044863513, 0.044712599, 0.04520041, 0.044936164, 0.044699173, 0.044697623, 0.044491113, 0.04505052, 0.04463654, 0.0445817, 0.044680747, 0.044545521 ], "watts": { "core": 8.366701776119957, "dram": 3.1493190519295386, "package-0": 23.552777545712487, "psys": 40.68503768154753, "uncore": 8.885504103320867 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 5.203982960432768, "compile_target": "NPU", "context_length": 4096, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 397910, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_1_5b_ov_fp16/openvino_model.xml", "key": "qwen25_1_5b_fp16", "size_b": 1.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "gpu_npu", "label": "iGPU -> NPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 4080, "prompt_sha256": "66f8de7a9ef70b33481de3e8dae7cef6e4434d9723c6203a6a3dbe07ef228a16", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "GPU", "NPUW_ONLINE_PIPELINE": "NONE" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 4096, "NPUW_LLM_MIN_RESPONSE_LEN": 4096, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.04610307049006224, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 21.868902, "dram": 6.653792, "package-0": 66.56519899999999, "psys": 114.673047, "uncore": 27.412283 }, "net_j": { "core": 11.143050800064472, "dram": 5.249531481085421, "package-0": 50.27996414701965, "psys": 85.79096747048017, "uncore": 27.408172559420812 } }, "first_decode": { "gpu_active_fraction": 0.03571428571428571, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 28, "gross_j": { "core": 3.917349, "dram": 0.332518, "package-0": 4.809131, "psys": 7.765117, "uncore": 0.008301 }, "logits": { "l2": 1147.84930727624, "sha256": "d711afeaae9e5c846617e5fbdf30231be31637936f41d7fa6839f09d6e1678c1", "sum": 302765.76277617365, "top1": 13 }, "net_j": { "core": 2.8615337185781744, "dram": 0.19428750553106047, "package-0": 3.206069434180092, "psys": 4.922066098203895, "uncore": 0.007754282126502436 }, "npu_busy_delta_us": 70916, "npu_utilization": 0.2533377885955956, "seconds": 0.279926656, "watts": { "core": 13.994197823018327, "dram": 1.1878754412012837, "package-0": 17.179968027053487, "psys": 27.739826963817265, "uncore": 0.029654196276327465 } }, "metrics": { "all_tokens_per_s_e2e": 1440.362704765708, "decode_tokens_per_s": 14.719507508358959, "end_to_end_ms": 2843.7281709999997, "handoff_estimate_ms": 228.932571, "handoff_plus_first_decode_ms": 279.926656, "output_tokens_per_s_e2e": 5.626416815491047, "prefill_tokens_per_s": 2236.0178960722633, "steady_tpot_ms": 50.994085, "steady_tpot_p95_ms": 60.5503181, "ttft_ms": 1824.672337 }, "prefill": { "gpu_active_fraction": 0.9888888888888889, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 180, "gross_j": { "core": 13.24545, "dram": 4.137441, "package-0": 50.295831, "psys": 85.70492, "uncore": 27.403982 }, "logits": { "l2": 3198.4374483838114, "sha256": "8ffcdb9ef1495d512db0d7eb0c62eab30d7c7aae955c1040aec2872d5443f1cc", "sum": 1194945.9496211372, "top1": 283 }, "net_j": { "core": 6.363229619414751, "dram": 3.2364001238238296, "package-0": 39.84644352309073, "psys": 67.17279302317412, "uncore": 27.40041827729431 }, "npu_busy_delta_us": 8610, "npu_utilization": 0.00471865541303485, "seconds": 1.824672337, "watts": { "core": 7.259084127826069, "dram": 2.26749806861351, "package-0": 27.564308385741697, "psys": 46.970033064078834, "uncore": 15.018577003833867 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 1257.9593452313686, "sha256": "2bbcc434bbf921b754cb1245e19feaa54c0a23ec8ece6bc4bb3fdb60eebe175d", "sum": 352825.7396939341, "top1": 198 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 73, "gross_j": { "core": 4.706103, "dram": 2.183833, "package-0": 11.460237, "psys": 21.20301, "uncore": 0.0 }, "net_j": { "core": 1.918287462071547, "dram": 1.8188438517305303, "package-0": 7.227451189748827, "psys": 13.69610834910215, "uncore": 0.0 }, "npu_busy_delta_us": 722397, "npu_utilization": 0.9773623089197, "seconds": 0.739129178, "token_latencies_s": [ 0.069330353, 0.055822607, 0.05318779, 0.051765133, 0.050952626, 0.050992144, 0.051133438, 0.050996026, 0.050801898, 0.050654539, 0.050718619, 0.050753798, 0.050704146, 0.051280826 ], "watts": { "core": 6.367091355714278, "dram": 2.9546026121025357, "package-0": 15.505052893474055, "psys": 28.686474071248206, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 22.45142, "dram": 6.555525, "package-0": 66.960827, "psys": 113.661268, "uncore": 27.36254 }, "net_j": { "core": 12.214156750693148, "dram": 5.2152318768935375, "package-0": 51.417423203036165, "psys": 86.09483566567036, "uncore": 27.358060290886876 } }, "first_decode": { "gpu_active_fraction": 0.07142857142857142, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 14, "gross_j": { "core": 2.605767, "dram": 0.229736, "package-0": 3.216911, "psys": 4.834765, "uncore": 0.024902 }, "logits": { "l2": 1290.0354581548856, "sha256": "0ba7257f7692ff17f9b89be8752da58a507ed8ad6c8b71db0317ad9ce49ea86e", "sum": 314371.7607283406, "top1": 13 }, "net_j": { "core": 2.0895980925147692, "dram": 0.1621576228286722, "package-0": 2.433203327246363, "psys": 3.4448490708340787, "uncore": 0.024634719565219912 }, "npu_busy_delta_us": 66808, "npu_utilization": 0.48818037618942456, "seconds": 0.136851056, "watts": { "core": 19.040898011046405, "dram": 1.67873019554924, "package-0": 23.506658216798854, "psys": 35.32866417925193, "uncore": 0.18196425170442238 } }, "metrics": { "all_tokens_per_s_e2e": 1509.1060637030785, "decode_tokens_per_s": 17.237192336887603, "end_to_end_ms": 2714.1896110000002, "handoff_estimate_ms": 85.8982135, "handoff_plus_first_decode_ms": 136.851056, "output_tokens_per_s_e2e": 5.89494556134015, "prefill_tokens_per_s": 2212.60740063433, "steady_tpot_ms": 50.952842499999996, "steady_tpot_p95_ms": 58.70467319999999, "ttft_ms": 1843.978285 }, "prefill": { "gpu_active_fraction": 0.994535519125683, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 183, "gross_j": { "core": 14.36648, "dram": 4.15038, "package-0": 51.473012, "psys": 86.677024, "uncore": 27.337027 }, "logits": { "l2": 3214.1518144681254, "sha256": "71fdd7f6aae8afe59eef177046b94bc6c134505127e15a4c245b1f55c8e9e449", "sum": 1201170.42534234, "top1": 283 }, "net_j": { "core": 7.4114422778582, "dram": 3.2398056596787375, "package-0": 40.91306476550008, "psys": 67.94881782994635, "uncore": 27.333425571321655 }, "npu_busy_delta_us": 6987, "npu_utilization": 0.0037890901735862905, "seconds": 1.843978285, "watts": { "core": 7.791024502221836, "dram": 2.2507748782952723, "package-0": 27.914109628465607, "psys": 47.00544724690183, "uncore": 14.825026532240319 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 1164.0301254401054, "sha256": "8389285ef6e290edb74f8be63da61a7e17fe2670ddaa25ba68a3f64e2f9d5ebc", "sum": 214924.83903697878, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 72, "gross_j": { "core": 5.479173, "dram": 2.175409, "package-0": 12.270904, "psys": 22.149479, "uncore": 0.000611 }, "net_j": { "core": 2.7131163803201783, "dram": 1.8132685943861275, "package-0": 8.071155110289723, "psys": 14.701168764889932, "uncore": 0.0 }, "npu_busy_delta_us": 717679, "npu_utilization": 0.9786172354278204, "seconds": 0.73336027, "token_latencies_s": [ 0.064988205, 0.055321233, 0.052661703, 0.051518109, 0.050851721, 0.051035125, 0.051041347, 0.050948068, 0.050688416, 0.050782734, 0.050854417, 0.050957617, 0.050938788, 0.050744871 ], "watts": { "core": 7.471325110099024, "dram": 2.966357858464299, "package-0": 16.732436296283133, "psys": 30.202725598974702, "uncore": 0.0008331512150228699 } } }, { "energy": { "gross_j": { "core": 19.914745, "dram": 6.549727, "package-0": 64.392231, "psys": 110.26857700000001, "uncore": 27.417654 }, "net_j": { "core": 9.789597155078177, "dram": 5.224112360960237, "package-0": 49.019053850508044, "psys": 83.0040438777303, "uncore": 27.414089163615547 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 0, "gpu_sample_count": 13, "gross_j": { "core": 2.630853, "dram": 0.222106, "package-0": 3.200309, "psys": 4.749622, "uncore": 0.0 }, "logits": { "l2": 1327.510740782216, "sha256": "a5cfa6d9c7d61c4d5314c5fb9e4394a7751d36c9ff305e9e941c701874e43519", "sum": 303319.145005431, "top1": 279 }, "net_j": { "core": 2.15504600199248, "dram": 0.15981192473544248, "package-0": 2.4778834728501735, "psys": 3.4683907670664413, "uncore": 0.0 }, "npu_busy_delta_us": 66405, "npu_utilization": 0.5263972881292731, "seconds": 0.126149966, "watts": { "core": 20.854964003716024, "dram": 1.7606504943489243, "package-0": 25.369083333720436, "psys": 37.65060071439099, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 1525.8163418326792, "decode_tokens_per_s": 17.45765204817119, "end_to_end_ms": 2684.4646289999996, "handoff_estimate_ms": 75.1438985, "handoff_plus_first_decode_ms": 126.149966, "output_tokens_per_s_e2e": 5.960220085283903, "prefill_tokens_per_s": 2235.319349104262, "steady_tpot_ms": 51.0060675, "steady_tpot_p95_ms": 58.60815495, "ttft_ms": 1825.242555 }, "prefill": { "gpu_active_fraction": 0.988950276243094, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 181, "gross_j": { "core": 12.878141, "dram": 4.142934, "package-0": 49.985224, "psys": 84.821499, "uncore": 27.417654 }, "logits": { "l2": 3223.526830340574, "sha256": "048b260b2c92587d73b6713099b0f5b80ad32e212c3094a362ae4de7628d322f", "sum": 1204873.0273589566, "top1": 283 }, "net_j": { "core": 5.993769895768977, "dram": 3.241611544651552, "package-0": 39.53257104429353, "psys": 66.28358065331216, "uncore": 27.414089163615547 }, "npu_busy_delta_us": 8724, "npu_utilization": 0.004779638725878819, "seconds": 1.825242555, "watts": { "core": 7.055577881811987, "dram": 2.269799150064195, "package-0": 27.385524111889886, "psys": 46.47135733694309, "uncore": 15.021375611089672 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 1042.1375812788901, "sha256": "c93af3e216090c756e1d61880a00a0f36db91d207f91589a55d400f2ecd8f9d2", "sum": 156904.19420131668, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 73, "gross_j": { "core": 4.405751, "dram": 2.184687, "package-0": 11.206698, "psys": 20.697456, "uncore": 0.0 }, "net_j": { "core": 1.6407812573167195, "dram": 1.8226888915732418, "package-0": 7.008599333364334, "psys": 13.2520724573517, "uncore": 0.0 }, "npu_busy_delta_us": 717462, "npu_utilization": 0.9787059037853887, "seconds": 0.733072108, "token_latencies_s": [ 0.062741529, 0.056382492, 0.052586517, 0.051468474, 0.051089063, 0.051095787, 0.050898616, 0.050934025, 0.050913882, 0.050999066, 0.050970306, 0.050998315, 0.050931558, 0.051013069 ], "watts": { "core": 6.009983127062311, "dram": 2.980180225326483, "package-0": 15.287306497821357, "psys": 28.233860999660347, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 2.1733186691999435, "compile_target": "AUTO:NPU,GPU", "context_length": 4096, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 397758, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_1_5b_ov_fp16/openvino_model.xml", "key": "qwen25_1_5b_fp16", "size_b": 1.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": null, "key": "openvino_auto", "label": "OpenVINO AUTO", "mode": "stock", "prefill_device": null }, "prompt_length": 4080, "prompt_sha256": "66f8de7a9ef70b33481de3e8dae7cef6e4434d9723c6203a6a3dbe07ef228a16", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.04595974553376436, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 18.286574, "dram": 5.473497, "package-0": 64.829669, "psys": 106.854402, "uncore": 34.603488 }, "net_j": { "core": 9.21583432980364, "dram": 4.285928632811989, "package-0": 51.05741698327603, "psys": 82.42913078723541, "uncore": 34.59879102734692 } }, "first_decode": { "gpu_active_fraction": 0.75, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 8, "gross_j": { "core": 2.01562, "dram": 0.165527, "package-0": 2.741937, "psys": 3.932363, "uncore": 0.366637 }, "logits": { "l2": 979.727719530536, "sha256": "8eca8a515a86885daba895c0e4490a5dfd1090aa41c110d1f8b0c9ea55c06524", "sum": 284299.187461914, "top1": 78 }, "net_j": { "core": 1.742072504885852, "dram": 0.12971334378975105, "package-0": 2.3266053756525378, "psys": 3.195766867747554, "uncore": 0.3664953527749021 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.07252522, "watts": { "core": 27.791987394178193, "dram": 2.282337095978475, "package-0": 37.80666918349231, "psys": 54.220628355212156, "uncore": 5.055303520623585 } }, "metrics": { "all_tokens_per_s_e2e": 1703.181505254159, "decode_tokens_per_s": 21.299049867702585, "end_to_end_ms": 2404.911037, "handoff_estimate_ms": 27.866270999999998, "handoff_plus_first_decode_ms": 72.52522, "output_tokens_per_s_e2e": 6.653052754899059, "prefill_tokens_per_s": 2399.07668005914, "steady_tpot_ms": 44.65894900000001, "steady_tpot_p95_ms": 47.0527045, "ttft_ms": 1700.65427 }, "prefill": { "gpu_active_fraction": 0.8452380952380952, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 168, "gross_j": { "core": 10.665378, "dram": 3.325919, "package-0": 46.885927, "psys": 76.701342, "uncore": 28.630786 }, "logits": { "l2": 1150.3575819756547, "sha256": "2c23050136374b58161b79874f63495a8c895d72d93c7555b2aaf74e81ff149c", "sum": 369774.01646265644, "top1": 220 }, "net_j": { "core": 4.25092361277194, "dram": 2.4861194467098215, "package-0": 37.14675626170307, "psys": 59.42879380806612, "uncore": 28.62746449393361 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.70065427, "watts": { "core": 6.271338147994066, "dram": 1.9556702727121602, "package-0": 27.569346590356663, "psys": 45.101078657215844, "uncore": 16.835159564795024 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 811.0208018948907, "sha256": "b95537717dfcf6809ff402f9c640cbc97a728f4a0d1eb86875f847d4d31b12aa", "sum": 202420.8223354963, "top1": 42 }, "gpu_active_fraction": 0.9841269841269841, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 63, "gross_j": { "core": 5.605576, "dram": 1.982051, "package-0": 15.201805, "psys": 26.220697, "uncore": 5.606065 }, "net_j": { "core": 3.222838212145847, "dram": 1.670095842312416, "package-0": 11.584055345920422, "psys": 19.804570111421736, "uncore": 5.604831180638413 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.631731547, "token_latencies_s": [ 0.049360224, 0.045810194, 0.044899891, 0.044527344, 0.0446105, 0.044837909, 0.044626674, 0.04505305, 0.044372493, 0.044386252, 0.044665544, 0.044652354, 0.04533809, 0.044561265 ], "watts": { "core": 8.873351388924702, "dram": 3.1374893487787148, "package-0": 24.06371040387508, "psys": 41.50607504804569, "uncore": 8.874125451898637 } } }, { "energy": { "gross_j": { "core": 19.070263999999998, "dram": 5.676011, "package-0": 66.26600499999999, "psys": 110.32820899999999, "uncore": 34.765109 }, "net_j": { "core": 9.503069146906086, "dram": 4.423445235425221, "package-0": 51.739976980163604, "psys": 84.56610607029198, "uncore": 34.76015495499094 } }, "first_decode": { "gpu_active_fraction": 0.625, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 8, "gross_j": { "core": 2.463556, "dram": 0.164978, "package-0": 3.143913, "psys": 4.352284, "uncore": 0.323791 }, "logits": { "l2": 979.727719530536, "sha256": "8eca8a515a86885daba895c0e4490a5dfd1090aa41c110d1f8b0c9ea55c06524", "sum": 284299.187461914, "top1": 78 }, "net_j": { "core": 2.182120867412432, "dram": 0.12813167070069112, "package-0": 2.7166054470017107, "psys": 3.594448399796637, "uncore": 0.3236452684313031 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.074616457, "watts": { "core": 33.01625538183889, "dram": 2.211013583772813, "package-0": 42.13431093357863, "psys": 58.32874107115539, "uncore": 4.339404643669962 } }, "metrics": { "all_tokens_per_s_e2e": 1614.8010239655146, "decode_tokens_per_s": 21.1610435402155, "end_to_end_ms": 2536.5354239999997, "handoff_estimate_ms": 29.993397499999997, "handoff_plus_first_decode_ms": 74.616457, "output_tokens_per_s_e2e": 6.307816499865291, "prefill_tokens_per_s": 2232.331310243325, "steady_tpot_ms": 44.6230595, "steady_tpot_p95_ms": 48.2103175, "ttft_ms": 1827.685694 }, "prefill": { "gpu_active_fraction": 0.8066298342541437, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 181, "gross_j": { "core": 10.954623, "dram": 3.527274, "package-0": 47.863586, "psys": 79.894815, "uncore": 28.836047 }, "logits": { "l2": 1150.3575819756547, "sha256": "2c23050136374b58161b79874f63495a8c895d72d93c7555b2aaf74e81ff149c", "sum": 369774.01646265644, "top1": 220 }, "net_j": { "core": 4.061036969517589, "dram": 2.624745098945908, "package-0": 37.39694187143839, "psys": 61.33208312309528, "uncore": 28.8324773919808 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.827685694, "watts": { "core": 5.993712724218544, "dram": 1.9299127916684344, "package-0": 26.188083737334324, "psys": 43.71365123789167, "uncore": 15.77735553474218 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 811.0208018948907, "sha256": "b95537717dfcf6809ff402f9c640cbc97a728f4a0d1eb86875f847d4d31b12aa", "sum": 202420.8223354963, "top1": 42 }, "gpu_active_fraction": 0.9841269841269841, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 63, "gross_j": { "core": 5.652085, "dram": 1.983759, "package-0": 15.258506, "psys": 26.08111, "uncore": 5.605271 }, "net_j": { "core": 3.259911309976064, "dram": 1.6705684657786222, "package-0": 11.626429661723503, "psys": 19.639574547400066, "uncore": 5.604032294578834 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.634233273, "token_latencies_s": [ 0.05384143, 0.04456289, 0.044729934, 0.044597111, 0.044737294, 0.044448042, 0.044603898, 0.044754349, 0.044638263, 0.044617655, 0.044386376, 0.044628464, 0.044482925, 0.04517818 ], "watts": { "core": 8.911681617183145, "dram": 3.1278065728349134, "package-0": 24.058192229217845, "psys": 41.12226701168357, "uncore": 8.837869658724134 } } }, { "energy": { "gross_j": { "core": 19.28236, "dram": 5.686386, "package-0": 66.565076, "psys": 109.25796, "uncore": 34.852450000000005 }, "net_j": { "core": 9.78819250742739, "dram": 4.443381195881322, "package-0": 52.149926612974575, "psys": 83.69250179294299, "uncore": 34.84753376971474 } }, "first_decode": { "gpu_active_fraction": 0.8571428571428571, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 7, "gross_j": { "core": 2.067866, "dram": 0.165405, "package-0": 2.747552, "psys": 3.844228, "uncore": 0.329223 }, "logits": { "l2": 979.727719530536, "sha256": "8eca8a515a86885daba895c0e4490a5dfd1090aa41c110d1f8b0c9ea55c06524", "sum": 284299.187461914, "top1": 78 }, "net_j": { "core": 1.806476744223326, "dram": 0.13118313684774463, "package-0": 2.350680428782, "psys": 3.140371016271821, "uncore": 0.3290876485037791 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.069301725, "watts": { "core": 29.838593483784138, "dram": 2.3867371266732538, "package-0": 39.646228142228786, "psys": 55.470884743489435, "uncore": 4.7505743904643065 } }, "metrics": { "all_tokens_per_s_e2e": 1627.221771402249, "decode_tokens_per_s": 21.359745537762972, "end_to_end_ms": 2517.1737940000003, "handoff_estimate_ms": 24.61765149999999, "handoff_plus_first_decode_ms": 69.30172499999999, "output_tokens_per_s_e2e": 6.356335044540035, "prefill_tokens_per_s": 2248.0351548751337, "steady_tpot_ms": 44.684073500000004, "steady_tpot_p95_ms": 48.42182755, "ttft_ms": 1814.918237 }, "prefill": { "gpu_active_fraction": 0.8166666666666667, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 180, "gross_j": { "core": 11.786407, "dram": 3.532278, "package-0": 48.796445, "psys": 79.77659, "uncore": 28.936205 }, "logits": { "l2": 1150.3575819756547, "sha256": "2c23050136374b58161b79874f63495a8c895d72d93c7555b2aaf74e81ff149c", "sum": 369774.01646265644, "top1": 220 }, "net_j": { "core": 4.940976712875935, "dram": 2.6360537928948147, "package-0": 38.40291652216784, "psys": 61.34352969727182, "uncore": 28.932660327787346 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.814918237, "watts": { "core": 6.494180707271167, "dram": 1.946246353135301, "package-0": 26.886304851208564, "psys": 43.95602422942648, "uncore": 15.943530904086673 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 811.0208018948907, "sha256": "b95537717dfcf6809ff402f9c640cbc97a728f4a0d1eb86875f847d4d31b12aa", "sum": 202420.8223354963, "top1": 42 }, "gpu_active_fraction": 0.9838709677419355, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 62, "gross_j": { "core": 5.428087, "dram": 1.988703, "package-0": 15.021079, "psys": 25.637142, "uncore": 5.587022 }, "net_j": { "core": 3.0407390503281286, "dram": 1.676144266138763, "package-0": 11.396329662024732, "psys": 19.208601079399354, "uncore": 5.5857857934236135 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.632953832, "token_latencies_s": [ 0.049069171, 0.048073258, 0.044983623, 0.044807323, 0.044667437, 0.044410077, 0.044877966, 0.044680398, 0.044728233, 0.044556619, 0.044687749, 0.044327617, 0.044534228, 0.044527339 ], "watts": { "core": 8.575802413342526, "dram": 3.141940058591825, "package-0": 23.731713500393187, "psys": 40.50396838422174, "uncore": 8.826902875911493 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 1.6647716462612152, "compile_target": "AUTO:GPU", "context_length": 8192, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 398324, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_1_5b_ov_fp16/openvino_model.xml", "key": "qwen25_1_5b_fp16", "size_b": 1.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "GPU", "key": "gpu_gpu", "label": "iGPU -> iGPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 8176, "prompt_sha256": "998ace73fce48442f97bcb84c31be293d0d94180b17f70a4d127bba1092ad744", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.047261254861950874, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 35.882416, "dram": 10.622043, "package-0": 137.941174, "psys": 223.042153, "uncore": 78.186018 }, "net_j": { "core": 16.498080529450466, "dram": 8.084187823820436, "package-0": 108.50962122880789, "psys": 170.84490466499898, "uncore": 78.17598048410674 } }, "first_decode": { "gpu_active_fraction": 0.6666666666666666, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 9, "gross_j": { "core": 2.396784, "dram": 0.181091, "package-0": 3.173209, "psys": 4.477527, "uncore": 0.386962 }, "logits": { "l2": 1104.323434566963, "sha256": "4cc0c5b7c789d0230c86bfae04035aec63fca0279550b97617a206b017f84bae", "sum": 343667.50144172134, "top1": 261 }, "net_j": { "core": 2.078594251096151, "dram": 0.13943264640362485, "package-0": 2.690096313452458, "psys": 3.6207202569254595, "uncore": 0.3867972363088975 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.084361151, "watts": { "core": 28.410992163916777, "dram": 2.1466160413102946, "package-0": 37.61457688029885, "psys": 53.07569831521147, "uncore": 4.586969184429454 } }, "metrics": { "all_tokens_per_s_e2e": 1593.9794344485392, "decode_tokens_per_s": 19.545877697122307, "end_to_end_ms": 5139.338578, "handoff_estimate_ms": 36.083645999999995, "handoff_plus_first_decode_ms": 84.36115099999999, "output_tokens_per_s_e2e": 3.113241082907303, "prefill_tokens_per_s": 1870.1194143434132, "steady_tpot_ms": 48.277505, "steady_tpot_p95_ms": 50.668234749999996, "ttft_ms": 4371.913332 }, "prefill": { "gpu_active_fraction": 0.8752886836027713, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 433, "gross_j": { "core": 27.451346, "dram": 8.360452, "package-0": 118.330446, "psys": 190.899841, "uncore": 71.609802 }, "logits": { "l2": 1063.3971159220778, "sha256": "084ff5e6e0b62105635ea609877c5eba14bc7d2005fb94d197b9e77e6987b4d0", "sum": 342411.9841043374, "top1": 220 }, "net_j": { "core": 10.961551955600694, "dram": 6.201558844581072, "package-0": 93.29372269621841, "psys": 146.49688088634434, "uncore": 71.60126332349759 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 4.371913332, "watts": { "core": 6.279023373832974, "dram": 1.9123096377062399, "package-0": 27.066054840082543, "psys": 43.66505612147391, "uncore": 16.37951088276514 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 1116.0949352903115, "sha256": "0b8aeac4667d48fb5fafee7dd6a8d3dfdcbf32877fc72e9ba5308bdb34b0f5fd", "sum": 350539.43440529774, "top1": 42 }, "gpu_active_fraction": 0.9852941176470589, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 68, "gross_j": { "core": 6.034286, "dram": 2.0805, "package-0": 16.437519, "psys": 27.664785, "uncore": 6.189254 }, "net_j": { "core": 3.457934322753622, "dram": 1.7431963328357385, "package-0": 12.525802219137024, "psys": 20.72730352172917, "uncore": 6.187919924300253 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.683064095, "token_latencies_s": [ 0.053622345, 0.048928247, 0.048392344, 0.04821759, 0.04907756, 0.048030054, 0.04819754, 0.048201542, 0.048895742, 0.048197996, 0.048193964, 0.048271342, 0.048522036, 0.048283668 ], "watts": { "core": 8.834143156067952, "dram": 3.0458342273136165, "package-0": 24.064387398374382, "psys": 40.50100891337291, "uncore": 9.061014984252687 } } }, { "energy": { "gross_j": { "core": 38.53457, "dram": 11.048067, "package-0": 142.17786, "psys": 231.643511, "uncore": 78.781109 }, "net_j": { "core": 18.291663057351624, "dram": 8.397805078802191, "package-0": 111.44272415641893, "psys": 177.1343408345819, "uncore": 78.77062690220143 } }, "first_decode": { "gpu_active_fraction": 0.625, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 8, "gross_j": { "core": 2.283198, "dram": 0.174743, "package-0": 3.013787, "psys": 4.228627, "uncore": 0.354919 }, "logits": { "l2": 1104.323434566963, "sha256": "4cc0c5b7c789d0230c86bfae04035aec63fca0279550b97617a206b017f84bae", "sum": 343667.50144172134, "top1": 261 }, "net_j": { "core": 1.981982111162069, "dram": 0.13530691493036826, "package-0": 2.5564460020935202, "psys": 3.4175266877912667, "uncore": 0.354763025642452 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.079860898, "watts": { "core": 28.589686031329126, "dram": 2.188092099840901, "package-0": 37.7379553132498, "psys": 52.949905471886886, "uncore": 4.444214989919097 } }, "metrics": { "all_tokens_per_s_e2e": 1526.3732712918795, "decode_tokens_per_s": 19.532305787252795, "end_to_end_ms": 5366.970291, "handoff_estimate_ms": 30.9977855, "handoff_plus_first_decode_ms": 79.860898, "output_tokens_per_s_e2e": 2.981197795491952, "prefill_tokens_per_s": 1777.773214865182, "steady_tpot_ms": 48.8631125, "steady_tpot_p95_ms": 50.528452200000004, "ttft_ms": 4599.011804 }, "prefill": { "gpu_active_fraction": 0.8549450549450549, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 455, "gross_j": { "core": 29.746994, "dram": 8.780312, "package-0": 122.208977, "psys": 198.94053, "uncore": 72.219846 }, "logits": { "l2": 1063.3971159220778, "sha256": "084ff5e6e0b62105635ea609877c5eba14bc7d2005fb94d197b9e77e6987b4d0", "sum": 342411.9841043374, "top1": 220 }, "net_j": { "core": 12.400639738894693, "dram": 6.5092754191342985, "package-0": 95.87172434570302, "psys": 152.23106387208279, "uncore": 72.21086378305218 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 4.599011804, "watts": { "core": 6.46812734294952, "dram": 1.9091736168981577, "package-0": 26.57287743721564, "psys": 43.25723404905616, "uncore": 15.703339995167363 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 1116.0949352903115, "sha256": "0b8aeac4667d48fb5fafee7dd6a8d3dfdcbf32877fc72e9ba5308bdb34b0f5fd", "sum": 350539.43440529774, "top1": 42 }, "gpu_active_fraction": 0.9855072463768116, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 69, "gross_j": { "core": 6.504378, "dram": 2.093012, "package-0": 16.955096, "psys": 28.474354, "uncore": 6.206344 }, "net_j": { "core": 3.9090412072948606, "dram": 1.7532227447375246, "package-0": 13.014553808622402, "psys": 21.485750274707847, "uncore": 6.205000093506804 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.688097589, "token_latencies_s": [ 0.053016543, 0.049174865, 0.048851492, 0.049009647, 0.049086684, 0.049188711, 0.048749696, 0.04889204, 0.048874733, 0.048674549, 0.048647884, 0.048798816, 0.048627863, 0.048472511 ], "watts": { "core": 9.452696977841033, "dram": 3.0417371510365805, "package-0": 24.64053975925209, "psys": 41.381272736882096, "uncore": 9.019569461098635 } } }, { "energy": { "gross_j": { "core": 38.108239999999995, "dram": 11.046907000000001, "package-0": 142.011294, "psys": 234.02643899999998, "uncore": 79.02335699999999 }, "net_j": { "core": 17.845775954967014, "dram": 8.394084604464897, "package-0": 111.24646428882369, "psys": 179.464606367435, "uncore": 79.01286477522409 } }, "first_decode": { "gpu_active_fraction": 0.6666666666666666, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 9, "gross_j": { "core": 2.442743, "dram": 0.175353, "package-0": 3.163383, "psys": 4.446522, "uncore": 0.345275 }, "logits": { "l2": 1104.323434566963, "sha256": "4cc0c5b7c789d0230c86bfae04035aec63fca0279550b97617a206b017f84bae", "sum": 343667.50144172134, "top1": 261 }, "net_j": { "core": 2.143483830722662, "dram": 0.13617309450757004, "package-0": 2.709012921323303, "psys": 3.6406906523622418, "uncore": 0.3451200388628287 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.079342116, "watts": { "core": 30.787469797251184, "dram": 2.210087263112569, "package-0": 39.87016176881393, "psys": 56.04239241615386, "uncore": 4.351724120894381 } }, "metrics": { "all_tokens_per_s_e2e": 1524.9000329790347, "decode_tokens_per_s": 19.58324790903004, "end_to_end_ms": 5372.155435, "handoff_estimate_ms": 30.770626000000004, "handoff_plus_first_decode_ms": 79.342116, "output_tokens_per_s_e2e": 2.978320376912177, "prefill_tokens_per_s": 1775.0009772871852, "steady_tpot_ms": 48.571490000000004, "steady_tpot_p95_ms": 50.694644, "ttft_ms": 4606.194646999999 }, "prefill": { "gpu_active_fraction": 0.8549450549450549, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 455, "gross_j": { "core": 29.327989, "dram": 8.78489, "package-0": 122.083733, "psys": 201.072117, "uncore": 72.471189 }, "logits": { "l2": 1063.3971159220778, "sha256": "084ff5e6e0b62105635ea609877c5eba14bc7d2005fb94d197b9e77e6987b4d0", "sum": 342411.9841043374, "top1": 220 }, "net_j": { "core": 11.95454280162862, "dram": 6.510306461678477, "package-0": 95.70534621538775, "psys": 154.28969894760223, "uncore": 72.462192754418 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 4.606194647, "watts": { "core": 6.367075481515143, "dram": 1.90719035412921, "package-0": 26.504249680267584, "psys": 43.65254454258846, "uncore": 15.733418701096415 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 1116.0949352903115, "sha256": "0b8aeac4667d48fb5fafee7dd6a8d3dfdcbf32877fc72e9ba5308bdb34b0f5fd", "sum": 350539.43440529774, "top1": 42 }, "gpu_active_fraction": 0.9852941176470589, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 68, "gross_j": { "core": 6.337508, "dram": 2.086664, "package-0": 16.764178, "psys": 28.5078, "uncore": 6.206893 }, "net_j": { "core": 3.7477493226157335, "dram": 1.7476050482788512, "package-0": 12.83210515211264, "psys": 21.534216767470532, "uncore": 6.205551981943257 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.686618672, "token_latencies_s": [ 0.052987064, 0.049460264, 0.048902616, 0.049254507, 0.048496768, 0.048907001, 0.049327183, 0.048546731, 0.048442555, 0.048596249, 0.048410434, 0.048469362, 0.048395614, 0.048400413 ], "watts": { "core": 9.230025716515906, "dram": 3.0390434823479424, "package-0": 24.415557985291727, "psys": 41.519115576862724, "uncore": 9.039796400992719 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 6.948272679001093, "compile_target": "NPU", "context_length": 8192, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 398393, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_1_5b_ov_fp16/openvino_model.xml", "key": "qwen25_1_5b_fp16", "size_b": 1.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "gpu_npu", "label": "iGPU -> NPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 8176, "prompt_sha256": "998ace73fce48442f97bcb84c31be293d0d94180b17f70a4d127bba1092ad744", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "GPU", "NPUW_ONLINE_PIPELINE": "NONE" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 8192, "NPUW_LLM_MIN_RESPONSE_LEN": 8192, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.04698196332901716, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 42.518811, "dram": 12.945523, "package-0": 141.026984, "psys": 243.361804, "uncore": 65.918471 }, "net_j": { "core": 21.012227852029998, "dram": 10.129816810487558, "package-0": 108.37318795890384, "psys": 185.44986458286223, "uncore": 65.90890867399104 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 0, "gpu_sample_count": 30, "gross_j": { "core": 4.55376, "dram": 0.391235, "package-0": 5.577196, "psys": 8.942543, "uncore": 0.002624 }, "logits": { "l2": 1241.46397566741, "sha256": "bdb717b7c1752b3326560db2337bf6d62440ead312748b3fc50ce706e9507034", "sum": 266600.9037211053, "top1": 45195 }, "net_j": { "core": 3.371212115696847, "dram": 0.23641229411882345, "package-0": 3.7817142557737102, "psys": 5.758232365323812, "uncore": 0.002011657977686571 }, "npu_busy_delta_us": 76496, "npu_utilization": 0.2439853096832885, "seconds": 0.313527073, "watts": { "core": 14.52429596087863, "dram": 1.24785077172586, "package-0": 17.788562712094723, "psys": 28.522394938442847, "uncore": 0.008369293199761412 } }, "metrics": { "all_tokens_per_s_e2e": 1436.6871705245176, "decode_tokens_per_s": 13.398854565866928, "end_to_end_ms": 5702.006789, "handoff_estimate_ms": 257.9817665, "handoff_plus_first_decode_ms": 313.527073, "output_tokens_per_s_e2e": 2.8060296299306984, "prefill_tokens_per_s": 1784.1757844230365, "steady_tpot_ms": 55.5453065, "steady_tpot_p95_ms": 66.3210663, "ttft_ms": 4582.508109 }, "prefill": { "gpu_active_fraction": 0.9933774834437086, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 453, "gross_j": { "core": 33.703588, "dram": 10.205967, "package-0": 123.838489, "psys": 212.656621, "uncore": 65.915847 }, "logits": { "l2": 3222.6951479846207, "sha256": "73a80de707a59e79c2b90653ba89bb9f702fb9496328d98e51493e37449336cc", "sum": 1204541.814432323, "top1": 283 }, "net_j": { "core": 16.41948166351151, "dram": 7.943080103601698, "package-0": 97.59574838512992, "psys": 166.1147732247727, "uncore": 65.90689701601335 }, "npu_busy_delta_us": 8560, "npu_utilization": 0.0018679726901493629, "seconds": 4.582508109, "watts": { "core": 7.354834339257686, "dram": 2.227157433710937, "package-0": 27.024172364645125, "psys": 46.40616359900041, "uncore": 14.384229210755118 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 1072.8463720276775, "sha256": "1b3c3c5e744d021148f21d2526bb88f65dc9872c0e17543459259b67a913c12f", "sum": 147567.20988509804, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 79, "gross_j": { "core": 4.261463, "dram": 2.348321, "package-0": 11.611299, "psys": 21.76264, "uncore": 0.0 }, "net_j": { "core": 1.2215340728216382, "dram": 1.9503244127670363, "package-0": 6.995725318000215, "psys": 13.576858992765713, "uncore": 0.0 }, "npu_busy_delta_us": 785168, "npu_utilization": 0.9741881639262262, "seconds": 0.805971607, "token_latencies_s": [ 0.077868211, 0.060103373, 0.057391334, 0.055917078, 0.055396267, 0.055286741, 0.055271382, 0.055393308, 0.05590076, 0.055726225, 0.055651875, 0.055282687, 0.055438738, 0.055307245 ], "watts": { "core": 5.287361196087395, "dram": 2.9136522671573464, "package-0": 14.40658566524416, "psys": 27.00174523641749, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 40.231648, "dram": 12.756132000000001, "package-0": 138.361402, "psys": 239.398252, "uncore": 65.90168700000001 }, "net_j": { "core": 19.430646481315193, "dram": 10.032802667709165, "package-0": 106.77890205714984, "psys": 183.3862703905657, "uncore": 65.89247136745512 } }, "first_decode": { "gpu_active_fraction": 0.058823529411764705, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 17, "gross_j": { "core": 2.967094, "dram": 0.284301, "package-0": 3.702078, "psys": 5.760239, "uncore": 0.015015 }, "logits": { "l2": 1528.6449015081357, "sha256": "36c32280eaaaedd81d9ef77d3977727da67f2c40dd6aecc130b0cea07b2b713d", "sum": 357628.61823196337, "top1": 311 }, "net_j": { "core": 2.323819259206406, "dram": 0.20008154697263714, "package-0": 2.725383466993919, "psys": 4.0280583202005715, "uncore": 0.014681902151778139 }, "npu_busy_delta_us": 72848, "npu_utilization": 0.42713466791848015, "seconds": 0.170550427, "watts": { "core": 17.397165472942497, "dram": 1.6669615256958579, "package-0": 21.706647500800454, "psys": 33.77440386003842, "uncore": 0.08803847791011395 } }, "metrics": { "all_tokens_per_s_e2e": 1485.4204045296813, "decode_tokens_per_s": 15.512472556393572, "end_to_end_ms": 5514.937034, "handoff_estimate_ms": 114.75934750000002, "handoff_plus_first_decode_ms": 170.550427, "output_tokens_per_s_e2e": 2.9012117275970337, "prefill_tokens_per_s": 1797.7238755916362, "steady_tpot_ms": 55.791079499999995, "steady_tpot_p95_ms": 62.431346199999986, "ttft_ms": 4547.973196 }, "prefill": { "gpu_active_fraction": 0.9933184855233853, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 449, "gross_j": { "core": 32.496377, "dram": 10.134923, "package-0": 122.486442, "psys": 210.835337, "uncore": 65.886672 }, "logits": { "l2": 3232.9150534350383, "sha256": "688fa4cf4457f077fd4509aff964fb4016666e8871d4f91399f7169f143c3ec9", "sum": 1208575.6573289484, "top1": 283 }, "net_j": { "core": 15.342527958079316, "dram": 7.889089778191488, "package-0": 96.44147316042393, "psys": 164.64424008555568, "uncore": 65.87778946530334 }, "npu_busy_delta_us": 6840, "npu_utilization": 0.0015039666473883062, "seconds": 4.547973196, "watts": { "core": 7.14524373815153, "dram": 2.2284482698609116, "package-0": 26.932094082640674, "psys": 46.3580869793675, "uncore": 14.487040525645174 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 1208.9667946533887, "sha256": "47f7337a58159a23820f7ac08b148be4ffd7e2a11bf4a8a77c0eaa9d27902624", "sum": 253333.13155601174, "top1": 311 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 79, "gross_j": { "core": 4.768177, "dram": 2.336908, "package-0": 12.172882, "psys": 22.802676, "uncore": 0.0 }, "net_j": { "core": 1.7642992640294697, "dram": 1.943631342545039, "package-0": 7.612045429731987, "psys": 14.713971984809458, "uncore": 0.0 }, "npu_busy_delta_us": 779709, "npu_utilization": 0.9790254523978628, "seconds": 0.796413411, "token_latencies_s": [ 0.068014997, 0.059424765, 0.05689953, 0.055778549, 0.055316883, 0.056162077, 0.055753213, 0.055873058, 0.05580361, 0.055914315, 0.055476847, 0.055436271, 0.055292128, 0.055237246 ], "watts": { "core": 5.987062666376922, "dram": 2.93429011581524, "package-0": 15.284627094256704, "psys": 28.63170770990445, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 42.356764999999996, "dram": 12.727018000000001, "package-0": 140.656135, "psys": 243.387622, "uncore": 66.045424 }, "net_j": { "core": 21.611620737261823, "dram": 10.01100166672049, "package-0": 109.15844404008067, "psys": 187.5260502441502, "uncore": 66.03624405269 } }, "first_decode": { "gpu_active_fraction": 0.058823529411764705, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 17, "gross_j": { "core": 3.010002, "dram": 0.284668, "package-0": 3.750662, "psys": 5.88353, "uncore": 0.014648 }, "logits": { "l2": 1558.9311317044221, "sha256": "a8d46c1c7b08272c8d75f1a3412e0a4d9699cbd3f7abd50b4eeebcbeea7b6ffd", "sum": 375173.26933249086, "top1": 1753 }, "net_j": { "core": 2.3694154959204443, "dram": 0.20080049896345376, "package-0": 2.7780490606513792, "psys": 4.158588080567628, "uncore": 0.01431629416332184 }, "npu_busy_delta_us": 73085, "npu_utilization": 0.4303225987535311, "seconds": 0.169837699, "watts": { "core": 17.722814296960063, "dram": 1.676117856495453, "package-0": 22.0838013119808, "psys": 34.64207319483291, "uncore": 0.08624704695274986 } }, "metrics": { "all_tokens_per_s_e2e": 1489.419967351387, "decode_tokens_per_s": 15.468493217695292, "end_to_end_ms": 5500.127687, "handoff_estimate_ms": 114.17112300000001, "handoff_plus_first_decode_ms": 169.83769900000001, "output_tokens_per_s_e2e": 2.9090233737331777, "prefill_tokens_per_s": 1804.691333085977, "steady_tpot_ms": 55.666576, "steady_tpot_p95_ms": 63.613831950000005, "ttft_ms": 4530.4146200000005 }, "prefill": { "gpu_active_fraction": 0.9955357142857143, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 448, "gross_j": { "core": 34.339451, "dram": 10.101414, "package-0": 124.467272, "psys": 214.224732, "uncore": 66.030776 }, "logits": { "l2": 3237.686448552048, "sha256": "e70dbb184774b1dcb7c98aa69c0e3c5953b75cf3f79612dccf368f3a8f764feb", "sum": 1210447.6801931784, "top1": 283 }, "net_j": { "core": 17.25182863733063, "dram": 7.864251373115647, "package-0": 98.52285620791979, "psys": 168.21196724600222, "uncore": 66.02192775852667 }, "npu_busy_delta_us": 8523, "npu_utilization": 0.0018812847641746307, "seconds": 4.53041462, "watts": { "core": 7.579759002278691, "dram": 2.22968863719586, "package-0": 27.473704382492034, "psys": 47.2858998499347, "uncore": 14.574996228490892 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 1151.2441687220637, "sha256": "53e96ef399ab02db12c66d59d43548dc5b1ae6a927dc046d7c6ddcad0edfdfbb", "sum": 109909.59551849216, "top1": 437 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 79, "gross_j": { "core": 5.007312, "dram": 2.340936, "package-0": 12.438201, "psys": 23.27936, "uncore": 0.0 }, "net_j": { "core": 1.9903766040107493, "dram": 1.9459497946413902, "package-0": 7.857538771509494, "psys": 15.15549491758037, "uncore": 0.0 }, "npu_busy_delta_us": 782538, "npu_utilization": 0.9783249132382333, "seconds": 0.799875368, "token_latencies_s": [ 0.068475622, 0.060995945, 0.056762271, 0.055456332, 0.055669597, 0.055801353, 0.055663555, 0.055581298, 0.055382226, 0.057817981, 0.055618211, 0.05545476, 0.055486474, 0.055681979 ], "watts": { "core": 6.2601152633568775, "dram": 2.9266259390550453, "package-0": 15.550173811578105, "psys": 29.10373407073088, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 2.1653368724510074, "compile_target": "AUTO:NPU,GPU", "context_length": 8192, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 398238, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_1_5b_ov_fp16/openvino_model.xml", "key": "qwen25_1_5b_fp16", "size_b": 1.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": null, "key": "openvino_auto", "label": "OpenVINO AUTO", "mode": "stock", "prefill_device": null }, "prompt_length": 8176, "prompt_sha256": "998ace73fce48442f97bcb84c31be293d0d94180b17f70a4d127bba1092ad744", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.046355665661394596, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 36.086394999999996, "dram": 10.645845000000001, "package-0": 138.196423, "psys": 224.759678, "uncore": 78.139143 }, "net_j": { "core": 16.602067007954226, "dram": 8.09489850383972, "package-0": 108.61304995219268, "psys": 172.29317439524198, "uncore": 78.12905370639554 } }, "first_decode": { "gpu_active_fraction": 0.6666666666666666, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 9, "gross_j": { "core": 2.50982, "dram": 0.182067, "package-0": 3.293937, "psys": 4.62804, "uncore": 0.386229 }, "logits": { "l2": 1104.323434566963, "sha256": "4cc0c5b7c789d0230c86bfae04035aec63fca0279550b97617a206b017f84bae", "sum": 343667.50144172134, "top1": 261 }, "net_j": { "core": 2.1798918196110164, "dram": 0.1388718158452579, "package-0": 2.793001661433737, "psys": 3.739624547706418, "uncore": 0.3860581579631749 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.087473343, "watts": { "core": 28.692398322995384, "dram": 2.0813998157129996, "package-0": 37.656466381992516, "psys": 52.908004213352186, "uncore": 4.415390869421785 } }, "metrics": { "all_tokens_per_s_e2e": 1585.7992178699267, "decode_tokens_per_s": 19.40710037826237, "end_to_end_ms": 5165.8494390000005, "handoff_estimate_ms": 38.863736499999995, "handoff_plus_first_decode_ms": 87.473343, "output_tokens_per_s_e2e": 3.0972640974022005, "prefill_tokens_per_s": 1861.1696519585068, "steady_tpot_ms": 48.6096065, "steady_tpot_p95_ms": 50.45135584999999, "ttft_ms": 4392.936448 }, "prefill": { "gpu_active_fraction": 0.8778801843317973, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 434, "gross_j": { "core": 26.490533, "dram": 8.377114, "package-0": 117.424565, "psys": 191.165648, "uncore": 71.575317 }, "logits": { "l2": 1063.3971159220778, "sha256": "084ff5e6e0b62105635ea609877c5eba14bc7d2005fb94d197b9e77e6987b4d0", "sum": 342411.9841043374, "top1": 220 }, "net_j": { "core": 9.921444873882717, "dram": 6.207839426242842, "package-0": 92.26744819838243, "psys": 146.5491684246523, "uncore": 71.5667372637663 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 4.392936448, "watts": { "core": 6.030256370328441, "dram": 1.9069508742412837, "package-0": 26.730312716784375, "psys": 43.516597670570256, "uncore": 16.29327395177468 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 1116.0949352903115, "sha256": "0b8aeac4667d48fb5fafee7dd6a8d3dfdcbf32877fc72e9ba5308bdb34b0f5fd", "sum": 350539.43440529774, "top1": 42 }, "gpu_active_fraction": 0.9852941176470589, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 68, "gross_j": { "core": 7.086042, "dram": 2.086664, "package-0": 17.477921, "psys": 28.96599, "uncore": 6.177597 }, "net_j": { "core": 4.500730314460494, "dram": 1.7481872617516214, "package-0": 13.552600092376515, "psys": 22.00438142288327, "uncore": 6.17625828466606 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.685439648, "token_latencies_s": [ 0.052803102, 0.049185031, 0.04858575, 0.048914197, 0.048747139, 0.048780339, 0.048633463, 0.048553321, 0.048760136, 0.048563576, 0.048401535, 0.048464501, 0.0485082, 0.048509397 ], "watts": { "core": 10.33795173751023, "dram": 3.0442709377675214, "package-0": 25.498847420043024, "psys": 42.2589940405665, "uncore": 9.012605293588152 } } }, { "energy": { "gross_j": { "core": 37.48672, "dram": 11.041720000000002, "package-0": 141.423221, "psys": 232.011674, "uncore": 79.04050799999999 }, "net_j": { "core": 17.190983884726204, "dram": 8.384541525515937, "package-0": 110.60787376179333, "psys": 177.36024786470094, "uncore": 79.02999854641918 } }, "first_decode": { "gpu_active_fraction": 0.625, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 8, "gross_j": { "core": 2.359491, "dram": 0.173034, "package-0": 3.068717, "psys": 4.300099, "uncore": 0.339477 }, "logits": { "l2": 1104.323434566963, "sha256": "4cc0c5b7c789d0230c86bfae04035aec63fca0279550b97617a206b017f84bae", "sum": 343667.50144172134, "top1": 261 }, "net_j": { "core": 2.071547591592009, "dram": 0.1353355877647577, "package-0": 2.6315278256508394, "psys": 3.5247382135581202, "uncore": 0.33932789834301275 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.076341986, "watts": { "core": 30.90685903822308, "dram": 2.266564037252057, "package-0": 40.19697627462822, "psys": 56.326789821789546, "uncore": 4.446792882752618 } }, "metrics": { "all_tokens_per_s_e2e": 1522.4001689327483, "decode_tokens_per_s": 19.82405508809927, "end_to_end_ms": 5380.976807, "handoff_estimate_ms": 28.140056499999996, "handoff_plus_first_decode_ms": 76.341986, "output_tokens_per_s_e2e": 2.973437829946774, "prefill_tokens_per_s": 1768.0436139441701, "steady_tpot_ms": 48.201929500000006, "steady_tpot_p95_ms": 50.6232435, "ttft_ms": 4624.3203140000005 }, "prefill": { "gpu_active_fraction": 0.8512035010940919, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 457, "gross_j": { "core": 29.064134, "dram": 8.792153, "package-0": 121.875054, "psys": 199.814491, "uncore": 72.484799 }, "logits": { "l2": 1063.3971159220778, "sha256": "084ff5e6e0b62105635ea609877c5eba14bc7d2005fb94d197b9e77e6987b4d0", "sum": 342411.9841043374, "top1": 220 }, "net_j": { "core": 11.622322193015183, "dram": 6.508618831724931, "package-0": 95.39286659654798, "psys": 152.84798117639912, "uncore": 72.47576735362316 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 4.624320314, "watts": { "core": 6.285060728169964, "dram": 1.9012854653216829, "package-0": 26.355236169740813, "psys": 43.20948321747247, "uncore": 15.674692512228077 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 1116.0949352903115, "sha256": "0b8aeac4667d48fb5fafee7dd6a8d3dfdcbf32877fc72e9ba5308bdb34b0f5fd", "sum": 350539.43440529774, "top1": 42 }, "gpu_active_fraction": 0.9852941176470589, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 68, "gross_j": { "core": 6.063095, "dram": 2.076533, "package-0": 16.47945, "psys": 27.897084, "uncore": 6.216232 }, "net_j": { "core": 3.4971141001190116, "dram": 1.7405871060262486, "package-0": 12.583479339594517, "psys": 20.98752847474369, "uncore": 6.214903294453014 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.680314507, "token_latencies_s": [ 0.053028419, 0.049328149, 0.04883711, 0.048416965, 0.048310765, 0.048325123, 0.04813078, 0.048040704, 0.048273079, 0.047893402, 0.04780936, 0.048052681, 0.047877043, 0.047967316 ], "watts": { "core": 8.912194194912265, "dram": 3.0523132736900465, "package-0": 24.223281776938503, "psys": 41.006157759325866, "uncore": 9.137291555653979 } } }, { "energy": { "gross_j": { "core": 37.480738, "dram": 11.055086000000001, "package-0": 141.684817, "psys": 233.68049399999998, "uncore": 79.26127199999999 }, "net_j": { "core": 17.140400181094368, "dram": 8.392068137077759, "package-0": 110.80175026755944, "psys": 178.90896644548164, "uncore": 79.2507394509507 } }, "first_decode": { "gpu_active_fraction": 0.75, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 8, "gross_j": { "core": 2.132685, "dram": 0.173523, "package-0": 2.874199, "psys": 4.100027, "uncore": 0.368163 }, "logits": { "l2": 1104.323434566963, "sha256": "4cc0c5b7c789d0230c86bfae04035aec63fca0279550b97617a206b017f84bae", "sum": 343667.50144172134, "top1": 261 }, "net_j": { "core": 1.8378931070757456, "dram": 0.13492796368883872, "package-0": 2.4266116598870795, "psys": 3.3062249289674286, "uncore": 0.3680103520892719 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.078157714, "watts": { "core": 27.28694188778346, "dram": 2.220164729997093, "package-0": 36.77434833879609, "psys": 52.45837922025201, "uncore": 4.710513923168223 } }, "metrics": { "all_tokens_per_s_e2e": 1519.0618939371043, "decode_tokens_per_s": 19.745044508515853, "end_to_end_ms": 5392.801987000001, "handoff_estimate_ms": 29.980421500000006, "handoff_plus_first_decode_ms": 78.157714, "output_tokens_per_s_e2e": 2.966917761595907, "prefill_tokens_per_s": 1764.686443149489, "steady_tpot_ms": 48.17729249999999, "steady_tpot_p95_ms": 50.75145075, "ttft_ms": 4633.117703 }, "prefill": { "gpu_active_fraction": 0.8518518518518519, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 459, "gross_j": { "core": 29.451218, "dram": 8.804664, "package-0": 122.497367, "psys": 201.727816, "uncore": 72.677488 }, "logits": { "l2": 1063.3971159220778, "sha256": "084ff5e6e0b62105635ea609877c5eba14bc7d2005fb94d197b9e77e6987b4d0", "sum": 342411.9841043374, "top1": 220 }, "net_j": { "core": 11.976224580818837, "dram": 6.516785595813701, "package-0": 95.9647994134422, "psys": 154.67195625493287, "uncore": 72.66843917165887 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 4.633117703, "watts": { "core": 6.356673818351298, "dram": 1.9003756356759238, "package-0": 26.439511113797405, "psys": 43.54040387736724, "uncore": 15.686518810635965 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 1116.0949352903115, "sha256": "0b8aeac4667d48fb5fafee7dd6a8d3dfdcbf32877fc72e9ba5308bdb34b0f5fd", "sum": 350539.43440529774, "top1": 42 }, "gpu_active_fraction": 0.9852941176470589, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 68, "gross_j": { "core": 5.896835, "dram": 2.076899, "package-0": 16.313251, "psys": 27.852651, "uncore": 6.215621 }, "net_j": { "core": 3.3262824931997836, "dram": 1.7403545775752192, "package-0": 12.410339194230167, "psys": 20.93078526158134, "uncore": 6.21428992720255 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.68152657, "token_latencies_s": [ 0.053352234, 0.049351029, 0.048413264, 0.048992454, 0.048325679, 0.048244088, 0.048131646, 0.048064406, 0.048222939, 0.048081485, 0.048072649, 0.048090432, 0.048070211, 0.048094232 ], "watts": { "core": 8.652391938292295, "dram": 3.047421907556737, "package-0": 23.936338975016042, "psys": 40.86803394913862, "uncore": 9.120144794941742 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 2.789744785055518, "compile_target": "AUTO:GPU", "context_length": 1024, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 393809, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 2.4565200143075967, "dram": 0.4907827289910671, "package-0": 4.295934585025856, "psys": 9.931950112327097, "uncore": 0.002123975024632864 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_3b_ov_fp16/openvino_model.xml", "key": "qwen25_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "GPU", "key": "gpu_gpu", "label": "iGPU -> iGPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 1008, "prompt_sha256": "2669029615323cd09063f0eb8d822f5daea575e1396736877003e47194c143f8", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.06067449878901243, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 14.149258, "dram": 5.502182, "package-0": 47.26111, "psys": 80.940162, "uncore": 21.727911000000002 }, "net_j": { "core": 9.25251221111787, "dram": 4.523871932324662, "package-0": 38.697736013214964, "psys": 61.14214097649398, "uncore": 21.723677138400266 } }, "first_decode": { "gpu_active_fraction": 0.9166666666666666, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 12, "gross_j": { "core": 2.629938, "dram": 0.29724, "package-0": 4.014516, "psys": 5.900986, "uncore": 0.761778 }, "logits": { "l2": 656.1393639359858, "sha256": "759589a4c665f287600ccbb81e67d5a11ba7b9e7dbbe102b502bb439588346fd", "sum": -851.9276055010268, "top1": 78 }, "net_j": { "core": 2.3513703327458884, "dram": 0.24158557864484417, "package-0": 3.5273599951407357, "psys": 4.7747097401035345, "uncore": 0.7615371427041213 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.113399307, "watts": { "core": 23.191834849572757, "dram": 2.6211800394864846, "package-0": 35.401592004437916, "psys": 52.03723158555104, "uncore": 6.7176600999863245 } }, "metrics": { "all_tokens_per_s_e2e": 513.7037132624425, "decode_tokens_per_s": 10.62354782209812, "end_to_end_ms": 1993.366942, "handoff_estimate_ms": 22.278633500000005, "handoff_plus_first_decode_ms": 113.39930700000001, "output_tokens_per_s_e2e": 8.026620519725665, "prefill_tokens_per_s": 1733.7185288979988, "steady_tpot_ms": 91.1206735, "steady_tpot_p95_ms": 97.85417325, "ttft_ms": 581.409256 }, "prefill": { "gpu_active_fraction": 0.8793103448275862, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 58, "gross_j": { "core": 2.795648, "dram": 1.294064, "package-0": 15.784567, "psys": 25.852656, "uncore": 10.118444 }, "logits": { "l2": 687.4985252735918, "sha256": "406295ad6f2a0cd6b351a0545dc583118a18409a1827387988bed8f0ee7f4e57", "sum": 121544.93127582863, "top1": 290 }, "net_j": { "core": 1.3674045261323107, "dram": 1.008718378679654, "package-0": 13.286870869095447, "psys": 20.078128274562786, "uncore": 10.117209101261166 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.581409256, "watts": { "core": 4.8083995415442775, "dram": 2.2257368396625594, "package-0": 27.14880583187688, "psys": 44.465504690898825, "uncore": 17.403307387318236 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 597.2912714442998, "sha256": "3ec855524c524afa8a2eb18bffab2133436dcb0e5f21c04bcd06f32483d1c0dd", "sum": -69328.94927807688, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 129, "gross_j": { "core": 8.723672, "dram": 3.910878, "package-0": 27.462027, "psys": 49.18652, "uncore": 10.847689 }, "net_j": { "core": 5.533737352239671, "dram": 3.2735679750001636, "package-0": 21.883505148978784, "psys": 36.28930296182766, "uncore": 10.844930894434977 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.298558379, "token_latencies_s": [ 0.098024314, 0.093376952, 0.09067414, 0.091202439, 0.090443544, 0.097762559, 0.09307435, 0.090874063, 0.091028076, 0.091038908, 0.090533161, 0.097218945, 0.09289667, 0.090351357 ], "watts": { "core": 6.717966739945853, "dram": 3.011707492898169, "package-0": 21.14808809839423, "psys": 37.877788781339035, "uncore": 8.353639832776437 } } }, { "energy": { "gross_j": { "core": 16.710773, "dram": 5.541489, "package-0": 50.114374, "psys": 85.053188, "uncore": 21.803411 }, "net_j": { "core": 11.759481709699099, "dram": 4.55228140660349, "package-0": 41.45561145352985, "psys": 65.03463419199078, "uncore": 21.79912997685383 } }, "first_decode": { "gpu_active_fraction": 0.9090909090909091, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 11, "gross_j": { "core": 2.620233, "dram": 0.294433, "package-0": 3.938162, "psys": 5.836472, "uncore": 0.702757 }, "logits": { "l2": 656.1393639359858, "sha256": "759589a4c665f287600ccbb81e67d5a11ba7b9e7dbbe102b502bb439588346fd", "sum": -851.9276055010268, "top1": 78 }, "net_j": { "core": 2.3485753835244947, "dram": 0.24015912248688048, "package-0": 3.463090214260142, "psys": 4.738133749648354, "uncore": 0.702522117324799 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.110586364, "watts": { "core": 23.69399720927618, "dram": 2.6624711162399732, "package-0": 35.6116419561457, "psys": 52.777501573340444, "uncore": 6.354825084944468 } }, "metrics": { "all_tokens_per_s_e2e": 508.0445377105066, "decode_tokens_per_s": 10.824217659499535, "end_to_end_ms": 2015.571321, "handoff_estimate_ms": 20.241441499999997, "handoff_plus_first_decode_ms": 110.586364, "output_tokens_per_s_e2e": 7.938195901726665, "prefill_tokens_per_s": 1600.5338288425166, "steady_tpot_ms": 90.34492250000001, "steady_tpot_p95_ms": 95.07992044999999, "ttft_ms": 629.789875 }, "prefill": { "gpu_active_fraction": 0.8253968253968254, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 63, "gross_j": { "core": 3.693227, "dram": 1.357723, "package-0": 17.143022, "psys": 28.166676, "uncore": 10.389378 }, "logits": { "l2": 687.4985252735918, "sha256": "406295ad6f2a0cd6b351a0545dc583118a18409a1827387988bed8f0ee7f4e57", "sum": 121544.93127582863, "top1": 290 }, "net_j": { "core": 2.14613556725422, "dram": 1.048633006456557, "package-0": 14.437485894688388, "psys": 21.91163438025128, "uncore": 10.388040342034733 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.629789875, "watts": { "core": 5.864220983228731, "dram": 2.1558349123983613, "package-0": 27.220224840864578, "psys": 44.72392637306212, "uncore": 16.496578323047824 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 597.2912714442998, "sha256": "3ec855524c524afa8a2eb18bffab2133436dcb0e5f21c04bcd06f32483d1c0dd", "sum": -69328.94927807688, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 126, "gross_j": { "core": 10.397313, "dram": 3.889333, "package-0": 29.03319, "psys": 51.05004, "uncore": 10.711276 }, "net_j": { "core": 7.264770758920383, "dram": 3.2634892776600526, "package-0": 23.55503534458132, "psys": 38.38486606209114, "uncore": 10.708567517494297 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.275195082, "token_latencies_s": [ 0.100339075, 0.092248068, 0.090524258, 0.090702288, 0.090640303, 0.0900161, 0.08998771, 0.089685271, 0.090268508, 0.090421337, 0.089856105, 0.089977806, 0.090461312, 0.090021764 ], "watts": { "core": 8.153507762665603, "dram": 3.0499905895967063, "package-0": 22.767645836952813, "psys": 40.03312177140282, "uncore": 8.399715581713638 } } }, { "energy": { "gross_j": { "core": 16.943255999999998, "dram": 5.523424, "package-0": 50.047846, "psys": 85.594995, "uncore": 21.611761 }, "net_j": { "core": 12.037326304696698, "dram": 4.543279099866579, "package-0": 41.46841130091035, "psys": 65.75984254671522, "uncore": 21.60751919774122 } }, "first_decode": { "gpu_active_fraction": 0.9090909090909091, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 11, "gross_j": { "core": 2.513971, "dram": 0.289795, "package-0": 3.798575, "psys": 5.668564, "uncore": 0.688597 }, "logits": { "l2": 656.1393639359858, "sha256": "759589a4c665f287600ccbb81e67d5a11ba7b9e7dbbe102b502bb439588346fd", "sum": -851.9276055010268, "top1": 78 }, "net_j": { "core": 2.2551898694181722, "dram": 0.23809368727361402, "package-0": 3.3460214680346123, "psys": 4.622286638537196, "uncore": 0.6883732506900002 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.105344605, "watts": { "core": 23.864259588803815, "dram": 2.750923979448212, "package-0": 36.05856227758413, "psys": 53.80972286146025, "uncore": 6.53661381140496 } }, "metrics": { "all_tokens_per_s_e2e": 512.7420592796457, "decode_tokens_per_s": 10.791038897709017, "end_to_end_ms": 1997.1055259999998, "handoff_estimate_ms": 14.368366999999992, "handoff_plus_first_decode_ms": 105.34460499999999, "output_tokens_per_s_e2e": 8.011594676244464, "prefill_tokens_per_s": 1660.4529404286563, "steady_tpot_ms": 90.976238, "steady_tpot_p95_ms": 95.74199429999999, "ttft_ms": 607.063275 }, "prefill": { "gpu_active_fraction": 0.8666666666666667, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 60, "gross_j": { "core": 4.003286, "dram": 1.336849, "package-0": 17.168413, "psys": 28.243458, "uncore": 10.17429 }, "logits": { "l2": 687.4985252735918, "sha256": "406295ad6f2a0cd6b351a0545dc583118a18409a1827387988bed8f0ee7f4e57", "sum": 121544.93127582863, "top1": 290 }, "net_j": { "core": 2.5120229150113835, "dram": 1.0389128292252452, "package-0": 14.560508881628438, "psys": 22.214135837674092, "uncore": 10.173000612765527 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.607063275, "watts": { "core": 6.594511914758804, "dram": 2.202157592221338, "package-0": 28.281093103515445, "psys": 46.52473500394172, "uncore": 16.759850939755825 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 597.2912714442998, "sha256": "3ec855524c524afa8a2eb18bffab2133436dcb0e5f21c04bcd06f32483d1c0dd", "sum": -69328.94927807688, "top1": 42 }, "gpu_active_fraction": 0.9921259842519685, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 127, "gross_j": { "core": 10.425999, "dram": 3.89678, "package-0": 29.080858, "psys": 51.682973, "uncore": 10.748874 }, "net_j": { "core": 7.270113520267143, "dram": 3.26627258336772, "package-0": 23.561880951247296, "psys": 38.92342007050394, "uncore": 10.746145334285693 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.284697646, "token_latencies_s": [ 0.100463359, 0.093199721, 0.090998013, 0.091349208, 0.091189707, 0.091013846, 0.091266685, 0.090549693, 0.090732896, 0.090680358, 0.090588082, 0.090954463, 0.090857153, 0.090827806 ], "watts": { "core": 8.115527441388338, "dram": 3.0332273217226713, "package-0": 22.636344116100297, "psys": 40.22967829116735, "uncore": 8.36685116802962 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 5.953581932932138, "compile_target": "NPU", "context_length": 1024, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 393877, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 2.4565200143075967, "dram": 0.4907827289910671, "package-0": 4.295934585025856, "psys": 9.931950112327097, "uncore": 0.002123975024632864 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_3b_ov_fp16/openvino_model.xml", "key": "qwen25_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "gpu_npu", "label": "iGPU -> NPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 1008, "prompt_sha256": "2669029615323cd09063f0eb8d822f5daea575e1396736877003e47194c143f8", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "GPU", "NPUW_ONLINE_PIPELINE": "NONE" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 1024, "NPUW_LLM_MIN_RESPONSE_LEN": 1024, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.05905019957572222, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 34.644442999999995, "dram": 14.288355, "package-0": 105.003394, "psys": 211.935675, "uncore": 31.031598 }, "net_j": { "core": 19.863919423877668, "dram": 11.335386713824334, "package-0": 79.15538053150351, "psys": 152.1765759749107, "uncore": 31.022432305796265 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 32, "gross_j": { "core": 4.542591, "dram": 0.465209, "package-0": 5.770127, "psys": 9.328345, "uncore": 0.0 }, "logits": { "l2": 657.1396888861135, "sha256": "d92c9a2b6fabdb458067c8b3cf593700f9212fa4e571c6361c9a75d8d32712e2", "sum": -1004.8725463692099, "top1": 78 }, "net_j": { "core": 3.7491938886851486, "dram": 0.3066979367085064, "package-0": 4.3826430510374355, "psys": 6.120563063379139, "uncore": 0.0 }, "npu_busy_delta_us": 128055, "npu_utilization": 0.3964845169537374, "seconds": 0.322976042, "watts": { "core": 14.064792459126116, "dram": 1.4403823798175097, "package-0": 17.865495422722404, "psys": 28.88246738747266, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 170.18859187875364, "decode_tokens_per_s": 8.815723876501492, "end_to_end_ms": 6016.85453, "handoff_estimate_ms": 225.1610995, "handoff_plus_first_decode_ms": 322.976042, "output_tokens_per_s_e2e": 2.6591967481055256, "prefill_tokens_per_s": 233.58479753314208, "steady_tpot_ms": 97.8149425, "steady_tpot_p95_ms": 101.43412945, "ttft_ms": 4315.349332000001 }, "prefill": { "gpu_active_fraction": 0.9976580796252927, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 427, "gross_j": { "core": 23.23578, "dram": 9.758581, "package-0": 79.169414, "psys": 164.841375, "uncore": 31.031598 }, "logits": { "l2": 687.8061052992338, "sha256": "d7e8d20b5dc6aa0d71807dad56aa5d9dc2be40a56a5857dab5cf9f63f533a459", "sum": 121705.75016708858, "top1": 290 }, "net_j": { "core": 12.63503799721308, "dram": 7.6406820782912614, "package-0": 60.63095555819298, "psys": 121.98154071731193, "uncore": 31.022432305796265 }, "npu_busy_delta_us": 11610, "npu_utilization": 0.0026903963287299403, "seconds": 4.315349332, "watts": { "core": 5.384449371849833, "dram": 2.2613652451346895, "package-0": 18.34600351191221, "psys": 38.19884841712277, "uncore": 7.190981682499857 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 597.4529443816389, "sha256": "6f77105b6ab09a679ccc06c0136f724e643c4d6fe6374032a2408f24c7dcdb18", "sum": -68897.5107829012, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 137, "gross_j": { "core": 6.866072, "dram": 4.064565, "package-0": 20.063853, "psys": 37.765955, "uncore": 0.0 }, "net_j": { "core": 3.4796875379794407, "dram": 3.3880066988245674, "package-0": 14.141781922273099, "psys": 24.07447219421962, "uncore": 0.0 }, "npu_busy_delta_us": 1364402, "npu_utilization": 0.9897520078276821, "seconds": 1.378529156, "token_latencies_s": [ 0.105533086, 0.099226999, 0.097978259, 0.097780159, 0.097753358, 0.097863056, 0.0977643, 0.097956971, 0.09779655, 0.097586589, 0.09765897, 0.097833297, 0.097944115, 0.097796588 ], "watts": { "core": 4.980723091793642, "dram": 2.9484795314695544, "package-0": 14.554536559979733, "psys": 27.395833331217553, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 29.673021000000002, "dram": 14.110742, "package-0": 99.67113099999999, "psys": 201.55435500000002, "uncore": 31.065655 }, "net_j": { "core": 15.38405315862766, "dram": 11.255980552582702, "package-0": 74.68274467440527, "psys": 143.7826637739639, "uncore": 31.056220958985488 } }, "first_decode": { "gpu_active_fraction": 0.07692307692307693, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 13, "gross_j": { "core": 2.351557, "dram": 0.312499, "package-0": 3.151786, "psys": 4.862475, "uncore": 0.002502 }, "logits": { "l2": 657.1396888861135, "sha256": "d92c9a2b6fabdb458067c8b3cf593700f9212fa4e571c6361c9a75d8d32712e2", "sum": -1004.8725463692099, "top1": 78 }, "net_j": { "core": 2.0346546918578547, "dram": 0.24918578508308087, "package-0": 2.597590806110611, "psys": 3.5812080464977907, "uncore": 0.002227997515256498 }, "npu_busy_delta_us": 124545, "npu_utilization": 0.9654309145792276, "seconds": 0.12900457, "watts": { "core": 18.22847826243675, "dram": 2.4223870518695576, "package-0": 24.431584090393073, "psys": 37.69226935138809, "uncore": 0.01939466175500604 } }, "metrics": { "all_tokens_per_s_e2e": 176.04326096722374, "decode_tokens_per_s": 9.972969920543374, "end_to_end_ms": 5816.752055, "handoff_estimate_ms": 31.265180500000017, "handoff_plus_first_decode_ms": 129.00457, "output_tokens_per_s_e2e": 2.750675952612871, "prefill_tokens_per_s": 233.72901957483646, "steady_tpot_ms": 97.7393895, "steady_tpot_p95_ms": 100.5588236, "ttft_ms": 4312.686554 }, "prefill": { "gpu_active_fraction": 0.9976580796252927, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 427, "gross_j": { "core": 20.750008, "dram": 9.745886, "package-0": 76.719591, "psys": 159.886432, "uncore": 31.063153 }, "logits": { "l2": 687.8061052992338, "sha256": "d7e8d20b5dc6aa0d71807dad56aa5d9dc2be40a56a5857dab5cf9f63f533a459", "sum": 121705.75016708858, "top1": 290 }, "net_j": { "core": 10.155807164663742, "dram": 7.629293923744799, "package-0": 58.192571678295415, "psys": 117.05304429556816, "uncore": 31.053992961470232 }, "npu_busy_delta_us": 11586, "npu_utilization": 0.0026864924809464836, "seconds": 4.312686554, "watts": { "core": 4.811387922628982, "dram": 2.259817836972346, "package-0": 17.789280542274252, "psys": 37.073510907419404, "uncore": 7.202738388485258 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 597.4529443816389, "sha256": "6f77105b6ab09a679ccc06c0136f724e643c4d6fe6374032a2408f24c7dcdb18", "sum": -68897.5107829012, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 136, "gross_j": { "core": 6.571456, "dram": 4.052357, "package-0": 19.799754, "psys": 36.805448, "uncore": 0.0 }, "net_j": { "core": 3.1935913021060633, "dram": 3.3775008437548224, "package-0": 13.89258218999925, "psys": 23.148411431897948, "uncore": 0.0 }, "npu_busy_delta_us": 1363223, "npu_utilization": 0.9913909771319073, "seconds": 1.375060931, "token_latencies_s": [ 0.104538979, 0.098415663, 0.097789404, 0.097858491, 0.097865387, 0.097795309, 0.097663261, 0.097792864, 0.097606031, 0.097689375, 0.097618236, 0.097591444, 0.097370825, 0.097432287 ], "watts": { "core": 4.779028951990492, "dram": 2.94703813383234, "package-0": 14.399183013367137, "psys": 26.76641243325384, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 34.1312, "dram": 14.117762, "package-0": 104.07700299999999, "psys": 204.64516700000001, "uncore": 31.006513 }, "net_j": { "core": 19.822727888387618, "dram": 11.259103837343407, "package-0": 79.0545078258749, "psys": 146.79461810625787, "uncore": 30.997352823388496 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 14, "gross_j": { "core": 2.519647, "dram": 0.316894, "package-0": 3.324942, "psys": 5.038073, "uncore": 0.0 }, "logits": { "l2": 657.1396888861135, "sha256": "d92c9a2b6fabdb458067c8b3cf593700f9212fa4e571c6361c9a75d8d32712e2", "sum": -1004.8725463692099, "top1": 78 }, "net_j": { "core": 2.185195971223005, "dram": 0.25007476479694, "package-0": 2.740057800745655, "psys": 3.685854856657956, "uncore": 0.0 }, "npu_busy_delta_us": 125828, "npu_utilization": 0.9241980851145687, "seconds": 0.136148302, "watts": { "core": 18.506635506919505, "dram": 2.327564834411229, "package-0": 24.421472402938967, "psys": 37.00430285204732, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 175.80329157642794, "decode_tokens_per_s": 9.921026729013935, "end_to_end_ms": 5824.691852, "handoff_estimate_ms": 38.46611899999999, "handoff_plus_first_decode_ms": 136.148302, "output_tokens_per_s_e2e": 2.7469264308816865, "prefill_tokens_per_s": 233.7254963120047, "steady_tpot_ms": 97.68218300000001, "steady_tpot_p95_ms": 100.82995355, "ttft_ms": 4312.7515650000005 }, "prefill": { "gpu_active_fraction": 0.9976580796252927, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 427, "gross_j": { "core": 23.51526, "dram": 9.735571, "package-0": 79.446513, "psys": 162.074475, "uncore": 31.006513 }, "logits": { "l2": 687.8061052992338, "sha256": "d7e8d20b5dc6aa0d71807dad56aa5d9dc2be40a56a5857dab5cf9f63f533a459", "sum": 121705.75016708858, "top1": 290 }, "net_j": { "core": 12.92089946384109, "dram": 7.6189470174688045, "package-0": 60.91921439529211, "psys": 119.24044160955938, "uncore": 30.997352823388496 }, "npu_busy_delta_us": 9091, "npu_utilization": 0.002107935006917098, "seconds": 4.312751565, "watts": { "core": 5.452495847624833, "dram": 2.2573920276346824, "package-0": 18.421305239268982, "psys": 37.580294750875595, "uncore": 7.189496666497645 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 597.4529443816389, "sha256": "6f77105b6ab09a679ccc06c0136f724e643c4d6fe6374032a2408f24c7dcdb18", "sum": -68897.5107829012, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 136, "gross_j": { "core": 8.096293, "dram": 4.065297, "package-0": 21.305548, "psys": 37.532619, "uncore": 0.0 }, "net_j": { "core": 4.716632453323522, "dram": 3.390082055077663, "package-0": 15.395235629837128, "psys": 23.868321640040527, "uncore": 0.0 }, "npu_busy_delta_us": 1363433, "npu_utilization": 0.9910168214855533, "seconds": 1.375791985, "token_latencies_s": [ 0.105289027, 0.098428914, 0.097896288, 0.0978713, 0.097644656, 0.097664091, 0.097479362, 0.0974379, 0.097747341, 0.097683766, 0.097648207, 0.0976806, 0.097716086, 0.097568683 ], "watts": { "core": 5.8848234967730235, "dram": 2.954877659067043, "package-0": 15.48602421898831, "psys": 27.280736775043792, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 23.223629924468696, "compile_target": "NPU", "context_length": 1024, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 394012, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 2.4565200143075967, "dram": 0.4907827289910671, "package-0": 4.295934585025856, "psys": 9.931950112327097, "uncore": 0.002123975024632864 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_3b_ov_fp16/openvino_model.xml", "key": "qwen25_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU+GPU", "key": "npu_gpu_combined", "label": "NPU + iGPU (subgraph combination)", "mode": "combined", "prefill_device": "NPU+GPU" }, "prompt_length": 1008, "prompt_sha256": "2669029615323cd09063f0eb8d822f5daea575e1396736877003e47194c143f8", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU,GPU", "NPUW_SUBMODEL_DEVICE": "0:GPU,last:NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "NPU,GPU", "NPUW_SUBMODEL_DEVICE": "0:GPU,last:NPU" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 1024, "NPUW_LLM_MIN_RESPONSE_LEN": 1024, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.06119983829557896, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 378.001717, "dram": 144.44207300000002, "package-0": 1133.3849169999999, "psys": 2402.3283229999997, "uncore": 369.285493 }, "net_j": { "core": 212.50720218401017, "dram": 111.37828799463438, "package-0": 843.9699699654745, "psys": 1733.2178515025864, "uncore": 369.14240187347315 } }, "first_decode": { "gpu_active_fraction": 0.85, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 500, "gross_j": { "core": 34.278232, "dram": 9.749242, "package-0": 83.419404, "psys": 173.291915, "uncore": 23.588441 }, "logits": { "l2": 3028.7603501122676, "sha256": "f451a70dcddf4cbe6d2ff66a3945a7621931ab3de01bc712804d7d9e5f08ec13", "sum": 1122318.0844391808, "top1": 141473 }, "net_j": { "core": 21.835099529569668, "dram": 7.263255940262374, "package-0": 61.65899382037975, "psys": 122.98311628572388, "uncore": 23.57768232413273 }, "npu_busy_delta_us": 11630, "npu_utilization": 0.0022959916109781073, "seconds": 5.065349518, "watts": { "core": 6.767199751604585, "dram": 1.9246928499910085, "package-0": 16.46863729809059, "psys": 34.2112453216106, "uncore": 4.656823959763718 } }, "metrics": { "all_tokens_per_s_e2e": 15.199757511286032, "decode_tokens_per_s": 0.22554565483834121, "end_to_end_ms": 67369.49581200001, "handoff_estimate_ms": 677.0697624999995, "handoff_plus_first_decode_ms": 5065.349518, "output_tokens_per_s_e2e": 0.23749621111384425, "prefill_tokens_per_s": 1166.5134038170588, "steady_tpot_ms": 4388.2797555, "steady_tpot_p95_ms": 4420.6474222, "ttft_ms": 864.113517 }, "prefill": { "gpu_active_fraction": 0.13953488372093023, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 86, "gross_j": { "core": 2.475396, "dram": 1.562984, "package-0": 18.398391, "psys": 31.476177, "uncore": 0.766905 }, "logits": { "l2": 690.102758738694, "sha256": "2f570dc830d3dd55b4b024060856ce9cff17aab7732962671d383c10d232e4d0", "sum": 123462.46132551879, "top1": 290 }, "net_j": { "core": 0.3526838508557719, "dram": 1.138892009968671, "package-0": 14.686215856931373, "psys": 22.893844657768486, "uncore": 0.7650696444714443 }, "npu_busy_delta_us": 708564, "npu_utilization": 0.8199894875617365, "seconds": 0.864113517, "watts": { "core": 2.864665291423742, "dram": 1.808771613047224, "package-0": 21.291636617229308, "psys": 36.42597457481966, "uncore": 0.8875049225737316 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 2808.0053142516367, "sha256": "c21cea26a3148945c34eeca07ac73bd0f71652d4375b55a5b70de31d97ad04c0", "sum": -711887.96540685, "top1": 220 }, "gpu_active_fraction": 0.9996698580389568, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 6058, "gross_j": { "core": 341.248089, "dram": 133.129847, "package-0": 1031.567122, "psys": 2197.560231, "uncore": 344.930147 }, "net_j": { "core": 190.31941880358474, "dram": 102.97614004440334, "package-0": 767.6247602881634, "psys": 1587.340890559094, "uncore": 344.799649904869 }, "npu_busy_delta_us": 163479, "npu_utilization": 0.0026607895961474515, "seconds": 61.440032777, "token_latencies_s": [ 4.357933267, 4.358327048, 4.376426719, 4.393156215, 4.383403296, 4.368133214, 4.365546084, 4.375708867, 4.419513096, 4.422754028, 4.419285121, 4.400771137, 4.400375828, 4.398633877 ], "watts": { "core": 5.554165152199362, "dram": 2.166825780891136, "package-0": 16.789820502605036, "psys": 35.76756280349274, "uncore": 5.614094449655375 } } }, { "energy": { "gross_j": { "core": 372.833519, "dram": 145.401666, "package-0": 1132.461397, "psys": 2418.4415010000002, "uncore": 372.374596 }, "net_j": { "core": 208.22301999969795, "dram": 112.51449656949946, "package-0": 844.5924068859165, "psys": 1752.9051917030906, "uncore": 372.23226921795356 } }, "first_decode": { "gpu_active_fraction": 0.9977011494252873, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 435, "gross_j": { "core": 27.147025, "dram": 9.56217, "package-0": 76.55852, "psys": 160.024553, "uncore": 24.625303 }, "logits": { "l2": 2808.823057621367, "sha256": "0bac2cc2cf3bdce12f64c9fb8052874bc0f3588cdfb36a3d50ac586531a65ff8", "sum": -712076.476965785, "top1": 220 }, "net_j": { "core": 16.332964979774964, "dram": 7.401652740915023, "package-0": 57.64701340740075, "psys": 116.30225315834392, "uncore": 24.615952865149048 }, "npu_busy_delta_us": 11686, "npu_utilization": 0.0026545897501502125, "seconds": 4.402186816, "watts": { "core": 6.166713530042065, "dram": 2.1721408926231267, "package-0": 17.391020236066236, "psys": 36.3511499372043, "uncore": 5.593879594227561 } }, "metrics": { "all_tokens_per_s_e2e": 15.281385512635882, "decode_tokens_per_s": 0.22676615859774746, "end_to_end_ms": 67009.630714, "handoff_estimate_ms": 0.0, "handoff_plus_first_decode_ms": 4402.186816, "output_tokens_per_s_e2e": 0.23877164863493566, "prefill_tokens_per_s": 1169.1096864390988, "steady_tpot_ms": 4408.0777025, "steady_tpot_p95_ms": 4430.79879175, "ttft_ms": 862.19455 }, "prefill": { "gpu_active_fraction": 0.13953488372093023, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 86, "gross_j": { "core": 3.31591, "dram": 1.56158, "package-0": 19.191724, "psys": 31.622051, "uncore": 0.779905 }, "logits": { "l2": 690.102758738694, "sha256": "2f570dc830d3dd55b4b024060856ce9cff17aab7732962671d383c10d232e4d0", "sum": 123462.46132551879, "top1": 290 }, "net_j": { "core": 1.1979118316980681, "dram": 1.1384298058297748, "package-0": 15.487792613634195, "psys": 23.058777742279688, "uncore": 0.7780737203094253 }, "npu_busy_delta_us": 712074, "npu_utilization": 0.8258855266482489, "seconds": 0.86219455, "watts": { "core": 3.845895337659001, "dram": 1.8111689525293333, "package-0": 22.259157170501716, "psys": 36.676236239257136, "uncore": 0.904558025795918 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 2806.3459274564684, "sha256": "1263a5ad985e73d4ff10f0150c722a4dc401619b5ca1690100a3cadbbd76a777", "sum": -709671.8661493696, "top1": 220 }, "gpu_active_fraction": 0.9998359310910583, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 6095, "gross_j": { "core": 342.370584, "dram": 134.277916, "package-0": 1036.711153, "psys": 2226.794897, "uncore": 346.969388 }, "net_j": { "core": 190.69214318822492, "dram": 103.97441402275466, "package-0": 771.4576008648816, "psys": 1613.544160802467, "uncore": 346.8382426324951 }, "npu_busy_delta_us": 162473, "npu_utilization": 0.00263134413927608, "seconds": 61.745249348, "token_latencies_s": [ 4.400253701, 4.402764825, 4.405152867, 4.404330115, 4.400407505, 4.401590521, 4.411365182, 4.418415966, 4.409351803, 4.406803602, 4.410526606, 4.412189366, 4.43028982, 4.431744025 ], "watts": { "core": 5.544889487292836, "dram": 2.174708458025677, "package-0": 16.790136309224902, "psys": 36.06423037422115, "uncore": 5.619369776036684 } } }, { "energy": { "gross_j": { "core": 355.299016, "dram": 145.67620100000002, "package-0": 1115.5413330000001, "psys": 2417.44126, "uncore": 372.594749 }, "net_j": { "core": 190.51473839882365, "dram": 112.75431272516659, "package-0": 827.3684408223124, "psys": 1751.2023468633415, "uncore": 372.45227196417756 } }, "first_decode": { "gpu_active_fraction": 0.9977011494252873, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 435, "gross_j": { "core": 24.970883, "dram": 9.614966, "package-0": 74.543083, "psys": 159.563923, "uncore": 24.720762 }, "logits": { "l2": 2805.755314989439, "sha256": "59e00002720744e4e9ec04d1081d48ec008f2032aa2805aeb701fc6b48dd571c", "sum": -709422.8614426963, "top1": 220 }, "net_j": { "core": 14.132508103934388, "dram": 7.449590925366763, "package-0": 55.589054825876566, "psys": 115.743315742801, "uncore": 24.711390841836106 }, "npu_busy_delta_us": 9454, "npu_utilization": 0.0021427511447029237, "seconds": 4.412084914, "watts": { "core": 5.6596560326309255, "dram": 2.1792341234165105, "package-0": 16.895205884063362, "psys": 36.165197658296925, "uncore": 5.602966053884972 } }, "metrics": { "all_tokens_per_s_e2e": 15.265270032248647, "decode_tokens_per_s": 0.22651740861171837, "end_to_end_ms": 67080.372495, "handoff_estimate_ms": 0.6553485000004855, "handoff_plus_first_decode_ms": 4412.084914, "output_tokens_per_s_e2e": 0.23851984425388512, "prefill_tokens_per_s": 1171.6889881071302, "steady_tpot_ms": 4411.4295655, "steady_tpot_p95_ms": 4431.34374395, "ttft_ms": 860.296555 }, "prefill": { "gpu_active_fraction": 0.13953488372093023, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 86, "gross_j": { "core": 2.698784, "dram": 1.561397, "package-0": 18.556043, "psys": 31.281902, "uncore": 0.779844 }, "logits": { "l2": 690.102758738694, "sha256": "2f570dc830d3dd55b4b024060856ce9cff17aab7732962671d383c10d232e4d0", "sum": 123462.46132551879, "top1": 290 }, "net_j": { "core": 0.5854482944026236, "dram": 1.1391783089954863, "package-0": 14.8602652759969, "psys": 22.737479533933133, "uncore": 0.7780167516034023 }, "npu_busy_delta_us": 711986, "npu_utilization": 0.8276053133794079, "seconds": 0.860296555, "watts": { "core": 3.137039180634636, "dram": 1.8149520545272901, "package-0": 21.56935639478412, "psys": 36.36176597266509, "uncore": 0.9064827651204531 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": null, "sha256": "b0ba3595a28b4c7ae5c9e581e843cac4c1d56ce6f3684ebf9bc0b9924c710c7a", "sum": null, "top1": 0 }, "gpu_active_fraction": 0.9995077932731747, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 6095, "gross_j": { "core": 327.629349, "dram": 134.499838, "package-0": 1022.442207, "psys": 2226.595435, "uncore": 347.094143 }, "net_j": { "core": 175.79678200048664, "dram": 104.16554349080434, "package-0": 756.919120720439, "psys": 1612.7215515866073, "uncore": 346.96286437073803 }, "npu_busy_delta_us": 164109, "npu_utilization": 0.0026551421147302185, "seconds": 61.807991026, "token_latencies_s": [ 4.429692753, 4.429075659, 4.43440987, 4.417047683, 4.410237764, 4.411556353, 4.408600612, 4.405753037, 4.413138022, 4.405813664, 4.41057777, 4.41405597, 4.406664255, 4.411302778 ], "watts": { "core": 5.300760363852955, "dram": 2.1760914044823365, "package-0": 16.54223329423378, "psys": 36.02439422539014, "uncore": 5.615683946983363 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 8.413142912089825, "compile_target": "AUTO:NPU,GPU", "context_length": 1024, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 393724, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 2.4565200143075967, "dram": 0.4907827289910671, "package-0": 4.295934585025856, "psys": 9.931950112327097, "uncore": 0.002123975024632864 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_3b_ov_fp16/openvino_model.xml", "key": "qwen25_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": null, "key": "openvino_auto", "label": "OpenVINO AUTO", "mode": "stock", "prefill_device": null }, "prompt_length": 1008, "prompt_sha256": "2669029615323cd09063f0eb8d822f5daea575e1396736877003e47194c143f8", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.06183239724487066, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 15.497032, "dram": 5.491015, "package-0": 48.586788999999996, "psys": 82.980745, "uncore": 21.675908999999997 }, "net_j": { "core": 10.580342034643227, "dram": 4.508720329186607, "package-0": 39.9885368616069, "psys": 63.102087724957705, "uncore": 21.671657894114656 } }, "first_decode": { "gpu_active_fraction": 0.9166666666666666, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 12, "gross_j": { "core": 2.616327, "dram": 0.298584, "package-0": 4.050954, "psys": 6.01268, "uncore": 0.810423 }, "logits": { "l2": 656.1393639359858, "sha256": "759589a4c665f287600ccbb81e67d5a11ba7b9e7dbbe102b502bb439588346fd", "sum": -851.9276055010268, "top1": 78 }, "net_j": { "core": 2.3313956586923443, "dram": 0.24165819420660734, "package-0": 3.5526692734957956, "psys": 4.860674784970748, "uncore": 0.8101766404958446 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.115989831, "watts": { "core": 22.55652049359396, "dram": 2.5742256663862197, "package-0": 34.92507890627067, "psys": 51.837992590919455, "uncore": 6.98701768088618 } }, "metrics": { "all_tokens_per_s_e2e": 511.61991347332133, "decode_tokens_per_s": 10.56431005256959, "end_to_end_ms": 2001.485816, "handoff_estimate_ms": 24.994537999999995, "handoff_plus_first_decode_ms": 115.989831, "output_tokens_per_s_e2e": 7.994061148020646, "prefill_tokens_per_s": 1733.1177749760836, "steady_tpot_ms": 90.995293, "steady_tpot_p95_ms": 100.76107875000001, "ttft_ms": 581.6107910000001 }, "prefill": { "gpu_active_fraction": 0.8793103448275862, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 58, "gross_j": { "core": 2.7445, "dram": 1.290097, "package-0": 15.724325, "psys": 26.085871, "uncore": 10.109715 }, "logits": { "l2": 687.4985252735918, "sha256": "406295ad6f2a0cd6b351a0545dc583118a18409a1827387988bed8f0ee7f4e57", "sum": 121544.93127582863, "top1": 290 }, "net_j": { "core": 1.3157614513712272, "dram": 1.0046524687823668, "package-0": 13.225763087918855, "psys": 20.309341638996898, "uncore": 10.108479673205858 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.581610791, "watts": { "core": 4.718791402204228, "dram": 2.218144883078691, "package-0": 27.035820592262706, "psys": 44.85107808118368, "uncore": 17.382268617502316 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 597.2912714442998, "sha256": "3ec855524c524afa8a2eb18bffab2133436dcb0e5f21c04bcd06f32483d1c0dd", "sum": -69328.94927807688, "top1": 42 }, "gpu_active_fraction": 0.9922480620155039, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 129, "gross_j": { "core": 10.136205, "dram": 3.902334, "package-0": 28.81151, "psys": 50.882194, "uncore": 10.755771 }, "net_j": { "core": 6.933184924579656, "dram": 3.262409666197633, "package-0": 23.21010450019225, "psys": 37.93207130099006, "uncore": 10.753001580412954 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.303885194, "token_latencies_s": [ 0.107377861, 0.093277404, 0.090525707, 0.097188518, 0.097198196, 0.093032097, 0.090834573, 0.091046992, 0.090177748, 0.090048414, 0.091091117, 0.09023503, 0.090943594, 0.090855458 ], "watts": { "core": 7.773847764084666, "dram": 2.992850918130757, "package-0": 22.09666167894226, "psys": 39.023523109351295, "uncore": 8.249016899259306 } } }, { "energy": { "gross_j": { "core": 14.58236, "dram": 5.532701, "package-0": 47.632873000000004, "psys": 81.486486, "uncore": 21.544745 }, "net_j": { "core": 9.619657569928716, "dram": 4.541213600094776, "package-0": 38.9541547805825, "psys": 61.42179584045928, "uncore": 21.54045411046734 } }, "first_decode": { "gpu_active_fraction": 0.9090909090909091, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 11, "gross_j": { "core": 2.490106, "dram": 0.290161, "package-0": 3.78594, "psys": 5.6046, "uncore": 0.701781 }, "logits": { "l2": 656.1393639359858, "sha256": "759589a4c665f287600ccbb81e67d5a11ba7b9e7dbbe102b502bb439588346fd", "sum": -851.9276055010268, "top1": 78 }, "net_j": { "core": 2.2310315437328234, "dram": 0.23840108438029334, "package-0": 3.332873503373618, "psys": 4.557136694170133, "uncore": 0.7015569970725144 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.105464012, "watts": { "core": 23.610954606961094, "dram": 2.7512797446014097, "package-0": 35.8979326521354, "psys": 53.1422984363614, "uncore": 6.6542224849174145 } }, "metrics": { "all_tokens_per_s_e2e": 506.87635015320603, "decode_tokens_per_s": 10.790380945119539, "end_to_end_ms": 2020.2165670000004, "handoff_estimate_ms": 15.022169000000002, "handoff_plus_first_decode_ms": 105.464012, "output_tokens_per_s_e2e": 7.919942971143844, "prefill_tokens_per_s": 1599.7725859785992, "steady_tpot_ms": 90.44184299999999, "steady_tpot_p95_ms": 97.91285529999999, "ttft_ms": 630.089557 }, "prefill": { "gpu_active_fraction": 0.8387096774193549, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 62, "gross_j": { "core": 3.291557, "dram": 1.35046, "package-0": 16.502094, "psys": 27.198722, "uncore": 10.167637 }, "logits": { "l2": 687.4985252735918, "sha256": "406295ad6f2a0cd6b351a0545dc583118a18409a1827387988bed8f0ee7f4e57", "sum": 121544.93127582863, "top1": 290 }, "net_j": { "core": 1.7437293924232928, "dram": 1.0412229277067675, "package-0": 13.79527048042008, "psys": 20.940703953577717, "uncore": 10.166298705517649 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.630089557, "watts": { "core": 5.223951045422579, "dram": 2.143282625457003, "package-0": 26.190076976628895, "psys": 43.166438322671645, "uncore": 16.136812437283417 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 597.2912714442998, "sha256": "3ec855524c524afa8a2eb18bffab2133436dcb0e5f21c04bcd06f32483d1c0dd", "sum": -69328.94927807688, "top1": 42 }, "gpu_active_fraction": 0.9921259842519685, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 127, "gross_j": { "core": 8.800697, "dram": 3.89208, "package-0": 27.344839, "psys": 48.683164, "uncore": 10.675327 }, "net_j": { "core": 5.6448966337726, "dram": 3.2615895880077144, "package-0": 21.8260107967888, "psys": 35.92395519271143, "uncore": 10.672598407877178 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.284662998, "token_latencies_s": [ 0.107271905, 0.092873367, 0.09061633, 0.090533028, 0.090398671, 0.090062548, 0.090588847, 0.09034045, 0.090045882, 0.090937211, 0.090102587, 0.089988632, 0.090485015, 0.090387272 ], "watts": { "core": 6.850588063718793, "dram": 3.0296505823389492, "package-0": 21.285612680190233, "psys": 37.895669195572175, "uncore": 8.309826792411437 } } }, { "energy": { "gross_j": { "core": 14.736291, "dram": 5.52678, "package-0": 48.065308, "psys": 81.769688, "uncore": 21.879095 }, "net_j": { "core": 9.78143912396, "dram": 4.536861045012433, "package-0": 39.40031874121712, "psys": 61.73673839642411, "uncore": 21.874810898273097 } }, "first_decode": { "gpu_active_fraction": 0.9090909090909091, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 11, "gross_j": { "core": 2.490839, "dram": 0.290893, "package-0": 3.784048, "psys": 5.581651, "uncore": 0.690367 }, "logits": { "l2": 656.1393639359858, "sha256": "759589a4c665f287600ccbb81e67d5a11ba7b9e7dbbe102b502bb439588346fd", "sum": -851.9276055010268, "top1": 78 }, "net_j": { "core": 2.229550008135565, "dram": 0.23869064718558702, "package-0": 3.327108748416759, "psys": 4.525234110667522, "uncore": 0.6901410823238975 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.106365505, "watts": { "core": 23.41773303290385, "dram": 2.7348434062340043, "package-0": 35.575894647423524, "psys": 52.47613876322028, "uncore": 6.490515886705939 } }, "metrics": { "all_tokens_per_s_e2e": 507.67945391364344, "decode_tokens_per_s": 10.60815137197216, "end_to_end_ms": 2017.020764, "handoff_estimate_ms": 15.092098999999997, "handoff_plus_first_decode_ms": 106.365505, "output_tokens_per_s_e2e": 7.932491467400679, "prefill_tokens_per_s": 1671.6035576181314, "steady_tpot_ms": 91.273406, "steady_tpot_p95_ms": 100.62771444999998, "ttft_ms": 603.0137920000001 }, "prefill": { "gpu_active_fraction": 0.8666666666666667, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 60, "gross_j": { "core": 3.405082, "dram": 1.330563, "package-0": 16.748798, "psys": 27.170401, "uncore": 10.388218 }, "logits": { "l2": 687.4985252735918, "sha256": "406295ad6f2a0cd6b351a0545dc583118a18409a1827387988bed8f0ee7f4e57", "sum": 121544.93127582863, "top1": 290 }, "net_j": { "core": 1.9237665510484818, "dram": 1.034614245542988, "package-0": 14.158290195699612, "psys": 21.181298100810807, "uncore": 10.386937213766283 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.603013792, "watts": { "core": 5.646773001172086, "dram": 2.2065216710665214, "package-0": 27.77514912959072, "psys": 45.05767755308654, "uncore": 17.227164847334038 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 597.2912714442998, "sha256": "3ec855524c524afa8a2eb18bffab2133436dcb0e5f21c04bcd06f32483d1c0dd", "sum": -69328.94927807688, "top1": 42 }, "gpu_active_fraction": 0.9922480620155039, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 129, "gross_j": { "core": 8.84037, "dram": 3.905324, "package-0": 27.532462, "psys": 49.017636, "uncore": 10.80051 }, "net_j": { "core": 5.628122564775953, "dram": 3.2635561522838574, "package-0": 21.91491979710075, "psys": 36.03020618494578, "uncore": 10.797732602182917 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.307641467, "token_latencies_s": [ 0.106311982, 0.093507582, 0.091153165, 0.091108806, 0.091988969, 0.091393647, 0.090797245, 0.090891614, 0.097223945, 0.093223965, 0.090617826, 0.090980076, 0.090845977, 0.097566955 ], "watts": { "core": 6.760545778868298, "dram": 2.9865403465367466, "package-0": 21.05505422917274, "psys": 37.48553195736183, "uncore": 8.25953464505726 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 2.800072302110493, "compile_target": "AUTO:GPU", "context_length": 2048, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 398915, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_3b_ov_fp16/openvino_model.xml", "key": "qwen25_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "GPU", "key": "gpu_gpu", "label": "iGPU -> iGPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 2032, "prompt_sha256": "cd1a97358e68d07c27508cfacc4ba6882a7a2962b1281e2e2b0c7e5e320da4fa", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.05963654909282923, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 19.056958, "dram": 6.960127, "package-0": 68.072764, "psys": 115.775948, "uncore": 34.181492 }, "net_j": { "core": 8.613791821207752, "dram": 5.59287644953272, "package-0": 52.21673442620212, "psys": 87.65506971318558, "uncore": 34.17608436316588 } }, "first_decode": { "gpu_active_fraction": 0.9166666666666666, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 12, "gross_j": { "core": 2.592889, "dram": 0.305725, "package-0": 4.024465, "psys": 6.088607, "uncore": 0.788206 }, "logits": { "l2": 597.2273383364098, "sha256": "fcc036c7b26185e79a5e482f5d15937fe9f09db4f7711fbfd4b32d0318d977c9", "sum": 14783.873326078989, "top1": 220 }, "net_j": { "core": 2.124192088020137, "dram": 0.2443617984636539, "package-0": 3.312834832456317, "psys": 4.826521480193037, "uncore": 0.7879633013162989 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.12426488, "watts": { "core": 20.865823070846726, "dram": 2.4602687420613134, "package-0": 32.386181839953494, "psys": 48.997005429048016, "uncore": 6.3429506389898735 } }, "metrics": { "all_tokens_per_s_e2e": 739.6758694038298, "decode_tokens_per_s": 9.826272362787046, "end_to_end_ms": 2768.7803329999997, "handoff_estimate_ms": 25.534078999999988, "handoff_plus_first_decode_ms": 124.26487999999999, "output_tokens_per_s_e2e": 5.77871772971742, "prefill_tokens_per_s": 1635.727819476029, "steady_tpot_ms": 98.73080100000001, "steady_tpot_p95_ms": 103.9432763, "ttft_ms": 1242.260464 }, "prefill": { "gpu_active_fraction": 0.8943089430894309, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 123, "gross_j": { "core": 6.861677, "dram": 2.652581, "package-0": 34.933871, "psys": 57.282263, "uncore": 22.105229 }, "logits": { "l2": 597.5347857334565, "sha256": "f657289f689ae92500218d2a246d30ac2917dc3da7ea94d5cbc4c64bf5fc7387", "sum": -5077.78165544162, "top1": 220 }, "net_j": { "core": 2.1761725891683, "dram": 2.03914074871527, "package-0": 27.819793216082186, "psys": 44.66535190297019, "uncore": 22.10280277161249 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.242260464, "watts": { "core": 5.52354131749942, "dram": 2.135285696414146, "package-0": 28.12121291175536, "psys": 46.111314543131115, "uncore": 17.79435926731707 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 683.1319075602754, "sha256": "8dff8948fa4761bb522f9e112d6d62b7d86c314faa7abb52288c95eeaa37d5b0", "sum": -135169.30241355416, "top1": 42 }, "gpu_active_fraction": 0.9928057553956835, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 139, "gross_j": { "core": 9.602392, "dram": 4.001821, "package-0": 29.114428, "psys": 52.405078, "uncore": 11.288057 }, "net_j": { "core": 4.3134271440193155, "dram": 3.309373902353796, "package-0": 21.084106377663623, "psys": 38.16319633002236, "uncore": 11.285318290237084 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.402254989, "token_latencies_s": [ 0.105260344, 0.10150523, 0.09848226, 0.102534748, 0.100741341, 0.098661505, 0.098800097, 0.103234086, 0.101156855, 0.09852638, 0.098390305, 0.09821515, 0.098378143, 0.098339222 ], "watts": { "core": 6.847821598301334, "dram": 2.853846861941883, "package-0": 20.76257758281365, "psys": 37.37200324555237, "uncore": 8.049931780274807 } } }, { "energy": { "gross_j": { "core": 22.018987, "dram": 7.075055000000001, "package-0": 71.567077, "psys": 120.049009, "uncore": 34.424839 }, "net_j": { "core": 11.349929749193517, "dram": 5.678230114769733, "package-0": 55.36807332649007, "psys": 91.3198616084489, "uncore": 34.419314393191385 } }, "first_decode": { "gpu_active_fraction": 0.8333333333333334, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 12, "gross_j": { "core": 2.479303, "dram": 0.297789, "package-0": 3.84844, "psys": 5.751145, "uncore": 0.74243 }, "logits": { "l2": 597.2273383364098, "sha256": "fcc036c7b26185e79a5e482f5d15937fe9f09db4f7711fbfd4b32d0318d977c9", "sum": 14783.873326078989, "top1": 220 }, "net_j": { "core": 2.049909865060237, "dram": 0.24157156650693667, "package-0": 3.1964853983182655, "psys": 4.5948948859918355, "uncore": 0.742207653434498 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.113844331, "watts": { "core": 21.77801018480226, "dram": 2.6157560713321772, "package-0": 33.8044061236567, "psys": 50.517623051428004, "uncore": 6.5214490126873335 } }, "metrics": { "all_tokens_per_s_e2e": 724.0150503497318, "decode_tokens_per_s": 9.859794128458285, "end_to_end_ms": 2828.6704799999998, "handoff_estimate_ms": 13.979001000000006, "handoff_plus_first_decode_ms": 113.84433100000001, "output_tokens_per_s_e2e": 5.65636758085728, "prefill_tokens_per_s": 1554.3004567468697, "steady_tpot_ms": 99.86533, "steady_tpot_p95_ms": 104.1105031, "ttft_ms": 1307.340541 }, "prefill": { "gpu_active_fraction": 0.8682170542635659, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 129, "gross_j": { "core": 8.67051, "dram": 2.768304, "package-0": 37.264614, "psys": 60.559842, "uncore": 22.334965 }, "logits": { "l2": 597.5347857334565, "sha256": "f657289f689ae92500218d2a246d30ac2917dc3da7ea94d5cbc4c64bf5fc7387", "sum": -5077.78165544162, "top1": 220 }, "net_j": { "core": 3.739539363141189, "dram": 2.122726576241825, "package-0": 29.77784084069321, "psys": 47.28195072249102, "uncore": 22.332411665313245 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.307340541, "watts": { "core": 6.632174041943062, "dram": 2.1175079584715486, "package-0": 28.504137086956597, "psys": 46.32292818952671, "uncore": 17.084274754392396 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 683.1319075602754, "sha256": "8dff8948fa4761bb522f9e112d6d62b7d86c314faa7abb52288c95eeaa37d5b0", "sum": -135169.30241355416, "top1": 42 }, "gpu_active_fraction": 0.9928057553956835, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 139, "gross_j": { "core": 10.869174, "dram": 4.008962, "package-0": 30.454023, "psys": 53.738022, "uncore": 11.347444 }, "net_j": { "core": 5.560480520992091, "dram": 3.3139319720209715, "package-0": 22.393747087478594, "psys": 39.44301599996605, "uncore": 11.344695074443642 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.407485608, "token_latencies_s": [ 0.106032921, 0.101961515, 0.100012575, 0.099967872, 0.099074879, 0.099419853, 0.103075355, 0.101230266, 0.099911529, 0.098849471, 0.099791116, 0.098938158, 0.099372649, 0.099819131 ], "watts": { "core": 7.722405073430775, "dram": 2.8483147374392197, "package-0": 21.63718252385853, "psys": 38.18015736328581, "uncore": 8.062209613727006 } } }, { "energy": { "gross_j": { "core": 21.703191, "dram": 7.0545469999999995, "package-0": 71.128358, "psys": 119.914489, "uncore": 34.367832 }, "net_j": { "core": 11.079000426287372, "dram": 5.663596194326064, "package-0": 54.99747613425398, "psys": 91.30615653603785, "uncore": 34.36233062586733 } }, "first_decode": { "gpu_active_fraction": 0.9090909090909091, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 11, "gross_j": { "core": 2.316766, "dram": 0.295531, "package-0": 3.672964, "psys": 5.557054, "uncore": 0.739806 }, "logits": { "l2": 597.2273383364098, "sha256": "fcc036c7b26185e79a5e482f5d15937fe9f09db4f7711fbfd4b32d0318d977c9", "sum": 14783.873326078989, "top1": 220 }, "net_j": { "core": 1.8915888707439317, "dram": 0.23986553858064646, "package-0": 3.0274106285256055, "psys": 4.4121565524780255, "uncore": 0.7395858365490091 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.112726548, "watts": { "core": 20.552088581653365, "dram": 2.6216628224967913, "package-0": 32.58295463815676, "psys": 49.29676370467762, "uncore": 6.562837353983376 } }, "metrics": { "all_tokens_per_s_e2e": 727.0726149942805, "decode_tokens_per_s": 9.858521320098015, "end_to_end_ms": 2816.7750480000004, "handoff_estimate_ms": 12.829017999999984, "handoff_plus_first_decode_ms": 112.726548, "output_tokens_per_s_e2e": 5.680254804642816, "prefill_tokens_per_s": 1568.8106920415082, "steady_tpot_ms": 99.89753000000002, "steady_tpot_p95_ms": 104.039077, "ttft_ms": 1295.2486940000001 }, "prefill": { "gpu_active_fraction": 0.875, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 128, "gross_j": { "core": 8.4954, "dram": 2.75042, "package-0": 36.951627, "psys": 60.003326, "uncore": 22.244877 }, "logits": { "l2": 597.5347857334565, "sha256": "f657289f689ae92500218d2a246d30ac2917dc3da7ea94d5cbc4c64bf5fc7387", "sum": -5077.78165544162, "top1": 220 }, "net_j": { "core": 3.6100368698561427, "dram": 2.110813647427889, "package-0": 29.534100462156353, "psys": 46.848244522061265, "uncore": 22.2423472816025 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.295248694, "watts": { "core": 6.558894858843224, "dram": 2.1234686533488216, "package-0": 28.528596223390593, "psys": 46.32571820219106, "uncore": 17.174213031864287 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 683.1319075602754, "sha256": "8dff8948fa4761bb522f9e112d6d62b7d86c314faa7abb52288c95eeaa37d5b0", "sum": -135169.30241355416, "top1": 42 }, "gpu_active_fraction": 0.9928571428571429, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 140, "gross_j": { "core": 10.891025, "dram": 4.008596, "package-0": 30.503767, "psys": 54.354109, "uncore": 11.383149 }, "net_j": { "core": 5.577374685687298, "dram": 3.3129170083175286, "package-0": 22.43596504357202, "psys": 40.04575546149855, "uncore": 11.38039750771582 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.408799806, "token_latencies_s": [ 0.106193723, 0.102547991, 0.100181598, 0.099243188, 0.099948852, 0.099824431, 0.100020096, 0.102878883, 0.100572106, 0.099846208, 0.099497948, 0.099507828, 0.099387479, 0.099125849 ], "watts": { "core": 7.730711598351825, "dram": 2.8453978932475805, "package-0": 21.65230778005942, "psys": 38.58185440437234, "uncore": 8.080033054746176 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 6.534896406345069, "compile_target": "NPU", "context_length": 2048, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 398983, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_3b_ov_fp16/openvino_model.xml", "key": "qwen25_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "gpu_npu", "label": "iGPU -> NPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 2032, "prompt_sha256": "cd1a97358e68d07c27508cfacc4ba6882a7a2962b1281e2e2b0c7e5e320da4fa", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "GPU", "NPUW_ONLINE_PIPELINE": "NONE" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 2048, "NPUW_LLM_MIN_RESPONSE_LEN": 2048, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.059766506776213646, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 68.32496, "dram": 25.172114999999998, "package-0": 200.704382, "psys": 396.91188700000004, "uncore": 65.266922 }, "net_j": { "core": 27.041020580213505, "dram": 19.76709817552495, "package-0": 138.02230329360532, "psys": 285.7443908835781, "uncore": 65.24898366649069 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 35, "gross_j": { "core": 4.564686, "dram": 0.488768, "package-0": 5.849412, "psys": 9.550818, "uncore": 0.0 }, "logits": { "l2": 924.1045417971341, "sha256": "eabdef562e197dcc4559d67137a6e297994f30355c212d87eb2e14c7047a635e", "sum": -80822.31826195121, "top1": 4710 }, "net_j": { "core": 3.238641773551792, "dram": 0.3151583238237132, "package-0": 3.8360574185680343, "psys": 5.9801070052243155, "uncore": 0.0 }, "npu_busy_delta_us": 133882, "npu_utilization": 0.380809583961391, "seconds": 0.351572034, "watts": { "core": 12.983643630767288, "dram": 1.3902357205124, "package-0": 16.637876265209425, "psys": 27.16603448612184, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 187.10806505361293, "decode_tokens_per_s": 8.518436657183571, "end_to_end_ms": 10945.546358, "handoff_estimate_ms": 251.839064, "handoff_plus_first_decode_ms": 351.572034, "output_tokens_per_s_e2e": 1.461781758231351, "prefill_tokens_per_s": 221.23845940293506, "steady_tpot_ms": 99.73297000000001, "steady_tpot_p95_ms": 104.46886035, "ttft_ms": 9184.659871 }, "prefill": { "gpu_active_fraction": 0.9988974641675854, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 907, "gross_j": { "core": 55.644572, "dram": 20.534615, "package-0": 173.266402, "psys": 349.057517, "uncore": 65.266922 }, "logits": { "l2": 2607.309964597262, "sha256": "4f901afd538c27dea9094e9b2e2d3c4f348a01d843d780b0f16c0dfd34e7033b", "sum": -638918.1009616219, "top1": 220 }, "net_j": { "core": 21.002268244320447, "dram": 15.999140981057852, "package-0": 120.66842707070805, "psys": 255.7743123846559, "uncore": 65.24898366649069 }, "npu_busy_delta_us": 11765, "npu_utilization": 0.0012809401943285092, "seconds": 9.184659871, "watts": { "core": 6.058424893413236, "dram": 2.2357512731458664, "package-0": 18.864759766126785, "psys": 38.00440320083357, "uncore": 7.106079366757641 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 852.6998990461121, "sha256": "e7b82e5d25e5c8a63909eb7510b07668bc1ad20828d519affcb9a9d9bc2c3791", "sum": -96127.76050041988, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 139, "gross_j": { "core": 8.115702, "dram": 4.148732, "package-0": 21.588568, "psys": 38.303552, "uncore": 0.0 }, "net_j": { "core": 2.800110562341267, "dram": 3.4527988706433836, "package-0": 13.517818804329234, "psys": 23.9899714936979, "uncore": 0.0 }, "npu_busy_delta_us": 1394879, "npu_utilization": 0.9897571099414533, "seconds": 1.409314453, "token_latencies_s": [ 0.109713659, 0.101644738, 0.099920969, 0.101382686, 0.099727762, 0.099479697, 0.099478436, 0.099566454, 0.099518227, 0.099624318, 0.099738178, 0.099830598, 0.09971947, 0.09991612 ], "watts": { "core": 5.758616881224875, "dram": 2.943794403845513, "package-0": 15.318489038443147, "psys": 27.178854171589204, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 68.854865, "dram": 25.014585, "package-0": 200.975681, "psys": 396.23214, "uncore": 65.414323 }, "net_j": { "core": 28.509491123068294, "dram": 19.732447983539863, "package-0": 139.71864166860587, "psys": 287.5919700740218, "uncore": 65.39617692235339 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 0, "gpu_sample_count": 16, "gross_j": { "core": 2.625482, "dram": 0.341369, "package-0": 3.504813, "psys": 5.45775, "uncore": 0.002625 }, "logits": { "l2": 918.1321595628507, "sha256": "cab5c133a903f9459bc22557fda52b8ad3eb3890e6cf5e2d76ac0ba74501ae02", "sum": -58615.71318984404, "top1": 1718 }, "net_j": { "core": 2.026030294952455, "dram": 0.2628869898822991, "package-0": 2.594655697074013, "psys": 3.843573978297077, "uncore": 0.002314594413705833 }, "npu_busy_delta_us": 122835, "npu_utilization": 0.7728791967699029, "seconds": 0.158931694, "watts": { "core": 16.519562171155112, "dram": 2.147897574161639, "package-0": 22.052322678949107, "psys": 34.34022417202701, "uncore": 0.016516529421752718 } }, "metrics": { "all_tokens_per_s_e2e": 191.4608114969905, "decode_tokens_per_s": 9.587053232222363, "end_to_end_ms": 10696.705941999999, "handoff_estimate_ms": 58.99234650000002, "handoff_plus_first_decode_ms": 158.93169400000002, "output_tokens_per_s_e2e": 1.4957875898202382, "prefill_tokens_per_s": 222.5118995479228, "steady_tpot_ms": 99.9393475, "steady_tpot_p95_ms": 102.39185755, "ttft_ms": 9132.095874999999 }, "prefill": { "gpu_active_fraction": 0.9988913525498891, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 902, "gross_j": { "core": 59.00052, "dram": 20.526498, "package-0": 176.60996, "psys": 352.033766, "uncore": 65.411698 }, "logits": { "l2": 2695.370592197745, "sha256": "07df53b12f154266b8efa14380ec825d8f02f7b9b1d08813636109b65b67466c", "sum": -667790.8857600205, "top1": 220 }, "net_j": { "core": 24.556474856138387, "dram": 16.016980591486348, "package-0": 124.31300435532205, "psys": 259.28442306958885, "uncore": 65.39386232793969 }, "npu_busy_delta_us": 11270, "npu_utilization": 0.0012341088129454183, "seconds": 9.132095875, "watts": { "core": 6.460786308816541, "dram": 2.2477313292552354, "package-0": 19.339477204076115, "psys": 38.54906593389221, "uncore": 7.162835223737728 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 836.8015178215198, "sha256": "568ae868543f3c53e0d210409c186a2477fc8751f54b903ed56f8bc232e53616", "sum": -19761.276002621278, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 139, "gross_j": { "core": 7.228863, "dram": 4.146718, "package-0": 20.860908, "psys": 38.740624, "uncore": 0.0 }, "net_j": { "core": 1.9269859719774542, "dram": 3.4525804021712156, "package-0": 12.810981616209808, "psys": 24.463973026135857, "uncore": 0.0 }, "npu_busy_delta_us": 1388243, "npu_utilization": 0.9875964706188163, "seconds": 1.405678373, "token_latencies_s": [ 0.105429823, 0.10075603, 0.099763821, 0.099695589, 0.100276127, 0.099792842, 0.100388974, 0.099979174, 0.099883471, 0.100276706, 0.099996648, 0.099744264, 0.099899521, 0.099754323 ], "watts": { "core": 5.142615223262029, "dram": 2.949976381261434, "package-0": 14.840455968230222, "psys": 27.560091087778297, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 65.757034, "dram": 25.076473, "package-0": 198.52274699999998, "psys": 399.781997, "uncore": 65.88197199999999 }, "net_j": { "core": 25.25754212515117, "dram": 19.77415839430289, "package-0": 137.03170779816514, "psys": 290.7268252070558, "uncore": 65.86374292190342 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 0, "gpu_sample_count": 17, "gross_j": { "core": 2.725274, "dram": 0.349913, "package-0": 3.628409, "psys": 5.779099, "uncore": 0.005371 }, "logits": { "l2": 911.9156589065695, "sha256": "108204db436c26fbbb8f3b21395d80af2310480d661184bea4fff12957b38ce5", "sum": -35362.86651572399, "top1": 1718 }, "net_j": { "core": 2.0690658106841706, "dram": 0.2640002612096108, "package-0": 2.632077401112863, "psys": 4.012091723984843, "uncore": 0.005031205007308373 }, "npu_busy_delta_us": 131007, "npu_utilization": 0.753002716665644, "seconds": 0.173979452, "watts": { "core": 15.664344085875154, "dram": 2.011231763162468, "package-0": 20.855388140893787, "psys": 33.21713532009516, "uncore": 0.030871461763197182 } }, "metrics": { "all_tokens_per_s_e2e": 190.732220702847, "decode_tokens_per_s": 9.505573428216362, "end_to_end_ms": 10737.567006, "handoff_estimate_ms": 74.2844115, "handoff_plus_first_decode_ms": 173.979452, "output_tokens_per_s_e2e": 1.4900954742409922, "prefill_tokens_per_s": 221.8450704149606, "steady_tpot_ms": 99.6950405, "steady_tpot_p95_ms": 102.94850089999998, "ttft_ms": 9159.545425999999 }, "prefill": { "gpu_active_fraction": 0.9988950276243094, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 905, "gross_j": { "core": 55.694987, "dram": 20.581612, "package-0": 173.92906, "psys": 354.108469, "uncore": 65.876601 }, "logits": { "l2": 2722.790925199642, "sha256": "0df34f45f9f926a49cf433f896f8897822b464feb4bddff4979e0fc2074631a3", "sum": -676477.0488090478, "top1": 220 }, "net_j": { "core": 21.147408824207446, "dram": 16.058539737290253, "package-0": 121.47490846395573, "psys": 261.0803370759303, "uncore": 65.85871171689611 }, "npu_busy_delta_us": 9406, "npu_utilization": 0.0010269068564582279, "seconds": 9.159545426, "watts": { "core": 6.0805405082555675, "dram": 2.247012383559743, "package-0": 18.988830985683023, "psys": 38.66004834637741, "uncore": 7.1921255844208964 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 858.7947249227907, "sha256": "5980de6d458600a7a9248f6eb7f4c5fa6366e9300635034843014fdbea2145dd", "sum": 37917.90754732303, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 139, "gross_j": { "core": 7.336773, "dram": 4.144948, "package-0": 20.965278, "psys": 39.894429, "uncore": 0.0 }, "net_j": { "core": 2.041067490259551, "dram": 3.451618395803028, "package-0": 12.924721933096551, "psys": 25.63439640714067, "uncore": 0.0 }, "npu_busy_delta_us": 1389452, "npu_utilization": 0.9896084827448995, "seconds": 1.404042128, "token_latencies_s": [ 0.107559285, 0.100465771, 0.099748129, 0.099710932, 0.099729064, 0.099807425, 0.099494863, 0.099565665, 0.099489828, 0.099675299, 0.099679149, 0.09965846, 0.09985043, 0.099574389 ], "watts": { "core": 5.225465001147032, "dram": 2.952153583813263, "package-0": 14.932086140366867, "psys": 28.413982888695777, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 7.723630679771304, "compile_target": "AUTO:NPU,GPU", "context_length": 2048, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 398828, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_3b_ov_fp16/openvino_model.xml", "key": "qwen25_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": null, "key": "openvino_auto", "label": "OpenVINO AUTO", "mode": "stock", "prefill_device": null }, "prompt_length": 2032, "prompt_sha256": "cd1a97358e68d07c27508cfacc4ba6882a7a2962b1281e2e2b0c7e5e320da4fa", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.06377757806330919, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 20.016244, "dram": 6.974348, "package-0": 69.264837, "psys": 116.913763, "uncore": 34.382968 }, "net_j": { "core": 9.507867702090076, "dram": 5.598559945701675, "package-0": 53.30979783854081, "psys": 88.61728989918167, "uncore": 34.37752659633347 } }, "first_decode": { "gpu_active_fraction": 0.9166666666666666, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 12, "gross_j": { "core": 2.293024, "dram": 0.300475, "package-0": 3.704519, "psys": 5.687364, "uncore": 0.777647 }, "logits": { "l2": 597.2273383364098, "sha256": "fcc036c7b26185e79a5e482f5d15937fe9f09db4f7711fbfd4b32d0318d977c9", "sum": 14783.873326078989, "top1": 220 }, "net_j": { "core": 1.8385880440822373, "dram": 0.24097888548422558, "package-0": 3.014541474691437, "psys": 4.463679727746709, "uncore": 0.7774116858651964 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.120483895, "watts": { "core": 19.03178843944247, "dram": 2.4939017783248127, "package-0": 30.747005647518286, "psys": 47.204350423764105, "uncore": 6.4543647099058346 } }, "metrics": { "all_tokens_per_s_e2e": 735.0857833443984, "decode_tokens_per_s": 9.772237343680946, "end_to_end_ms": 2786.0693899999997, "handoff_estimate_ms": 20.43445149999999, "handoff_plus_first_decode_ms": 120.48389499999999, "output_tokens_per_s_e2e": 5.742857682378112, "prefill_tokens_per_s": 1624.1594121798773, "steady_tpot_ms": 100.0494435, "steady_tpot_p95_ms": 105.15026845, "ttft_ms": 1251.108718 }, "prefill": { "gpu_active_fraction": 0.8870967741935484, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 124, "gross_j": { "core": 7.416301, "dram": 2.667535, "package-0": 35.762909, "psys": 58.124607, "uncore": 22.332523 }, "logits": { "l2": 597.5347857334565, "sha256": "f657289f689ae92500218d2a246d30ac2917dc3da7ea94d5cbc4c64bf5fc7387", "sum": -5077.78165544162, "top1": 220 }, "net_j": { "core": 2.69742312655993, "dram": 2.0497253951742955, "package-0": 28.598159743366086, "psys": 45.417829375525145, "uncore": 22.330079490305017 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.251108718, "watts": { "core": 5.927783008222951, "dram": 2.1321368491974653, "package-0": 28.584973060670496, "psys": 46.458478119245264, "uncore": 17.850185742211412 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 683.1319075602754, "sha256": "8dff8948fa4761bb522f9e112d6d62b7d86c314faa7abb52288c95eeaa37d5b0", "sum": -135169.30241355416, "top1": 42 }, "gpu_active_fraction": 0.9928571428571429, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 140, "gross_j": { "core": 10.306919, "dram": 4.006338, "package-0": 29.797409, "psys": 53.101792, "uncore": 11.272798 }, "net_j": { "core": 4.971856531447909, "dram": 3.3078556650431543, "package-0": 21.697096620483286, "psys": 38.73578079590982, "uncore": 11.270035420163255 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.414476777, "token_latencies_s": [ 0.109264903, 0.10290229, 0.101456903, 0.099853759, 0.102934696, 0.101404243, 0.100399433, 0.099736826, 0.099262246, 0.099457353, 0.09852163, 0.099424135, 0.100245128, 0.099584195 ], "watts": { "core": 7.2867361045405135, "dram": 2.832381602260834, "package-0": 21.066029138490407, "psys": 37.5416499326521, "uncore": 7.969588602160557 } } }, { "energy": { "gross_j": { "core": 21.576423, "dram": 7.061261000000001, "package-0": 71.315247, "psys": 119.056703, "uncore": 34.658418999999995 }, "net_j": { "core": 10.893718929440029, "dram": 5.662649432311498, "package-0": 55.095523137431016, "psys": 90.29080807894302, "uncore": 34.652887326651985 } }, "first_decode": { "gpu_active_fraction": 0.8333333333333334, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 12, "gross_j": { "core": 2.466547, "dram": 0.298094, "package-0": 3.806386, "psys": 5.717454, "uncore": 0.719236 }, "logits": { "l2": 597.2273383364098, "sha256": "fcc036c7b26185e79a5e482f5d15937fe9f09db4f7711fbfd4b32d0318d977c9", "sum": 14783.873326078989, "top1": 220 }, "net_j": { "core": 2.0352238872556896, "dram": 0.24162388804041796, "package-0": 3.1515010815329374, "psys": 4.556006930393431, "uncore": 0.719012654061426 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.114356023, "watts": { "core": 21.569016963802596, "dram": 2.606718843309198, "package-0": 33.285400280140905, "psys": 49.99696430506332, "uncore": 6.289445725128094 } }, "metrics": { "all_tokens_per_s_e2e": 723.0901438068121, "decode_tokens_per_s": 9.79450211911987, "end_to_end_ms": 2832.2886399999998, "handoff_estimate_ms": 14.458865500000002, "handoff_plus_first_decode_ms": 114.35602300000001, "output_tokens_per_s_e2e": 5.649141748490719, "prefill_tokens_per_s": 1562.094933725843, "steady_tpot_ms": 99.8971575, "steady_tpot_p95_ms": 106.0088787, "ttft_ms": 1300.817227 }, "prefill": { "gpu_active_fraction": 0.8682170542635659, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 129, "gross_j": { "core": 8.781655, "dram": 2.758233, "package-0": 37.653712, "psys": 60.378264, "uncore": 22.656131 }, "logits": { "l2": 597.5347857334565, "sha256": "f657289f689ae92500218d2a246d30ac2917dc3da7ea94d5cbc4c64bf5fc7387", "sum": -5077.78165544162, "top1": 220 }, "net_j": { "core": 3.875288717385728, "dram": 2.1158768518883866, "package-0": 30.204296033125694, "psys": 47.16662619765273, "uncore": 22.653590405837523 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.300817227, "watts": { "core": 6.75087538643121, "dram": 2.120384741798934, "package-0": 28.94619722006495, "psys": 46.41563991218576, "uncore": 17.41684421896113 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 683.1319075602754, "sha256": "8dff8948fa4761bb522f9e112d6d62b7d86c314faa7abb52288c95eeaa37d5b0", "sum": -135169.30241355416, "top1": 42 }, "gpu_active_fraction": 0.9928571428571429, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 140, "gross_j": { "core": 10.328221, "dram": 4.004934, "package-0": 29.855149, "psys": 52.960985, "uncore": 11.283052 }, "net_j": { "core": 4.983206324798611, "dram": 3.3051486923826934, "package-0": 21.739726022772388, "psys": 38.568174950896854, "uncore": 11.28028426675304 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.41711539, "token_latencies_s": [ 0.11166375, 0.102615279, 0.099494242, 0.100003437, 0.100070121, 0.099772248, 0.09946119, 0.099695025, 0.100263253, 0.099171416, 0.102963948, 0.102512974, 0.099790878, 0.099609278 ], "watts": { "core": 7.288200433699333, "dram": 2.826117074347771, "package-0": 21.067549763890433, "psys": 37.372387156136945, "uncore": 7.9619853680369665 } } }, { "energy": { "gross_j": { "core": 21.425297999999998, "dram": 7.067181, "package-0": 71.02679599999999, "psys": 119.97704900000001, "uncore": 34.505893 }, "net_j": { "core": 10.739367010413103, "dram": 5.668146954421139, "package-0": 54.802172653630116, "psys": 91.20246477958266, "uncore": 34.50035965570221 } }, "first_decode": { "gpu_active_fraction": 0.9166666666666666, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 12, "gross_j": { "core": 2.453424, "dram": 0.299011, "package-0": 3.805166, "psys": 5.726304, "uncore": 0.725218 }, "logits": { "l2": 597.2273383364098, "sha256": "fcc036c7b26185e79a5e482f5d15937fe9f09db4f7711fbfd4b32d0318d977c9", "sum": 14783.873326078989, "top1": 220 }, "net_j": { "core": 2.0189440550144955, "dram": 0.24212758612166932, "package-0": 3.1454880116421657, "psys": 4.556356357510966, "uncore": 0.7249930194036973 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.115192989, "watts": { "core": 21.298379539400614, "dram": 2.595739572310256, "package-0": 33.0329652267292, "psys": 49.71052535150381, "uncore": 6.295678289934816 } }, "metrics": { "all_tokens_per_s_e2e": 722.8717862911677, "decode_tokens_per_s": 9.807851600405176, "end_to_end_ms": 2833.144188, "handoff_estimate_ms": 15.073428, "handoff_plus_first_decode_ms": 115.192989, "output_tokens_per_s_e2e": 5.647435830399748, "prefill_tokens_per_s": 1558.5723337212964, "steady_tpot_ms": 100.11956099999999, "steady_tpot_p95_ms": 104.67509199999999, "ttft_ms": 1303.757263 }, "prefill": { "gpu_active_fraction": 0.8682170542635659, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 129, "gross_j": { "core": 8.331643, "dram": 2.757561, "package-0": 37.016507, "psys": 59.999358, "uncore": 22.451481 }, "logits": { "l2": 597.5347857334565, "sha256": "f657289f689ae92500218d2a246d30ac2917dc3da7ea94d5cbc4c64bf5fc7387", "sum": -5077.78165544162, "top1": 220 }, "net_j": { "core": 3.4141876164723417, "dram": 2.1137530336305477, "package-0": 29.55025427030691, "psys": 46.75785997597698, "uncore": 22.448934663725456 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.303757263, "watts": { "core": 6.3904863554344, "dram": 2.1150877377700943, "package-0": 28.39217701830743, "psys": 46.02034420267694, "uncore": 17.220598985073497 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 683.1319075602754, "sha256": "8dff8948fa4761bb522f9e112d6d62b7d86c314faa7abb52288c95eeaa37d5b0", "sum": -135169.30241355416, "top1": 42 }, "gpu_active_fraction": 0.9928571428571429, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 140, "gross_j": { "core": 10.640231, "dram": 4.010609, "package-0": 30.205123, "psys": 54.251387, "uncore": 11.329194 }, "net_j": { "core": 5.306235338926266, "dram": 3.3122663346689216, "package-0": 22.106430371681036, "psys": 39.888248446094714, "uncore": 11.326431972573062 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.414193936, "token_latencies_s": [ 0.106336986, 0.102244007, 0.100121094, 0.100007831, 0.103780226, 0.101792269, 0.099423881, 0.101088198, 0.099824522, 0.100225698, 0.099312594, 0.099770934, 0.100118028, 0.100114329 ], "watts": { "core": 7.523883909512111, "dram": 2.835968177988284, "package-0": 21.35854371249404, "psys": 38.36205602284523, "uncore": 8.01106107981501 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 2.763699364848435, "compile_target": "AUTO:GPU", "context_length": 4096, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 399323, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_3b_ov_fp16/openvino_model.xml", "key": "qwen25_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "GPU", "key": "gpu_gpu", "label": "iGPU -> iGPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 4080, "prompt_sha256": "66f8de7a9ef70b33481de3e8dae7cef6e4434d9723c6203a6a3dbe07ef228a16", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.06079309061169624, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 31.347149, "dram": 10.57413, "package-0": 121.269038, "psys": 200.139563, "uncore": 66.86823700000001 }, "net_j": { "core": 14.18038655644472, "dram": 8.32660611702695, "package-0": 95.20446253205941, "psys": 153.91369347067837, "uncore": 66.85934777853947 } }, "first_decode": { "gpu_active_fraction": 0.8461538461538461, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 13, "gross_j": { "core": 2.160577, "dram": 0.309143, "package-0": 3.626272, "psys": 5.606187, "uncore": 0.795775 }, "logits": { "l2": 670.568131117269, "sha256": "f155fefe53ba4dfd4365752c28829b31e15f6eec90450cbf0ac8b52f5d1cfc31", "sum": -55835.01430409169, "top1": 1718 }, "net_j": { "core": 1.692335878432797, "dram": 0.24783947190769742, "package-0": 2.9153338664753936, "psys": 4.345328811684205, "uncore": 0.795532537331793 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.124144037, "watts": { "core": 17.40379201620453, "dram": 2.490196125972607, "package-0": 29.21019879513021, "psys": 45.15872961340866, "uncore": 6.410094429263647 } }, "metrics": { "all_tokens_per_s_e2e": 899.9434864931993, "decode_tokens_per_s": 10.48851653020552, "end_to_end_ms": 4551.396906, "handoff_estimate_ms": 31.45649399999999, "handoff_plus_first_decode_ms": 124.144037, "output_tokens_per_s_e2e": 3.51540424411406, "prefill_tokens_per_s": 1307.1638322479619, "steady_tpot_ms": 92.687543, "steady_tpot_p95_ms": 96.28903589999999, "ttft_ms": 3121.26139 }, "prefill": { "gpu_active_fraction": 0.9158576051779935, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 309, "gross_j": { "core": 20.009715, "dram": 6.305403, "package-0": 89.136979, "psys": 143.999875, "uncore": 54.78294 }, "logits": { "l2": 662.3364776849841, "sha256": "d5055a398340419d05f270e2706e72a6086d083d704644e6d8b567ea276b1166", "sum": 8132.857491835253, "top1": 36 }, "net_j": { "core": 8.237075981035225, "dram": 4.764093888993046, "package-0": 71.26238912461984, "psys": 112.29905330483116, "uncore": 54.77684394107459 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 3.12126139, "watts": { "core": 6.410778368036648, "dram": 2.0201457718989695, "package-0": 28.557998790354436, "psys": 46.1351540314283, "uncore": 17.551538674561314 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 706.6311889009895, "sha256": "b5b9231e8728eabecddc2a8f58f01d40f25e30ef87c52a6cee95dc6ef8f92e49", "sum": -130420.66216149065, "top1": 42 }, "gpu_active_fraction": 0.9922480620155039, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 129, "gross_j": { "core": 9.176857, "dram": 3.959584, "package-0": 28.505787, "psys": 50.533501, "uncore": 11.289522 }, "net_j": { "core": 4.250974696976697, "dram": 3.314672756126206, "package-0": 21.026739540964172, "psys": 37.26931135416299, "uncore": 11.286971300133086 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.305991479, "token_latencies_s": [ 0.101734693, 0.093356759, 0.092773563, 0.092729729, 0.092555365, 0.09269699, 0.092534318, 0.092817279, 0.092678096, 0.09214686, 0.092513595, 0.092289964, 0.092254252, 0.0928761 ], "watts": { "core": 7.026735738755919, "dram": 3.0318605164498167, "package-0": 21.82693184325133, "psys": 38.69359166010302, "uncore": 8.64440708958102 } } }, { "energy": { "gross_j": { "core": 30.602338, "dram": 10.823580999999999, "package-0": 121.397212, "psys": 203.488066, "uncore": 67.27296 }, "net_j": { "core": 12.984179572425761, "dram": 8.51695900472438, "package-0": 94.64727464567737, "psys": 156.04669809514434, "uncore": 67.26383703855025 } }, "first_decode": { "gpu_active_fraction": 0.8333333333333334, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 12, "gross_j": { "core": 2.451471, "dram": 0.301208, "package-0": 3.807119, "psys": 5.721238, "uncore": 0.725706 }, "logits": { "l2": 670.568131117269, "sha256": "f155fefe53ba4dfd4365752c28829b31e15f6eec90450cbf0ac8b52f5d1cfc31", "sum": -55835.01430409169, "top1": 1718 }, "net_j": { "core": 2.0180797235955388, "dram": 0.2444671178683852, "package-0": 3.1490939549087535, "psys": 4.554221874269205, "uncore": 0.7254815831335296 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.114904352, "watts": { "core": 21.334883817107293, "dram": 2.621380258947894, "package-0": 33.13293999517094, "psys": 49.79130816559498, "uncore": 6.3157398946908465 } }, "metrics": { "all_tokens_per_s_e2e": 876.8859758392337, "decode_tokens_per_s": 10.48239379444412, "end_to_end_ms": 4671.074817999999, "handoff_estimate_ms": 22.046770000000006, "handoff_plus_first_decode_ms": 114.904352, "output_tokens_per_s_e2e": 3.4253358431220065, "prefill_tokens_per_s": 1259.2188538799617, "steady_tpot_ms": 92.857582, "steady_tpot_p95_ms": 100.4056106, "ttft_ms": 3240.103964 }, "prefill": { "gpu_active_fraction": 0.897196261682243, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 321, "gross_j": { "core": 19.836985, "dram": 6.541548, "package-0": 89.900161, "psys": 147.508778, "uncore": 55.218059 }, "logits": { "l2": 662.3364776849841, "sha256": "d5055a398340419d05f270e2706e72a6086d083d704644e6d8b567ea276b1166", "sum": 8132.857491835253, "top1": 36 }, "net_j": { "core": 7.616100688837314, "dram": 4.941553274687056, "package-0": 71.3449929850285, "psys": 114.60094196753823, "uncore": 55.21173083256349 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 3.240103964, "watts": { "core": 6.122329783366173, "dram": 2.0189315135197927, "package-0": 27.746072965206867, "psys": 45.52593979666512, "uncore": 17.04206396261179 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 706.6311889009895, "sha256": "b5b9231e8728eabecddc2a8f58f01d40f25e30ef87c52a6cee95dc6ef8f92e49", "sum": -130420.66216149065, "top1": 42 }, "gpu_active_fraction": 0.9923076923076923, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 130, "gross_j": { "core": 8.313882, "dram": 3.980825, "package-0": 27.689932, "psys": 50.25805, "uncore": 11.329195 }, "net_j": { "core": 3.349999159992908, "dram": 3.33093861216894, "package-0": 20.153187705740116, "psys": 36.89153425333689, "uncore": 11.326624622853227 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.316066502, "token_latencies_s": [ 0.101705296, 0.09275046, 0.092622302, 0.09970578, 0.093185036, 0.092897502, 0.09279048, 0.092689397, 0.092686889, 0.093199939, 0.092817662, 0.093313978, 0.09301514, 0.092660661 ], "watts": { "core": 6.317220282839476, "dram": 3.024790156082857, "package-0": 21.03991854356916, "psys": 38.188077824049046, "uncore": 8.608375779478658 } } }, { "energy": { "gross_j": { "core": 33.585913000000005, "dram": 10.885531, "package-0": 124.720994, "psys": 209.14826799999997, "uncore": 67.397411 }, "net_j": { "core": 15.91777577302597, "dram": 8.572365630825972, "package-0": 97.89517301921977, "psys": 161.57231947938268, "uncore": 67.38826215873642 } }, "first_decode": { "gpu_active_fraction": 0.8333333333333334, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 12, "gross_j": { "core": 2.552911, "dram": 0.303527, "package-0": 3.927052, "psys": 5.948593, "uncore": 0.737852 }, "logits": { "l2": 670.568131117269, "sha256": "f155fefe53ba4dfd4365752c28829b31e15f6eec90450cbf0ac8b52f5d1cfc31", "sum": -55835.01430409169, "top1": 1718 }, "net_j": { "core": 2.1162812743079327, "dram": 0.24636213040181287, "package-0": 3.2641099645249554, "psys": 4.772856526752401, "uncore": 0.7376259062132017 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.115762957, "watts": { "core": 22.05291801590728, "dram": 2.6219699968445, "package-0": 33.923217769912355, "psys": 51.38598005923432, "uncore": 6.373817835354706 } }, "metrics": { "all_tokens_per_s_e2e": 874.4054818448777, "decode_tokens_per_s": 10.568191756125048, "end_to_end_ms": 4684.325619, "handoff_estimate_ms": 23.90276799999999, "handoff_plus_first_decode_ms": 115.762957, "output_tokens_per_s_e2e": 3.4156464134565536, "prefill_tokens_per_s": 1249.627824156777, "steady_tpot_ms": 91.860189, "steady_tpot_p95_ms": 100.3072394, "ttft_ms": 3264.972115 }, "prefill": { "gpu_active_fraction": 0.8978328173374613, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 323, "gross_j": { "core": 20.82795, "dram": 6.616072, "package-0": 91.281566, "psys": 150.646709, "uncore": 55.399883 }, "logits": { "l2": 662.3364776849841, "sha256": "d5055a398340419d05f270e2706e72a6086d083d704644e6d8b567ea276b1166", "sum": 8132.857491835253, "top1": 36 }, "net_j": { "core": 8.51326907003119, "dram": 5.003797140817782, "package-0": 72.58398504854449, "psys": 117.48630174000012, "uncore": 55.39350626318884 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 3.264972115, "watts": { "core": 6.379212215722094, "dram": 2.02637932789818, "package-0": 27.9578393887753, "psys": 46.140274309815965, "uncore": 16.967949816625005 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 706.6311889009895, "sha256": "b5b9231e8728eabecddc2a8f58f01d40f25e30ef87c52a6cee95dc6ef8f92e49", "sum": -130420.66216149065, "top1": 42 }, "gpu_active_fraction": 0.9922480620155039, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 129, "gross_j": { "core": 10.205052, "dram": 3.965932, "package-0": 29.512376, "psys": 52.552966, "uncore": 11.259676 }, "net_j": { "core": 5.288225428686845, "dram": 3.322206359606377, "package-0": 22.04707800615032, "psys": 39.31316121263016, "uncore": 11.25712998933438 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.303590547, "token_latencies_s": [ 0.102212015, 0.091879994, 0.092006547, 0.091841512, 0.091815969, 0.091823258, 0.091361638, 0.091772552, 0.091610665, 0.091878866, 0.099281591, 0.092324194, 0.091923527, 0.09183308 ], "watts": { "core": 7.8284182280128185, "dram": 3.0423141753573946, "package-0": 22.639298871810553, "psys": 40.31401280175132, "uncore": 8.637432992984108 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 8.387475336901844, "compile_target": "NPU", "context_length": 4096, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 399393, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_3b_ov_fp16/openvino_model.xml", "key": "qwen25_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "gpu_npu", "label": "iGPU -> NPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 4080, "prompt_sha256": "66f8de7a9ef70b33481de3e8dae7cef6e4434d9723c6203a6a3dbe07ef228a16", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "GPU", "NPUW_ONLINE_PIPELINE": "NONE" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 4096, "NPUW_LLM_MIN_RESPONSE_LEN": 4096, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.05893102381378412, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 128.731178, "dram": 46.372623000000004, "package-0": 384.775261, "psys": 768.031237, "uncore": 134.87911 }, "net_j": { "core": 51.864434261035726, "dram": 36.308999009877816, "package-0": 268.0672300159029, "psys": 561.0479995430668, "uncore": 134.84290250755205 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 38, "gross_j": { "core": 5.061267, "dram": 0.532408, "package-0": 6.469527, "psys": 10.818393, "uncore": 0.0 }, "logits": { "l2": 913.718460033083, "sha256": "050909a6c91a93f5f72fdfebadeb65e20a07ce9afa0f13e33f5908e164cd68b6", "sum": -33577.34618644975, "top1": 4710 }, "net_j": { "core": 3.5794586640221815, "dram": 0.3384052207310998, "package-0": 4.21967323001223, "psys": 6.828247564505421, "uncore": 0.0 }, "npu_busy_delta_us": 134433, "npu_utilization": 0.342182302461029, "seconds": 0.392869529, "watts": { "core": 12.882818916709624, "dram": 1.355177637103029, "package-0": 16.467367719933304, "psys": 27.536859444245675, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 200.98569672364553, "decode_tokens_per_s": 8.14839636520181, "end_to_end_ms": 20379.559674, "handoff_estimate_ms": 290.36244899999997, "handoff_plus_first_decode_ms": 392.869529, "output_tokens_per_s_e2e": 0.7851003778267404, "prefill_tokens_per_s": 220.08007756998083, "steady_tpot_ms": 102.50708, "steady_tpot_p95_ms": 108.01841569999999, "ttft_ms": 18538.706661 }, "prefill": { "gpu_active_fraction": 0.9989071038251366, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 1830, "gross_j": { "core": 115.901254, "dram": 41.582779, "package-0": 356.59296, "psys": 717.111445, "uncore": 134.87911 }, "logits": { "l2": 2726.901179322358, "sha256": "13b2c92fc478d8b1d1227a16736da6fb25c66c7424aaab7c23a4956313737475", "sum": -678237.9385230131, "top1": 220 }, "net_j": { "core": 45.97776021711283, "dram": 32.42818606061, "package-0": 250.42697869288713, "psys": 528.8246721580113, "uncore": 134.84290250755205 }, "npu_busy_delta_us": 11648, "npu_utilization": 0.0006283070449841021, "seconds": 18.538706661, "watts": { "core": 6.25185219872011, "dram": 2.243024810758669, "package-0": 19.235050563163988, "psys": 38.68184863772574, "uncore": 7.275540439061269 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 844.4514378557863, "sha256": "b94aa7470bacc216bd3da9fb2a4994eff6fed67bd3d68a29a4cda171a3d91cd1", "sum": 14698.156317738816, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 143, "gross_j": { "core": 7.768657, "dram": 4.257436, "package-0": 21.712774, "psys": 40.101399, "uncore": 0.0 }, "net_j": { "core": 2.307215379900712, "dram": 3.5424077285367144, "package-0": 13.420578093003542, "psys": 25.39507982055013, "uncore": 0.0 }, "npu_busy_delta_us": 1430313, "npu_utilization": 0.9877964878776199, "seconds": 1.447983484, "token_latencies_s": [ 0.114787439, 0.104373557, 0.102584854, 0.102702579, 0.102476318, 0.102493548, 0.102288936, 0.10253398, 0.102520612, 0.10205644, 0.102034329, 0.102316004, 0.102579795, 0.102189578 ], "watts": { "core": 5.365155808641806, "dram": 2.9402517687832965, "package-0": 14.995180704699251, "psys": 27.69465221331212, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 123.883654, "dram": 46.325627, "package-0": 380.198673, "psys": 768.226855, "uncore": 135.395466 }, "net_j": { "core": 47.725245250188685, "dram": 36.3547403452083, "package-0": 264.5661185853531, "psys": 563.150989471775, "uncore": 135.3592353454075 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 20, "gross_j": { "core": 2.852043, "dram": 0.389464, "package-0": 3.882436, "psys": 6.33818, "uncore": 0.0 }, "logits": { "l2": 928.4021349618147, "sha256": "dc9ab7845381a164647b92cacba6a85b45904ffc066588b6c31732bd1fd90879", "sum": 4324.664099577814, "top1": 1718 }, "net_j": { "core": 2.085854865644957, "dram": 0.28915235786738547, "package-0": 2.7191200564025593, "psys": 4.275023778780952, "uncore": 0.0 }, "npu_busy_delta_us": 126123, "npu_utilization": 0.6208726910301483, "seconds": 0.203138263, "watts": { "core": 14.03991034421713, "dram": 1.9172360452840929, "package-0": 19.112283144805662, "psys": 31.201310409944778, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 202.85502676409223, "decode_tokens_per_s": 9.1396876314944, "end_to_end_ms": 20191.759925, "handoff_estimate_ms": 100.81320700000002, "handoff_plus_first_decode_ms": 203.13826300000002, "output_tokens_per_s_e2e": 0.7924024482972353, "prefill_tokens_per_s": 219.93938106248552, "steady_tpot_ms": 102.325056, "steady_tpot_p95_ms": 104.5934553, "ttft_ms": 18550.56598 }, "prefill": { "gpu_active_fraction": 0.9989077007099946, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 1831, "gross_j": { "core": 112.40291, "dram": 41.694779, "package-0": 353.672862, "psys": 719.603505, "uncore": 135.395466 }, "logits": { "l2": 2748.16653849303, "sha256": "41fbb18485a75aa399f63fa651920f76f60f246ee8d8f1c1884ec0ebb413bcc3", "sum": -685130.2997232471, "top1": 220 }, "net_j": { "core": 42.434685749474, "dram": 32.53432981400291, "package-0": 247.43896569416086, "psys": 531.1962840089875, "uncore": 135.3592353454075 }, "npu_busy_delta_us": 11830, "npu_utilization": 0.0006377163916591186, "seconds": 18.55056598, "watts": { "core": 6.059271189956438, "dram": 2.2476284036267447, "package-0": 19.065340776195555, "psys": 38.791458210807654, "uncore": 7.298724262428138 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 931.1000582231685, "sha256": "0405a79d99d43e746aae0ab37db1bb0fa2c2f655e168c01217b32984b173a4fe", "sum": 51959.973022663034, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 142, "gross_j": { "core": 8.628701, "dram": 4.241384, "package-0": 22.643375, "psys": 42.28517, "uncore": 0.0 }, "net_j": { "core": 3.2047046350697306, "dram": 3.5312581733380046, "package-0": 14.40803283478964, "psys": 27.679681684006567, "uncore": 0.0 }, "npu_busy_delta_us": 1423531, "npu_utilization": 0.989899777747271, "seconds": 1.438055682, "token_latencies_s": [ 0.10803081, 0.102742572, 0.102346792, 0.10263592, 0.102321815, 0.102043492, 0.102112601, 0.101988104, 0.10227752, 0.10246868, 0.102328297, 0.102100183, 0.102135059, 0.102470484 ], "watts": { "core": 6.000255141719888, "dram": 2.949387880517411, "package-0": 15.745826314950717, "psys": 29.40440382752857, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 125.19163499999999, "dram": 46.294376, "package-0": 382.10504, "psys": 772.525121, "uncore": 136.046954 }, "net_j": { "core": 48.81744977781478, "dram": 36.29523924408186, "package-0": 266.14486864727934, "psys": 566.8682224964397, "uncore": 136.0106195597234 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 0, "gpu_sample_count": 21, "gross_j": { "core": 2.849114, "dram": 0.393981, "package-0": 3.859914, "psys": 6.339278, "uncore": 0.0 }, "logits": { "l2": 949.3787553822775, "sha256": "d8dc07559dcd42c867154c906a223ecc99445f3647bcf48618c4d6fa20fd761c", "sum": 19532.202365728095, "top1": 1718 }, "net_j": { "core": 2.0728924757012868, "dram": 0.29235575644721523, "package-0": 2.6813641967714297, "psys": 4.249104327121218, "uncore": 0.0 }, "npu_busy_delta_us": 131611, "npu_utilization": 0.6395142041920637, "seconds": 0.2057984, "watts": { "core": 13.844198983082475, "dram": 1.9144026386988433, "package-0": 18.755801794377412, "psys": 30.80333957892773, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 202.28190978809923, "decode_tokens_per_s": 9.117084489125128, "end_to_end_ms": 20248.968404000003, "handoff_estimate_ms": 103.5047355, "handoff_plus_first_decode_ms": 205.7984, "output_tokens_per_s_e2e": 0.7901637101097626, "prefill_tokens_per_s": 219.31114628148657, "steady_tpot_ms": 102.29366449999999, "steady_tpot_p95_ms": 105.2186293, "ttft_ms": 18603.705599 }, "prefill": { "gpu_active_fraction": 0.9989136338946225, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 1841, "gross_j": { "core": 114.189466, "dram": 41.656998, "package-0": 356.117558, "psys": 725.548374, "uncore": 136.046954 }, "logits": { "l2": 2756.8856185329796, "sha256": "69616f1e9441c7b5b9cc435f33c7c8f0372778126e2b065fc191ee8537c523a8", "sum": -687922.0755873509, "top1": 220 }, "net_j": { "core": 44.02081202766452, "dram": 32.47030795536078, "package-0": 249.57934597782827, "psys": 536.6014450587769, "uncore": 136.0106195597234 }, "npu_busy_delta_us": 9123, "npu_utilization": 0.0004903861734132357, "seconds": 18.603705599, "watts": { "core": 6.137995755326186, "dram": 2.2391774465749004, "package-0": 19.142291631358766, "psys": 39.00020725112959, "uncore": 7.312895448491342 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 994.7934300990154, "sha256": "0f720c6fde57b284973add3ed988f92a790012d4227c244f5e98a893d0af1f6e", "sum": 63108.52525581047, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 142, "gross_j": { "core": 8.153055, "dram": 4.243397, "package-0": 22.127568, "psys": 40.637469, "uncore": 0.0 }, "net_j": { "core": 2.7237452744489747, "dram": 3.532575532273869, "package-0": 13.884158472679665, "psys": 26.017673110541573, "uncore": 0.0 }, "npu_busy_delta_us": 1426629, "npu_utilization": 0.99108320778519, "seconds": 1.439464405, "token_latencies_s": [ 0.109740327, 0.102783869, 0.102348338, 0.10224748, 0.102184676, 0.10277247, 0.102080464, 0.102401105, 0.102170907, 0.102339849, 0.1023559, 0.102009959, 0.101968036, 0.102022179 ], "watts": { "core": 5.663950405220336, "dram": 2.9478999169833586, "package-0": 15.372084174599648, "psys": 28.23096483584115, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 3.5447567850351334, "compile_target": "AUTO:NPU,GPU", "context_length": 4096, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 399235, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_3b_ov_fp16/openvino_model.xml", "key": "qwen25_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": null, "key": "openvino_auto", "label": "OpenVINO AUTO", "mode": "stock", "prefill_device": null }, "prompt_length": 4080, "prompt_sha256": "66f8de7a9ef70b33481de3e8dae7cef6e4434d9723c6203a6a3dbe07ef228a16", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.05975502822548151, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 32.33439, "dram": 10.583469, "package-0": 122.42882399999999, "psys": 203.42898300000002, "uncore": 67.017651 }, "net_j": { "core": 15.058093561304554, "dram": 8.321604598779293, "package-0": 96.19794128035583, "psys": 156.9081653588825, "uncore": 67.00870506010212 } }, "first_decode": { "gpu_active_fraction": 0.8461538461538461, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 13, "gross_j": { "core": 2.697381, "dram": 0.312988, "package-0": 4.187428, "psys": 6.295333, "uncore": 0.818418 }, "logits": { "l2": 670.568131117269, "sha256": "f155fefe53ba4dfd4365752c28829b31e15f6eec90450cbf0ac8b52f5d1cfc31", "sum": -55835.01430409169, "top1": 1718 }, "net_j": { "core": 2.2101573730297734, "dram": 0.24919922552578705, "package-0": 3.4476684188509785, "psys": 4.983359592901106, "uncore": 0.8181657078898723 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.129176839, "watts": { "core": 20.88130520053986, "dram": 2.422942087938845, "package-0": 32.416244525073104, "psys": 48.73422394242052, "uncore": 6.335640400675852 } }, "metrics": { "all_tokens_per_s_e2e": 894.2377262438449, "decode_tokens_per_s": 10.384095135058907, "end_to_end_ms": 4580.437483, "handoff_estimate_ms": 36.29455049999999, "handoff_plus_first_decode_ms": 129.176839, "output_tokens_per_s_e2e": 3.493116118140019, "prefill_tokens_per_s": 1301.0533199167141, "steady_tpot_ms": 92.88228849999999, "steady_tpot_p95_ms": 100.38342, "ttft_ms": 3135.920671 }, "prefill": { "gpu_active_fraction": 0.9129032258064517, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 310, "gross_j": { "core": 19.354565, "dram": 6.297408, "package-0": 88.555437, "psys": 144.900325, "uncore": 54.865155 }, "logits": { "l2": 662.3364776849841, "sha256": "d5055a398340419d05f270e2706e72a6086d083d704644e6d8b567ea276b1166", "sum": 8132.857491835253, "top1": 36 }, "net_j": { "core": 7.526634737551589, "dram": 4.748859994026255, "package-0": 70.59689752764523, "psys": 113.05061758361306, "uncore": 54.859030310393074 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 3.135920671, "watts": { "core": 6.171892413920059, "dram": 2.008152839526979, "package-0": 28.239055221942504, "psys": 46.20662963192668, "uncore": 17.495708838356645 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 706.6311889009895, "sha256": "b5b9231e8728eabecddc2a8f58f01d40f25e30ef87c52a6cee95dc6ef8f92e49", "sum": -130420.66216149065, "top1": 42 }, "gpu_active_fraction": 0.9923076923076923, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 130, "gross_j": { "core": 10.282444, "dram": 3.973073, "package-0": 29.685959, "psys": 52.233325, "uncore": 11.334078 }, "net_j": { "core": 5.321301450723192, "dram": 3.3235453792272516, "package-0": 22.15337533385963, "psys": 38.874188182368336, "uncore": 11.331509041819166 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.315339973, "token_latencies_s": [ 0.102089293, 0.093226358, 0.099464873, 0.092745762, 0.093175623, 0.092567746, 0.09278905, 0.09293055, 0.092304634, 0.093116181, 0.092834027, 0.092379771, 0.093125818, 0.092559209 ], "watts": { "core": 7.8173279996562535, "dram": 3.020567367794881, "package-0": 22.56903888680041, "psys": 39.710893055935436, "uncore": 8.616842970376299 } } }, { "energy": { "gross_j": { "core": 31.975198, "dram": 10.745334, "package-0": 122.58867599999999, "psys": 205.40725, "uncore": 67.15998499999999 }, "net_j": { "core": 14.39978398377925, "dram": 8.444308230902525, "package-0": 95.90363818274609, "psys": 158.08098228301205, "uncore": 67.15088417228338 } }, "first_decode": { "gpu_active_fraction": 0.9166666666666666, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 12, "gross_j": { "core": 2.796623, "dram": 0.302306, "package-0": 4.156789, "psys": 6.949201, "uncore": 0.732236 }, "logits": { "l2": 670.568131117269, "sha256": "f155fefe53ba4dfd4365752c28829b31e15f6eec90450cbf0ac8b52f5d1cfc31", "sum": -55835.01430409169, "top1": 1718 }, "net_j": { "core": 2.3612751511334626, "dram": 0.2453089575497613, "package-0": 3.4957932590210485, "psys": 5.776916305798361, "uncore": 0.7320105699893228 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.115423095, "watts": { "core": 24.229319097707435, "dram": 2.6191118857105677, "package-0": 36.013494526377066, "psys": 60.20633045752239, "uncore": 6.343929696218941 } }, "metrics": { "all_tokens_per_s_e2e": 879.0186126480612, "decode_tokens_per_s": 10.527165091603338, "end_to_end_ms": 4659.742059, "handoff_estimate_ms": 22.689726500000006, "handoff_plus_first_decode_ms": 115.423095, "output_tokens_per_s_e2e": 3.433666455656489, "prefill_tokens_per_s": 1261.2613076033124, "steady_tpot_ms": 92.7333685, "steady_tpot_p95_ms": 97.10337725, "ttft_ms": 3234.857024 }, "prefill": { "gpu_active_fraction": 0.896875, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 320, "gross_j": { "core": 19.519847, "dram": 6.479781, "package-0": 89.433853, "psys": 146.950368, "uncore": 55.144574 }, "logits": { "l2": 662.3364776849841, "sha256": "d5055a398340419d05f270e2706e72a6086d083d704644e6d8b567ea276b1166", "sum": 8132.857491835253, "top1": 36 }, "net_j": { "core": 7.318752870708037, "dram": 4.882377264487787, "package-0": 70.90873274100984, "psys": 114.09582206042123, "uncore": 55.13825608023308 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 3.234857024, "watts": { "core": 6.034222488097204, "dram": 2.0031120237850732, "package-0": 27.646926073230986, "psys": 45.42716012168332, "uncore": 17.046989585898928 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 706.6311889009895, "sha256": "b5b9231e8728eabecddc2a8f58f01d40f25e30ef87c52a6cee95dc6ef8f92e49", "sum": -130420.66216149065, "top1": 42 }, "gpu_active_fraction": 0.9923076923076923, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 130, "gross_j": { "core": 9.658728, "dram": 3.963247, "package-0": 28.998034, "psys": 51.507681, "uncore": 11.283175 }, "net_j": { "core": 4.71975596193775, "dram": 3.3166220088649765, "package-0": 21.49911218271521, "psys": 38.20824391679247, "uncore": 11.280617522060982 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.30946194, "token_latencies_s": [ 0.104420762, 0.093163247, 0.092847174, 0.092836554, 0.092812207, 0.092546024, 0.092616328, 0.092965256, 0.09265453, 0.092817805, 0.092487615, 0.092370358, 0.092244352, 0.092647497 ], "watts": { "core": 7.37610441736092, "dram": 3.0266225225301318, "package-0": 22.144999494983413, "psys": 39.334996632280884, "uncore": 8.616649827943835 } } }, { "energy": { "gross_j": { "core": 37.197904, "dram": 10.775973, "package-0": 128.108802, "psys": 212.037726, "uncore": 67.417186 }, "net_j": { "core": 19.638541837883924, "dram": 8.477048787651206, "package-0": 101.44813597469508, "psys": 164.7546819785683, "uncore": 67.40809348418765 } }, "first_decode": { "gpu_active_fraction": 0.9166666666666666, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 12, "gross_j": { "core": 2.517145, "dram": 0.300292, "package-0": 3.875844, "psys": 5.881333, "uncore": 0.728391 }, "logits": { "l2": 670.568131117269, "sha256": "f155fefe53ba4dfd4365752c28829b31e15f6eec90450cbf0ac8b52f5d1cfc31", "sum": -55835.01430409169, "top1": 1718 }, "net_j": { "core": 2.0826394938183435, "dram": 0.2434052395734184, "package-0": 3.216127201661056, "psys": 4.711316527495859, "uncore": 0.7281660061677051 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.115199766, "watts": { "core": 21.850261397232355, "dram": 2.6067066837618404, "package-0": 33.64454750715379, "psys": 51.05334154932224, "uncore": 6.322851384958542 } }, "metrics": { "all_tokens_per_s_e2e": 879.8221656698196, "decode_tokens_per_s": 10.586823056760185, "end_to_end_ms": 4655.486255999999, "handoff_estimate_ms": 23.738090000000003, "handoff_plus_first_decode_ms": 115.199766, "output_tokens_per_s_e2e": 3.4368053346477327, "prefill_tokens_per_s": 1259.7917110035642, "steady_tpot_ms": 91.461676, "steady_tpot_p95_ms": 101.17716695, "ttft_ms": 3238.630612 }, "prefill": { "gpu_active_fraction": 0.897196261682243, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 321, "gross_j": { "core": 22.002263, "dram": 6.509139, "package-0": 92.24189, "psys": 151.165323, "uncore": 55.40324 }, "logits": { "l2": 662.3364776849841, "sha256": "d5055a398340419d05f270e2706e72a6086d083d704644e6d8b567ea276b1166", "sum": 8132.857491835253, "top1": 36 }, "net_j": { "core": 9.786935814331333, "dram": 4.909871830177473, "package-0": 73.69515945932221, "psys": 118.27245093985911, "uncore": 55.39691471013108 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 3.238630612, "watts": { "core": 6.793693272235394, "dram": 2.0098429798946147, "package-0": 28.48175696796631, "psys": 46.67569139867069, "uncore": 17.10699571439733 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 706.6311889009895, "sha256": "b5b9231e8728eabecddc2a8f58f01d40f25e30ef87c52a6cee95dc6ef8f92e49", "sum": -130420.66216149065, "top1": 42 }, "gpu_active_fraction": 0.9922480620155039, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 129, "gross_j": { "core": 12.678496, "dram": 3.966542, "package-0": 31.991068, "psys": 54.99107, "uncore": 11.285555 }, "net_j": { "core": 7.7689665297342465, "dram": 3.3237717179003146, "package-0": 24.536849313711805, "psys": 41.77091451121332, "uncore": 11.283012767888867 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.301655878, "token_latencies_s": [ 0.105705234, 0.092246253, 0.091822925, 0.091223243, 0.091637262, 0.098738977, 0.091638716, 0.091076312, 0.090928838, 0.091139372, 0.09125447, 0.09129244, 0.091312382, 0.09161097 ], "watts": { "core": 9.74028252342744, "dram": 3.0473046425254955, "package-0": 24.577208570021146, "psys": 42.2470108493606, "uncore": 8.670152527056771 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 2.785680193454027, "compile_target": "AUTO:GPU", "context_length": 8192, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 399811, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_3b_ov_fp16/openvino_model.xml", "key": "qwen25_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "GPU", "key": "gpu_gpu", "label": "iGPU -> iGPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 8176, "prompt_sha256": "998ace73fce48442f97bcb84c31be293d0d94180b17f70a4d127bba1092ad744", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.06051105260848999, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 59.252901, "dram": 19.136485999999998, "package-0": 247.554787, "psys": 406.515866, "uncore": 145.329156 }, "net_j": { "core": 25.151504512799242, "dram": 14.67182917844012, "package-0": 195.77808040052733, "psys": 314.689191636203, "uncore": 145.3114977568406 } }, "first_decode": { "gpu_active_fraction": 0.8571428571428571, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 14, "gross_j": { "core": 2.945855, "dram": 0.32849, "package-0": 4.493031, "psys": 6.787702, "uncore": 0.846067 }, "logits": { "l2": 641.8176091039514, "sha256": "ae4c116f8db1f114b722fdb50e703656b8ec36d44932a98d79948cfafc69699b", "sum": -12608.67373151111, "top1": 78 }, "net_j": { "core": 2.4178898663994683, "dram": 0.25936722562446657, "package-0": 3.6914129246978282, "psys": 5.366021734889127, "uncore": 0.8457936112925225 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.139978571, "watts": { "core": 21.045042672995997, "dram": 2.3467163413177006, "package-0": 32.09799162758991, "psys": 48.49100795578204, "uncore": 6.04426087475918 } }, "metrics": { "all_tokens_per_s_e2e": 906.0694069260272, "decode_tokens_per_s": 9.686000326255487, "end_to_end_ms": 9041.249972, "handoff_estimate_ms": 40.392893, "handoff_plus_first_decode_ms": 139.978571, "output_tokens_per_s_e2e": 1.769666810402397, "prefill_tokens_per_s": 1091.206623799312, "steady_tpot_ms": 99.585678, "steady_tpot_p95_ms": 107.27231370000001, "ttft_ms": 7492.62314 }, "prefill": { "gpu_active_fraction": 0.9284750337381916, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 741, "gross_j": { "core": 45.050178, "dram": 14.695336, "package-0": 211.082345, "psys": 343.780187, "uncore": 132.189908 }, "logits": { "l2": 630.9416903237038, "sha256": "14983026d2c3865bd74d1e89295c4c951cf5e5701b07ca9e53c79ce2d506866a", "sum": 80776.56900331145, "top1": 198 }, "net_j": { "core": 16.7898253698211, "dram": 10.995406110109062, "package-0": 168.17419057648246, "psys": 267.6820072861132, "uncore": 132.17527434185342 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 7.49262314, "watts": { "core": 6.012604285339781, "dram": 1.9613072385220751, "package-0": 28.172022141767563, "psys": 45.882487424824625, "uncore": 17.642674071553532 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 572.7403064888838, "sha256": "1f72cc218133f7bc0627789756871cfa6b42dada579c1e1db76b598cec8f6f75", "sum": -71473.40795485722, "top1": 42 }, "gpu_active_fraction": 0.9928057553956835, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 139, "gross_j": { "core": 11.256868, "dram": 4.11266, "package-0": 31.979411, "psys": 55.947977, "uncore": 12.293181 }, "net_j": { "core": 5.943789276578674, "dram": 3.417055842706592, "package-0": 23.91247689934704, "psys": 41.6411626152007, "uncore": 12.290429803694638 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.408648261, "token_latencies_s": [ 0.109898315, 0.099871449, 0.099669116, 0.09866253, 0.099311326, 0.099029114, 0.099677494, 0.09904256, 0.105858313, 0.099663692, 0.099588846, 0.09958251, 0.099226278, 0.099535382 ], "watts": { "core": 7.991255384086263, "dram": 2.9195790843346665, "package-0": 22.702197479232893, "psys": 39.71749268357632, "uncore": 8.726934423837692 } } }, { "energy": { "gross_j": { "core": 61.178005, "dram": 19.557933, "package-0": 250.28762899999998, "psys": 417.939299, "uncore": 145.322382 }, "net_j": { "core": 26.25683161547912, "dram": 14.985948535182136, "package-0": 197.26624176389998, "psys": 323.90516705512033, "uncore": 145.30429926338044 } }, "first_decode": { "gpu_active_fraction": 0.8461538461538461, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 13, "gross_j": { "core": 2.570673, "dram": 0.318481, "package-0": 4.025747, "psys": 6.196822, "uncore": 0.782835 }, "logits": { "l2": 641.8176091039514, "sha256": "ae4c116f8db1f114b722fdb50e703656b8ec36d44932a98d79948cfafc69699b", "sum": -12608.67373151111, "top1": 78 }, "net_j": { "core": 2.0807812737255413, "dram": 0.25434290998489095, "package-0": 3.2819364001513387, "psys": 4.877664057626371, "uncore": 0.7825813263057984 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.129884228, "watts": { "core": 19.79203356392125, "dram": 2.452037517596055, "package-0": 30.994887231419664, "psys": 47.71035017431062, "uncore": 6.027175216378081 } }, "metrics": { "all_tokens_per_s_e2e": 884.7993665700598, "decode_tokens_per_s": 9.855231543392565, "end_to_end_ms": 9258.59614, "handoff_estimate_ms": 32.16617049999999, "handoff_plus_first_decode_ms": 129.88422799999998, "output_tokens_per_s_e2e": 1.728123762832148, "prefill_tokens_per_s": 1056.8001809501457, "steady_tpot_ms": 97.7180575, "steady_tpot_p95_ms": 107.81119354999998, "ttft_ms": 7736.561885 }, "prefill": { "gpu_active_fraction": 0.9137254901960784, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 765, "gross_j": { "core": 48.498899, "dram": 15.138999, "package-0": 215.589353, "psys": 356.844668, "uncore": 132.32565 }, "logits": { "l2": 630.9416903237038, "sha256": "14983026d2c3865bd74d1e89295c4c951cf5e5701b07ca9e53c79ce2d506866a", "sum": 80776.56900331145, "top1": 198 }, "net_j": { "core": 19.31846873812223, "dram": 11.318609794717954, "package-0": 171.28422969859236, "psys": 278.2689454937781, "uncore": 132.3105399110846 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 7.736561885, "watts": { "core": 6.26879222591522, "dram": 1.956812241023003, "package-0": 27.86629981180587, "psys": 46.124450796660305, "uncore": 17.103934792605873 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 572.7403064888838, "sha256": "1f72cc218133f7bc0627789756871cfa6b42dada579c1e1db76b598cec8f6f75", "sum": -71473.40795485722, "top1": 42 }, "gpu_active_fraction": 0.9927536231884058, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 138, "gross_j": { "core": 10.108433, "dram": 4.100453, "package-0": 30.672529, "psys": 54.897809, "uncore": 12.213897 }, "net_j": { "core": 4.857581603631346, "dram": 3.41299583047929, "package-0": 22.700075665156294, "psys": 40.75855750371584, "uncore": 12.211178025990067 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.392150027, "token_latencies_s": [ 0.113735042, 0.098577988, 0.097677854, 0.097593822, 0.097758261, 0.097606737, 0.097638762, 0.097464358, 0.097638063, 0.097637551, 0.104621429, 0.098021256, 0.098232888, 0.097920396 ], "watts": { "core": 7.261022737458166, "dram": 2.9454102794051806, "package-0": 22.03248816946652, "psys": 39.4338310780351, "uncore": 8.773405712831265 } } }, { "energy": { "gross_j": { "core": 59.900603000000004, "dram": 19.610851, "package-0": 249.049716, "psys": 412.77097599999996, "uncore": 145.30926 }, "net_j": { "core": 25.00429956929731, "dram": 15.042122583919257, "package-0": 196.06608922049801, "psys": 318.80381272461636, "uncore": 145.2911901414364 } }, "first_decode": { "gpu_active_fraction": 0.7857142857142857, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 14, "gross_j": { "core": 2.978203, "dram": 0.320922, "package-0": 4.455494, "psys": 6.617048, "uncore": 0.798399 }, "logits": { "l2": 641.8176091039514, "sha256": "ae4c116f8db1f114b722fdb50e703656b8ec36d44932a98d79948cfafc69699b", "sum": -12608.67373151111, "top1": 78 }, "net_j": { "core": 2.4825701657042822, "dram": 0.25603226695173686, "package-0": 3.702966582185124, "psys": 5.282430665605476, "uncore": 0.7981423534691439 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.131406359, "watts": { "core": 22.664070617769724, "dram": 2.4422105782567187, "package-0": 33.90622823664112, "psys": 50.35561482987288, "uncore": 6.075801856742716 } }, "metrics": { "all_tokens_per_s_e2e": 885.4299468098451, "decode_tokens_per_s": 9.70446882113418, "end_to_end_ms": 9252.002408, "handoff_estimate_ms": 31.965154999999996, "handoff_plus_first_decode_ms": 131.406359, "output_tokens_per_s_e2e": 1.7293553648629787, "prefill_tokens_per_s": 1060.9469989685113, "steady_tpot_ms": 99.441204, "steady_tpot_p95_ms": 107.64871504999999, "ttft_ms": 7706.322755 }, "prefill": { "gpu_active_fraction": 0.9145860709592641, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 761, "gross_j": { "core": 45.479925, "dram": 15.16677, "package-0": 212.387945, "psys": 349.058188, "uncore": 132.189909 }, "logits": { "l2": 630.9416903237038, "sha256": "14983026d2c3865bd74d1e89295c4c951cf5e5701b07ca9e53c79ce2d506866a", "sum": 80776.56900331145, "top1": 198 }, "net_j": { "core": 16.413549383746624, "dram": 11.361313170047763, "package-0": 168.25599272767374, "psys": 270.7895866055883, "uncore": 132.17485797038637 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 7.706322755, "watts": { "core": 5.901637713070324, "dram": 1.9680943145236847, "package-0": 27.560219283859983, "psys": 45.29503877494941, "uncore": 17.153435328702372 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 572.7403064888838, "sha256": "1f72cc218133f7bc0627789756871cfa6b42dada579c1e1db76b598cec8f6f75", "sum": -71473.40795485722, "top1": 42 }, "gpu_active_fraction": 0.9928571428571429, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 140, "gross_j": { "core": 11.442475, "dram": 4.123159, "package-0": 32.206277, "psys": 57.09574, "uncore": 12.320952 }, "net_j": { "core": 6.108180019846403, "dram": 3.424777146919757, "package-0": 24.107129910639138, "psys": 42.7317954534226, "uncore": 12.318189817580903 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.414273294, "token_latencies_s": [ 0.111278265, 0.099954819, 0.105694342, 0.099467568, 0.099617773, 0.105665746, 0.099892691, 0.09941484, 0.099150481, 0.098806237, 0.09911801, 0.098637142, 0.098750954, 0.098797943 ], "watts": { "core": 8.090709941667045, "dram": 2.9153905525136787, "package-0": 22.772315037435757, "psys": 40.37107979216356, "uncore": 8.711860750161348 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 11.432788781821728, "compile_target": "NPU", "context_length": 8192, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 399882, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_3b_ov_fp16/openvino_model.xml", "key": "qwen25_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "gpu_npu", "label": "iGPU -> NPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 8176, "prompt_sha256": "998ace73fce48442f97bcb84c31be293d0d94180b17f70a4d127bba1092ad744", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "GPU", "NPUW_ONLINE_PIPELINE": "NONE" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 8192, "NPUW_LLM_MIN_RESPONSE_LEN": 8192, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.058648111298680305, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 243.79735000000002, "dram": 91.704476, "package-0": 769.0989219999999, "psys": 1562.7052279999998, "uncore": 289.146354 }, "net_j": { "core": 90.593715480322, "dram": 71.64659795378061, "package-0": 536.487344422071, "psys": 1150.1655167987642, "uncore": 289.07095152272984 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 46, "gross_j": { "core": 6.469832, "dram": 0.63281, "package-0": 8.125956, "psys": 13.474514, "uncore": 0.0 }, "logits": { "l2": 918.8020555783207, "sha256": "1bb6018b81cc8c62cfc72ed7206d2c94b7a2abbf7d1eaf57f9ec872a0d5d3e65", "sum": -25677.505140628666, "top1": 1825 }, "net_j": { "core": 4.696840984083592, "dram": 0.4006847133787733, "package-0": 5.433994816854322, "psys": 8.700285216448568, "uncore": 0.0 }, "npu_busy_delta_us": 145249, "npu_utilization": 0.3089941829088565, "seconds": 0.470070338, "watts": { "core": 13.763540212996807, "dram": 1.3462027889111352, "package-0": 17.286681041338117, "psys": 28.664888870312, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 201.68080337897354, "decode_tokens_per_s": 7.45678042608262, "end_to_end_ms": 40618.640261, "handoff_estimate_ms": 361.84099299999997, "handoff_plus_first_decode_ms": 470.070338, "output_tokens_per_s_e2e": 0.3939078190995577, "prefill_tokens_per_s": 211.77480207661233, "steady_tpot_ms": 108.22934500000001, "steady_tpot_p95_ms": 117.97439245, "ttft_ms": 38607.048241000004 }, "prefill": { "gpu_active_fraction": 0.9992128050380478, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 3811, "gross_j": { "core": 228.675562, "dram": 86.60067, "package-0": 737.592777, "psys": 1506.485767, "uncore": 289.146354 }, "logits": { "l2": 2756.7075776903853, "sha256": "84cd27dc25b56e5e390e1b1b887e9cf001acecb3b20c00fa1bc22fbf10010345", "sum": -687907.677328907, "top1": 220 }, "net_j": { "core": 83.05916345509891, "dram": 67.53613558017099, "package-0": 516.5010235769098, "psys": 1114.3766163899886, "uncore": 289.07095152272984 }, "npu_busy_delta_us": 11610, "npu_utilization": 0.00030072229110927954, "seconds": 38.607048241, "watts": { "core": 5.923155807523006, "dram": 2.2431310847544053, "package-0": 19.105132627484572, "psys": 39.02100356380363, "uncore": 7.489470632280343 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 997.8621615342367, "sha256": "29494008627b5f3ecc3f195d249808708e3e02693b135cdb6ada818d59389de2", "sum": 88720.19780683517, "top1": 220 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 151, "gross_j": { "core": 8.651956, "dram": 4.470996, "package-0": 23.380189, "psys": 42.744947, "uncore": 0.0 }, "net_j": { "core": 2.837711041139503, "dram": 3.709777660230846, "package-0": 14.552326028306952, "psys": 27.088615192326966, "uncore": 0.0 }, "npu_busy_delta_us": 1515885, "npu_utilization": 0.9833692368395762, "seconds": 1.541521682, "token_latencies_s": [ 0.13144969, 0.110718463, 0.108562676, 0.108409888, 0.108237745, 0.10857501, 0.108192256, 0.108145068, 0.108166442, 0.108141932, 0.108181916, 0.108220945, 0.10806269, 0.108403835 ], "watts": { "core": 5.612607400224683, "dram": 2.9003782770017508, "package-0": 15.166954362695758, "psys": 27.729059862811585, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 220.631943, "dram": 91.64277100000001, "package-0": 750.822663, "psys": 1555.000536, "uncore": 289.427665 }, "net_j": { "core": 67.98900070301652, "dram": 71.6583004565434, "package-0": 519.0623935706379, "psys": 1143.9706310664053, "uncore": 289.35160010883976 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 0, "gpu_sample_count": 26, "gross_j": { "core": 3.697134, "dram": 0.481139, "package-0": 4.926074, "psys": 8.250284, "uncore": 0.003113 }, "logits": { "l2": 928.0207259122993, "sha256": "96bb7937c1ddec1e5746531cc2c98ec321f0f8ff76d31787766875d670a08fba", "sum": -8464.025744725019, "top1": 1825 }, "net_j": { "core": 2.7011726789501522, "dram": 0.3507447650394482, "package-0": 3.41388967864014, "psys": 5.5684050939174465, "uncore": 0.0025972754550806203 }, "npu_busy_delta_us": 139865, "npu_utilization": 0.5296759625640716, "seconds": 0.264057669, "watts": { "core": 14.0012369797902, "dram": 1.822098187195616, "package-0": 18.655296089885578, "psys": 31.244250664047183, "uncore": 0.01178909142002613 } }, "metrics": { "all_tokens_per_s_e2e": 202.42162281169493, "decode_tokens_per_s": 8.390060790330883, "end_to_end_ms": 40469.984808, "handoff_estimate_ms": 155.709177, "handoff_plus_first_decode_ms": 264.057669, "output_tokens_per_s_e2e": 0.39535473205409166, "prefill_tokens_per_s": 211.36361147807168, "steady_tpot_ms": 108.348492, "steady_tpot_p95_ms": 111.32695169999998, "ttft_ms": 38682.155092 }, "prefill": { "gpu_active_fraction": 0.998949027850762, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 3806, "gross_j": { "core": 209.477858, "dram": 86.706382, "package-0": 723.630899, "psys": 1504.622125, "uncore": 289.424552 }, "logits": { "l2": 2763.2184793149227, "sha256": "0d0b42e7fc1606224918b1e28631651539a24fa3768396a1ab8522ae44b7e5fe", "sum": -690004.62714516, "top1": 220 }, "net_j": { "core": 63.57817467558411, "dram": 67.6047590895639, "package-0": 502.10902967480774, "psys": 1111.7501581454844, "uncore": 289.3490028333847 }, "npu_busy_delta_us": 8960, "npu_utilization": 0.00023163135504446208, "seconds": 38.682155092, "watts": { "core": 5.415361618342792, "dram": 2.2415085662570045, "package-0": 18.70709884904155, "psys": 38.897060451297776, "uncore": 7.482120665501829 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 1053.16157417548, "sha256": "976fdf28a39fa930aa5a083ec7fdcd6fd44fbeebbe2fcfebf1c014f2501fccd0", "sum": 84184.23420181125, "top1": 220 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 150, "gross_j": { "core": 7.456951, "dram": 4.45525, "package-0": 22.26569, "psys": 42.128127, "uncore": 0.0 }, "net_j": { "core": 1.7096533484822505, "dram": 3.7027966019400527, "package-0": 13.539474217189998, "psys": 26.652067827003457, "uncore": 0.0 }, "npu_busy_delta_us": 1508805, "npu_utilization": 0.9901776338334417, "seconds": 1.523772047, "token_latencies_s": [ 0.115988844, 0.108816702, 0.108491134, 0.10849455, 0.108279471, 0.108737424, 0.108417513, 0.108208627, 0.108577581, 0.107911744, 0.107922596, 0.107959962, 0.107831916, 0.108088828 ], "watts": { "core": 4.893744451265682, "dram": 2.9238297216250224, "package-0": 14.612218437683415, "psys": 27.647263304863603, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 235.62830300000002, "dram": 91.64258799999999, "package-0": 761.199041, "psys": 1555.684311, "uncore": 289.522086 }, "net_j": { "core": 83.14381982333549, "dram": 71.67886339850902, "package-0": 529.6793626380402, "psys": 1145.08109750841, "uncore": 289.4466305862844 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 27, "gross_j": { "core": 3.603873, "dram": 0.482482, "package-0": 4.839954, "psys": 8.271402, "uncore": 0.0 }, "logits": { "l2": 937.7241911868175, "sha256": "f634f53d4b1a8bdc58612b054b7078b9f79e875e90249ee418cda8638c28c679", "sum": 8204.341230405495, "top1": 1825 }, "net_j": { "core": 2.589732372939451, "dram": 0.3497076759242601, "package-0": 3.3001677417560775, "psys": 5.54057069357129, "uncore": 0.0 }, "npu_busy_delta_us": 136832, "npu_utilization": 0.5089008556758483, "seconds": 0.26887752, "watts": { "core": 13.403400180126626, "dram": 1.794430415752124, "package-0": 18.000590008417216, "psys": 30.762713074711492, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 202.63197570541928, "decode_tokens_per_s": 8.362041149715992, "end_to_end_ms": 40427.972789, "handoff_estimate_ms": 160.46092999999996, "handoff_plus_first_decode_ms": 268.87752, "output_tokens_per_s_e2e": 0.39576557754964703, "prefill_tokens_per_s": 211.62622949967678, "steady_tpot_ms": 108.41659000000001, "steady_tpot_p95_ms": 111.1662418, "ttft_ms": 38634.152389 }, "prefill": { "gpu_active_fraction": 0.9992128050380478, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 3811, "gross_j": { "core": 225.904512, "dram": 86.696677, "package-0": 735.47529, "psys": 1506.959949, "uncore": 289.522086 }, "logits": { "l2": null, "sha256": "b0ba3595a28b4c7ae5c9e581e843cac4c1d56ce6f3684ebf9bc0b9924c710c7a", "sum": null, "top1": 0 }, "net_j": { "core": 80.18588319929822, "dram": 67.61875828932233, "package-0": 514.2283187117948, "psys": 1114.575517452064, "uncore": 289.4466305862844 }, "npu_busy_delta_us": 8911, "npu_utilization": 0.00023065084773380868, "seconds": 38.634152389, "watts": { "core": 5.8472749634937, "dram": 2.244042424616114, "package-0": 19.03692056175163, "psys": 39.0059016651046, "uncore": 7.493941709523136 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 1073.8044177318684, "sha256": "f44e69db153b06ae69976129b2fb2b8c01efb46054fd94323dfac83613b8fa55", "sum": 74667.62392079458, "top1": 220 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 151, "gross_j": { "core": 6.119918, "dram": 4.463429, "package-0": 20.883797, "psys": 40.45296, "uncore": 0.0 }, "net_j": { "core": 0.36820425109782384, "dram": 3.71039743326243, "package-0": 12.150876184489242, "psys": 24.965009362774687, "uncore": 0.0 }, "npu_busy_delta_us": 1508335, "npu_utilization": 0.9891091789615096, "seconds": 1.52494288, "token_latencies_s": [ 0.115011387, 0.109095779, 0.108347609, 0.108177885, 0.108474542, 0.108210081, 0.108682487, 0.108365508, 0.108171777, 0.108375109, 0.108572627, 0.108611577, 0.108458071, 0.10833528 ], "watts": { "core": 4.013211301396417, "dram": 2.926948319533122, "package-0": 13.694806063817945, "psys": 26.527524755550186, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 3.51444427203387, "compile_target": "AUTO:NPU,GPU", "context_length": 8192, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 399723, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_3b_ov_fp16/openvino_model.xml", "key": "qwen25_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": null, "key": "openvino_auto", "label": "OpenVINO AUTO", "mode": "stock", "prefill_device": null }, "prompt_length": 8176, "prompt_sha256": "998ace73fce48442f97bcb84c31be293d0d94180b17f70a4d127bba1092ad744", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.061123667284846306, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 54.979962, "dram": 19.41206, "package-0": 243.268175, "psys": 403.04382899999996, "uncore": 145.02385900000002 }, "net_j": { "core": 20.75228509830278, "dram": 14.930870168883676, "package-0": 191.29973478712247, "psys": 310.8771125354391, "uncore": 145.00613536684207 } }, "first_decode": { "gpu_active_fraction": 0.7857142857142857, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 14, "gross_j": { "core": 2.786309, "dram": 0.327819, "package-0": 4.329212, "psys": 6.560408, "uncore": 0.841734 }, "logits": { "l2": 641.8176091039514, "sha256": "ae4c116f8db1f114b722fdb50e703656b8ec36d44932a98d79948cfafc69699b", "sum": -12608.67373151111, "top1": 78 }, "net_j": { "core": 2.2585994859045027, "dram": 0.25872969209459495, "package-0": 3.527982035962369, "psys": 5.1394160553530295, "uncore": 0.8414607436563504 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.139910799, "watts": { "core": 19.914895918791803, "dram": 2.3430571645867024, "package-0": 30.942657971669508, "psys": 46.88993306370868, "uncore": 6.016218948188552 } }, "metrics": { "all_tokens_per_s_e2e": 902.7265326609174, "decode_tokens_per_s": 9.73978923886958, "end_to_end_ms": 9074.730501, "handoff_estimate_ms": 41.047672000000006, "handoff_plus_first_decode_ms": 139.910799, "output_tokens_per_s_e2e": 1.7631377591033544, "prefill_tokens_per_s": 1085.1191989132783, "steady_tpot_ms": 98.86312699999999, "steady_tpot_p95_ms": 106.38641039999999, "ttft_ms": 7534.656108 }, "prefill": { "gpu_active_fraction": 0.9288590604026845, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 745, "gross_j": { "core": 41.544877, "dram": 14.973412, "package-0": 207.61336, "psys": 341.30418, "uncore": 131.899198 }, "logits": { "l2": 630.9416903237038, "sha256": "14983026d2c3865bd74d1e89295c4c951cf5e5701b07ca9e53c79ce2d506866a", "sum": 80776.56900331145, "top1": 198 }, "net_j": { "core": 13.12598623496796, "dram": 11.252725823324294, "package-0": 164.4644945400239, "psys": 264.77909607739946, "uncore": 131.88448224829645 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 7.534656108, "watts": { "core": 5.513838508951894, "dram": 1.987272117715077, "package-0": 27.554457300256125, "psys": 45.29791076166259, "uncore": 17.50566928462132 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 572.7403064888838, "sha256": "1f72cc218133f7bc0627789756871cfa6b42dada579c1e1db76b598cec8f6f75", "sum": -71473.40795485722, "top1": 42 }, "gpu_active_fraction": 0.9928057553956835, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 139, "gross_j": { "core": 10.648776, "dram": 4.110829, "package-0": 31.325603, "psys": 55.179241, "uncore": 12.282927 }, "net_j": { "core": 5.367699377430318, "dram": 3.4194146534647865, "package-0": 23.3072582111362, "psys": 40.95860040268663, "uncore": 12.280192374889271 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.400163594, "token_latencies_s": [ 0.108406522, 0.09943424, 0.098498826, 0.098789552, 0.098829543, 0.105298658, 0.098887598, 0.099213776, 0.098774743, 0.098893934, 0.098722326, 0.098463872, 0.099079368, 0.098838656 ], "watts": { "core": 7.60537986106215, "dram": 2.93596335286518, "package-0": 22.37281638676859, "psys": 39.409138500997194, "uncore": 8.772494194703366 } } }, { "energy": { "gross_j": { "core": 58.368625, "dram": 19.586986, "package-0": 248.11380599999998, "psys": 413.946328, "uncore": 145.851861 }, "net_j": { "core": 23.45702970533852, "dram": 15.016255527353522, "package-0": 195.10696133398565, "psys": 319.9379874956227, "uncore": 145.8337832230671 } }, "first_decode": { "gpu_active_fraction": 0.8461538461538461, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 13, "gross_j": { "core": 2.289179, "dram": 0.322448, "package-0": 3.777395, "psys": 5.901169, "uncore": 0.802427 }, "logits": { "l2": 641.8176091039514, "sha256": "ae4c116f8db1f114b722fdb50e703656b8ec36d44932a98d79948cfafc69699b", "sum": -12608.67373151111, "top1": 78 }, "net_j": { "core": 1.7911000305187375, "dram": 0.2572380116171292, "package-0": 3.021153575227899, "psys": 4.559964825115906, "uncore": 0.8021690868216058 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.132054899, "watts": { "core": 17.33505547567758, "dram": 2.441772341971198, "package-0": 28.604732036484307, "psys": 44.68724026664092, "uncore": 6.076465213153508 } }, "metrics": { "all_tokens_per_s_e2e": 885.0421136507651, "decode_tokens_per_s": 9.861322321369133, "end_to_end_ms": 9256.056716, "handoff_estimate_ms": 34.0448985, "handoff_plus_first_decode_ms": 132.054899, "output_tokens_per_s_e2e": 1.7285978782241507, "prefill_tokens_per_s": 1057.0186944027648, "steady_tpot_ms": 98.0100005, "steady_tpot_p95_ms": 106.1401223, "ttft_ms": 7734.962535000001 }, "prefill": { "gpu_active_fraction": 0.912303664921466, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 764, "gross_j": { "core": 46.320377, "dram": 15.164085, "package-0": 213.901064, "psys": 353.616038, "uncore": 132.733486 }, "logits": { "l2": 630.9416903237038, "sha256": "14983026d2c3865bd74d1e89295c4c951cf5e5701b07ca9e53c79ce2d506866a", "sum": 80776.56900331145, "top1": 198 }, "net_j": { "core": 17.145979097435458, "dram": 11.344485569241353, "package-0": 169.6050997281465, "psys": 275.0565591539024, "uncore": 132.7183790347358 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 7.734962535, "watts": { "core": 5.9884423215244444, "dram": 1.9604600450724743, "package-0": 27.653794447240976, "psys": 45.716580578111355, "uncore": 17.160197660866885 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 572.7403064888838, "sha256": "1f72cc218133f7bc0627789756871cfa6b42dada579c1e1db76b598cec8f6f75", "sum": -71473.40795485722, "top1": 42 }, "gpu_active_fraction": 0.9927536231884058, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 138, "gross_j": { "core": 9.759069, "dram": 4.100453, "package-0": 30.435347, "psys": 54.429121, "uncore": 12.315948 }, "net_j": { "core": 4.519950577384324, "dram": 3.4145319464950394, "package-0": 22.480708030611247, "psys": 40.321463516604375, "uncore": 12.313235101509692 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.389039282, "token_latencies_s": [ 0.108032674, 0.099217043, 0.097602912, 0.105121056, 0.098332319, 0.098163488, 0.097957261, 0.09808941, 0.09806274, 0.097918848, 0.097561998, 0.09749588, 0.097647183, 0.097783248 ], "watts": { "core": 7.025768908384263, "dram": 2.95200650776124, "package-0": 21.91107724194657, "psys": 39.18472407895517, "uncore": 8.866522465993155 } } }, { "energy": { "gross_j": { "core": 61.078823, "dram": 19.575633999999997, "package-0": 251.020964, "psys": 415.161291, "uncore": 146.055046 }, "net_j": { "core": 26.13484856081367, "dram": 15.000664352909933, "package-0": 197.96495755063657, "psys": 321.0657614222492, "uncore": 146.03695145663326 } }, "first_decode": { "gpu_active_fraction": 0.8461538461538461, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 13, "gross_j": { "core": 2.646721, "dram": 0.3208, "package-0": 4.119923, "psys": 6.784163, "uncore": 0.793516 }, "logits": { "l2": 641.8176091039514, "sha256": "ae4c116f8db1f114b722fdb50e703656b8ec36d44932a98d79948cfafc69699b", "sum": -12608.67373151111, "top1": 78 }, "net_j": { "core": 2.149070430433175, "dram": 0.2556460990152235, "package-0": 3.3643320218066894, "psys": 5.44411240073136, "uncore": 0.793258308653866 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.131941318, "watts": { "core": 20.059834478840052, "dram": 2.4313839278155456, "package-0": 31.225419470192044, "psys": 51.418032674192325, "uncore": 6.014158506435414 } }, "metrics": { "all_tokens_per_s_e2e": 884.2220321640885, "decode_tokens_per_s": 9.705495964586772, "end_to_end_ms": 9264.641348, "handoff_estimate_ms": 33.072184500000006, "handoff_plus_first_decode_ms": 131.941318, "output_tokens_per_s_e2e": 1.7269961565704854, "prefill_tokens_per_s": 1059.1873700275985, "steady_tpot_ms": 98.8691335, "steady_tpot_p95_ms": 107.23170945000001, "ttft_ms": 7719.125276 }, "prefill": { "gpu_active_fraction": 0.9136125654450262, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 764, "gross_j": { "core": 47.70575, "dram": 15.140892, "package-0": 215.360533, "psys": 352.979565, "uncore": 132.884852 }, "logits": { "l2": 630.9416903237038, "sha256": "14983026d2c3865bd74d1e89295c4c951cf5e5701b07ca9e53c79ce2d506866a", "sum": 80776.56900331145, "top1": 198 }, "net_j": { "core": 18.5910863875062, "dram": 11.32911316115598, "package-0": 171.15526427523594, "psys": 274.5809359069246, "uncore": 132.8697759660972 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 7.719125276, "watts": { "core": 6.1802015505985946, "dram": 1.961477688032278, "package-0": 27.89960329696818, "psys": 45.72792283828715, "uncore": 17.215014298726352 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 572.7403064888838, "sha256": "1f72cc218133f7bc0627789756871cfa6b42dada579c1e1db76b598cec8f6f75", "sum": -71473.40795485722, "top1": 42 }, "gpu_active_fraction": 0.9928571428571429, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 140, "gross_j": { "core": 10.726352, "dram": 4.113942, "package-0": 31.540508, "psys": 55.397563, "uncore": 12.376678 }, "net_j": { "core": 5.394691742874296, "dram": 3.4159050927387296, "package-0": 23.44536125359393, "psys": 41.0407131145932, "uncore": 12.373917181882211 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.413574754, "token_latencies_s": [ 0.109016081, 0.106270894, 0.099583976, 0.098923403, 0.105318778, 0.09895458, 0.098755646, 0.098749229, 0.098366066, 0.098175806, 0.105128852, 0.098814864, 0.098753034, 0.098734115 ], "watts": { "core": 7.5881038265911185, "dram": 2.9103108897203747, "package-0": 22.312585811786505, "psys": 39.18969466824532, "uncore": 8.755587891604351 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 10.92683682218194, "compile_target": "AUTO:GPU", "context_length": 1024, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 394304, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 2.4565200143075967, "dram": 0.4907827289910671, "package-0": 4.295934585025856, "psys": 9.931950112327097, "uncore": 0.002123975024632864 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_7b_ov_fp16/openvino_model.xml", "key": "qwen25_7b_fp16", "size_b": 7.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "GPU", "key": "gpu_gpu", "label": "iGPU -> iGPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 1008, "prompt_sha256": "2669029615323cd09063f0eb8d822f5daea575e1396736877003e47194c143f8", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.05495245289057493, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 24.365965, "dram": 12.233795, "package-0": 97.665644, "psys": 168.58648300000002, "uncore": 48.311949 }, "net_j": { "core": 14.7783067474509, "dram": 10.318297851833647, "package-0": 80.89885503160497, "psys": 129.8226451709811, "uncore": 48.30365924597621 } }, "first_decode": { "gpu_active_fraction": 0.95, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 20, "gross_j": { "core": 2.811028, "dram": 0.633727, "package-0": 5.787522, "psys": 9.31919, "uncore": 1.68365 }, "logits": { "l2": 826.5092086937404, "sha256": "4dd5fd8e1442cc127c6d0044f29c595ddad6c04dc29eeda373180bd92ad1842c", "sum": 6180.617078131763, "top1": 78 }, "net_j": { "core": 2.3141447120341847, "dram": 0.5344557811059611, "package-0": 4.9185780919263, "psys": 7.310242383460212, "uncore": 1.683220380991138 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.202271215, "watts": { "core": 13.897320980644723, "dram": 3.133055783542903, "package-0": 28.612682234592796, "psys": 46.07274445847374, "uncore": 8.323725152884458 } }, "metrics": { "all_tokens_per_s_e2e": 262.3660990401052, "decode_tokens_per_s": 5.34787218891652, "end_to_end_ms": 3902.943268, "handoff_estimate_ms": 16.331582499999996, "handoff_plus_first_decode_ms": 202.271215, "output_tokens_per_s_e2e": 4.099470297501644, "prefill_tokens_per_s": 917.9580468748337, "steady_tpot_ms": 185.93963250000002, "steady_tpot_p95_ms": 186.1239023, "ttft_ms": 1098.089399 }, "prefill": { "gpu_active_fraction": 0.9357798165137615, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 109, "gross_j": { "core": 5.378038, "dram": 2.915642, "package-0": 34.537327, "psys": 56.04368, "uncore": 22.934512 }, "logits": { "l2": 848.6664321595354, "sha256": "0a12f44f7c57878debefd7f9c698d7b450eec350fae83b41138241ea19af3147", "sum": -208252.0210649966, "top1": 290 }, "net_j": { "core": 2.6805594138575, "dram": 2.376718688082619, "package-0": 29.820006773385643, "psys": 45.137510870256754, "uncore": 22.932179685541712 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.098089399, "watts": { "core": 4.897632200891505, "dram": 2.6551954719307878, "package-0": 31.45219964007684, "psys": 51.03744745285534, "uncore": 20.88583317613833 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 760.3157061515358, "sha256": "779808bb2a5fa209d3cc4c2c6cffdb0cd54061013fde8ed3d471ca149f7ac757", "sum": -163608.12077951845, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 258, "gross_j": { "core": 16.176899, "dram": 8.684426, "package-0": 57.340795, "psys": 103.223613, "uncore": 23.693787 }, "net_j": { "core": 9.783602621559215, "dram": 7.407123382645066, "package-0": 46.16027016629302, "psys": 77.37489191726415, "uncore": 23.68825917944336 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.602582654, "token_latencies_s": [ 0.186020491, 0.186125084, 0.185978366, 0.185934993, 0.185759083, 0.185862061, 0.185765606, 0.185662852, 0.18571649, 0.185975506, 0.186123266, 0.1857271, 0.185944272, 0.185955699 ], "watts": { "core": 6.21570998912836, "dram": 3.3368492588132037, "package-0": 22.0322666455457, "psys": 39.66199222966158, "uncore": 9.10395178557891 } } }, { "energy": { "gross_j": { "core": 27.678397, "dram": 12.288909, "package-0": 101.639205, "psys": 174.949991, "uncore": 48.541684000000004 }, "net_j": { "core": 18.03140653745829, "dram": 10.361558000137896, "package-0": 84.7686565297777, "psys": 135.94626725665168, "uncore": 48.53334294570939 } }, "first_decode": { "gpu_active_fraction": 0.95, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 20, "gross_j": { "core": 2.854973, "dram": 0.628721, "package-0": 5.816331, "psys": 9.372535, "uncore": 1.65234 }, "logits": { "l2": 826.5092086937404, "sha256": "4dd5fd8e1442cc127c6d0044f29c595ddad6c04dc29eeda373180bd92ad1842c", "sum": 6180.617078131763, "top1": 78 }, "net_j": { "core": 2.377992729515987, "dram": 0.5334261611793915, "package-0": 4.982193265150395, "psys": 7.444057225172348, "uncore": 1.6519275896895405 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.194169096, "watts": { "core": 14.703539640520344, "dram": 3.238007556053101, "package-0": 29.95497800535673, "psys": 48.269962589721274, "uncore": 8.50979910829888 } }, "metrics": { "all_tokens_per_s_e2e": 260.752459994474, "decode_tokens_per_s": 5.348148215410686, "end_to_end_ms": 3927.09622, "handoff_estimate_ms": 7.846010000000014, "handoff_plus_first_decode_ms": 194.16909600000002, "output_tokens_per_s_e2e": 4.074257187413656, "prefill_tokens_per_s": 898.0858630919743, "steady_tpot_ms": 186.323086, "steady_tpot_p95_ms": 187.25145110000003, "ttft_ms": 1122.3871139999999 }, "prefill": { "gpu_active_fraction": 0.9279279279279279, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 111, "gross_j": { "core": 7.446454, "dram": 2.953911, "package-0": 37.08334, "psys": 59.790313, "uncore": 23.110963 }, "logits": { "l2": 848.6664321595354, "sha256": "0a12f44f7c57878debefd7f9c698d7b450eec350fae83b41138241ea19af3147", "sum": -208252.0210649966, "top1": 290 }, "net_j": { "core": 4.689287590658058, "dram": 2.4030627892066723, "package-0": 32.26163837918004, "psys": 48.642820177033215, "uncore": 23.108579077801895 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.122387114, "watts": { "core": 6.6344792336951235, "dram": 2.631811220170513, "package-0": 33.03970576412017, "psys": 53.270669499151076, "uncore": 20.590901937243736 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 760.3157061515358, "sha256": "779808bb2a5fa209d3cc4c2c6cffdb0cd54061013fde8ed3d471ca149f7ac757", "sum": -163608.12077951845, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 257, "gross_j": { "core": 17.37697, "dram": 8.706277, "package-0": 58.739534, "psys": 105.787143, "uncore": 23.778381 }, "net_j": { "core": 10.964126217284246, "dram": 7.425069049751833, "package-0": 47.52482488544726, "psys": 79.85938985444612, "uncore": 23.772836278217955 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.61054001, "token_latencies_s": [ 0.186082539, 0.18625113, 0.185821196, 0.185915456, 0.186131708, 0.185764101, 0.186256107, 0.186390065, 0.186693132, 0.1866024, 0.187131021, 0.187003835, 0.186996032, 0.187475107 ], "watts": { "core": 6.656465686576473, "dram": 3.3350482914069572, "package-0": 22.500913134826845, "psys": 40.523088171324375, "uncore": 9.108606230478728 } } }, { "energy": { "gross_j": { "core": 27.060295, "dram": 12.279632000000001, "package-0": 100.71428, "psys": 171.998462, "uncore": 48.559934 }, "net_j": { "core": 17.37805025475524, "dram": 10.345237624560717, "package-0": 83.78207923651233, "psys": 132.8522017517532, "uncore": 48.55156246388326 } }, "first_decode": { "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 19, "gross_j": { "core": 2.759576, "dram": 0.627257, "package-0": 5.606797, "psys": 9.031288, "uncore": 1.585445 }, "logits": { "l2": 826.5092086937404, "sha256": "4dd5fd8e1442cc127c6d0044f29c595ddad6c04dc29eeda373180bd92ad1842c", "sum": 6180.617078131763, "top1": 78 }, "net_j": { "core": 2.2820517773761786, "dram": 0.5318534861781455, "package-0": 4.771708007763364, "psys": 7.100610973595076, "uncore": 1.5850321193735029 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.194390528, "watts": { "core": 14.196041486136608, "dram": 3.226787881351914, "package-0": 28.84295370605712, "psys": 46.45950650434984, "uncore": 8.155978669907208 } }, "metrics": { "all_tokens_per_s_e2e": 259.8030271736732, "decode_tokens_per_s": 5.332785433951368, "end_to_end_ms": 3941.447531, "handoff_estimate_ms": 7.608663500000001, "handoff_plus_first_decode_ms": 194.39052800000002, "output_tokens_per_s_e2e": 4.059422299588643, "prefill_tokens_per_s": 893.0955975463864, "steady_tpot_ms": 186.7818645, "steady_tpot_p95_ms": 188.26474005, "ttft_ms": 1128.65857 }, "prefill": { "gpu_active_fraction": 0.9285714285714286, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 112, "gross_j": { "core": 5.90416, "dram": 2.952996, "package-0": 35.417817, "psys": 57.262793, "uncore": 23.223451 }, "logits": { "l2": 848.6664321595354, "sha256": "0a12f44f7c57878debefd7f9c698d7b450eec350fae83b41138241ea19af3147", "sum": -208252.0210649966, "top1": 290 }, "net_j": { "core": 3.1315876334752084, "dram": 2.399069866916245, "package-0": 30.569173614451174, "psys": 46.053012388909565, "uncore": 23.221053757385985 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.12865857, "watts": { "core": 5.231130261120509, "dram": 2.616376713464374, "package-0": 31.380452814884485, "psys": 50.73526620189487, "uncore": 20.576152626918873 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 760.3157061515358, "sha256": "779808bb2a5fa209d3cc4c2c6cffdb0cd54061013fde8ed3d471ca149f7ac757", "sum": -163608.12077951845, "top1": 42 }, "gpu_active_fraction": 0.9961538461538462, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 260, "gross_j": { "core": 18.396559, "dram": 8.699379, "package-0": 59.689666, "psys": 105.704381, "uncore": 23.751038 }, "net_j": { "core": 11.964410843903853, "dram": 7.414314271466327, "package-0": 48.44119761429779, "psys": 79.69857838924855, "uncore": 23.745476587123772 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.618398433, "token_latencies_s": [ 0.188787433, 0.187816799, 0.187697824, 0.18798329, 0.187250099, 0.186775363, 0.186820822, 0.186772544, 0.186788366, 0.186538855, 0.186446257, 0.186078308, 0.186285922, 0.186326829 ], "watts": { "core": 7.025882221798596, "dram": 3.322404600598843, "package-0": 22.796250275635575, "psys": 40.36986108294085, "uncore": 9.070826540629847 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 9.113449837081134, "compile_target": "NPU", "context_length": 1024, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 394373, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 2.4565200143075967, "dram": 0.4907827289910671, "package-0": 4.295934585025856, "psys": 9.931950112327097, "uncore": 0.002123975024632864 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_7b_ov_fp16/openvino_model.xml", "key": "qwen25_7b_fp16", "size_b": 7.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "gpu_npu", "label": "iGPU -> NPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 1008, "prompt_sha256": "2669029615323cd09063f0eb8d822f5daea575e1396736877003e47194c143f8", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "GPU", "NPUW_ONLINE_PIPELINE": "NONE" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 1024, "NPUW_LLM_MIN_RESPONSE_LEN": 1024, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.047103749588131905, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 50.566337, "dram": 26.820427000000002, "package-0": 185.419019, "psys": 372.549583, "uncore": 57.687352 }, "net_j": { "core": 24.173351449699716, "dram": 21.54743048494852, "package-0": 139.26326257740027, "psys": 265.8401671295015, "uncore": 57.67187235103964 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 55, "gross_j": { "core": 6.187545, "dram": 0.917357, "package-0": 8.97721, "psys": 15.674887, "uncore": 0.0 }, "logits": { "l2": 828.5850497589859, "sha256": "b1caee57e8f1958b838d2101b7e06beaac2d30c9baf03c086a8b6dbdc5604ef3", "sum": 8582.001019639429, "top1": 78 }, "net_j": { "core": 4.81910563361967, "dram": 0.6439595097376434, "package-0": 6.5840986199565, "psys": 10.142153116508794, "uncore": 0.0 }, "npu_busy_delta_us": 241460, "npu_utilization": 0.43345093485848896, "seconds": 0.557064204, "watts": { "core": 11.107418059840011, "dram": 1.6467706835458413, "package-0": 16.115216047879464, "psys": 28.138384924837137, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 95.30852392037775, "decode_tokens_per_s": 4.340279358915426, "end_to_end_ms": 10744.054759, "handoff_estimate_ms": 350.250513, "handoff_plus_first_decode_ms": 557.064204, "output_tokens_per_s_e2e": 1.4891956862559024, "prefill_tokens_per_s": 138.30848685993183, "steady_tpot_ms": 206.81369100000003, "steady_tpot_p95_ms": 209.1745115, "ttft_ms": 7288.056018 }, "prefill": { "gpu_active_fraction": 0.9986033519553073, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 716, "gross_j": { "core": 34.430881, "dram": 16.89168, "package-0": 136.398943, "psys": 280.123245, "uncore": 57.687352 }, "logits": { "l2": 848.6442854813025, "sha256": "61eaa50892be757f703f9f7b089ab795b4ea224866c02f6fed0de86c01524847", "sum": -208360.92914666422, "top1": 290 }, "net_j": { "core": 16.527625526388075, "dram": 13.314827978446191, "package-0": 105.08993109466797, "psys": 207.73863621337873, "uncore": 57.67187235103964 }, "npu_busy_delta_us": 19331, "npu_utilization": 0.002652421983620379, "seconds": 7.288056018, "watts": { "core": 4.724288742424976, "dram": 2.3177209338513625, "package-0": 18.715408150420725, "psys": 38.43593467286107, "uncore": 7.9153277441232754 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 764.336809503711, "sha256": "bf80027ec35e469a6666cab0a8823ab8cf2b37944db579c08097a81568e0a72a", "sum": -165352.43837253097, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 286, "gross_j": { "core": 9.947911, "dram": 9.01139, "package-0": 40.042866, "psys": 76.751451, "uncore": 0.0 }, "net_j": { "core": 2.8266202896919728, "dram": 7.588642996764685, "package-0": 27.58923286277578, "psys": 47.95937779961395, "uncore": 0.0 }, "npu_busy_delta_us": 2871402, "npu_utilization": 0.9905025323446963, "seconds": 2.898934537, "token_latencies_s": [ 0.211506094, 0.206843467, 0.206877097, 0.206751202, 0.207661107, 0.206783915, 0.206998221, 0.20741524, 0.205842218, 0.206379571, 0.205783449, 0.207919044, 0.206221665, 0.205899449 ], "watts": { "core": 3.4315749020999706, "dram": 3.10851793477391, "package-0": 13.812959723277807, "psys": 26.47574480223594, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 53.838423999999996, "dram": 26.516778000000002, "package-0": 188.008918, "psys": 372.533104, "uncore": 57.885106 }, "net_j": { "core": 28.314402024978012, "dram": 21.4173898058191, "package-0": 143.37279530903015, "psys": 269.3369926533101, "uncore": 57.86969713431484 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 25, "gross_j": { "core": 3.183646, "dram": 0.677062, "package-0": 5.087938, "psys": 8.479104, "uncore": 0.0 }, "logits": { "l2": 828.5850497589859, "sha256": "b1caee57e8f1958b838d2101b7e06beaac2d30c9baf03c086a8b6dbdc5604ef3", "sum": 8582.001019639429, "top1": 78 }, "net_j": { "core": 2.582704359641274, "dram": 0.5570011918234568, "package-0": 4.0370180150769865, "psys": 6.049438311261378, "uncore": 0.0 }, "npu_busy_delta_us": 235304, "npu_utilization": 0.9618720797939488, "seconds": 0.244631282, "watts": { "core": 13.014059256738882, "dram": 2.7676836521667743, "package-0": 20.798394867586886, "psys": 34.660751195343856, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 98.55329607193744, "decode_tokens_per_s": 4.7837933363467595, "end_to_end_ms": 10390.317126, "handoff_estimate_ms": 38.373330499999994, "handoff_plus_first_decode_ms": 244.631282, "output_tokens_per_s_e2e": 1.5398952511240225, "prefill_tokens_per_s": 138.9438306865149, "steady_tpot_ms": 206.25795150000002, "steady_tpot_p95_ms": 208.01400879999997, "ttft_ms": 7254.730167 }, "prefill": { "gpu_active_fraction": 0.9972144846796658, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 718, "gross_j": { "core": 36.730802, "dram": 16.842852, "package-0": 138.514172, "psys": 283.452338, "uncore": 57.885106 }, "logits": { "l2": 848.6442854813025, "sha256": "61eaa50892be757f703f9f7b089ab795b4ea224866c02f6fed0de86c01524847", "sum": -208360.92914666422, "top1": 290 }, "net_j": { "core": 18.909412146363405, "dram": 13.28235573054592, "package-0": 107.3483257705543, "psys": 211.39871990296155, "uncore": 57.86969713431484 }, "npu_busy_delta_us": 16959, "npu_utilization": 0.00233764724663949, "seconds": 7.254730167, "watts": { "core": 5.063014220305459, "dram": 2.3216372783393147, "package-0": 19.09294609330437, "psys": 39.071382597984915, "uncore": 7.97894679299104 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 764.336809503711, "sha256": "bf80027ec35e469a6666cab0a8823ab8cf2b37944db579c08097a81568e0a72a", "sum": -165352.43837253097, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 285, "gross_j": { "core": 13.923976, "dram": 8.996864, "package-0": 44.406808, "psys": 80.601662, "uncore": 0.0 }, "net_j": { "core": 6.822285518973332, "dram": 7.578032883449723, "package-0": 31.98745152339886, "psys": 51.88883443908719, "uncore": 0.0 }, "npu_busy_delta_us": 2867776, "npu_utilization": 0.9919820019433664, "seconds": 2.890955677, "token_latencies_s": [ 0.208843232, 0.206246533, 0.205858169, 0.205696034, 0.206504411, 0.206781482, 0.206689356, 0.205705822, 0.20597667, 0.206553449, 0.2061037, 0.207567504, 0.20626937, 0.206126496 ], "watts": { "core": 4.8163920708909576, "dram": 3.11207261722401, "package-0": 15.360598003384746, "psys": 27.88062876274945, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 51.952137, "dram": 26.533012, "package-0": 186.069957, "psys": 369.59732, "uncore": 57.989049 }, "net_j": { "core": 26.443544274883642, "dram": 21.43670638171136, "package-0": 141.46081680781074, "psys": 266.4635906161848, "uncore": 57.97365151646549 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 25, "gross_j": { "core": 2.897148, "dram": 0.676878, "package-0": 4.688159, "psys": 7.937113, "uncore": 0.0 }, "logits": { "l2": 828.5850497589859, "sha256": "b1caee57e8f1958b838d2101b7e06beaac2d30c9baf03c086a8b6dbdc5604ef3", "sum": 8582.001019639429, "top1": 78 }, "net_j": { "core": 2.2888436041587914, "dram": 0.5553462031046428, "package-0": 3.6243631110607097, "psys": 5.477678972426208, "uncore": 0.0 }, "npu_busy_delta_us": 238245, "npu_utilization": 0.9621064960403268, "seconds": 0.247628512, "watts": { "core": 11.699573593528681, "dram": 2.7334412928992604, "package-0": 18.932226188880865, "psys": 32.052500481043154, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 98.61290749192062, "decode_tokens_per_s": 4.785200466747688, "end_to_end_ms": 10384.036188, "handoff_estimate_ms": 41.660278, "handoff_plus_first_decode_ms": 247.628512, "output_tokens_per_s_e2e": 1.5408266795612597, "prefill_tokens_per_s": 139.04654095136732, "steady_tpot_ms": 205.968234, "steady_tpot_p95_ms": 207.05076935, "ttft_ms": 7249.371276 }, "prefill": { "gpu_active_fraction": 0.99860529986053, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 717, "gross_j": { "core": 38.973655, "dram": 16.839495, "package-0": 140.998662, "psys": 284.459782, "uncore": 57.989049 }, "logits": { "l2": 848.6442854813025, "sha256": "61eaa50892be757f703f9f7b089ab795b4ea224866c02f6fed0de86c01524847", "sum": -208360.92914666422, "top1": 290 }, "net_j": { "core": 21.1654293693594, "dram": 13.281628781695265, "package-0": 109.85583721573857, "psys": 212.45938814103098, "uncore": 57.97365151646549 }, "npu_busy_delta_us": 16659, "npu_utilization": 0.002297992386615901, "seconds": 7.249371276, "watts": { "core": 5.376142773791629, "dram": 2.3228904078550054, "package-0": 19.449778005824406, "psys": 39.239234848095265, "uncore": 7.999183210822765 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 764.336809503711, "sha256": "bf80027ec35e469a6666cab0a8823ab8cf2b37944db579c08097a81568e0a72a", "sum": -165352.43837253097, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 285, "gross_j": { "core": 10.081334, "dram": 9.016639, "package-0": 40.383136, "psys": 77.200425, "uncore": 0.0 }, "net_j": { "core": 2.9892713013654477, "dram": 7.599731396911453, "package-0": 27.98061648101146, "psys": 48.52652350272758, "uncore": 0.0 }, "npu_busy_delta_us": 2867373, "npu_utilization": 0.9931890709795, "seconds": 2.8870364, "token_latencies_s": [ 0.207562021, 0.206765845, 0.205783042, 0.205991599, 0.205944869, 0.205916718, 0.20677548, 0.205843808, 0.206392005, 0.205663847, 0.206223223, 0.205597092, 0.206696886, 0.205818322 ], "watts": { "core": 3.49193172625049, "dram": 3.123146975216523, "package-0": 13.987747435397766, "psys": 26.740371198645086, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 475.13566965516657, "compile_target": "NPU", "context_length": 1024, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 394514, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 2.4565200143075967, "dram": 0.4907827289910671, "package-0": 4.295934585025856, "psys": 9.931950112327097, "uncore": 0.002123975024632864 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_7b_ov_fp16/openvino_model.xml", "key": "qwen25_7b_fp16", "size_b": 7.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU+GPU", "key": "npu_gpu_combined", "label": "NPU + iGPU (subgraph combination)", "mode": "combined", "prefill_device": "NPU+GPU" }, "prompt_length": 1008, "prompt_sha256": "2669029615323cd09063f0eb8d822f5daea575e1396736877003e47194c143f8", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU,GPU", "NPUW_SUBMODEL_DEVICE": "0:GPU,last:NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "NPU,GPU", "NPUW_SUBMODEL_DEVICE": "0:GPU,last:NPU" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 1024, "NPUW_LLM_MIN_RESPONSE_LEN": 1024, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.04873363580554724, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 675.580913, "dram": 253.428733, "package-0": 2005.351312, "psys": 4124.793944, "uncore": 621.436885 }, "net_j": { "core": 385.5892213462486, "dram": 195.49193094439786, "package-0": 1498.2171016985849, "psys": 2952.3292387587444, "uncore": 621.1861501772278 } }, "first_decode": { "gpu_active_fraction": 0.9058971141781681, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 797, "gross_j": { "core": 48.156676, "dram": 16.220113, "package-0": 130.372469, "psys": 263.966243, "uncore": 38.74276 }, "logits": { "l2": 899.2846808864034, "sha256": "80ccdd393fadb75c2187ae4c3e4330c0cb47061d068a1bf13053982ef530897e", "sum": -50935.694061841816, "top1": 15 }, "net_j": { "core": 28.31012213431361, "dram": 12.255013668205944, "package-0": 95.6650387130083, "psys": 183.72468919724412, "uncore": 38.72560012159875 }, "npu_busy_delta_us": 19528, "npu_utilization": 0.0024170908039777054, "seconds": 8.079133795, "watts": { "core": 5.960623653714351, "dram": 2.0076549555396985, "package-0": 16.13693649691563, "psys": 32.67259209933656, "uncore": 4.795410124780585 } }, "metrics": { "all_tokens_per_s_e2e": 8.674305392357397, "decode_tokens_per_s": 0.1293517003838133, "end_to_end_ms": 118049.798074, "handoff_estimate_ms": 375.76810600000067, "handoff_plus_first_decode_ms": 8079.133795000001, "output_tokens_per_s_e2e": 0.13553602175558432, "prefill_tokens_per_s": 483.01657559932204, "steady_tpot_ms": 7703.365689, "steady_tpot_p95_ms": 7740.94827255, "ttft_ms": 2086.884904 }, "prefill": { "gpu_active_fraction": 0.13043478260869565, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 207, "gross_j": { "core": 6.064987, "dram": 4.264454, "package-0": 48.252074, "psys": 74.647941, "uncore": 1.921748 }, "logits": { "l2": 851.3160897084861, "sha256": "de7287cd18d1bf5585ee00e95918aa2c864ab04ccd6f05685b99b54687e84540", "sum": -209094.41266852617, "top1": 290 }, "net_j": { "core": 0.9385124657676123, "dram": 3.2402469317246183, "package-0": 39.286952965938035, "psys": 53.921104243303475, "uncore": 1.9173155085846207 }, "npu_busy_delta_us": 1799508, "npu_utilization": 0.8622938411940326, "seconds": 2.086884904, "watts": { "core": 2.9062393370976247, "dram": 2.0434543332151103, "package-0": 23.121578917703456, "psys": 35.77003257674626, "uncore": 0.9208691846476647 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": null, "sha256": "7b769dbc741082009e7e9a93ca58b6e7f8d647e22fffd35e2edcda73c4a3253b", "sum": null, "top1": 0 }, "gpu_active_fraction": 0.9819531910893881, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 10639, "gross_j": { "core": 621.35925, "dram": 232.944166, "package-0": 1826.726769, "psys": 3786.17976, "uncore": 580.772377 }, "net_j": { "core": 356.3405867461674, "dram": 179.9966703444673, "package-0": 1363.2651100196385, "psys": 2714.683445318197, "uncore": 580.5432345470444 }, "npu_busy_delta_us": 269916, "npu_utilization": 0.002501914574773859, "seconds": 107.883779375, "token_latencies_s": [ 7.685417873, 7.709529459, 7.669118025, 7.697201919, 7.726413474, 7.722269014, 7.673825334, 7.690132589, 7.709883637, 7.731335754, 7.691409223, 7.734213127, 7.7534564, 7.689507587 ], "watts": { "core": 5.759524310324524, "dram": 2.15921399259007, "package-0": 16.932357946511736, "psys": 35.09498630780611, "uncore": 5.38331508559092 } } }, { "energy": { "gross_j": { "core": 662.5741710000001, "dram": 254.030416, "package-0": 1998.688121, "psys": 4229.072914, "uncore": 632.834379 }, "net_j": { "core": 373.52309784656296, "dram": 196.2815380450911, "package-0": 1493.1988537575787, "psys": 3060.41122113619, "uncore": 632.5844574619521 } }, "first_decode": { "gpu_active_fraction": 0.9802371541501976, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 759, "gross_j": { "core": 44.132334, "dram": 16.673297, "package-0": 130.489901, "psys": 277.516746, "uncore": 42.20558 }, "logits": { "l2": null, "sha256": "7b769dbc741082009e7e9a93ca58b6e7f8d647e22fffd35e2edcda73c4a3253b", "sum": null, "top1": 0 }, "net_j": { "core": 25.195354986903162, "dram": 12.889919626820383, "package-0": 97.37312498985148, "psys": 200.95269207785998, "uncore": 42.18920656512809 }, "npu_busy_delta_us": 16451, "npu_utilization": 0.0021340368348840193, "seconds": 7.708864126, "watts": { "core": 5.724881549170529, "dram": 2.1628733789411716, "package-0": 16.92725398543365, "psys": 35.99969352994665, "uncore": 5.474941484265045 } }, "metrics": { "all_tokens_per_s_e2e": 8.702532971796606, "decode_tokens_per_s": 0.12980586119120155, "end_to_end_ms": 117666.89115899999, "handoff_estimate_ms": 4.4758590000002485, "handoff_plus_first_decode_ms": 7708.8641259999995, "output_tokens_per_s_e2e": 0.13597707768432196, "prefill_tokens_per_s": 477.7918032740029, "steady_tpot_ms": 7704.388266999999, "steady_tpot_p95_ms": 7729.5312370500005, "ttft_ms": 2109.70551 }, "prefill": { "gpu_active_fraction": 0.1291866028708134, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 209, "gross_j": { "core": 6.548079, "dram": 4.27776, "package-0": 48.778683, "psys": 77.211411, "uncore": 1.999385 }, "logits": { "l2": 851.3160897084861, "sha256": "de7287cd18d1bf5585ee00e95918aa2c864ab04ccd6f05685b99b54687e84540", "sum": -209094.41266852617, "top1": 290 }, "net_j": { "core": 1.3655451903899847, "dram": 3.2423529724347087, "package-0": 39.71552613537139, "psys": 56.2579211229784, "uncore": 1.9949040381874297 }, "npu_busy_delta_us": 1820083, "npu_utilization": 0.8627189867840843, "seconds": 2.10970551, "watts": { "core": 3.10378816804626, "dram": 2.02765740513234, "package-0": 23.12108622212396, "psys": 36.59819374505971, "uncore": 0.947708099790667 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": null, "sha256": "7b769dbc741082009e7e9a93ca58b6e7f8d647e22fffd35e2edcda73c4a3253b", "sum": null, "top1": 0 }, "gpu_active_fraction": 0.981457078313253, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 10624, "gross_j": { "core": 611.893758, "dram": 233.079359, "package-0": 1819.419537, "psys": 3874.344757, "uncore": 588.629414 }, "net_j": { "core": 346.9621976692698, "dram": 180.14926544583602, "package-0": 1356.1102026323558, "psys": 2803.2006079353514, "uncore": 588.4003468586366 }, "npu_busy_delta_us": 267738, "npu_utilization": 0.002482542113025853, "seconds": 107.848321523, "token_latencies_s": [ 7.709477718, 7.722819643, 7.715957626, 7.678374276, 7.699298816, 7.716738099, 7.684040405, 7.681785127, 7.741995626, 7.721207771, 7.71757316, 7.676434782, 7.693536295, 7.689013931 ], "watts": { "core": 5.673651192332243, "dram": 2.1611774361299907, "package-0": 16.870170173320556, "psys": 35.92401534198886, "uncore": 5.457937645088593 } } }, { "energy": { "gross_j": { "core": 649.812545, "dram": 255.059405, "package-0": 1994.857687, "psys": 4243.751891, "uncore": 639.47993 }, "net_j": { "core": 357.4451683484226, "dram": 196.64797006550197, "package-0": 1483.5689054816412, "psys": 3061.6820595055137, "uncore": 639.2271410943902 } }, "first_decode": { "gpu_active_fraction": 0.9841479524438573, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 757, "gross_j": { "core": 41.928237, "dram": 16.657062, "package-0": 127.922158, "psys": 271.235145, "uncore": 41.866775 }, "logits": { "l2": null, "sha256": "7b769dbc741082009e7e9a93ca58b6e7f8d647e22fffd35e2edcda73c4a3253b", "sum": null, "top1": 0 }, "net_j": { "core": 22.970031583133174, "dram": 12.86944385031206, "package-0": 94.768261493564, "psys": 194.5852706562555, "uncore": 41.850383212193364 }, "npu_busy_delta_us": 17043, "npu_utilization": 0.0022083562068905744, "seconds": 7.717504969, "watts": { "core": 5.432874636092768, "dram": 2.158348075823571, "package-0": 16.57558479247414, "psys": 35.14544481532681, "uncore": 5.42491066324832 } }, "metrics": { "all_tokens_per_s_e2e": 8.603820725349753, "decode_tokens_per_s": 0.1283144284148036, "end_to_end_ms": 119016.891761, "handoff_estimate_ms": 7.636519500000105, "handoff_plus_first_decode_ms": 7717.5049690000005, "output_tokens_per_s_e2e": 0.13443469883358988, "prefill_tokens_per_s": 476.2457848669109, "steady_tpot_ms": 7709.8684495, "steady_tpot_p95_ms": 8335.74722145, "ttft_ms": 2116.554166 }, "prefill": { "gpu_active_fraction": 0.12980769230769232, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 208, "gross_j": { "core": 6.021652, "dram": 4.231617, "package-0": 48.119384, "psys": 75.321462, "uncore": 2.060908 }, "logits": { "l2": 851.3160897084861, "sha256": "de7287cd18d1bf5585ee00e95918aa2c864ab04ccd6f05685b99b54687e84540", "sum": -209094.41266852617, "top1": 290 }, "net_j": { "core": 0.822294329854877, "dram": 3.192848770353108, "package-0": 39.02680575720004, "psys": 54.29995161324992, "uncore": 2.0564124918131332 }, "npu_busy_delta_us": 1818856, "npu_utilization": 0.8593477215078276, "seconds": 2.116554166, "watts": { "core": 2.8450261735470277, "dram": 1.9992953962511537, "package-0": 22.734775595627255, "psys": 35.586834114596435, "uncore": 0.973709075395333 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": null, "sha256": "7b769dbc741082009e7e9a93ca58b6e7f8d647e22fffd35e2edcda73c4a3253b", "sum": null, "top1": 0 }, "gpu_active_fraction": 0.9822688451541032, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 10772, "gross_j": { "core": 601.862656, "dram": 234.170726, "package-0": 1818.816145, "psys": 3897.195284, "uncore": 595.552247 }, "net_j": { "core": 333.65284243543454, "dram": 180.5856774448368, "package-0": 1349.7738382308771, "psys": 2812.7968372360083, "uncore": 595.3203453903836 }, "npu_busy_delta_us": 270769, "npu_utilization": 0.0024799594724520917, "seconds": 109.182832626, "token_latencies_s": [ 7.699136824, 7.693529823, 7.680483029, 7.711910971, 7.724094863, 7.708874295, 7.687142327, 7.710862604, 7.678736704, 7.727380883, 7.714119771, 7.676723884, 8.221086883, 8.54868785 ], "watts": { "core": 5.512429395028141, "dram": 2.144757745955716, "package-0": 16.65844438411172, "psys": 35.69421300278621, "uncore": 5.454632680579304 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 18.58357109222561, "compile_target": "AUTO:NPU,GPU", "context_length": 1024, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 394219, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 2.4565200143075967, "dram": 0.4907827289910671, "package-0": 4.295934585025856, "psys": 9.931950112327097, "uncore": 0.002123975024632864 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_7b_ov_fp16/openvino_model.xml", "key": "qwen25_7b_fp16", "size_b": 7.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": null, "key": "openvino_auto", "label": "OpenVINO AUTO", "mode": "stock", "prefill_device": null }, "prompt_length": 1008, "prompt_sha256": "2669029615323cd09063f0eb8d822f5daea575e1396736877003e47194c143f8", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.04962586984038353, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 25.492611, "dram": 12.250333000000001, "package-0": 98.718008, "psys": 171.27605, "uncore": 48.225462 }, "net_j": { "core": 15.889626312733956, "dram": 10.331773817161723, "package-0": 81.92441633487049, "psys": 132.45024589994648, "uncore": 48.217158994317614 } }, "first_decode": { "gpu_active_fraction": 0.9523809523809523, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 21, "gross_j": { "core": 3.152702, "dram": 0.637327, "package-0": 6.115218, "psys": 9.851659, "uncore": 1.656307 }, "logits": { "l2": 826.5092086937404, "sha256": "4dd5fd8e1442cc127c6d0044f29c595ddad6c04dc29eeda373180bd92ad1842c", "sum": 6180.617078131763, "top1": 78 }, "net_j": { "core": 2.6514924437457474, "dram": 0.5371914454829739, "package-0": 5.23870836257903, "psys": 7.825219858635238, "uncore": 1.6558736403801348 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.204032352, "watts": { "core": 15.451971067803992, "dram": 3.123656585598739, "package-0": 29.97180564776315, "psys": 48.28478867900322, "uncore": 8.117864562968915 } }, "metrics": { "all_tokens_per_s_e2e": 261.9473608019604, "decode_tokens_per_s": 5.3361626834722005, "end_to_end_ms": 3909.1823520000003, "handoff_estimate_ms": 17.914383000000004, "handoff_plus_first_decode_ms": 204.032352, "output_tokens_per_s_e2e": 4.092927512530631, "prefill_tokens_per_s": 917.8876628268174, "steady_tpot_ms": 186.117969, "steady_tpot_p95_ms": 186.7896282, "ttft_ms": 1098.173601 }, "prefill": { "gpu_active_fraction": 0.9357798165137615, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 109, "gross_j": { "core": 6.445235, "dram": 2.901176, "package-0": 35.665558, "psys": 57.799229, "uncore": 22.978151 }, "logits": { "l2": 848.6664321595354, "sha256": "0a12f44f7c57878debefd7f9c698d7b450eec350fae83b41138241ea19af3147", "sum": -208252.0210649966, "top1": 290 }, "net_j": { "core": 3.747549569959255, "dram": 2.362211363195273, "package-0": 30.947876047101712, "psys": 46.892223580193395, "uncore": 22.975818506698765 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.098173601, "watts": { "core": 5.8690492961504, "dram": 2.6418191052472766, "package-0": 32.47715840876418, "psys": 52.632142083335324, "uncore": 20.923969560983828 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 760.3157061515358, "sha256": "779808bb2a5fa209d3cc4c2c6cffdb0cd54061013fde8ed3d471ca149f7ac757", "sum": -163608.12077951845, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 256, "gross_j": { "core": 15.894674, "dram": 8.71183, "package-0": 56.937232, "psys": 103.625162, "uncore": 23.591004 }, "net_j": { "core": 9.490584299028953, "dram": 7.432371008483476, "package-0": 45.73783192518974, "psys": 77.73280246111786, "uncore": 23.585466847238717 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.606976399, "token_latencies_s": [ 0.187316019, 0.186506187, 0.186109384, 0.185969027, 0.185738382, 0.186212148, 0.186049341, 0.185983335, 0.186126554, 0.185974281, 0.186091623, 0.186274996, 0.186458708, 0.186130159 ], "watts": { "core": 6.096976561083167, "dram": 3.34173719537382, "package-0": 21.84033274019678, "psys": 39.74917534341668, "uncore": 9.049182036726217 } } }, { "energy": { "gross_j": { "core": 25.780575, "dram": 12.320037, "package-0": 99.296926, "psys": 171.48100599999998, "uncore": 48.269591 }, "net_j": { "core": 16.08147532906803, "dram": 10.382275216040725, "package-0": 82.33524953195223, "psys": 132.26659964258357, "uncore": 48.26120489064917 } }, "first_decode": { "gpu_active_fraction": 0.95, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 20, "gross_j": { "core": 2.924126, "dram": 0.630431, "package-0": 5.812973, "psys": 9.278601, "uncore": 1.61871 }, "logits": { "l2": 826.5092086937404, "sha256": "4dd5fd8e1442cc127c6d0044f29c595ddad6c04dc29eeda373180bd92ad1842c", "sum": 6180.617078131763, "top1": 78 }, "net_j": { "core": 2.4446889810453176, "dram": 0.5346453328076066, "package-0": 4.974538931043453, "psys": 7.340190351388662, "uncore": 1.6182954655169863 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.195169189, "watts": { "core": 14.982518577765878, "dram": 3.2301768697722055, "package-0": 29.784276041645082, "psys": 47.54132067434066, "uncore": 8.29388085431866 } }, "metrics": { "all_tokens_per_s_e2e": 259.3515460192473, "decode_tokens_per_s": 5.3618570554846094, "end_to_end_ms": 3948.308833, "handoff_estimate_ms": 9.275330999999998, "handoff_plus_first_decode_ms": 195.169189, "output_tokens_per_s_e2e": 4.052367906550739, "prefill_tokens_per_s": 875.934770670514, "steady_tpot_ms": 185.893858, "steady_tpot_p95_ms": 186.3148617, "ttft_ms": 1150.770621 }, "prefill": { "gpu_active_fraction": 0.9122807017543859, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 114, "gross_j": { "core": 6.793196, "dram": 2.982414, "package-0": 36.264251, "psys": 58.664584, "uncore": 23.06592 }, "logits": { "l2": 848.6664321595354, "sha256": "0a12f44f7c57878debefd7f9c698d7b450eec350fae83b41138241ea19af3147", "sum": -208252.0210649966, "top1": 290 }, "net_j": { "core": 3.9663049376363184, "dram": 2.4176356541828747, "package-0": 31.32061568981442, "psys": 47.23518760149632, "uncore": 23.06347579194191 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.150770621, "watts": { "core": 5.903171210694299, "dram": 2.591666788823939, "package-0": 31.5130142690704, "psys": 50.978520766389984, "uncore": 20.043890223714705 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 760.3157061515358, "sha256": "779808bb2a5fa209d3cc4c2c6cffdb0cd54061013fde8ed3d471ca149f7ac757", "sum": -163608.12077951845, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 257, "gross_j": { "core": 16.063253, "dram": 8.707192, "package-0": 57.219702, "psys": 103.537821, "uncore": 23.584961 }, "net_j": { "core": 9.670481410386394, "dram": 7.4299942290502425, "package-0": 46.04009491109435, "psys": 77.69122168969858, "uncore": 23.57943363319027 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.602369023, "token_latencies_s": [ 0.186620168, 0.186150466, 0.18606484, 0.186137962, 0.185880094, 0.185697566, 0.185919071, 0.185907622, 0.185325541, 0.186063856, 0.185693032, 0.185505621, 0.185874779, 0.185497544 ], "watts": { "core": 6.172550033462337, "dram": 3.345871366837277, "package-0": 21.987543463008702, "psys": 39.78598733881409, "uncore": 9.06288108702253 } } }, { "energy": { "gross_j": { "core": 26.783989000000002, "dram": 12.310272, "package-0": 100.133228, "psys": 172.820297, "uncore": 48.199706 }, "net_j": { "core": 17.160754385495736, "dram": 10.387667128751303, "package-0": 83.30422348976428, "psys": 133.9126204682395, "uncore": 48.19138548567146 } }, "first_decode": { "gpu_active_fraction": 0.9, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 20, "gross_j": { "core": 2.781426, "dram": 0.628478, "package-0": 5.621262, "psys": 9.084327, "uncore": 1.568417 }, "logits": { "l2": 826.5092086937404, "sha256": "4dd5fd8e1442cc127c6d0044f29c595ddad6c04dc29eeda373180bd92ad1842c", "sum": 6180.617078131763, "top1": 78 }, "net_j": { "core": 2.3061283720301873, "dram": 0.5335193326252685, "package-0": 4.7900668514630995, "psys": 7.162652313040789, "uncore": 1.568006044548713 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.193484126, "watts": { "core": 14.375473882544762, "dram": 3.2482147915328206, "package-0": 29.052832995715626, "psys": 46.9512780598859, "uncore": 8.106179211828467 } }, "metrics": { "all_tokens_per_s_e2e": 261.3961516494277, "decode_tokens_per_s": 5.382832361145902, "end_to_end_ms": 3917.425691, "handoff_estimate_ms": 8.476140500000007, "handoff_plus_first_decode_ms": 193.484126, "output_tokens_per_s_e2e": 4.0843148695223075, "prefill_tokens_per_s": 891.4132564637025, "steady_tpot_ms": 185.0079855, "steady_tpot_p95_ms": 186.48864155, "ttft_ms": 1130.788658 }, "prefill": { "gpu_active_fraction": 0.9285714285714286, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 112, "gross_j": { "core": 6.34233, "dram": 2.978692, "package-0": 35.780487, "psys": 57.950169, "uncore": 23.066225 }, "logits": { "l2": 848.6664321595354, "sha256": "0a12f44f7c57878debefd7f9c698d7b450eec350fae83b41138241ea19af3147", "sum": -208252.0210649966, "top1": 290 }, "net_j": { "core": 3.5645250296709716, "dram": 2.4237204565146135, "package-0": 30.922692895742827, "psys": 46.7192324611587, "uncore": 23.06382323313227 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 1.130788658, "watts": { "core": 5.608766903638328, "dram": 2.6341721584547413, "package-0": 31.64206392314204, "psys": 51.24756831439674, "uncore": 20.398351926165144 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 760.3157061515358, "sha256": "779808bb2a5fa209d3cc4c2c6cffdb0cd54061013fde8ed3d471ca149f7ac757", "sum": -163608.12077951845, "top1": 42 }, "gpu_active_fraction": 0.9961089494163424, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 257, "gross_j": { "core": 17.660233, "dram": 8.703102, "package-0": 58.731479, "psys": 105.785801, "uncore": 23.565064 }, "net_j": { "core": 11.290100983794577, "dram": 7.430427339611421, "package-0": 47.59146374255836, "psys": 80.03073569404002, "uncore": 23.55955620799048 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.593152907, "token_latencies_s": [ 0.187072259, 0.185068887, 0.184964879, 0.186174386, 0.185016371, 0.184959004, 0.184984993, 0.1849996, 0.185153331, 0.184718199, 0.184785292, 0.184963859, 0.185235775, 0.185024983 ], "watts": { "core": 6.810332299467446, "dram": 3.3561854283666426, "package-0": 22.64867561085938, "psys": 40.79427816016559, "uncore": 9.087417844272922 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 9.288249555043876, "compile_target": "AUTO:GPU", "context_length": 2048, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 400412, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_7b_ov_fp16/openvino_model.xml", "key": "qwen25_7b_fp16", "size_b": 7.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "GPU", "key": "gpu_gpu", "label": "iGPU -> iGPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 2032, "prompt_sha256": "cd1a97358e68d07c27508cfacc4ba6882a7a2962b1281e2e2b0c7e5e320da4fa", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.04650996718555689, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 33.736913, "dram": 15.269613, "package-0": 139.270579, "psys": 235.656014, "uncore": 74.074701 }, "net_j": { "core": 14.68363870787872, "dram": 12.775101343467147, "package-0": 110.34168181678035, "psys": 184.35023200238007, "uncore": 74.06483491282789 } }, "first_decode": { "gpu_active_fraction": 0.9523809523809523, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 21, "gross_j": { "core": 3.063713, "dram": 0.640867, "package-0": 6.03508, "psys": 9.682349, "uncore": 1.671627 }, "logits": { "l2": 653.054767718728, "sha256": "7147004b0f99126b2e87e0a57bfb6d03ac73ede7f2bfe0eabe33e7fdde6a26d6", "sum": -38815.08208623878, "top1": 220 }, "net_j": { "core": 2.2969966046962385, "dram": 0.54048619636401, "package-0": 4.870961989186338, "psys": 7.617770301958478, "uncore": 1.671229982108147 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.20327832, "watts": { "core": 15.0715186941726, "dram": 3.1526578928830182, "package-0": 29.688753822837572, "psys": 47.630996753613466, "uncore": 8.223341279089674 } }, "metrics": { "all_tokens_per_s_e2e": 405.4189271720613, "decode_tokens_per_s": 5.386155441135341, "end_to_end_ms": 5051.564845, "handoff_estimate_ms": 19.185620000000014, "handoff_plus_first_decode_ms": 203.27832, "output_tokens_per_s_e2e": 3.1673353685317287, "prefill_tokens_per_s": 896.4783368403306, "steady_tpot_ms": 184.0927, "steady_tpot_p95_ms": 185.77264455, "ttft_ms": 2266.647075 }, "prefill": { "gpu_active_fraction": 0.9377777777777778, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 225, "gross_j": { "core": 13.091214, "dram": 5.904342, "package-0": 74.396111, "psys": 120.887997, "uncore": 48.632016 }, "logits": { "l2": 637.7699802581292, "sha256": "af4cff367047f08fb39636f2515d91bc86b19e3f9a6df0c7debf7735673bb18f", "sum": -27833.03061964945, "top1": 220 }, "net_j": { "core": 4.541972375168518, "dram": 4.785049716653374, "package-0": 61.4156579730092, "psys": 97.86699177414451, "uncore": 48.62758906729369 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.266647075, "watts": { "core": 5.77558550882916, "dram": 2.60487927967348, "package-0": 32.82209737040779, "psys": 53.33340083391677, "uncore": 21.455486624445054 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 838.660960244918, "sha256": "f7d1eadbc4a56468606230a00858318b02ca9435aea82c0d341aed77f8519197", "sum": -213878.9870355043, "top1": 42 }, "gpu_active_fraction": 0.99609375, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 256, "gross_j": { "core": 17.581986, "dram": 8.724404, "package-0": 58.839388, "psys": 105.085668, "uncore": 23.771058 }, "net_j": { "core": 7.8446697280139635, "dram": 7.449565430449764, "package-0": 44.05506185458481, "psys": 78.86546992627707, "uncore": 23.76601586342605 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.58163945, "token_latencies_s": [ 0.187462159, 0.184862906, 0.184317839, 0.184206702, 0.184031236, 0.183943354, 0.184059122, 0.183863952, 0.184126278, 0.184272005, 0.184053799, 0.184319923, 0.184058968, 0.184018031 ], "watts": { "core": 6.81039561895446, "dram": 3.3794045097970593, "package-0": 22.791481591281077, "psys": 40.70501324265091, "uncore": 9.207737354648806 } } }, { "energy": { "gross_j": { "core": 35.284333, "dram": 15.452231000000001, "package-0": 141.766423, "psys": 240.821282, "uncore": 74.542717 }, "net_j": { "core": 15.749090477170023, "dram": 12.894618621189798, "package-0": 112.10574558950907, "psys": 188.21767808498163, "uncore": 74.53260134204518 } }, "first_decode": { "gpu_active_fraction": 0.95, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 20, "gross_j": { "core": 3.246634, "dram": 0.634154, "package-0": 6.144394, "psys": 10.77457, "uncore": 1.602657 }, "logits": { "l2": 653.054767718728, "sha256": "7147004b0f99126b2e87e0a57bfb6d03ac73ede7f2bfe0eabe33e7fdde6a26d6", "sum": -38815.08208623878, "top1": 220 }, "net_j": { "core": 2.49492581368135, "dram": 0.5357381159722121, "package-0": 5.003063197724773, "psys": 8.750404717865855, "uncore": 1.6022677535964422 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.199299217, "watts": { "core": 16.290249650102737, "dram": 3.1819191743237005, "package-0": 30.829995684328253, "psys": 54.06227963253865, "uncore": 8.041461597914857 } }, "metrics": { "all_tokens_per_s_e2e": 395.4165408286831, "decode_tokens_per_s": 5.346936994274535, "end_to_end_ms": 5179.3483289999995, "handoff_estimate_ms": 13.26480100000002, "handoff_plus_first_decode_ms": 199.299217, "output_tokens_per_s_e2e": 3.0891917252240866, "prefill_tokens_per_s": 855.937942878251, "steady_tpot_ms": 186.034416, "steady_tpot_p95_ms": 186.86304305000002, "ttft_ms": 2374.003883 }, "prefill": { "gpu_active_fraction": 0.9234042553191489, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 235, "gross_j": { "core": 13.419704, "dram": 6.096847, "package-0": 75.58904, "psys": 122.892019, "uncore": 49.05853 }, "logits": { "l2": 637.7699802581292, "sha256": "af4cff367047f08fb39636f2515d91bc86b19e3f9a6df0c7debf7735673bb18f", "sum": -27833.03061964945, "top1": 220 }, "net_j": { "core": 4.465538599523981, "dram": 4.924540884864355, "package-0": 61.993784598726165, "psys": 98.78065362709816, "uncore": 49.05389339134995 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.374003883, "watts": { "core": 5.652772556985746, "dram": 2.5681706098540533, "package-0": 31.840318603219405, "psys": 51.76571945817665, "uncore": 20.664890378361694 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 838.660960244918, "sha256": "f7d1eadbc4a56468606230a00858318b02ca9435aea82c0d341aed77f8519197", "sum": -213878.9870355043, "top1": 42 }, "gpu_active_fraction": 0.9961240310077519, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 258, "gross_j": { "core": 18.617995, "dram": 8.72123, "package-0": 60.032989, "psys": 107.154693, "uncore": 23.88153 }, "net_j": { "core": 8.788626063964692, "dram": 7.434339620353231, "package-0": 45.108897793058134, "psys": 80.68661974001762, "uncore": 23.876440197098784 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.606045229, "token_latencies_s": [ 0.187855491, 0.185879076, 0.186328648, 0.186126218, 0.186108277, 0.186163195, 0.186042763, 0.186025639, 0.186026069, 0.185977507, 0.185984257, 0.185834711, 0.186313657, 0.185347007 ], "watts": { "core": 7.144156514560631, "dram": 3.346538234620947, "package-0": 23.036050307935774, "psys": 41.1177410919755, "uncore": 9.163896978550868 } } }, { "energy": { "gross_j": { "core": 34.883395, "dram": 15.353293, "package-0": 141.180304, "psys": 239.899166, "uncore": 74.595268 }, "net_j": { "core": 15.54623908556046, "dram": 12.821614712415121, "package-0": 111.82038471851315, "psys": 187.82896060698423, "uncore": 74.58525491442806 } }, "first_decode": { "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 20, "gross_j": { "core": 2.763421, "dram": 0.631468, "package-0": 5.656053, "psys": 9.213477, "uncore": 1.614681 }, "logits": { "l2": 653.054767718728, "sha256": "7147004b0f99126b2e87e0a57bfb6d03ac73ede7f2bfe0eabe33e7fdde6a26d6", "sum": -38815.08208623878, "top1": 220 }, "net_j": { "core": 2.0250140952465743, "dram": 0.5347935595373738, "package-0": 4.53491775051564, "psys": 7.225128798007116, "uncore": 1.614298641210687 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.195772669, "watts": { "core": 14.115458578132783, "dram": 3.225516632252687, "package-0": 28.89092246068321, "psys": 47.06212081115367, "uncore": 8.247734519060982 } }, "metrics": { "all_tokens_per_s_e2e": 399.4671220941392, "decode_tokens_per_s": 5.359118528725976, "end_to_end_ms": 5126.829936, "handoff_estimate_ms": 10.113841000000011, "handoff_plus_first_decode_ms": 195.772669, "output_tokens_per_s_e2e": 3.1208368913604625, "prefill_tokens_per_s": 872.9039131132442, "steady_tpot_ms": 185.658828, "steady_tpot_p95_ms": 187.4788681, "ttft_ms": 2327.862173 }, "prefill": { "gpu_active_fraction": 0.9264069264069265, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 231, "gross_j": { "core": 14.699303, "dram": 5.99028, "package-0": 76.688281, "psys": 124.866868, "uncore": 49.079647 }, "logits": { "l2": 637.7699802581292, "sha256": "af4cff367047f08fb39636f2515d91bc86b19e3f9a6df0c7debf7735673bb18f", "sum": -27833.03061964945, "top1": 220 }, "net_j": { "core": 5.919172910148113, "dram": 4.840759108251377, "package-0": 63.357266249970934, "psys": 101.22413691902466, "uncore": 49.075100509589966 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.327862173, "watts": { "core": 6.31450743540219, "dram": 2.57329667945079, "package-0": 32.94365185769097, "psys": 53.64014650363924, "uncore": 21.083570827025934 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 838.660960244918, "sha256": "f7d1eadbc4a56468606230a00858318b02ca9435aea82c0d341aed77f8519197", "sum": -213878.9870355043, "top1": 42 }, "gpu_active_fraction": 0.9961240310077519, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 258, "gross_j": { "core": 17.420671, "dram": 8.731545, "package-0": 58.83597, "psys": 105.818821, "uncore": 23.90094 }, "net_j": { "core": 7.602052080165773, "dram": 7.446062044626371, "package-0": 43.928200718026574, "psys": 79.37969488995246, "uncore": 23.895855763627413 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.603195094, "token_latencies_s": [ 0.189998441, 0.186122175, 0.185868193, 0.185666476, 0.185416776, 0.185625292, 0.185678032, 0.185706653, 0.185225968, 0.185371103, 0.185848897, 0.185551696, 0.185421231, 0.18565118 ], "watts": { "core": 6.692034354302605, "dram": 3.3541646648478203, "package-0": 22.601444715230397, "psys": 40.64959297284231, "uncore": 9.181386387477572 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 8.87854654993862, "compile_target": "NPU", "context_length": 2048, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 400486, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_7b_ov_fp16/openvino_model.xml", "key": "qwen25_7b_fp16", "size_b": 7.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "gpu_npu", "label": "iGPU -> NPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 2032, "prompt_sha256": "cd1a97358e68d07c27508cfacc4ba6882a7a2962b1281e2e2b0c7e5e320da4fa", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "GPU", "NPUW_ONLINE_PIPELINE": "NONE" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 2048, "NPUW_LLM_MIN_RESPONSE_LEN": 2048, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.046284615993499756, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 110.79360100000001, "dram": 45.207525, "package-0": 355.566961, "psys": 705.92672, "uncore": 120.297788 }, "net_j": { "core": 40.40845321994294, "dram": 35.992490938545146, "package-0": 248.70004289779396, "psys": 516.3968264757608, "uncore": 120.26814113426224 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 55, "gross_j": { "core": 6.811567, "dram": 0.932981, "package-0": 9.547278, "psys": 16.596576, "uncore": 0.0 }, "logits": { "l2": 959.3573524085323, "sha256": "dd8740ccd3404cbdedc788dbda4aa438ecf1b13d968cf307f013f36b6ff4bf1f", "sum": -210567.55291402992, "top1": 82 }, "net_j": { "core": 4.6954852812920285, "dram": 0.6559372524234559, "package-0": 6.33439660579748, "psys": 10.898488328314416, "uncore": 0.0 }, "npu_busy_delta_us": 246964, "npu_utilization": 0.4401947948058141, "seconds": 0.56103344, "watts": { "core": 12.141106954337696, "dram": 1.6629686102133234, "package-0": 17.017306490679058, "psys": 29.58215111027963, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 109.74698876480163, "decode_tokens_per_s": 4.308458346813925, "end_to_end_ms": 18661.104264, "handoff_estimate_ms": 352.94955649999997, "handoff_plus_first_decode_ms": 561.03344, "output_tokens_per_s_e2e": 0.8573983497250127, "prefill_tokens_per_s": 133.86404164804205, "steady_tpot_ms": 208.0838835, "steady_tpot_p95_ms": 210.72838845, "ttft_ms": 15179.580528 }, "prefill": { "gpu_active_fraction": 0.9979933110367893, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 1495, "gross_j": { "core": 91.141124, "dram": 35.196565, "package-0": 302.682758, "psys": 606.59415, "uncore": 120.297788 }, "logits": { "l2": 1074.7595910452458, "sha256": "750c1cd564e3f8f2852142f0c33196393ada994ce8a31a875ac2c64a687452ad", "sum": -321836.47732180264, "top1": 61262 }, "net_j": { "core": 33.887437209144366, "dram": 27.70074108597973, "package-0": 215.7535513377196, "psys": 452.42405156887025, "uncore": 120.26814113426224 }, "npu_busy_delta_us": 19128, "npu_utilization": 0.0012601138723640493, "seconds": 15.179580528, "watts": { "core": 6.004192529028231, "dram": 2.318678367631899, "package-0": 19.940126635362315, "psys": 39.96119318851312, "uncore": 7.924974460137466 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 924.3563894551908, "sha256": "a6f365aaf639e7fe3d581f0c0d352c6d68d8793d9d25068091ec29dcd4ec6d3d", "sum": -206137.09938866645, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 289, "gross_j": { "core": 12.84091, "dram": 9.077979, "package-0": 43.336925, "psys": 82.735994, "uncore": 0.0 }, "net_j": { "core": 1.8255307295065446, "dram": 7.635812600141961, "package-0": 26.61209495427686, "psys": 53.07428657857611, "uncore": 0.0 }, "npu_busy_delta_us": 2904074, "npu_utilization": 0.9943789246543691, "seconds": 2.920490296, "token_latencies_s": [ 0.21146587, 0.207985041, 0.207970445, 0.210331283, 0.20968208, 0.20808403, 0.207899494, 0.208435323, 0.207993434, 0.208083737, 0.208112642, 0.207966865, 0.208606846, 0.207813705 ], "watts": { "core": 4.396833647277423, "dram": 3.108374991840753, "package-0": 14.838921074093513, "psys": 28.329487727905807, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 113.63166700000001, "dram": 44.944648, "package-0": 357.45581200000004, "psys": 698.216719, "uncore": 120.310238 }, "net_j": { "core": 45.199778345544594, "dram": 35.85579692350612, "package-0": 252.05224027609128, "psys": 511.28208964350364, "uncore": 120.28055127727191 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 29, "gross_j": { "core": 3.204765, "dram": 0.718565, "package-0": 5.130786, "psys": 8.59269, "uncore": 0.0 }, "logits": { "l2": 1160.1109936426578, "sha256": "ee26dafc622fa1586c41834e36486e7569534a9d8a43aa7dc2efbf4ca2a106ad", "sum": -332855.33728590235, "top1": 82 }, "net_j": { "core": 2.123772498588445, "dram": 0.5770382285877615, "package-0": 3.489497450341238, "psys": 5.681843036472221, "uncore": 0.0 }, "npu_busy_delta_us": 233955, "npu_utilization": 0.8163066552120303, "seconds": 0.286601853, "watts": { "core": 11.18194096253802, "dram": 2.5071889538690457, "package-0": 17.902138267054397, "psys": 29.981278592780065, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 111.27063598008455, "decode_tokens_per_s": 4.679330396542526, "end_to_end_ms": 18405.574678, "handoff_estimate_ms": 78.151252, "handoff_plus_first_decode_ms": 286.601853, "output_tokens_per_s_e2e": 0.8693018435944105, "prefill_tokens_per_s": 133.68431760569518, "steady_tpot_ms": 208.450601, "steady_tpot_p95_ms": 209.06685405, "ttft_ms": 15199.987824999998 }, "prefill": { "gpu_active_fraction": 0.9973333333333333, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 1500, "gross_j": { "core": 100.406664, "dram": 35.132295, "package-0": 311.894892, "psys": 613.700517, "uncore": 120.310238 }, "logits": { "l2": 1099.655919480962, "sha256": "53b6512b8a4023eb3df87a5f0f2dea980cdd9c64787beb63d76b70823888e6a9", "sum": -334396.9358225651, "top1": 81842 }, "net_j": { "core": 43.07600584695615, "dram": 27.626393765029327, "package-0": 224.84881846283736, "psys": 459.3231536196577, "uncore": 120.28055127727191 }, "npu_busy_delta_us": 19386, "npu_utilization": 0.0012753957584173263, "seconds": 15.199987825, "watts": { "core": 6.605706870031655, "dram": 2.311337048718985, "package-0": 20.51941722525689, "psys": 40.37506635305479, "uncore": 7.915153576775974 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 934.8850718863815, "sha256": "cff6f4d6adb86e0011bd27d99cdcbdb1ddbe055e012bb2cc498d9ba52a16da5a", "sum": -179314.85746893287, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 286, "gross_j": { "core": 10.020238, "dram": 9.093788, "package-0": 40.430134, "psys": 75.923512, "uncore": 0.0 }, "net_j": { "core": 0.0, "dram": 7.65236492988903, "package-0": 23.713924362912692, "psys": 46.27709298737369, "uncore": 0.0 }, "npu_busy_delta_us": 2899695, "npu_utilization": 0.9933915384971145, "seconds": 2.918985, "token_latencies_s": [ 0.209580447, 0.208711221, 0.208790304, 0.20868243, 0.208524595, 0.208054801, 0.208055263, 0.208466085, 0.208239135, 0.208435117, 0.208543522, 0.208278713, 0.208287669, 0.208276503 ], "watts": { "core": 3.4327816004535823, "dram": 3.1153938783515502, "package-0": 13.850750860316172, "psys": 26.010243971791564, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 110.363609, "dram": 45.050301, "package-0": 355.26935499999996, "psys": 704.231841, "uncore": 121.057064 }, "net_j": { "core": 40.9080154736985, "dram": 35.95696696163944, "package-0": 249.81379428554956, "psys": 517.205008480759, "uncore": 121.02736083404473 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 29, "gross_j": { "core": 3.170829, "dram": 0.717406, "package-0": 5.09685, "psys": 8.829385, "uncore": 0.0 }, "logits": { "l2": 1051.2152043955496, "sha256": "88a3b6b82b708f14d483f7b8106973fef372581f06ddb88a53c39b31214058f2", "sum": -237789.0732052317, "top1": 82 }, "net_j": { "core": 2.0810303063112947, "dram": 0.5747262955565027, "package-0": 3.4421908649470683, "psys": 5.894825126330188, "uncore": 0.0 }, "npu_busy_delta_us": 235245, "npu_utilization": 0.8141750807811229, "seconds": 0.288936625, "watts": { "core": 10.974133168475959, "dram": 2.4829181831829037, "package-0": 17.640027462769734, "psys": 30.558206319465384, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 111.21578019057145, "decode_tokens_per_s": 4.678368371722701, "end_to_end_ms": 18414.652997, "handoff_estimate_ms": 80.7656015, "handoff_plus_first_decode_ms": 288.936625, "output_tokens_per_s_e2e": 0.8688732827388395, "prefill_tokens_per_s": 133.61031197249645, "steady_tpot_ms": 208.1710235, "steady_tpot_p95_ms": 209.62374945000002, "ttft_ms": 15208.406971 }, "prefill": { "gpu_active_fraction": 0.9973368841544608, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 1502, "gross_j": { "core": 95.607849, "dram": 35.240938, "package-0": 308.008855, "psys": 613.770646, "uncore": 121.057064 }, "logits": { "l2": null, "sha256": "7b769dbc741082009e7e9a93ca58b6e7f8d647e22fffd35e2edcda73c4a3253b", "sum": null, "top1": 0 }, "net_j": { "core": 38.245435875384864, "dram": 27.730879309175656, "package-0": 220.9145673714661, "psys": 459.3077742905487, "uncore": 121.02736083404473 }, "npu_busy_delta_us": 16935, "npu_utilization": 0.0011135288549479466, "seconds": 15.208406971, "watts": { "core": 6.286513057041995, "dram": 2.3172011419209673, "package-0": 20.252538979744795, "psys": 40.357326521466874, "uncore": 7.95987799582405 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 965.9590353683316, "sha256": "dbeca01a9b1e044fd4348fb2ab0d9cd6a49005015a4454119b4d452579f76fc8", "sum": -210379.17602261715, "top1": 271 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 288, "gross_j": { "core": 11.584931, "dram": 9.091957, "package-0": 42.16365, "psys": 81.63181, "uncore": 0.0 }, "net_j": { "core": 0.5815492920023395, "dram": 7.651361356907277, "package-0": 25.45703604913639, "psys": 52.00240906388014, "uncore": 0.0 }, "npu_busy_delta_us": 2900361, "npu_utilization": 0.9941903998958115, "seconds": 2.917309401, "token_latencies_s": [ 0.208650638, 0.208241794, 0.207759065, 0.207900049, 0.207908723, 0.211129791, 0.208812804, 0.208250814, 0.208139036, 0.20862718, 0.208203011, 0.207872886, 0.207792791, 0.207987724 ], "watts": { "core": 3.9711012469328413, "dram": 3.116555616926832, "package-0": 14.45292363763236, "psys": 27.9818828856542, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 10.806858167983592, "compile_target": "AUTO:NPU,GPU", "context_length": 2048, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 400326, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_7b_ov_fp16/openvino_model.xml", "key": "qwen25_7b_fp16", "size_b": 7.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": null, "key": "openvino_auto", "label": "OpenVINO AUTO", "mode": "stock", "prefill_device": null }, "prompt_length": 2032, "prompt_sha256": "cd1a97358e68d07c27508cfacc4ba6882a7a2962b1281e2e2b0c7e5e320da4fa", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.05028646066784859, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 36.02719, "dram": 15.214623, "package-0": 141.730838, "psys": 236.40179999999998, "uncore": 74.245109 }, "net_j": { "core": 16.975593615401422, "dram": 12.720331020138454, "package-0": 112.80448841114459, "psys": 185.10053619469977, "uncore": 74.23524378167498 } }, "first_decode": { "gpu_active_fraction": 0.95, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 20, "gross_j": { "core": 3.109001, "dram": 0.636168, "package-0": 6.015365, "psys": 9.582312, "uncore": 1.611079 }, "logits": { "l2": 653.054767718728, "sha256": "7147004b0f99126b2e87e0a57bfb6d03ac73ede7f2bfe0eabe33e7fdde6a26d6", "sum": -38815.08208623878, "top1": 220 }, "net_j": { "core": 2.348080885976656, "dram": 0.53654606284527, "package-0": 4.860047577612, "psys": 7.533341261997814, "uncore": 1.6106849835143888 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.201741561, "watts": { "core": 15.41081066583003, "dram": 3.1533809733929834, "package-0": 29.81718278664454, "psys": 47.497957052092005, "uncore": 7.985855725583485 } }, "metrics": { "all_tokens_per_s_e2e": 405.4546331283506, "decode_tokens_per_s": 5.39349008766773, "end_to_end_ms": 5051.119984, "handoff_estimate_ms": 17.535054500000008, "handoff_plus_first_decode_ms": 201.74156100000002, "output_tokens_per_s_e2e": 3.1676143213152392, "prefill_tokens_per_s": 895.1583477055146, "steady_tpot_ms": 184.20650650000002, "steady_tpot_p95_ms": 184.4843545, "ttft_ms": 2269.9894440000003 }, "prefill": { "gpu_active_fraction": 0.9422222222222222, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 225, "gross_j": { "core": 14.193506, "dram": 5.882676, "package-0": 75.624501, "psys": 121.158076, "uncore": 48.775754 }, "logits": { "l2": 637.7699802581292, "sha256": "af4cff367047f08fb39636f2515d91bc86b19e3f9a6df0c7debf7735673bb18f", "sum": -27833.03061964945, "top1": 220 }, "net_j": { "core": 5.631657772007612, "dram": 4.761733222462611, "package-0": 62.624907161772235, "psys": 98.10312429210748, "uncore": 48.77132053939493 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.269989444, "watts": { "core": 6.252674891293458, "dram": 2.5914992757120503, "package-0": 33.3149130714636, "psys": 53.37384996227321, "uncore": 21.48721621984776 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 838.660960244918, "sha256": "f7d1eadbc4a56468606230a00858318b02ca9435aea82c0d341aed77f8519197", "sum": -213878.9870355043, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 254, "gross_j": { "core": 18.724683, "dram": 8.695779, "package-0": 60.090972, "psys": 105.661412, "uncore": 23.858276 }, "net_j": { "core": 8.995854957417153, "dram": 7.422051734830573, "package-0": 45.31953367176036, "psys": 79.46407064059447, "uncore": 23.85323825876566 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.579388979, "token_latencies_s": [ 0.184430114, 0.184394104, 0.184389861, 0.184090122, 0.184158812, 0.184108165, 0.184100188, 0.184089887, 0.184023195, 0.184024977, 0.184305586, 0.184254201, 0.184456505, 0.184536075 ], "watts": { "core": 7.259348300099874, "dram": 3.371255390635675, "package-0": 23.296591746816176, "psys": 40.963737094419834, "uncore": 9.249584376083357 } } }, { "energy": { "gross_j": { "core": 38.191735, "dram": 15.395040999999999, "package-0": 144.725277, "psys": 241.94530500000002, "uncore": 74.830131 }, "net_j": { "core": 18.822873067621305, "dram": 12.85921166571738, "package-0": 115.31721795424016, "psys": 189.78972309796757, "uncore": 74.82010149654985 } }, "first_decode": { "gpu_active_fraction": 0.95, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 20, "gross_j": { "core": 3.275138, "dram": 0.632811, "package-0": 6.159408, "psys": 9.686011, "uncore": 1.606503 }, "logits": { "l2": 653.054767718728, "sha256": "7147004b0f99126b2e87e0a57bfb6d03ac73ede7f2bfe0eabe33e7fdde6a26d6", "sum": -38815.08208623878, "top1": 220 }, "net_j": { "core": 2.5273248428573662, "dram": 0.5349050648415319, "package-0": 5.023991084064256, "psys": 7.672334073552924, "uncore": 1.6061157705042344 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.198266534, "watts": { "core": 16.51886444940829, "dram": 3.1917186790585648, "package-0": 31.066301890363405, "psys": 48.85348426981631, "uncore": 8.102744157518787 } }, "metrics": { "all_tokens_per_s_e2e": 398.8132111011527, "decode_tokens_per_s": 5.377329538516507, "end_to_end_ms": 5135.236103, "handoff_estimate_ms": 13.366095500000007, "handoff_plus_first_decode_ms": 198.266534, "output_tokens_per_s_e2e": 3.1157282117277556, "prefill_tokens_per_s": 866.2484289655399, "steady_tpot_ms": 184.90043849999998, "steady_tpot_p95_ms": 186.01432140000003, "ttft_ms": 2345.7474 }, "prefill": { "gpu_active_fraction": 0.9310344827586207, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 232, "gross_j": { "core": 13.889674, "dram": 6.032516, "package-0": 76.030445, "psys": 123.071036, "uncore": 49.209773 }, "logits": { "l2": 637.7699802581292, "sha256": "af4cff367047f08fb39636f2515d91bc86b19e3f9a6df0c7debf7735673bb18f", "sum": -27833.03061964945, "top1": 220 }, "net_j": { "core": 5.0420851827509665, "dram": 4.874163209899049, "package-0": 62.597006563676246, "psys": 99.24665515591252, "uncore": 49.20519157839258 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.3457474, "watts": { "core": 5.921214705385584, "dram": 2.571681844345218, "package-0": 32.41203421988233, "psys": 52.46559625303219, "uncore": 20.97829160974451 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 838.660960244918, "sha256": "f7d1eadbc4a56468606230a00858318b02ca9435aea82c0d341aed77f8519197", "sum": -213878.9870355043, "top1": 42 }, "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 2450, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 256, "gross_j": { "core": 21.026923, "dram": 8.729714, "package-0": 62.535424, "psys": 109.188258, "uncore": 24.013855 }, "net_j": { "core": 11.253463042012973, "dram": 7.450143390976799, "package-0": 47.69622030649966, "psys": 82.87073386850211, "uncore": 24.008794147653035 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.591222169, "token_latencies_s": [ 0.186240784, 0.185870879, 0.185493526, 0.18589238, 0.185625148, 0.184926683, 0.184856874, 0.184846511, 0.184874194, 0.184971079, 0.184532287, 0.18444911, 0.184151947, 0.184458015 ], "watts": { "core": 8.11467393709227, "dram": 3.368956203152953, "package-0": 24.133563207408635, "psys": 42.13774461575317, "uncore": 9.26738559406019 } } }, { "energy": { "gross_j": { "core": 37.497951, "dram": 15.420187, "package-0": 144.016417, "psys": 244.079149, "uncore": 74.733269 }, "net_j": { "core": 18.05887825143846, "dram": 12.875165455670038, "package-0": 114.50175572664597, "psys": 191.73450663656214, "uncore": 74.72320314027732 } }, "first_decode": { "gpu_active_fraction": 1.0, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 20, "gross_j": { "core": 2.898857, "dram": 0.632078, "package-0": 5.825852, "psys": 9.335608, "uncore": 1.641537 }, "logits": { "l2": 653.054767718728, "sha256": "7147004b0f99126b2e87e0a57bfb6d03ac73ede7f2bfe0eabe33e7fdde6a26d6", "sum": -38815.08208623878, "top1": 220 }, "net_j": { "core": 2.1585769879917653, "dram": 0.5351583267333204, "package-0": 4.701872781213739, "psys": 7.342215980813286, "uncore": 1.6411536712860864 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.196269283, "watts": { "core": 14.769794619364866, "dram": 3.220463183737213, "package-0": 29.682953496090374, "psys": 47.56530343059337, "uncore": 8.363697950636524 } }, "metrics": { "all_tokens_per_s_e2e": 397.37276168167176, "decode_tokens_per_s": 5.357202494286338, "end_to_end_ms": 5153.850987, "handoff_estimate_ms": 10.983376999999988, "handoff_plus_first_decode_ms": 196.269283, "output_tokens_per_s_e2e": 3.1044747006380606, "prefill_tokens_per_s": 863.2547699795491, "steady_tpot_ms": 185.285906, "steady_tpot_p95_ms": 189.0231269, "ttft_ms": 2353.882157 }, "prefill": { "gpu_active_fraction": 0.9227467811158798, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 233, "gross_j": { "core": 15.558615, "dram": 6.047897, "package-0": 77.693039, "psys": 126.280256, "uncore": 49.119625 }, "logits": { "l2": 637.7699802581292, "sha256": "af4cff367047f08fb39636f2515d91bc86b19e3f9a6df0c7debf7735673bb18f", "sum": -27833.03061964945, "top1": 220 }, "net_j": { "core": 6.680343857337652, "dram": 4.88552718808854, "package-0": 64.21301508846526, "psys": 102.3732551991166, "uncore": 49.11502769059857 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.353882157, "watts": { "core": 6.609768018221143, "dram": 2.569328707477857, "package-0": 33.006341786888356, "psys": 53.647654205826065, "uncore": 20.867495364594838 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 838.660960244918, "sha256": "f7d1eadbc4a56468606230a00858318b02ca9435aea82c0d341aed77f8519197", "sum": -213878.9870355043, "top1": 42 }, "gpu_active_fraction": 0.9961089494163424, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 257, "gross_j": { "core": 19.040479, "dram": 8.740212, "package-0": 60.497526, "psys": 108.463285, "uncore": 23.972107 }, "net_j": { "core": 9.219957406109042, "dram": 7.454479940848177, "package-0": 45.586867856966975, "psys": 82.01903545663225, "uncore": 23.96702177839266 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.603699547, "token_latencies_s": [ 0.195117783, 0.185645203, 0.185324044, 0.184661946, 0.184984062, 0.185136193, 0.185247768, 0.185070948, 0.18500667, 0.185058867, 0.185633582, 0.185741389, 0.185335556, 0.18569932 ], "watts": { "core": 7.312855671822261, "dram": 3.3568435382917086, "package-0": 23.235217776838212, "psys": 41.65737368774831, "uncore": 9.206940573316464 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 5.165887873619795, "compile_target": "AUTO:GPU", "context_length": 4096, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 400833, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_7b_ov_fp16/openvino_model.xml", "key": "qwen25_7b_fp16", "size_b": 7.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "GPU", "key": "gpu_gpu", "label": "iGPU -> iGPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 4080, "prompt_sha256": "66f8de7a9ef70b33481de3e8dae7cef6e4434d9723c6203a6a3dbe07ef228a16", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.046541888266801834, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 52.242725, "dram": 22.619206000000002, "package-0": 238.720031, "psys": 398.251788, "uncore": 138.88233499999998 }, "net_j": { "core": 21.227217440000445, "dram": 18.558562986337286, "package-0": 191.62867990033, "psys": 314.7346536207989, "uncore": 138.8662746789978 } }, "first_decode": { "gpu_active_fraction": 0.9523809523809523, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 21, "gross_j": { "core": 3.197868, "dram": 0.653136, "package-0": 6.228011, "psys": 9.966955, "uncore": 1.698054 }, "logits": { "l2": 728.3785859021272, "sha256": "c0ed3cbfd820a639c3f89b51903efbb9d58551be551f5f0d06ab29ea3c24d06a", "sum": -48270.276146711665, "top1": 78 }, "net_j": { "core": 2.395075716024573, "dram": 0.5480320331201751, "package-0": 5.009118378892325, "psys": 7.8052328057183455, "uncore": 1.6976383014416647 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.212843064, "watts": { "core": 15.024534696606324, "dram": 3.068627127074247, "package-0": 29.261047472986952, "psys": 46.82771809749929, "uncore": 7.977962579978645 } }, "metrics": { "all_tokens_per_s_e2e": 498.109405927577, "decode_tokens_per_s": 5.255019366956979, "end_to_end_ms": 8223.093062, "handoff_estimate_ms": 24.365791499999983, "handoff_plus_first_decode_ms": 212.843064, "output_tokens_per_s_e2e": 1.9457398669045975, "prefill_tokens_per_s": 759.963452791308, "steady_tpot_ms": 188.47727250000003, "steady_tpot_p95_ms": 189.7162846, "ttft_ms": 5368.679224 }, "prefill": { "gpu_active_fraction": 0.9510357815442562, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 531, "gross_j": { "core": 30.181746, "dram": 13.134427, "package-0": 171.24413, "psys": 279.519731, "uncore": 112.567095 }, "logits": { "l2": 680.59636228507, "sha256": "8abfd8de30eb1ada57815fb449ec5279eb380a6c63325e4eb5ea55f3d3b40f0e", "sum": 21399.900736933458, "top1": 49 }, "net_j": { "core": 9.93239337760783, "dram": 10.483321191131829, "package-0": 140.49920757571238, "psys": 224.99320420418383, "uncore": 112.55660956452637 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 5.368679224, "watts": { "core": 5.621819583683885, "dram": 2.4464912973910247, "package-0": 31.896882427706768, "psys": 52.06489703285725, "uncore": 20.96737210462921 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 811.5720666048574, "sha256": "8248a9d79387f416fa16b5c5c188e01f3867aea1d96b374479ac88edb673436c", "sum": -176094.05456782482, "top1": 42 }, "gpu_active_fraction": 0.9961685823754789, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 261, "gross_j": { "core": 18.863111, "dram": 8.831643, "package-0": 61.24789, "psys": 108.765102, "uncore": 24.617186 }, "net_j": { "core": 8.89974834636804, "dram": 7.527209762085283, "package-0": 46.12035394572532, "psys": 81.93621661089674, "uncore": 24.61202681302977 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.641570774, "token_latencies_s": [ 0.191071571, 0.188986515, 0.188773333, 0.188380882, 0.188574069, 0.188533096, 0.188342663, 0.188324014, 0.188261792, 0.188255816, 0.188421449, 0.18863195, 0.188624163, 0.188356265 ], "watts": { "core": 7.140869056268564, "dram": 3.343330069717072, "package-0": 23.18616279481899, "psys": 41.17440390790756, "uncore": 9.319146866060837 } } }, { "energy": { "gross_j": { "core": 52.845262, "dram": 22.969302, "package-0": 240.888543, "psys": 405.645873, "uncore": 139.672922 }, "net_j": { "core": 21.220425088392957, "dram": 18.828883765189023, "package-0": 192.87203720549314, "psys": 320.48796452606746, "uncore": 139.6565461586097 } }, "first_decode": { "gpu_active_fraction": 0.9047619047619048, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 21, "gross_j": { "core": 2.450555, "dram": 0.643919, "package-0": 5.419114, "psys": 9.094276, "uncore": 1.659236 }, "logits": { "l2": 728.3785859021272, "sha256": "c0ed3cbfd820a639c3f89b51903efbb9d58551be551f5f0d06ab29ea3c24d06a", "sum": -48270.276146711665, "top1": 78 }, "net_j": { "core": 1.674865436743472, "dram": 0.542363402377894, "package-0": 4.241371881899924, "psys": 7.005534767385454, "uncore": 1.6588343356596744 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.205657362, "watts": { "core": 11.915717366830757, "dram": 3.1310282001964023, "package-0": 26.35020670935184, "psys": 44.22052248243853, "uncore": 8.067963061784289 } }, "metrics": { "all_tokens_per_s_e2e": 488.51211749912517, "decode_tokens_per_s": 5.252636397745488, "end_to_end_ms": 8384.643601, "handoff_estimate_ms": 16.53028850000002, "handoff_plus_first_decode_ms": 205.657362, "output_tokens_per_s_e2e": 1.9082504589809577, "prefill_tokens_per_s": 737.9359947373357, "steady_tpot_ms": 189.1270735, "steady_tpot_p95_ms": 190.72192264999998, "ttft_ms": 5528.934798 }, "prefill": { "gpu_active_fraction": 0.9396709323583181, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 547, "gross_j": { "core": 32.6511, "dram": 13.490811, "package-0": 175.292642, "psys": 288.125178, "uncore": 113.326737 }, "logits": { "l2": 680.59636228507, "sha256": "8abfd8de30eb1ada57815fb449ec5279eb380a6c63325e4eb5ea55f3d3b40f0e", "sum": 21399.900736933458, "top1": 49 }, "net_j": { "core": 11.7973023191057, "dram": 10.760569436245293, "package-0": 143.62998078277724, "psys": 231.97102930961674, "uncore": 113.31593857330257 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 5.528934798, "watts": { "core": 5.905495577884368, "dram": 2.4400379987986254, "package-0": 31.70459562362884, "psys": 52.11224015595635, "uncore": 20.497029019223387 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 811.5720666048574, "sha256": "8248a9d79387f416fa16b5c5c188e01f3867aea1d96b374479ac88edb673436c", "sum": -176094.05456782482, "top1": 42 }, "gpu_active_fraction": 0.9961977186311787, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 263, "gross_j": { "core": 17.743607, "dram": 8.834572, "package-0": 60.176787, "psys": 108.426419, "uncore": 24.686949 }, "net_j": { "core": 7.748257332543785, "dram": 7.525950926565837, "package-0": 45.00068454081597, "psys": 81.51140044906529, "uncore": 24.68177324964744 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.650051441, "token_latencies_s": [ 0.192755135, 0.189123625, 0.188554996, 0.188807943, 0.18857951, 0.188662776, 0.188701032, 0.188920403, 0.189178376, 0.189627116, 0.189189275, 0.189130522, 0.189423868, 0.189375524 ], "watts": { "core": 6.695570782318259, "dram": 3.3337360412393595, "package-0": 22.707780712849942, "psys": 40.91483558488403, "uncore": 9.31564897875505 } } }, { "energy": { "gross_j": { "core": 60.733304000000004, "dram": 22.895631, "package-0": 248.924534, "psys": 407.235835, "uncore": 139.902901 }, "net_j": { "core": 29.099991449114118, "dram": 18.75410310914495, "package-0": 200.8951595041088, "psys": 322.0551037138587, "uncore": 139.88652076978946 } }, "first_decode": { "gpu_active_fraction": 0.9523809523809523, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 21, "gross_j": { "core": 3.448172, "dram": 0.646787, "package-0": 6.452315, "psys": 10.056493, "uncore": 1.687007 }, "logits": { "l2": 728.3785859021272, "sha256": "c0ed3cbfd820a639c3f89b51903efbb9d58551be551f5f0d06ab29ea3c24d06a", "sum": -48270.276146711665, "top1": 78 }, "net_j": { "core": 2.659387330722775, "dram": 0.5435169519316696, "package-0": 5.254690368801754, "psys": 7.932489867281941, "uncore": 1.6865985548063918 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.209129247, "watts": { "core": 16.488234187540492, "dram": 3.0927620563755966, "package-0": 30.853240723426886, "psys": 48.087453784022856, "uncore": 8.066815255161321 } }, "metrics": { "all_tokens_per_s_e2e": 488.3812285040317, "decode_tokens_per_s": 5.254028638369279, "end_to_end_ms": 8386.890734, "handoff_estimate_ms": 20.805042999999994, "handoff_plus_first_decode_ms": 209.129247, "output_tokens_per_s_e2e": 1.9077391738438738, "prefill_tokens_per_s": 737.5352939832277, "steady_tpot_ms": 188.324204, "steady_tpot_p95_ms": 191.93740460000004, "ttft_ms": 5531.938652 }, "prefill": { "gpu_active_fraction": 0.9397810218978102, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 548, "gross_j": { "core": 37.900171, "dram": 13.417629, "package-0": 180.586513, "psys": 288.725396, "uncore": 113.474563 }, "logits": { "l2": 680.59636228507, "sha256": "8abfd8de30eb1ada57815fb449ec5279eb380a6c63325e4eb5ea55f3d3b40f0e", "sum": 21399.900736933458, "top1": 49 }, "net_j": { "core": 17.035043512213853, "dram": 10.685904104040828, "package-0": 148.90664955214461, "psys": 232.54073892593652, "uncore": 113.46375870654914 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 5.531938652, "watts": { "core": 6.851155333455784, "dram": 2.4254840561453137, "package-0": 32.64434484187768, "psys": 52.1924435831578, "uncore": 20.512621368093942 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 811.5720666048574, "sha256": "8248a9d79387f416fa16b5c5c188e01f3867aea1d96b374479ac88edb673436c", "sum": -176094.05456782482, "top1": 42 }, "gpu_active_fraction": 0.9961832061068703, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 262, "gross_j": { "core": 19.384961, "dram": 8.831215, "package-0": 61.885706, "psys": 108.453946, "uncore": 24.741331 }, "net_j": { "core": 9.40556060617749, "dram": 7.524682053172452, "package-0": 46.733819583162415, "psys": 81.58187492064027, "uncore": 24.736163508433915 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.645822835, "token_latencies_s": [ 0.197799427, 0.188780931, 0.188150261, 0.188389867, 0.188369775, 0.188309607, 0.188420428, 0.188143914, 0.188330012, 0.188234266, 0.188356951, 0.188318396, 0.187895585, 0.188292806 ], "watts": { "core": 7.326628504209731, "dram": 3.3377952911953, "package-0": 23.389965942296357, "psys": 40.99063042518491, "uncore": 9.351091340172818 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 12.078973207622766, "compile_target": "NPU", "context_length": 4096, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 400902, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_7b_ov_fp16/openvino_model.xml", "key": "qwen25_7b_fp16", "size_b": 7.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "gpu_npu", "label": "iGPU -> NPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 4080, "prompt_sha256": "66f8de7a9ef70b33481de3e8dae7cef6e4434d9723c6203a6a3dbe07ef228a16", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "GPU", "NPUW_ONLINE_PIPELINE": "NONE" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 4096, "NPUW_LLM_MIN_RESPONSE_LEN": 4096, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.04649261571466923, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 217.701347, "dram": 82.94302499999999, "package-0": 688.033577, "psys": 1369.036046, "uncore": 248.241003 }, "net_j": { "core": 85.4405387400066, "dram": 65.62704440534506, "package-0": 487.21983418283855, "psys": 1012.890215557502, "uncore": 248.17957328269597 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 61, "gross_j": { "core": 7.139874, "dram": 1.003538, "package-0": 9.827428, "psys": 17.088335, "uncore": 0.0 }, "logits": { "l2": 1172.8521223979067, "sha256": "f34cfb4134e278790a81609bd0bc150951ab37a8c8a15deb20eea3ad3a0caeae", "sum": -332944.55895423517, "top1": 82 }, "net_j": { "core": 4.830995527933675, "dram": 0.7012527248402955, "package-0": 6.321820149984649, "psys": 10.871093085007123, "uncore": 0.0 }, "npu_busy_delta_us": 257571, "npu_utilization": 0.42076497078061054, "seconds": 0.612149342, "watts": { "core": 11.6636145955377, "dram": 1.639367930579267, "package-0": 16.053971352631134, "psys": 27.915304040300676, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 116.80796638475361, "decode_tokens_per_s": 4.151343714530478, "end_to_end_ms": 35066.101455, "handoff_estimate_ms": 397.9884635000001, "handoff_plus_first_decode_ms": 612.149342, "output_tokens_per_s_e2e": 0.4562811186904438, "prefill_tokens_per_s": 129.71812497716138, "steady_tpot_ms": 214.1608785, "steady_tpot_p95_ms": 215.79169374999998, "ttft_ms": 31452.813557999998 }, "prefill": { "gpu_active_fraction": 0.9977426636568849, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 3101, "gross_j": { "core": 197.483625, "dram": 72.595334, "package-0": 633.880884, "psys": 1267.20867, "uncore": 248.241003 }, "logits": { "l2": null, "sha256": "7b769dbc741082009e7e9a93ca58b6e7f8d647e22fffd35e2edcda73c4a3253b", "sum": null, "top1": 0 }, "net_j": { "core": 78.85126010564711, "dram": 57.06362997192164, "package-0": 453.7594290724311, "psys": 947.7608859701123, "uncore": 248.17957328269597 }, "npu_busy_delta_us": 19029, "npu_utilization": 0.0006050015196545108, "seconds": 31.452813558, "watts": { "core": 6.2787268501698215, "dram": 2.3080712275908755, "package-0": 20.153392091016066, "psys": 40.289199173333934, "uncore": 7.892489571473013 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 826.7915182092064, "sha256": "ff3906e0aa910a2693afc8b2817af7b14250e5373ac7ea4a033206fe356d605d", "sum": -21699.585335694253, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 296, "gross_j": { "core": 13.077848, "dram": 9.344153, "package-0": 44.325265, "psys": 84.739041, "uncore": 0.0 }, "net_j": { "core": 1.7582831064258162, "dram": 7.8621617085831215, "package-0": 27.138584960422794, "psys": 54.25823650238256, "uncore": 0.0 }, "npu_busy_delta_us": 2986510, "npu_utilization": 0.9951256648995335, "seconds": 3.001138555, "token_latencies_s": [ 0.216395339, 0.215466654, 0.214270064, 0.213999575, 0.214034035, 0.214142197, 0.21418248, 0.214227899, 0.21417956, 0.213963466, 0.214273048, 0.214073312, 0.213934869, 0.213939565 ], "watts": { "core": 4.357628866621921, "dram": 3.113536022664572, "package-0": 14.769483043744378, "psys": 28.23563106035936, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 212.465277, "dram": 82.555391, "package-0": 682.927145, "psys": 1356.3416200000001, "uncore": 248.884861 }, "net_j": { "core": 81.11902542490313, "dram": 65.35914690186827, "package-0": 483.5019885171326, "psys": 1002.6584657978357, "uncore": 248.82336498299983 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 33, "gross_j": { "core": 3.821524, "dram": 0.793577, "package-0": 5.950363, "psys": 10.358677, "uncore": 0.0 }, "logits": { "l2": 1057.9006125110022, "sha256": "d8d60d86e793894196a424af650e125bbb024e6c0c26858765388ce2d8633963", "sum": -231576.80951821245, "top1": 82 }, "net_j": { "core": 2.543966654530762, "dram": 0.6263153713003077, "package-0": 4.010626840948888, "psys": 6.918529251578265, "uncore": 0.0 }, "npu_busy_delta_us": 241875, "npu_utilization": 0.7140921623597257, "seconds": 0.338716783, "watts": { "core": 11.282357981062898, "dram": 2.3428924689568746, "package-0": 17.56736984597542, "psys": 30.582119103321784, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 117.62129379398894, "decode_tokens_per_s": 4.495235132572752, "end_to_end_ms": 34823.62647, "handoff_estimate_ms": 124.68929550000001, "handoff_plus_first_decode_ms": 338.716783, "output_tokens_per_s_e2e": 0.4594581788827693, "prefill_tokens_per_s": 129.57827409433824, "steady_tpot_ms": 214.0274875, "steady_tpot_p95_ms": 214.77195360000002, "ttft_ms": 31486.759864 }, "prefill": { "gpu_active_fraction": 0.9980707395498393, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 3110, "gross_j": { "core": 196.680649, "dram": 72.418149, "package-0": 633.6188, "psys": 1261.939335, "uncore": 248.884861 }, "logits": { "l2": null, "sha256": "7b769dbc741082009e7e9a93ca58b6e7f8d647e22fffd35e2edcda73c4a3253b", "sum": null, "top1": 0 }, "net_j": { "core": 77.92024689356174, "dram": 56.86968195717047, "package-0": 453.30294408458735, "psys": 942.1467782454814, "uncore": 248.82336498299983 }, "npu_busy_delta_us": 16748, "npu_utilization": 0.0005319061114048962, "seconds": 31.486759864, "watts": { "core": 6.246455648327042, "dram": 2.2999555785604477, "package-0": 20.12334081807002, "psys": 40.078412019866896, "uncore": 7.9044291021052135 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 996.0453905212725, "sha256": "00b38d5947b99dea850265253a33360c423c4e3867809feaf8734e71936f6b65", "sum": -159259.00955941156, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 295, "gross_j": { "core": 11.963104, "dram": 9.343665, "package-0": 43.357982, "psys": 84.043608, "uncore": 0.0 }, "net_j": { "core": 0.654811876810621, "dram": 7.863149573397492, "package-0": 26.188417591596323, "psys": 53.593158300776, "uncore": 0.0 }, "npu_busy_delta_us": 2983086, "npu_utilization": 0.994975627006883, "seconds": 2.998149823, "token_latencies_s": [ 0.21544929, 0.21400437, 0.213782058, 0.214050605, 0.21398043, 0.21436247, 0.213995224, 0.21408853, 0.214407234, 0.214144598, 0.213863075, 0.21415792, 0.213957722, 0.213877195 ], "watts": { "core": 3.990162168756968, "dram": 3.116477011362484, "package-0": 14.461579493921109, "psys": 28.031823945310556, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 210.614574, "dram": 82.62387299999999, "package-0": 680.556912, "psys": 1357.55524, "uncore": 248.698704 }, "net_j": { "core": 79.7855965848623, "dram": 65.40994807070463, "package-0": 480.9267105956118, "psys": 1003.508435915967, "uncore": 248.63716569851644 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 34, "gross_j": { "core": 3.490104, "dram": 0.793943, "package-0": 5.607468, "psys": 9.764013, "uncore": 0.0 }, "logits": { "l2": 1037.37398392076, "sha256": "f24b0a92963b9d23a311db8388f6d69f6cb2c438ee75a7260dd6294ce7f2dab9", "sum": -205472.12200375088, "top1": 82 }, "net_j": { "core": 2.1852290002479373, "dram": 0.6231048622975575, "package-0": 3.626255000804863, "psys": 6.250305526680981, "uncore": 0.0 }, "npu_busy_delta_us": 243407, "npu_utilization": 0.7035708547015638, "seconds": 0.34595947, "watts": { "core": 10.088187497801405, "dram": 2.2949017698518266, "package-0": 16.208453550931846, "psys": 28.222996757394732, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 117.50048216936753, "decode_tokens_per_s": 4.476247218038092, "end_to_end_ms": 34859.43142, "handoff_estimate_ms": 131.443986, "handoff_plus_first_decode_ms": 345.95947, "output_tokens_per_s_e2e": 0.4589862584740919, "prefill_tokens_per_s": 129.4892376690935, "steady_tpot_ms": 214.51548400000001, "steady_tpot_p95_ms": 215.484725, "ttft_ms": 31508.410069 }, "prefill": { "gpu_active_fraction": 0.9980713596914176, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 3111, "gross_j": { "core": 196.442429, "dram": 72.496946, "package-0": 633.068814, "psys": 1269.667526, "uncore": 248.698704 }, "logits": { "l2": null, "sha256": "7b769dbc741082009e7e9a93ca58b6e7f8d647e22fffd35e2edcda73c4a3253b", "sum": null, "top1": 0 }, "net_j": { "core": 77.60036758461436, "dram": 56.93778787620923, "package-0": 452.62897342366387, "psys": 949.6550808085283, "uncore": 248.63716569851644 }, "npu_busy_delta_us": 19616, "npu_utilization": 0.0006225639426757202, "seconds": 31.508410069, "watts": { "core": 6.23460303359682, "dram": 2.3008760467836855, "package-0": 20.0920583619944, "psys": 40.296147067388226, "uncore": 7.893089605453808 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 868.6798604910134, "sha256": "eb0b0977dcabeb495be6d6188eaaa16de8d04d91612c28c0136baa3d5e134f7f", "sum": -136328.59814876132, "top1": 198 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 296, "gross_j": { "core": 10.682041, "dram": 9.332984, "package-0": 41.88063, "psys": 78.123701, "uncore": 0.0 }, "net_j": { "core": 0.0, "dram": 7.849055332197847, "package-0": 24.671482171143076, "psys": 47.6030495807577, "uncore": 0.0 }, "npu_busy_delta_us": 2985494, "npu_utilization": 0.9934883600488492, "seconds": 3.005061881, "token_latencies_s": [ 0.215764355, 0.21479991, 0.214122807, 0.214506947, 0.214260313, 0.21434419, 0.214436798, 0.215334155, 0.215167918, 0.214775767, 0.214524021, 0.214526227, 0.214384702, 0.214059003 ], "watts": { "core": 3.5546825399965862, "dram": 3.1057543470267057, "package-0": 13.936694703292865, "psys": 25.997368471494713, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 6.344236670061946, "compile_target": "AUTO:NPU,GPU", "context_length": 4096, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 400748, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_7b_ov_fp16/openvino_model.xml", "key": "qwen25_7b_fp16", "size_b": 7.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": null, "key": "openvino_auto", "label": "OpenVINO AUTO", "mode": "stock", "prefill_device": null }, "prompt_length": 4080, "prompt_sha256": "66f8de7a9ef70b33481de3e8dae7cef6e4434d9723c6203a6a3dbe07ef228a16", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.04840262979269028, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 54.762800000000006, "dram": 22.649478000000002, "package-0": 241.683646, "psys": 398.893754, "uncore": 138.984142 }, "net_j": { "core": 23.604732300709408, "dram": 18.570170586519904, "package-0": 194.37584351519035, "psys": 314.99274022474987, "uncore": 138.96800785909997 } }, "first_decode": { "gpu_active_fraction": 0.9523809523809523, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 21, "gross_j": { "core": 3.131828, "dram": 0.650511, "package-0": 6.156295, "psys": 9.829443, "uncore": 1.696468 }, "logits": { "l2": 728.3785859021272, "sha256": "c0ed3cbfd820a639c3f89b51903efbb9d58551be551f5f0d06ab29ea3c24d06a", "sum": -48270.276146711665, "top1": 78 }, "net_j": { "core": 2.335063040042409, "dram": 0.5461961484076409, "package-0": 4.946553763275324, "psys": 7.683950907030431, "uncore": 1.6960554224854951 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.211245049, "watts": { "core": 14.825568763980831, "dram": 3.079414183098795, "package-0": 29.142907865263155, "psys": 46.53099822472052, "uncore": 8.030805966960202 } }, "metrics": { "all_tokens_per_s_e2e": 495.83036388374614, "decode_tokens_per_s": 5.226943087414251, "end_to_end_ms": 8260.889809, "handoff_estimate_ms": 21.54377900000001, "handoff_plus_first_decode_ms": 211.245049, "output_tokens_per_s_e2e": 1.9368373589208834, "prefill_tokens_per_s": 756.7967553502504, "steady_tpot_ms": 189.70126999999997, "steady_tpot_p95_ms": 190.83485255, "ttft_ms": 5391.143621 }, "prefill": { "gpu_active_fraction": 0.9512195121951219, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 533, "gross_j": { "core": 32.187662, "dram": 13.156704, "package-0": 173.71214, "psys": 280.83888, "uncore": 112.717485 }, "logits": { "l2": 680.59636228507, "sha256": "8abfd8de30eb1ada57815fb449ec5279eb380a6c63325e4eb5ea55f3d3b40f0e", "sum": 21399.900736933458, "top1": 49 }, "net_j": { "core": 11.853579134409678, "dram": 10.494505054273663, "package-0": 142.83857026521824, "psys": 226.08419549651217, "uncore": 112.70695568986459 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 5.391143621, "watts": { "core": 5.97047013821337, "dram": 2.440429141741093, "package-0": 32.22176076395795, "psys": 52.092635578479985, "uncore": 20.907898754715813 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 811.5720666048574, "sha256": "8248a9d79387f416fa16b5c5c188e01f3867aea1d96b374479ac88edb673436c", "sum": -176094.05456782482, "top1": 42 }, "gpu_active_fraction": 0.9961977186311787, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 263, "gross_j": { "core": 19.44331, "dram": 8.842263, "package-0": 61.815211, "psys": 108.225431, "uncore": 24.570189 }, "net_j": { "core": 9.416090126257322, "dram": 7.529469383838599, "package-0": 46.590719486696784, "psys": 81.22459382120726, "uncore": 24.5649967467499 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.658501139, "token_latencies_s": [ 0.192278823, 0.189756143, 0.189596364, 0.189646397, 0.189956999, 0.189913206, 0.189472156, 0.189847858, 0.189479864, 0.189825747, 0.189606108, 0.18951307, 0.19005733, 0.189516134 ], "watts": { "core": 7.313636136832213, "dram": 3.326033180984844, "package-0": 23.251903146918302, "psys": 40.70919113493748, "uncore": 9.24212092278513 } } }, { "energy": { "gross_j": { "core": 57.066809, "dram": 22.827029, "package-0": 244.579024, "psys": 403.873782, "uncore": 139.383982 }, "net_j": { "core": 25.56360958713523, "dram": 18.702535910328493, "package-0": 196.74720240416625, "psys": 319.0434133965132, "uncore": 139.3676691444329 } }, "first_decode": { "gpu_active_fraction": 0.9523809523809523, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 21, "gross_j": { "core": 3.328116, "dram": 0.64514, "package-0": 6.30046, "psys": 9.924474, "uncore": 1.657466 }, "logits": { "l2": 728.3785859021272, "sha256": "c0ed3cbfd820a639c3f89b51903efbb9d58551be551f5f0d06ab29ea3c24d06a", "sum": -48270.276146711665, "top1": 78 }, "net_j": { "core": 2.5566841769166784, "dram": 0.5441418384557939, "package-0": 5.129182478263525, "psys": 7.847197814588357, "uncore": 1.6570665403849654 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.204528514, "watts": { "core": 16.272136998951648, "dram": 3.154279016567832, "package-0": 30.804800156128845, "psys": 48.523669418533984, "uncore": 8.10383827459872 } }, "metrics": { "all_tokens_per_s_e2e": 490.39831931942734, "decode_tokens_per_s": 5.265535644353759, "end_to_end_ms": 8352.394041, "handoff_estimate_ms": 15.836338999999978, "handoff_plus_first_decode_ms": 204.528514, "output_tokens_per_s_e2e": 1.915618434841513, "prefill_tokens_per_s": 741.3220337162451, "steady_tpot_ms": 188.69217500000002, "steady_tpot_p95_ms": 189.93227535000003, "ttft_ms": 5503.681011000001 }, "prefill": { "gpu_active_fraction": 0.9357798165137615, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 545, "gross_j": { "core": 34.04752, "dram": 13.337063, "package-0": 176.172156, "psys": 284.549686, "uncore": 113.102738 }, "logits": { "l2": 680.59636228507, "sha256": "8abfd8de30eb1ada57815fb449ec5279eb380a6c63325e4eb5ea55f3d3b40f0e", "sum": 21399.900736933458, "top1": 49 }, "net_j": { "core": 13.288973463135267, "dram": 10.619292000907143, "package-0": 144.65411614863638, "psys": 228.65202521587787, "uncore": 113.09198889585345 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 5.503681011, "watts": { "core": 6.186317835635913, "dram": 2.423298692882766, "package-0": 32.009877688748915, "psys": 51.70170390167621, "uncore": 20.550380331626382 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 811.5720666048574, "sha256": "8248a9d79387f416fa16b5c5c188e01f3867aea1d96b374479ac88edb673436c", "sum": -176094.05456782482, "top1": 42 }, "gpu_active_fraction": 0.9961832061068703, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 262, "gross_j": { "core": 19.691173, "dram": 8.844826, "package-0": 62.106408, "psys": 109.399622, "uncore": 24.623778 }, "net_j": { "core": 9.717951947083284, "dram": 7.539102070965555, "package-0": 46.96390377726633, "psys": 82.54419036604693, "uncore": 24.6186137081945 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.644184516, "token_latencies_s": [ 0.191668738, 0.188904542, 0.188871058, 0.188997257, 0.18868876, 0.188934233, 0.188523841, 0.188769778, 0.188291469, 0.18869559, 0.188642575, 0.188393052, 0.188472906, 0.18830558 ], "watts": { "core": 7.44697387071455, "dram": 3.34501088955019, "package-0": 23.48792515204336, "psys": 41.37367167004467, "uncore": 9.312428028755615 } } }, { "energy": { "gross_j": { "core": 55.82255, "dram": 22.86365, "package-0": 243.433154, "psys": 404.320864, "uncore": 139.325571 }, "net_j": { "core": 24.320423361532924, "dram": 18.73929736116118, "package-0": 195.60296121503495, "psys": 319.49338411414436, "uncore": 139.30925869993249 } }, "first_decode": { "gpu_active_fraction": 0.9047619047619048, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 21, "gross_j": { "core": 3.177299, "dram": 0.645933, "package-0": 6.207503, "psys": 9.969884, "uncore": 1.664913 }, "logits": { "l2": 728.3785859021272, "sha256": "c0ed3cbfd820a639c3f89b51903efbb9d58551be551f5f0d06ab29ea3c24d06a", "sum": -48270.276146711665, "top1": 78 }, "net_j": { "core": 2.4035227811003246, "dram": 0.5446279031436917, "package-0": 5.032665943850265, "psys": 7.886294933199791, "uncore": 1.664512326419684 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.20515008, "watts": { "core": 15.487681018696167, "dram": 3.14858760961731, "package-0": 30.25835037451606, "psys": 48.59800200906575, "uncore": 8.115585428969855 } }, "metrics": { "all_tokens_per_s_e2e": 490.41501935900476, "decode_tokens_per_s": 5.250463004729424, "end_to_end_ms": 8352.109618, "handoff_estimate_ms": 15.860385000000004, "handoff_plus_first_decode_ms": 205.15008, "output_tokens_per_s_e2e": 1.9156836693711123, "prefill_tokens_per_s": 742.4636238054565, "steady_tpot_ms": 189.289695, "steady_tpot_p95_ms": 190.30641555, "ttft_ms": 5495.218714 }, "prefill": { "gpu_active_fraction": 0.9393382352941176, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 544, "gross_j": { "core": 34.733676, "dram": 13.364956, "package-0": 176.808447, "psys": 286.563536, "uncore": 113.034745 }, "logits": { "l2": 680.59636228507, "sha256": "8abfd8de30eb1ada57815fb449ec5279eb380a6c63325e4eb5ea55f3d3b40f0e", "sum": 21399.900736933458, "top1": 49 }, "net_j": { "core": 14.007047189786249, "dram": 10.65136376514263, "package-0": 145.33886835370024, "psys": 230.75182180439108, "uncore": 113.02401242335777 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 5.495218714, "watts": { "core": 6.320708566432503, "dram": 2.432106290137372, "package-0": 32.17496085270465, "psys": 52.14779445810426, "uncore": 20.569653526623945 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 811.5720666048574, "sha256": "8248a9d79387f416fa16b5c5c188e01f3867aea1d96b374479ac88edb673436c", "sum": -176094.05456782482, "top1": 42 }, "gpu_active_fraction": 0.9961977186311787, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 263, "gross_j": { "core": 17.911575, "dram": 8.852761, "package-0": 60.417204, "psys": 107.787444, "uncore": 24.625913 }, "net_j": { "core": 7.909853390646353, "dram": 7.543305692874861, "package-0": 45.231426917484455, "psys": 80.85526737655351, "uncore": 24.620733950155017 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.651740824, "token_latencies_s": [ 0.191527748, 0.189013559, 0.189173941, 0.189315444, 0.189422637, 0.18911767, 0.188783522, 0.188939389, 0.189095198, 0.189648775, 0.189266767, 0.189312623, 0.189542561, 0.189549645 ], "watts": { "core": 6.7546476781925495, "dram": 3.3384714372825144, "package-0": 22.78397777534838, "psys": 40.64780502847513, "uncore": 9.286696790696617 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 5.547535102814436, "compile_target": "AUTO:GPU", "context_length": 8192, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 401325, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_7b_ov_fp16/openvino_model.xml", "key": "qwen25_7b_fp16", "size_b": 7.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "GPU", "key": "gpu_gpu", "label": "iGPU -> iGPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 8176, "prompt_sha256": "998ace73fce48442f97bcb84c31be293d0d94180b17f70a4d127bba1092ad744", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.04683802369982004, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 88.304279, "dram": 41.468399999999995, "package-0": 464.982573, "psys": 772.664464, "uncore": 287.207883 }, "net_j": { "core": 26.533168689207045, "dram": 33.381141487642005, "package-0": 371.1944885124335, "psys": 606.3300549757737, "uncore": 287.1758969408219 } }, "first_decode": { "gpu_active_fraction": 0.9230769230769231, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 26, "gross_j": { "core": 3.228386, "dram": 0.704893, "package-0": 6.784772, "psys": 11.126131, "uncore": 1.946162 }, "logits": { "l2": 842.3149968162743, "sha256": "d805104637e3082ed7ef6942f1d19e2c0a800de34252afd57ab996e71250bf5f", "sum": -27488.31816223578, "top1": 78 }, "net_j": { "core": 2.233616222199197, "dram": 0.5746547654453855, "package-0": 5.2743968181934004, "psys": 8.44746062685719, "uncore": 1.9456468924550452 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.263741757, "watts": { "core": 12.240708626203622, "dram": 2.672663623758296, "package-0": 25.725058015746818, "psys": 42.18570137151244, "uncore": 7.379043887995331 } }, "metrics": { "all_tokens_per_s_e2e": 500.2052243362155, "decode_tokens_per_s": 4.952966834959828, "end_to_end_ms": 16377.277968000002, "handoff_estimate_ms": 66.8084195, "handoff_plus_first_decode_ms": 263.741757, "output_tokens_per_s_e2e": 0.9769633287816709, "prefill_tokens_per_s": 612.4899666880626, "steady_tpot_ms": 196.93333750000002, "steady_tpot_p95_ms": 200.4805697, "ttft_ms": 13348.790094 }, "prefill": { "gpu_active_fraction": 0.9587786259541985, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 1310, "gross_j": { "core": 67.152294, "dram": 31.687907, "package-0": 395.736108, "psys": 649.431382, "uncore": 259.146736 }, "logits": { "l2": 1025.1392851150267, "sha256": "4ff86f4268bae4a3871f14f73062d317d9c93b8e3d5513020b94dee3023132a8", "sum": -317470.8328210273, "top1": 8 }, "net_j": { "core": 16.80390356397988, "dram": 25.0961451377224, "package-0": 319.29132544278286, "psys": 513.8555484693997, "uncore": 259.1206648060728 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 13.348790094, "watts": { "core": 5.030590302725903, "dram": 2.37384113293107, "package-0": 29.645840949875677, "psys": 48.650954687788946, "uncore": 19.413499963302364 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 727.8313868092002, "sha256": "474801b73ad22489ab599f54a4d0eb647b9530fa6cbcb6caa0057490e9977584", "sum": -118284.7835280532, "top1": 42 }, "gpu_active_fraction": 0.9963235294117647, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 272, "gross_j": { "core": 17.923599, "dram": 9.0756, "package-0": 62.461693, "psys": 112.106951, "uncore": 26.114985 }, "net_j": { "core": 7.49564890302797, "dram": 7.710341584474218, "package-0": 46.62876625145723, "psys": 84.02704587951683, "uncore": 26.109585242294013 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.764746117, "token_latencies_s": [ 0.205273177, 0.196863468, 0.197899935, 0.197039062, 0.196485823, 0.197023833, 0.197219045, 0.196131999, 0.196767983, 0.196370331, 0.197443275, 0.197003207, 0.196450385, 0.196733348 ], "watts": { "core": 6.482909548110236, "dram": 3.2826160580154276, "package-0": 22.592198472016154, "psys": 40.54873259814764, "uncore": 9.445708175308742 } } }, { "energy": { "gross_j": { "core": 119.203919, "dram": 43.560619, "package-0": 502.080136, "psys": 836.679561, "uncore": 288.51897499999995 }, "net_j": { "core": 51.37893799813362, "dram": 34.68076963031782, "package-0": 399.10036088238843, "psys": 654.0435670011412, "uncore": 288.48385415070385 } }, "first_decode": { "gpu_active_fraction": 0.9565217391304348, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 23, "gross_j": { "core": 2.824029, "dram": 0.680601, "package-0": 6.076828, "psys": 9.977208, "uncore": 1.801876 }, "logits": { "l2": 842.3149968162743, "sha256": "d805104637e3082ed7ef6942f1d19e2c0a800de34252afd57ab996e71250bf5f", "sum": -27488.31816223578, "top1": 78 }, "net_j": { "core": 1.9392499431477224, "dram": 0.5647630796018874, "package-0": 4.733453524142605, "psys": 7.594715588934793, "uncore": 1.8014178473922577 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.234580089, "watts": { "core": 12.038656017391144, "dram": 2.9013587764475615, "package-0": 25.90513127480312, "psys": 42.53220314875062, "uncore": 7.681282787815807 } }, "metrics": { "all_tokens_per_s_e2e": 455.5582859604053, "decode_tokens_per_s": 4.994483062377571, "end_to_end_ms": 17982.331246, "handoff_estimate_ms": 37.42046249999997, "handoff_plus_first_decode_ms": 234.580089, "output_tokens_per_s_e2e": 0.8897622772664167, "prefill_tokens_per_s": 545.8301947938577, "steady_tpot_ms": 197.15962650000003, "steady_tpot_p95_ms": 200.97298634999999, "ttft_ms": 14979.017427 }, "prefill": { "gpu_active_fraction": 0.8695064232589588, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 1479, "gross_j": { "core": 99.275564, "dram": 33.811498, "package-0": 434.594835, "psys": 715.445557, "uncore": 260.794925 }, "logits": { "l2": 1025.1392851150267, "sha256": "4ff86f4268bae4a3871f14f73062d317d9c93b8e3d5513020b94dee3023132a8", "sum": -317470.8328210273, "top1": 8 }, "net_j": { "core": 42.77835245859381, "dram": 26.414714088170353, "package-0": 348.8141970497138, "psys": 563.312460379677, "uncore": 260.76566984912654 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 14.979017427, "watts": { "core": 6.627641932043798, "dram": 2.257257404551386, "package-0": 29.013574296043842, "psys": 47.76318343220524, "uncore": 17.410683061888395 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 727.8313868092002, "sha256": "474801b73ad22489ab599f54a4d0eb647b9530fa6cbcb6caa0057490e9977584", "sum": -118284.7835280532, "top1": 42 }, "gpu_active_fraction": 0.9963369963369964, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 273, "gross_j": { "core": 17.104326, "dram": 9.06852, "package-0": 61.408473, "psys": 111.256796, "uncore": 25.922174 }, "net_j": { "core": 6.661335596392092, "dram": 7.701292462545577, "package-0": 45.55271030853201, "psys": 83.13639103252953, "uncore": 25.916766454185044 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.76873373, "token_latencies_s": [ 0.206184388, 0.197772983, 0.198166847, 0.197165967, 0.197465865, 0.1964097, 0.196840054, 0.196136329, 0.19656939, 0.197153286, 0.197059042, 0.197635826, 0.197367107, 0.196766749 ], "watts": { "core": 6.177670974521627, "dram": 3.2753312107047576, "package-0": 22.1792627924535, "psys": 40.183277573607626, "uncore": 9.362465490677573 } } }, { "energy": { "gross_j": { "core": 98.41021, "dram": 41.596268, "package-0": 475.068426, "psys": 784.449713, "uncore": 287.320493 }, "net_j": { "core": 36.06931586162302, "dram": 33.43441168477512, "package-0": 380.41522943184714, "psys": 616.5810162520157, "uncore": 287.28821189773237 } }, "first_decode": { "gpu_active_fraction": 0.9130434782608695, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 23, "gross_j": { "core": 3.272147, "dram": 0.681761, "package-0": 6.495162, "psys": 10.508335, "uncore": 1.788021 }, "logits": { "l2": 842.3149968162743, "sha256": "d805104637e3082ed7ef6942f1d19e2c0a800de34252afd57ab996e71250bf5f", "sum": -27488.31816223578, "top1": 78 }, "net_j": { "core": 2.3839941330503915, "dram": 0.565481370293257, "package-0": 5.146665013233194, "psys": 8.116757748045972, "uncore": 1.787561100379959 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.235474582, "watts": { "core": 13.89596691162191, "dram": 2.8952636594976524, "package-0": 27.583282853008736, "psys": 44.626196639771514, "uncore": 7.593265416646966 } }, "metrics": { "all_tokens_per_s_e2e": 495.6334444277145, "decode_tokens_per_s": 5.00980860741616, "end_to_end_ms": 16528.343864, "handoff_estimate_ms": 38.78060449999998, "handoff_plus_first_decode_ms": 235.474582, "output_tokens_per_s_e2e": 0.9680340711478799, "prefill_tokens_per_s": 604.0984634963498, "steady_tpot_ms": 196.69397750000002, "steady_tpot_p95_ms": 198.88595655000003, "ttft_ms": 13534.217505999999 }, "prefill": { "gpu_active_fraction": 0.9514200298953662, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 1338, "gross_j": { "core": 78.102462, "dram": 31.843119, "package-0": 407.319148, "psys": 664.025033, "uncore": 259.633918 }, "logits": { "l2": 1025.1392851150267, "sha256": "4ff86f4268bae4a3871f14f73062d317d9c93b8e3d5513020b94dee3023132a8", "sum": -317470.8328210273, "top1": 8 }, "net_j": { "core": 27.05468445315004, "dram": 25.15979128538408, "package-0": 329.8124745829619, "psys": 526.5659219765113, "uncore": 259.60748465235076 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 13.534217506, "watts": { "core": 5.770740862216494, "dram": 2.3527861131153895, "package-0": 30.095507761673474, "psys": 49.062683727790244, "uncore": 19.183518950016794 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 727.8313868092002, "sha256": "474801b73ad22489ab599f54a4d0eb647b9530fa6cbcb6caa0057490e9977584", "sum": -118284.7835280532, "top1": 42 }, "gpu_active_fraction": 0.9963235294117647, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 272, "gross_j": { "core": 17.035601, "dram": 9.071388, "package-0": 61.254116, "psys": 109.916345, "uncore": 25.898554 }, "net_j": { "core": 6.630637275422586, "dram": 7.709139029097786, "package-0": 45.456089835652044, "psys": 81.89833652745844, "uncore": 25.893166145001672 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.758651776, "token_latencies_s": [ 0.200611, 0.196887108, 0.196252684, 0.197060868, 0.196389091, 0.196500847, 0.197099661, 0.19613736, 0.197644193, 0.197957087, 0.19644485, 0.196186573, 0.196951478, 0.196468223 ], "watts": { "core": 6.175335773876231, "dram": 3.288341094341876, "package-0": 22.204366833431028, "psys": 39.84422606588531, "uncore": 9.388120032152981 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 15.612135223113, "compile_target": "NPU", "context_length": 8192, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 401394, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_7b_ov_fp16/openvino_model.xml", "key": "qwen25_7b_fp16", "size_b": 7.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "gpu_npu", "label": "iGPU -> NPU", "mode": "staged", "prefill_device": "GPU" }, "prompt_length": 8176, "prompt_sha256": "998ace73fce48442f97bcb84c31be293d0d94180b17f70a4d127bba1092ad744", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "GPU", "NPUW_ONLINE_PIPELINE": "NONE" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 8192, "NPUW_LLM_MIN_RESPONSE_LEN": 8192, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.0465422049164772, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 388.932598, "dram": 158.13967000000002, "package-0": 1322.617158, "psys": 2648.284846, "uncore": 517.074421 }, "net_j": { "core": 133.16714723749644, "dram": 124.65409222037042, "package-0": 934.2843016789285, "psys": 1959.5713861423321, "uncore": 516.9494647944705 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 67, "gross_j": { "core": 8.604042, "dram": 1.106992, "package-0": 11.576936, "psys": 19.474925, "uncore": 0.0 }, "logits": { "l2": 963.664214914989, "sha256": "37bca8e9d81de23e1b20414b6d4ee54ba3041b1458dcaeccb8b11c2c6d324175", "sum": -155958.32397764642, "top1": 644 }, "net_j": { "core": 6.023795146479088, "dram": 0.7691783647078411, "package-0": 7.6593051094554605, "psys": 12.52695476994197, "uncore": 0.0 }, "npu_busy_delta_us": 265546, "npu_utilization": 0.38817020252266426, "seconds": 0.684096817, "watts": { "core": 12.577228524073076, "dram": 1.6181803108725763, "package-0": 16.922949664886396, "psys": 28.468083049127824, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 120.8069033498919, "decode_tokens_per_s": 3.914922688470836, "end_to_end_ms": 67810.694363, "handoff_estimate_ms": 459.60501500000004, "handoff_plus_first_decode_ms": 684.096817, "output_tokens_per_s_e2e": 0.2359509831052576, "prefill_tokens_per_s": 127.791530098787, "steady_tpot_ms": 224.49180199999998, "steady_tpot_p95_ms": 226.46343520000002, "ttft_ms": 63979.201076000005 }, "prefill": { "gpu_active_fraction": 0.99857120177806, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 6299, "gross_j": { "core": 364.959943, "dram": 147.338184, "package-0": 1262.941347, "psys": 2541.350367, "uncore": 517.074421 }, "logits": { "l2": null, "sha256": "7b769dbc741082009e7e9a93ca58b6e7f8d647e22fffd35e2edcda73c4a3253b", "sum": null, "top1": 0 }, "net_j": { "core": 123.64595327691865, "dram": 115.74463472235213, "package-0": 896.5503797289184, "psys": 1891.551137750049, "uncore": 516.9494647944705 }, "npu_busy_delta_us": 19467, "npu_utilization": 0.00030427075788075917, "seconds": 63.979201076, "watts": { "core": 5.704352928172223, "dram": 2.302907531230017, "package-0": 19.739873673942405, "psys": 39.72150830675684, "uncore": 8.081914314399995 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 1793.833741861425, "sha256": "abce275f1b23bba1147d8eaa404898b4767f46363c829f7f936abbf8b0cc514d", "sum": -631544.1516896337, "top1": 15 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 309, "gross_j": { "core": 15.368613, "dram": 9.694494, "package-0": 48.098875, "psys": 87.459554, "uncore": 0.0 }, "net_j": { "core": 3.4973988140987053, "dram": 8.14027913331044, "package-0": 30.074616840554654, "psys": 55.493293622341014, "uncore": 0.0 }, "npu_busy_delta_us": 3129592, "npu_utilization": 0.9943431117847064, "seconds": 3.14739647, "token_latencies_s": [ 0.228600032, 0.22531296, 0.225197055, 0.224492072, 0.224491532, 0.224765094, 0.224612395, 0.224616439, 0.224083261, 0.224131079, 0.224280993, 0.224314108, 0.224252064, 0.224197104 ], "watts": { "core": 4.882960614110367, "dram": 3.0801629513170297, "package-0": 15.282115061913379, "psys": 27.787904966418164, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 405.725036, "dram": 157.771631, "package-0": 1335.555419, "psys": 2675.2137940000002, "uncore": 513.700101 }, "net_j": { "core": 151.32154861726053, "dram": 124.46436553952897, "package-0": 949.290453896807, "psys": 1990.16776658358, "uncore": 513.5753062151273 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 40, "gross_j": { "core": 4.874621, "dram": 0.902342, "package-0": 7.28941, "psys": 12.405242, "uncore": 0.0 }, "logits": { "l2": 989.2704500041049, "sha256": "cb3068c96c42c3db93aa7cf355e8b3d774627ba1dca0013bcfc7356782d3aa5c", "sum": -167179.82651147246, "top1": 644 }, "net_j": { "core": 3.3163420704470203, "dram": 0.6983274619052634, "package-0": 4.9234496802844285, "psys": 8.209180062016493, "uncore": 0.0 }, "npu_busy_delta_us": 253173, "npu_utilization": 0.6127959366657525, "seconds": 0.413144058, "watts": { "core": 11.798840877919634, "dram": 2.184085629521507, "package-0": 17.64374885430399, "psys": 30.02643208776344, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 121.45364990229947, "decode_tokens_per_s": 4.221727922177888, "end_to_end_ms": 67449.59914, "handoff_estimate_ms": 189.06075549999997, "handoff_plus_first_decode_ms": 413.144058, "output_tokens_per_s_e2e": 0.23721415996542866, "prefill_tokens_per_s": 127.95682701199303, "steady_tpot_ms": 224.0833025, "steady_tpot_p95_ms": 225.22429145, "ttft_ms": 63896.551602 }, "prefill": { "gpu_active_fraction": 0.9982520260607024, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 6293, "gross_j": { "core": 384.148187, "dram": 147.189443, "package-0": 1278.945456, "psys": 2573.526615, "uncore": 513.700101 }, "logits": { "l2": null, "sha256": "7b769dbc741082009e7e9a93ca58b6e7f8d647e22fffd35e2edcda73c4a3253b", "sum": null, "top1": 0 }, "net_j": { "core": 143.1459309963779, "dram": 115.63670683326436, "package-0": 913.0277991206399, "psys": 1924.5668079935267, "uncore": 513.5753062151273 }, "npu_busy_delta_us": 18955, "npu_utilization": 0.00029665137671383655, "seconds": 63.896551602, "watts": { "core": 6.012033159360292, "dram": 2.303558475531141, "package-0": 20.01587603610158, "psys": 40.276455465547336, "uncore": 8.039559070413448 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 1742.1929938026972, "sha256": "e2843afb7cd2280dfd2dae620c6d9141e0428f52bd1f1c78f70eb6d15dc51199", "sum": -606370.0193425957, "top1": 15 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 309, "gross_j": { "core": 16.702228, "dram": 9.679846, "package-0": 49.320553, "psys": 89.281937, "uncore": 0.0 }, "net_j": { "core": 4.859275550435603, "dram": 8.129331244359351, "package-0": 31.339205095882647, "psys": 57.391778528036795, "uncore": 0.0 }, "npu_busy_delta_us": 3125576, "npu_utilization": 0.9954369680178832, "seconds": 3.13990348, "token_latencies_s": [ 0.225399757, 0.22439506, 0.224244435, 0.224206925, 0.224036196, 0.224085891, 0.223933938, 0.224080714, 0.223920374, 0.223989555, 0.223982291, 0.224072247, 0.22512981, 0.224385639 ], "watts": { "core": 5.31934440226806, "dram": 3.082848266405947, "package-0": 15.70766531969957, "psys": 28.434611945460183, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 384.387932, "dram": 151.061503, "package-0": 1251.380952, "psys": 2532.750842, "uncore": 465.706999 }, "net_j": { "core": 135.3611980303917, "dram": 118.45817817764305, "package-0": 873.2795993176032, "psys": 1862.1830892485877, "uncore": 465.58499751665295 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 41, "gross_j": { "core": 5.410326, "dram": 0.902525, "package-0": 7.822673, "psys": 13.118253, "uncore": 0.0 }, "logits": { "l2": null, "sha256": "7b769dbc741082009e7e9a93ca58b6e7f8d647e22fffd35e2edcda73c4a3253b", "sum": null, "top1": 0 }, "net_j": { "core": 3.8428971579684044, "dram": 0.6973125279972001, "package-0": 5.442820218901095, "psys": 8.897552597901203, "uncore": 0.0 }, "npu_busy_delta_us": 253201, "npu_utilization": 0.6092860995053637, "seconds": 0.41556996, "watts": { "core": 13.019049788873094, "dram": 2.171776323774702, "package-0": 18.823961674226886, "psys": 31.566894296209473, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 124.07596404600552, "decode_tokens_per_s": 4.216178673446935, "end_to_end_ms": 66024.068908, "handoff_estimate_ms": 191.29923050000002, "handoff_plus_first_decode_ms": 415.56996000000004, "output_tokens_per_s_e2e": 0.24233586727735454, "prefill_tokens_per_s": 130.8864799171463, "steady_tpot_ms": 224.2707295, "steady_tpot_p95_ms": 225.40313005000002, "ttft_ms": 62466.344921 }, "prefill": { "gpu_active_fraction": 0.9980456026058632, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 6140, "gross_j": { "core": 364.317732, "dram": 140.47132, "package-0": 1196.604908, "psys": 2433.9248, "uncore": 465.706999 }, "logits": { "l2": null, "sha256": "7b769dbc741082009e7e9a93ca58b6e7f8d647e22fffd35e2edcda73c4a3253b", "sum": null, "top1": 0 }, "net_j": { "core": 128.7098678380444, "dram": 109.62483374719685, "package-0": 838.8776442552257, "psys": 1799.4907636088546, "uncore": 465.58499751665295 }, "npu_busy_delta_us": 17037, "npu_utilization": 0.00027273886476864256, "seconds": 62.466344921, "watts": { "core": 5.8322242554890265, "dram": 2.2487520308359867, "package-0": 19.155993671685504, "psys": 38.96377806446236, "uncore": 7.45532653765753 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": null, "sha256": "7b769dbc741082009e7e9a93ca58b6e7f8d647e22fffd35e2edcda73c4a3253b", "sum": null, "top1": 0 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 309, "gross_j": { "core": 14.659874, "dram": 9.687658, "package-0": 46.953371, "psys": 85.707789, "uncore": 0.0 }, "net_j": { "core": 2.808433034378888, "dram": 8.136031902449007, "package-0": 28.959134843476377, "psys": 53.794773041831974, "uncore": 0.0 }, "npu_busy_delta_us": 3125760, "npu_utilization": 0.9947825514411042, "seconds": 3.142154027, "token_latencies_s": [ 0.226267502, 0.224937699, 0.224006437, 0.224226763, 0.224302905, 0.22430406, 0.224367411, 0.224536623, 0.224238554, 0.223990205, 0.224109749, 0.224024072, 0.22414712, 0.224649712 ], "watts": { "core": 4.665549134138611, "dram": 3.0831263893353373, "package-0": 14.94305199443999, "psys": 27.276762457704944, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 6.129528114572167, "compile_target": "AUTO:NPU,GPU", "context_length": 8192, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "GPU.0" ], "host": { "hostname": "wuklab-NUC16", "pid": 401237, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 3.7717568469857548, "dram": 0.4938096873094483, "package-0": 5.7267199513143465, "psys": 10.156413620702507, "uncore": 0.0019530754280777179 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_7b_ov_fp16/openvino_model.xml", "key": "qwen25_7b_fp16", "size_b": 7.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": null, "key": "openvino_auto", "label": "OpenVINO AUTO", "mode": "stock", "prefill_device": null }, "prompt_length": 8176, "prompt_sha256": "998ace73fce48442f97bcb84c31be293d0d94180b17f70a4d127bba1092ad744", "properties": { "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.04655962623655796, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 95.417542, "dram": 41.425066, "package-0": 470.951858, "psys": 788.19183, "uncore": 286.679137 }, "net_j": { "core": 33.85747181940967, "dram": 33.365437492685516, "package-0": 377.48419918492226, "psys": 622.4257001464254, "uncore": 286.6472602207436 } }, "first_decode": { "gpu_active_fraction": 0.9166666666666666, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 24, "gross_j": { "core": 3.162224, "dram": 0.687926, "package-0": 6.386031, "psys": 10.566563, "uncore": 1.796749 }, "logits": { "l2": 842.3149968162743, "sha256": "d805104637e3082ed7ef6942f1d19e2c0a800de34252afd57ab996e71250bf5f", "sum": -27488.31816223578, "top1": 78 }, "net_j": { "core": 2.265249997072613, "dram": 0.570491480788017, "package-0": 5.024140738574577, "psys": 8.151232598193172, "uncore": 1.7962845326560506 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.237813316, "watts": { "core": 13.297085517280287, "dram": 2.8927143844207617, "package-0": 26.853126256395164, "psys": 44.43217553049048, "uncore": 7.555291815534837 } }, "metrics": { "all_tokens_per_s_e2e": 501.9200270543129, "decode_tokens_per_s": 5.0185961621677, "end_to_end_ms": 16321.325228, "handoff_estimate_ms": 41.48733100000002, "handoff_plus_first_decode_ms": 237.813316, "output_tokens_per_s_e2e": 0.9803125528404549, "prefill_tokens_per_s": 613.2410144329137, "steady_tpot_ms": 196.32598499999997, "steady_tpot_p95_ms": 197.32361790000002, "ttft_ms": 13332.441581 }, "prefill": { "gpu_active_fraction": 0.9604863221884499, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 1316, "gross_j": { "core": 72.611569, "dram": 31.694499, "package-0": 400.913158, "psys": 663.302989, "uncore": 259.087532 }, "logits": { "l2": 1025.1392851150267, "sha256": "4ff86f4268bae4a3871f14f73062d317d9c93b8e3d5513020b94dee3023132a8", "sum": -317470.8328210273, "top1": 8 }, "net_j": { "core": 22.324841179825675, "dram": 25.110810191814906, "package-0": 324.56199879835435, "psys": 527.8931977295111, "uncore": 259.0614927359519 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 13.332441581, "watts": { "core": 5.446231926752142, "dram": 2.377246418628054, "package-0": 30.07049800775722, "psys": 49.751051596226006, "uncore": 19.432864597676126 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 727.8313868092002, "sha256": "474801b73ad22489ab599f54a4d0eb647b9530fa6cbcb6caa0057490e9977584", "sum": -118284.7835280532, "top1": 42 }, "gpu_active_fraction": 0.9963235294117647, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 272, "gross_j": { "core": 19.643749, "dram": 9.042641, "package-0": 63.652669, "psys": 114.322278, "uncore": 25.794856 }, "net_j": { "core": 9.267380642511382, "dram": 7.684135820082589, "package-0": 47.898059647993335, "psys": 86.38126981872104, "uncore": 25.78948295213561 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.751070331, "token_latencies_s": [ 0.198835878, 0.19628435, 0.196502363, 0.196204352, 0.196407612, 0.196509324, 0.196200959, 0.19615562, 0.196468163, 0.196266558, 0.196319751, 0.196047689, 0.196503537, 0.196332219 ], "watts": { "core": 7.140402329467017, "dram": 3.2869537714483092, "package-0": 23.137419746322, "psys": 41.55556356076307, "uncore": 9.376298275378405 } } }, { "energy": { "gross_j": { "core": 95.01428399999999, "dram": 41.655656, "package-0": 471.34614400000004, "psys": 780.6806190000001, "uncore": 287.188657 }, "net_j": { "core": 32.700097034429604, "dram": 33.49729626771712, "package-0": 376.73349736800674, "psys": 612.88383810043, "uncore": 287.1563897271294 } }, "first_decode": { "gpu_active_fraction": 0.9545454545454546, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 22, "gross_j": { "core": 2.889397, "dram": 0.679625, "package-0": 6.096908, "psys": 10.03037, "uncore": 1.799922 }, "logits": { "l2": 842.3149968162743, "sha256": "d805104637e3082ed7ef6942f1d19e2c0a800de34252afd57ab996e71250bf5f", "sum": -27488.31816223578, "top1": 78 }, "net_j": { "core": 2.009652838732223, "dram": 0.5644462631906187, "package-0": 4.761178088245295, "psys": 7.661435324538587, "uncore": 1.7994664545409282 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.233245195, "watts": { "core": 12.387809318001173, "dram": 2.913779209899694, "package-0": 26.139479529256757, "psys": 43.00354397439999, "uncore": 7.716866364599708 } }, "metrics": { "all_tokens_per_s_e2e": 495.84586745196697, "decode_tokens_per_s": 5.011159632009462, "end_to_end_ms": 16521.263033, "handoff_estimate_ms": 36.44838249999999, "handoff_plus_first_decode_ms": 233.245195, "output_tokens_per_s_e2e": 0.968448959867123, "prefill_tokens_per_s": 604.3786150972744, "steady_tpot_ms": 196.7968125, "steady_tpot_p95_ms": 199.01717010000002, "ttft_ms": 13527.943901 }, "prefill": { "gpu_active_fraction": 0.9506357516828721, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 1337, "gross_j": { "core": 70.247745, "dram": 31.93516, "package-0": 399.082962, "psys": 655.58725, "uncore": 259.349922 }, "logits": { "l2": 1025.1392851150267, "sha256": "4ff86f4268bae4a3871f14f73062d317d9c93b8e3d5513020b94dee3023132a8", "sum": -317470.8328210273, "top1": 8 }, "net_j": { "core": 19.22362996576406, "dram": 25.25493025230743, "package-0": 321.61221576188206, "psys": 518.1918563037842, "uncore": 259.3235009051745 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 13.527943901, "watts": { "core": 5.192788018200401, "dram": 2.360680989935161, "package-0": 29.50063697192737, "psys": 48.46170673072782, "uncore": 19.171422050384802 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 727.8313868092002, "sha256": "474801b73ad22489ab599f54a4d0eb647b9530fa6cbcb6caa0057490e9977584", "sum": -118284.7835280532, "top1": 42 }, "gpu_active_fraction": 0.9963369963369964, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 273, "gross_j": { "core": 21.877142, "dram": 9.040871, "package-0": 66.166274, "psys": 115.062999, "uncore": 26.038813 }, "net_j": { "core": 11.46681422993332, "dram": 7.677919752219071, "package-0": 50.36010351787937, "psys": 87.0305464721072, "uncore": 26.03342236741397 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.760073937, "token_latencies_s": [ 0.201836562, 0.197499036, 0.197044282, 0.196866754, 0.196715114, 0.196646921, 0.19685332, 0.196865295, 0.196312369, 0.196651672, 0.196740305, 0.196678852, 0.196972966, 0.196358737 ], "watts": { "core": 7.926288389136004, "dram": 3.2755901495257658, "package-0": 23.972645483518438, "psys": 41.68837561107697, "uncore": 9.434099808319736 } } }, { "energy": { "gross_j": { "core": 96.90227900000001, "dram": 41.682266, "package-0": 474.959294, "psys": 790.577383, "uncore": 288.63939799999997 }, "net_j": { "core": 34.54050157181344, "dram": 33.517675582013666, "package-0": 380.27438999224984, "psys": 622.6524526882756, "uncore": 288.60710608403417 } }, "first_decode": { "gpu_active_fraction": 0.9090909090909091, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 22, "gross_j": { "core": 3.508414, "dram": 0.681578, "package-0": 6.713057, "psys": 10.765841, "uncore": 1.788814 }, "logits": { "l2": 842.3149968162743, "sha256": "d805104637e3082ed7ef6942f1d19e2c0a800de34252afd57ab996e71250bf5f", "sum": -27488.31816223578, "top1": 78 }, "net_j": { "core": 2.6371966645232305, "dram": 0.567515620725958, "package-0": 5.3902735069461105, "psys": 8.419866969436566, "uncore": 1.7883628698663607 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 0.230984491, "watts": { "core": 15.18895915830124, "dram": 2.95075222171518, "package-0": 29.06280404774016, "psys": 46.60850152056313, "uncore": 7.744303490921388 } }, "metrics": { "all_tokens_per_s_e2e": 495.4674700554924, "decode_tokens_per_s": 5.0314153237436505, "end_to_end_ms": 16533.880618, "handoff_estimate_ms": 34.78480849999998, "handoff_plus_first_decode_ms": 230.984491, "output_tokens_per_s_e2e": 0.9677099024521336, "prefill_tokens_per_s": 603.2785360492352, "steady_tpot_ms": 196.1996825, "steady_tpot_p95_ms": 197.8425613, "ttft_ms": 13552.612121 }, "prefill": { "gpu_active_fraction": 0.9507462686567164, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 1340, "gross_j": { "core": 76.778429, "dram": 31.930155, "package-0": 407.529169, "psys": 670.526787, "uncore": 260.990238 }, "logits": { "l2": 1025.1392851150267, "sha256": "4ff86f4268bae4a3871f14f73062d317d9c93b8e3d5513020b94dee3023132a8", "sum": -317470.8328210273, "top1": 8 }, "net_j": { "core": 25.66127143807612, "dram": 25.23774384630275, "package-0": 329.9171547742447, "psys": 532.8808526581777, "uncore": 260.9637687262802 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 13.552612121, "watts": { "core": 5.665212603630155, "dram": 2.356014819499164, "package-0": 30.070156613463965, "psys": 49.47583395831181, "uncore": 19.2575597729674 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 727.8313868092002, "sha256": "474801b73ad22489ab599f54a4d0eb647b9530fa6cbcb6caa0057490e9977584", "sum": -118284.7835280532, "top1": 42 }, "gpu_active_fraction": 0.9963235294117647, "gpu_freq_mhz_after": 2450, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 2450, "gpu_sample_count": 272, "gross_j": { "core": 16.615436, "dram": 9.070533, "package-0": 60.717068, "psys": 109.284755, "uncore": 25.860346 }, "net_j": { "core": 6.242033469214089, "dram": 7.712416114984963, "package-0": 44.96696171105906, "psys": 81.35173306066136, "uncore": 25.854974487887645 }, "npu_busy_delta_us": 0, "npu_utilization": 0.0, "seconds": 2.750284006, "token_latencies_s": [ 0.200131574, 0.196610016, 0.19598051, 0.196225125, 0.1963334, 0.196521574, 0.196343023, 0.195932347, 0.196058054, 0.196261632, 0.195909876, 0.19617424, 0.196118327, 0.195653409 ], "watts": { "core": 6.041352807110787, "dram": 3.298035032095518, "package-0": 22.07665385376204, "psys": 39.73580719721497, "uncore": 9.402791109421155 } } } ] } ]