[ { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 6.715313421562314, "compile_target": "NPU", "context_length": 1024, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 408986, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 1.5259465821328744, "dram": 0.48698386485903, "package-0": 3.248422500499841, "psys": 8.010065800945089, "uncore": 0.0023192603038971636 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_1b_ov_fp16/openvino_model.xml", "key": "llama32_1b_fp16", "size_b": 1.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "npu_npu", "label": "NPU -> NPU", "mode": "staged", "prefill_device": "NPU" }, "prompt_length": 1008, "prompt_sha256": "2669029615323cd09063f0eb8d822f5daea575e1396736877003e47194c143f8", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "NPU" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 1024, "NPUW_LLM_MIN_RESPONSE_LEN": 1024, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.025821217335760593, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 7.257427, "dram": 2.338556, "package-0": 18.084488, "psys": 30.446029000000003, "uncore": 0.0 }, "net_j": { "core": 5.718181396657727, "dram": 1.8473279441429187, "package-0": 14.807754652818453, "psys": 22.366153392982568, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 11, "gross_j": { "core": 2.491815, "dram": 0.163513, "package-0": 3.020195, "psys": 4.277272, "uncore": 0.0 }, "logits": { "l2": 403.47644126655484, "sha256": "82fc2ee3733521e21ac618cc56a1a447e6bfe4441ae27196716ae874e9aa61fb", "sum": 41930.551351014525, "top1": 546 }, "net_j": { "core": 2.330040537078452, "dram": 0.11188501111003939, "package-0": 2.675810857387284, "psys": 3.4280783859595076, "uncore": 0.0 }, "npu_busy_delta_us": 39835, "npu_utilization": 0.3757458439453512, "seconds": 0.10601581, "watts": { "core": 23.504183008175854, "dram": 1.5423454294222718, "package-0": 28.48815662494113, "psys": 40.34560505645337, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 1015.1526804501798, "decode_tokens_per_s": 23.534573459649497, "end_to_end_ms": 1008.7152600000001, "handoff_estimate_ms": 68.2995795, "handoff_plus_first_decode_ms": 106.01581, "output_tokens_per_s_e2e": 15.86176063203406, "prefill_tokens_per_s": 2714.383233196603, "steady_tpot_ms": 37.7162305, "steady_tpot_p95_ms": 38.885700449999995, "ttft_ms": 371.355079 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 37, "gross_j": { "core": 0.991452, "dram": 0.557799, "package-0": 6.010361, "psys": 10.145238, "uncore": 0.0 }, "logits": { "l2": 628.0526414006035, "sha256": "701f4560c1677db1b1d9f45ca56d23a3429b2db7c259cf40c52b2ab0d3b85d4b", "sum": -187921.24455487914, "top1": 5809 }, "net_j": { "core": 0.42478398644226645, "dram": 0.37695506839354964, "package-0": 4.804042805701504, "psys": 7.17065938169484, "uncore": 0.0 }, "npu_busy_delta_us": 366875, "npu_utilization": 0.987935861784726, "seconds": 0.371355079, "watts": { "core": 2.669822108451626, "dram": 1.5020637431486432, "package-0": 16.184943575256714, "psys": 27.31950786110024, "uncore": 0.0 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 515.7828349073951, "sha256": "a5e5c388249e29831855bd73b45711046f70e06f6e5b7ebe55336c2dfcae2631", "sum": 92985.67104252055, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 53, "gross_j": { "core": 3.77416, "dram": 1.617244, "package-0": 9.053932, "psys": 16.023519, "uncore": 0.0 }, "net_j": { "core": 2.963356873137008, "dram": 1.3584878646393297, "package-0": 7.327900989729665, "psys": 11.767415625328221, "uncore": 0.0 }, "npu_busy_delta_us": 520848, "npu_utilization": 0.9802456343326915, "seconds": 0.531344371, "token_latencies_s": [ 0.038971297, 0.03883961, 0.037966017, 0.038545887, 0.037585356, 0.037704661, 0.0376612, 0.037537949, 0.03772751, 0.037843312, 0.038023015, 0.037704951, 0.037586166, 0.037621356 ], "watts": { "core": 7.103039395894909, "dram": 3.0436833215270855, "package-0": 17.039668610698428, "psys": 30.15656112032097, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 5.938949, "dram": 2.293145, "package-0": 16.629473, "psys": 28.042774, "uncore": 0.0 }, "net_j": { "core": 4.491686849301956, "dram": 1.831272137697043, "package-0": 13.548553268276429, "psys": 20.44574197145735, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 4, "gross_j": { "core": 1.322628, "dram": 0.118713, "package-0": 1.708247, "psys": 2.344537, "uncore": 0.0 }, "logits": { "l2": 403.47644126655484, "sha256": "82fc2ee3733521e21ac618cc56a1a447e6bfe4441ae27196716ae874e9aa61fb", "sum": 41930.551351014525, "top1": 546 }, "net_j": { "core": 1.2589365598608067, "dram": 0.09838679458718103, "package-0": 1.5726611874558898, "psys": 2.010205095826261, "uncore": 0.0 }, "npu_busy_delta_us": 37245, "npu_utilization": 0.8923315335205556, "seconds": 0.041738971, "watts": { "core": 31.688083541877443, "dram": 2.8441764891616517, "package-0": 40.9269073739264, "psys": 56.1714135214306, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 1079.6726075855745, "decode_tokens_per_s": 26.146813717555577, "end_to_end_ms": 948.435658, "handoff_estimate_ms": 3.849347000000003, "handoff_plus_first_decode_ms": 41.738971, "output_tokens_per_s_e2e": 16.8698844935246, "prefill_tokens_per_s": 2689.7789121813603, "steady_tpot_ms": 37.889624, "steady_tpot_p95_ms": 38.696261199999995, "ttft_ms": 374.75199 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 38, "gross_j": { "core": 0.90344, "dram": 0.558531, "package-0": 5.928146, "psys": 9.847509, "uncore": 0.0 }, "logits": { "l2": 628.0526414006035, "sha256": "701f4560c1677db1b1d9f45ca56d23a3429b2db7c259cf40c52b2ab0d3b85d4b", "sum": -187921.24455487914, "top1": 5809 }, "net_j": { "core": 0.33158848171200694, "dram": 0.37603282754618744, "package-0": 4.710793203576909, "psys": 6.845720901064885, "uncore": 0.0 }, "npu_busy_delta_us": 371268, "npu_utilization": 0.9907032114759418, "seconds": 0.37475199, "watts": { "core": 2.4107677186717544, "dram": 1.490401692063063, "package-0": 15.818851288821708, "psys": 26.27740282313111, "uncore": 0.0 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 515.7828349073951, "sha256": "a5e5c388249e29831855bd73b45711046f70e06f6e5b7ebe55336c2dfcae2631", "sum": 92985.67104252055, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 53, "gross_j": { "core": 3.712881, "dram": 1.615901, "package-0": 8.99308, "psys": 15.850728, "uncore": 0.0 }, "net_j": { "core": 2.9011618077291423, "dram": 1.3568525155636744, "package-0": 7.265098877243631, "psys": 11.589815974566203, "uncore": 0.0 }, "npu_busy_delta_us": 519663, "npu_utilization": 0.9769117032855766, "seconds": 0.531944697, "token_latencies_s": [ 0.038830624, 0.038623912, 0.037920425, 0.037767377, 0.037724127, 0.037836459, 0.037921237, 0.037796923, 0.037858823, 0.037717425, 0.037835252, 0.037922573, 0.038166028, 0.037974961 ], "watts": { "core": 6.979825197881425, "dram": 3.037723675248895, "package-0": 16.906043148316225, "psys": 29.797699064194262, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 6.222091000000001, "dram": 2.299615, "package-0": 16.932331, "psys": 28.363757, "uncore": 0.0 }, "net_j": { "core": 4.772175370108659, "dram": 1.8368953180455525, "package-0": 13.845762563651352, "psys": 20.752796211965624, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 5, "gross_j": { "core": 1.419979, "dram": 0.123535, "package-0": 1.821956, "psys": 2.481256, "uncore": 0.0 }, "logits": { "l2": 403.47644126655484, "sha256": "82fc2ee3733521e21ac618cc56a1a447e6bfe4441ae27196716ae874e9aa61fb", "sum": 41930.551351014525, "top1": 546 }, "net_j": { "core": 1.351889310293606, "dram": 0.10180515661064307, "package-0": 1.6770072300468963, "psys": 2.1238366110776052, "uncore": 0.0 }, "npu_busy_delta_us": 37224, "npu_utilization": 0.8342208022719205, "seconds": 0.04462128, "watts": { "core": 31.82291050368793, "dram": 2.7685221042516037, "package-0": 40.83154943112344, "psys": 55.60701082532819, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 1077.6967072361067, "decode_tokens_per_s": 25.956409774464063, "end_to_end_ms": 950.1745649999999, "handoff_estimate_ms": 6.635176999999999, "handoff_plus_first_decode_ms": 44.62128, "output_tokens_per_s_e2e": 16.839011050564167, "prefill_tokens_per_s": 2707.6203193742926, "steady_tpot_ms": 37.986103, "steady_tpot_p95_ms": 38.633267800000006, "ttft_ms": 372.282625 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 37, "gross_j": { "core": 1.010984, "dram": 0.558897, "package-0": 6.023972, "psys": 9.9791, "uncore": 0.0 }, "logits": { "l2": 628.0526414006035, "sha256": "701f4560c1677db1b1d9f45ca56d23a3429b2db7c259cf40c52b2ab0d3b85d4b", "sum": -187921.24455487914, "top1": 5809 }, "net_j": { "core": 0.44290060079379556, "dram": 0.37760136845763503, "package-0": 4.814640744404855, "psys": 6.9970916772014355, "uncore": 0.0 }, "npu_busy_delta_us": 367567, "npu_utilization": 0.987333212233582, "seconds": 0.372282625, "watts": { "core": 2.715635735081647, "dram": 1.5012707079735457, "package-0": 16.181179554108926, "psys": 26.805172548678577, "uncore": 0.0 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 515.7828349073951, "sha256": "a5e5c388249e29831855bd73b45711046f70e06f6e5b7ebe55336c2dfcae2631", "sum": 92985.67104252055, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 53, "gross_j": { "core": 3.791128, "dram": 1.617183, "package-0": 9.086403, "psys": 15.903401, "uncore": 0.0 }, "net_j": { "core": 2.977385459021258, "dram": 1.3574887929772743, "package-0": 7.3541145891996, "psys": 11.631867923686585, "uncore": 0.0 }, "npu_busy_delta_us": 520296, "npu_utilization": 0.9756696533801428, "seconds": 0.53327066, "token_latencies_s": [ 0.038574199, 0.038742967, 0.038039353, 0.0379759, 0.038109088, 0.037935448, 0.037874742, 0.038142298, 0.037996306, 0.037838648, 0.037940378, 0.037896361, 0.037962355, 0.03819538 ], "watts": { "core": 7.1092004199143455, "dram": 3.032574490409804, "package-0": 17.039007921418367, "psys": 29.822381377591636, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 14.859274039976299, "compile_target": "NPU", "context_length": 2048, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 409111, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 1.5259465821328744, "dram": 0.48698386485903, "package-0": 3.248422500499841, "psys": 8.010065800945089, "uncore": 0.0023192603038971636 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_1b_ov_fp16/openvino_model.xml", "key": "llama32_1b_fp16", "size_b": 1.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "npu_npu", "label": "NPU -> NPU", "mode": "staged", "prefill_device": "NPU" }, "prompt_length": 2032, "prompt_sha256": "cd1a97358e68d07c27508cfacc4ba6882a7a2962b1281e2e2b0c7e5e320da4fa", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "NPU" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 2048, "NPUW_LLM_MIN_RESPONSE_LEN": 2048, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.02583260927349329, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 8.978614, "dram": 2.921929, "package-0": 26.566399, "psys": 43.076733000000004, "uncore": 0.0 }, "net_j": { "core": 6.875950641690203, "dram": 2.2508942857336987, "package-0": 22.09026658220391, "psys": 32.03934017478309, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 11, "gross_j": { "core": 2.564385, "dram": 0.172119, "package-0": 3.118522, "psys": 4.519947, "uncore": 0.0 }, "logits": { "l2": 392.44545785960474, "sha256": "da8a47e01f0c4f258fd33ddaf9c7ae00cfdff7bbd7a3dcc5f722a75b5d939ae9", "sum": 47131.24965239875, "top1": 3862 }, "net_j": { "core": 2.393072971142117, "dram": 0.11744723436489787, "package-0": 2.753834365672465, "psys": 3.620688376066361, "uncore": 0.0 }, "npu_busy_delta_us": 40347, "npu_utilization": 0.35938729556691, "seconds": 0.112266072, "watts": { "core": 22.842030137119256, "dram": 1.533134605439834, "package-0": 27.777955925989822, "psys": 40.261023829176104, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 1486.2762447718853, "decode_tokens_per_s": 22.261778896810455, "end_to_end_ms": 1377.940344, "handoff_estimate_ms": 72.4623745, "handoff_plus_first_decode_ms": 112.266072, "output_tokens_per_s_e2e": 11.611533162280354, "prefill_tokens_per_s": 2885.79081098679, "steady_tpot_ms": 39.8036975, "steady_tpot_p95_ms": 41.9399647, "ttft_ms": 704.1397430000001 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 70, "gross_j": { "core": 2.140863, "dram": 1.07446, "package-0": 13.678982, "psys": 21.48194, "uncore": 0.0 }, "logits": { "l2": 443.1691514737982, "sha256": "4995d5054aceead0555f3f6020af75f0389032b26365d691afaab466921c7c37", "sum": -56908.99897800945, "top1": 8 }, "net_j": { "core": 1.0663833658252293, "dram": 0.7315553065530158, "package-0": 11.391638615342623, "psys": 15.841734325509437, "uncore": 0.0 }, "npu_busy_delta_us": 685860, "npu_utilization": 0.9740396090666338, "seconds": 0.704139743, "watts": { "core": 3.0403950654437066, "dram": 1.5259186982149933, "package-0": 19.426516023254774, "psys": 30.50806351091022, "uncore": 0.0 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 453.91327201082566, "sha256": "e94495b9cabc339c33632c206c7636bb049ae1092943ac3356384433083bc8a8", "sum": 9498.106287172064, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 56, "gross_j": { "core": 4.273366, "dram": 1.67535, "package-0": 9.768895, "psys": 17.074846, "uncore": 0.0 }, "net_j": { "core": 3.416494304722857, "dram": 1.4018917448157848, "package-0": 7.94479360118882, "psys": 12.576917473207292, "uncore": 0.0 }, "npu_busy_delta_us": 549591, "npu_utilization": 0.9787305528276785, "seconds": 0.561534529, "token_latencies_s": [ 0.042701948, 0.040468577, 0.03970427, 0.039797149, 0.039703644, 0.039811891, 0.039641551, 0.041529666, 0.03946634, 0.039810246, 0.039702334, 0.039452351, 0.039886598, 0.039829901 ], "watts": { "core": 7.610157130693561, "dram": 2.9835208940463924, "package-0": 17.396784161068037, "psys": 30.407472948115004, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 9.011207, "dram": 2.929984, "package-0": 26.477105, "psys": 42.398208000000004, "uncore": 0.0 }, "net_j": { "core": 6.970217923886795, "dram": 2.278631744253657, "package-0": 22.132264285554072, "psys": 31.684558524914166, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 8, "gross_j": { "core": 2.282526, "dram": 0.149047, "package-0": 2.761773, "psys": 3.796805, "uncore": 0.0 }, "logits": { "l2": 392.44545785960474, "sha256": "da8a47e01f0c4f258fd33ddaf9c7ae00cfdff7bbd7a3dcc5f722a75b5d939ae9", "sum": 47131.24965239875, "top1": 3862 }, "net_j": { "core": 2.1696686769564257, "dram": 0.11303021147219683, "package-0": 2.521523261369895, "psys": 3.204389384484811, "uncore": 0.0 }, "npu_busy_delta_us": 39529, "npu_utilization": 0.5344725607379613, "seconds": 0.073958895, "watts": { "core": 30.862089002276196, "dram": 2.015268075597939, "package-0": 37.34199922808473, "psys": 51.33669182050381, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 1531.1883031532643, "decode_tokens_per_s": 23.642482614634385, "end_to_end_ms": 1337.523279, "handoff_estimate_ms": 34.0508135, "handoff_plus_first_decode_ms": 73.958895, "output_tokens_per_s_e2e": 11.962408618384877, "prefill_tokens_per_s": 2890.172830784942, "steady_tpot_ms": 39.908081499999994, "steady_tpot_p95_ms": 40.625068549999995, "ttft_ms": 703.072141 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 70, "gross_j": { "core": 2.691277, "dram": 1.107602, "package-0": 14.195825, "psys": 21.975652, "uncore": 0.0 }, "logits": { "l2": 443.1691514737982, "sha256": "4995d5054aceead0555f3f6020af75f0389032b26365d691afaab466921c7c37", "sum": -56908.99897800945, "top1": 8 }, "net_j": { "core": 1.6184264694482076, "dram": 0.7652172115011071, "package-0": 11.911949637701003, "psys": 16.343997887778656, "uncore": 0.0 }, "npu_busy_delta_us": 685677, "npu_utilization": 0.9752583839045899, "seconds": 0.703072141, "watts": { "core": 3.8278817251557116, "dram": 1.5753746100999326, "package-0": 20.191135691721282, "psys": 31.256610408063374, "uncore": 0.0 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 453.91327201082566, "sha256": "e94495b9cabc339c33632c206c7636bb049ae1092943ac3356384433083bc8a8", "sum": 9498.106287172064, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 56, "gross_j": { "core": 4.037404, "dram": 1.673335, "package-0": 9.519507, "psys": 16.625751, "uncore": 0.0 }, "net_j": { "core": 3.182122777482162, "dram": 1.4003843212803533, "package-0": 7.698791386483176, "psys": 12.136171252650698, "uncore": 0.0 }, "npu_busy_delta_us": 547796, "npu_utilization": 0.9773480486865541, "seconds": 0.560492243, "token_latencies_s": [ 0.040758457, 0.040553244, 0.040447942, 0.039903241, 0.039844073, 0.039704511, 0.039912922, 0.039817207, 0.039810117, 0.039626854, 0.039889025, 0.040001374, 0.039966706, 0.040220469 ], "watts": { "core": 7.203318244673728, "dram": 2.985473966675396, "package-0": 16.984190448466208, "psys": 29.662767340029006, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 8.037578, "dram": 2.924308, "package-0": 25.467098, "psys": 41.432634, "uncore": 0.0 }, "net_j": { "core": 6.000539839161551, "dram": 2.2742166219362443, "package-0": 21.1306679614102, "psys": 30.73972384285314, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 8, "gross_j": { "core": 2.283869, "dram": 0.148559, "package-0": 2.762932, "psys": 3.767447, "uncore": 0.0 }, "logits": { "l2": 392.44545785960474, "sha256": "da8a47e01f0c4f258fd33ddaf9c7ae00cfdff7bbd7a3dcc5f722a75b5d939ae9", "sum": 47131.24965239875, "top1": 3862 }, "net_j": { "core": 2.17011036745201, "dram": 0.11225457153051463, "package-0": 2.52076356112944, "psys": 3.1703001910590887, "uncore": 0.0 }, "npu_busy_delta_us": 39461, "npu_utilization": 0.5293257903055647, "seconds": 0.074549551, "watts": { "core": 30.635583573132454, "dram": 1.9927551273917128, "package-0": 37.06168532121676, "psys": 50.536146086245374, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 1534.1581028220958, "decode_tokens_per_s": 23.685173976863762, "end_to_end_ms": 1334.934122, "handoff_estimate_ms": 34.713420000000006, "handoff_plus_first_decode_ms": 74.54955100000001, "output_tokens_per_s_e2e": 11.985610178297623, "prefill_tokens_per_s": 2896.1275725724354, "steady_tpot_ms": 39.836130999999995, "steady_tpot_p95_ms": 40.53517215000001, "ttft_ms": 701.6265510000001 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 70, "gross_j": { "core": 1.936458, "dram": 1.104977, "package-0": 13.428738, "psys": 21.299751, "uncore": 0.0 }, "logits": { "l2": 443.1691514737982, "sha256": "4995d5054aceead0555f3f6020af75f0389032b26365d691afaab466921c7c37", "sum": -56908.99897800945, "top1": 8 }, "net_j": { "core": 0.865813362567873, "dram": 0.7632961905063087, "package-0": 11.149558524783501, "psys": 15.679676158799845, "uncore": 0.0 }, "npu_busy_delta_us": 682896, "npu_utilization": 0.9733041017713708, "seconds": 0.701626551, "watts": { "core": 2.759955416795508, "dram": 1.5748791125779389, "package-0": 19.139438182407094, "psys": 30.357675275603984, "uncore": 0.0 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 453.91327201082566, "sha256": "e94495b9cabc339c33632c206c7636bb049ae1092943ac3356384433083bc8a8", "sum": 9498.106287172064, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 56, "gross_j": { "core": 3.817251, "dram": 1.670772, "package-0": 9.275428, "psys": 16.365436, "uncore": 0.0 }, "net_j": { "core": 2.964616109141668, "dram": 1.3986658598994208, "package-0": 7.46034587549726, "psys": 11.889747492994207, "uncore": 0.0 }, "npu_busy_delta_us": 547307, "npu_utilization": 0.9795062986299508, "seconds": 0.55875802, "token_latencies_s": [ 0.040762106, 0.040412977, 0.039854143, 0.039657194, 0.039522151, 0.039703482, 0.040054239, 0.03978252, 0.03966908, 0.039658331, 0.039858988, 0.039916003, 0.039818119, 0.040055482 ], "watts": { "core": 6.831671069347695, "dram": 2.9901530540894967, "package-0": 16.600080299518567, "psys": 29.288950519224763, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 11.439364975318313, "compile_target": "NPU", "context_length": 4096, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 409297, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 1.5259465821328744, "dram": 0.48698386485903, "package-0": 3.248422500499841, "psys": 8.010065800945089, "uncore": 0.0023192603038971636 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_1b_ov_fp16/openvino_model.xml", "key": "llama32_1b_fp16", "size_b": 1.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "npu_npu", "label": "NPU -> NPU", "mode": "staged", "prefill_device": "NPU" }, "prompt_length": 4080, "prompt_sha256": "66f8de7a9ef70b33481de3e8dae7cef6e4434d9723c6203a6a3dbe07ef228a16", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "NPU" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 4096, "NPUW_LLM_MIN_RESPONSE_LEN": 4096, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.02579568326473236, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 11.048434, "dram": 4.764087, "package-0": 45.430243, "psys": 75.806384, "uncore": 0.0 }, "net_j": { "core": 7.387809437801369, "dram": 3.5958514156287014, "package-0": 37.63753561891491, "psys": 56.59087281560286, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 15, "gross_j": { "core": 2.946282, "dram": 0.217407, "package-0": 3.609183, "psys": 5.358567, "uncore": 0.0 }, "logits": { "l2": 388.9912025260317, "sha256": "dae6a98394073cc6ade52b2f16f4e8282f7be5042a0c7bf4ba934676eabeec04", "sum": 48757.71511605475, "top1": 78 }, "net_j": { "core": 2.7204490747413774, "dram": 0.14533567585106066, "package-0": 3.1284317310149308, "psys": 4.173114892749259, "uncore": 0.0 }, "npu_busy_delta_us": 43861, "npu_utilization": 0.2963675157742507, "seconds": 0.147995302, "watts": { "core": 19.907942753480107, "dram": 1.4690128474483601, "package-0": 24.38714574872113, "psys": 36.20768313307675, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 1707.4346451585393, "decode_tokens_per_s": 19.95787873726198, "end_to_end_ms": 2398.920516, "handoff_estimate_ms": 105.0421315, "handoff_plus_first_decode_ms": 147.99530199999998, "output_tokens_per_s_e2e": 6.669666582650544, "prefill_tokens_per_s": 2476.723601352069, "steady_tpot_ms": 42.9531705, "steady_tpot_p95_ms": 44.112806049999996, "ttft_ms": 1647.3376349999999 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 163, "gross_j": { "core": 4.281483, "dram": 2.775505, "package-0": 32.174051, "psys": 52.893053, "uncore": 0.0 }, "logits": { "l2": 384.67176099469424, "sha256": "2a1e5f084e13626a7729e5e9158bdf8240bf1f797678c5c26a29afe86cd5ec20", "sum": 17386.329117944464, "top1": 72 }, "net_j": { "core": 1.7677337662528974, "dram": 1.9732781517799658, "package-0": 26.822802360545804, "psys": 39.69777014727674, "uncore": 0.0 }, "npu_busy_delta_us": 1604811, "npu_utilization": 0.9741846273062292, "seconds": 1.647337635, "watts": { "core": 2.5990318614920733, "dram": 1.6848428282281063, "package-0": 19.530939084020865, "psys": 32.108204096241636, "uncore": 0.0 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 441.38267318488283, "sha256": "d3c8964f9838d4f83cf8587ff10448c8cc4639e4ad36480f4b6e161a894e35c1", "sum": -5983.314064975828, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 60, "gross_j": { "core": 3.820669, "dram": 1.771175, "package-0": 9.647009, "psys": 17.554764, "uncore": 0.0 }, "net_j": { "core": 2.899626596807094, "dram": 1.477237587997675, "package-0": 7.686301527354175, "psys": 12.719987775576858, "uncore": 0.0 }, "npu_busy_delta_us": 589383, "npu_utilization": 0.9764664159863369, "seconds": 0.603587579, "token_latencies_s": [ 0.044939218, 0.043667815, 0.043048877, 0.042954279, 0.042972128, 0.043029527, 0.042938416, 0.042922471, 0.042952661, 0.042695012, 0.042724293, 0.042807169, 0.04294501, 0.04295368 ], "watts": { "core": 6.329933108182797, "dram": 2.9344126049353307, "package-0": 15.982782508518122, "psys": 29.084037860891762, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 11.895233000000001, "dram": 4.741383, "package-0": 46.125187000000004, "psys": 75.202079, "uncore": 0.0 }, "net_j": { "core": 8.29823999521884, "dram": 3.5934545103337117, "package-0": 38.467937953614836, "psys": 56.32058538076437, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 11, "gross_j": { "core": 2.454095, "dram": 0.19397, "package-0": 3.033134, "psys": 4.391347, "uncore": 0.0 }, "logits": { "l2": 388.9912025260317, "sha256": "dae6a98394073cc6ade52b2f16f4e8282f7be5042a0c7bf4ba934676eabeec04", "sum": 48757.71511605475, "top1": 78 }, "net_j": { "core": 2.2836638329623327, "dram": 0.13957934872279093, "package-0": 2.6703215370514135, "psys": 3.496712234610351, "uncore": 0.0 }, "npu_busy_delta_us": 42783, "npu_utilization": 0.38305536339466617, "seconds": 0.111688816, "watts": { "core": 21.97261183250434, "dram": 1.736700297727214, "package-0": 27.157007376638322, "psys": 39.31769676920919, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 1737.63952059632, "decode_tokens_per_s": 21.032105365822613, "end_to_end_ms": 2357.220788, "handoff_estimate_ms": 68.881819, "handoff_plus_first_decode_ms": 111.688816, "output_tokens_per_s_e2e": 6.787654377329375, "prefill_tokens_per_s": 2481.713442794406, "steady_tpot_ms": 42.806997, "steady_tpot_p95_ms": 43.764470949999996, "ttft_ms": 1644.025426 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 163, "gross_j": { "core": 5.126818, "dram": 2.775506, "package-0": 32.946083, "psys": 52.935107, "uncore": 0.0 }, "logits": { "l2": 384.67176099469424, "sha256": "2a1e5f084e13626a7729e5e9158bdf8240bf1f797678c5c26a29afe86cd5ec20", "sum": 17386.329117944464, "top1": 72 }, "net_j": { "core": 2.6181230202557573, "dram": 1.9748921441200067, "package-0": 27.605593814787767, "psys": 39.76635515931322, "uncore": 0.0 }, "npu_busy_delta_us": 1603712, "npu_utilization": 0.9754788305810546, "seconds": 1.644025426, "watts": { "core": 3.118454203274591, "dram": 1.688237880087385, "package-0": 20.039886536402022, "psys": 32.19847221511281, "uncore": 0.0 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 441.38267318488283, "sha256": "d3c8964f9838d4f83cf8587ff10448c8cc4639e4ad36480f4b6e161a894e35c1", "sum": -5983.314064975828, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 60, "gross_j": { "core": 4.31432, "dram": 1.771907, "package-0": 10.14597, "psys": 17.875625, "uncore": 0.0 }, "net_j": { "core": 3.39645314200075, "dram": 1.478983017490914, "package-0": 8.192022601775658, "psys": 13.057517986840796, "uncore": 0.0 }, "npu_busy_delta_us": 588688, "npu_utilization": 0.9786892660017702, "seconds": 0.601506546, "token_latencies_s": [ 0.044581649, 0.043324452, 0.042817876, 0.042752824, 0.042770664, 0.04286276, 0.042796118, 0.042584708, 0.042708335, 0.042969792, 0.042929798, 0.042736589, 0.042690708, 0.042946417 ], "watts": { "core": 7.172523771669811, "dram": 2.945781740503286, "package-0": 16.867596982061773, "psys": 29.71808888676666, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 12.009611, "dram": 4.748646, "package-0": 46.410463, "psys": 75.039602, "uncore": 0.0 }, "net_j": { "core": 8.388613804104695, "dram": 3.5930569186075965, "package-0": 38.702114029518874, "psys": 56.03210452678017, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 11, "gross_j": { "core": 2.476006, "dram": 0.195679, "package-0": 3.065055, "psys": 4.396595, "uncore": 0.0 }, "logits": { "l2": 388.9912025260317, "sha256": "dae6a98394073cc6ade52b2f16f4e8282f7be5042a0c7bf4ba934676eabeec04", "sum": 48757.71511605475, "top1": 78 }, "net_j": { "core": 2.2985648617300547, "dram": 0.1390512174082244, "package-0": 2.6873197680990697, "psys": 3.4651631858206136, "uncore": 0.0 }, "npu_busy_delta_us": 43042, "npu_utilization": 0.37014974897333575, "seconds": 0.116282667, "watts": { "core": 21.292992875713797, "dram": 1.6827873409542626, "package-0": 26.35865756329789, "psys": 37.809547316282305, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 1726.1204199499105, "decode_tokens_per_s": 20.805680993969492, "end_to_end_ms": 2372.951477, "handoff_estimate_ms": 73.232411, "handoff_plus_first_decode_ms": 116.282667, "output_tokens_per_s_e2e": 6.742657890429338, "prefill_tokens_per_s": 2469.741815890334, "steady_tpot_ms": 43.050256000000005, "steady_tpot_p95_ms": 44.141504, "ttft_ms": 1651.994542 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 164, "gross_j": { "core": 5.281297, "dram": 2.783379, "package-0": 33.277441, "psys": 52.980455, "uncore": 0.0 }, "logits": { "l2": 384.67176099469424, "sha256": "2a1e5f084e13626a7729e5e9158bdf8240bf1f797678c5c26a29afe86cd5ec20", "sum": 17386.329117944464, "top1": 72 }, "net_j": { "core": 2.7604415749329374, "dram": 1.978884313210817, "package-0": 27.911064759064274, "psys": 39.747870015777856, "uncore": 0.0 }, "npu_busy_delta_us": 1598586, "npu_utilization": 0.9676702672786434, "seconds": 1.651994542, "watts": { "core": 3.1969215791755325, "dram": 1.684859682787015, "package-0": 20.143795971451826, "psys": 32.0705992986265, "uncore": 0.0 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 441.38267318488283, "sha256": "d3c8964f9838d4f83cf8587ff10448c8cc4639e4ad36480f4b6e161a894e35c1", "sum": -5983.314064975828, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 60, "gross_j": { "core": 4.252308, "dram": 1.769588, "package-0": 10.067967, "psys": 17.662552, "uncore": 0.0 }, "net_j": { "core": 3.3296073674417026, "dram": 1.475121387988555, "package-0": 8.10372950235553, "psys": 12.819071325181696, "uncore": 0.0 }, "npu_busy_delta_us": 592436, "npu_utilization": 0.9797605609372482, "seconds": 0.604674268, "token_latencies_s": [ 0.043877501, 0.043118939, 0.043086678, 0.043013834, 0.042913365, 0.042845764, 0.043220444, 0.043998312, 0.042664596, 0.044407432, 0.042774459, 0.043160995, 0.042820934, 0.042737878 ], "watts": { "core": 7.0323945056646595, "dram": 2.926514478370361, "package-0": 16.6502322536404, "psys": 29.210027505255113, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 25.7384000280872, "compile_target": "NPU", "context_length": 8192, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 409469, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 1.5259465821328744, "dram": 0.48698386485903, "package-0": 3.248422500499841, "psys": 8.010065800945089, "uncore": 0.0023192603038971636 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_1b_ov_fp16/openvino_model.xml", "key": "llama32_1b_fp16", "size_b": 1.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "npu_npu", "label": "NPU -> NPU", "mode": "staged", "prefill_device": "NPU" }, "prompt_length": 8176, "prompt_sha256": "998ace73fce48442f97bcb84c31be293d0d94180b17f70a4d127bba1092ad744", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "NPU" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 8192, "NPUW_LLM_MIN_RESPONSE_LEN": 8192, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.026572737842798233, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 20.187692, "dram": 11.0795, "package-0": 103.328838, "psys": 172.386338, "uncore": 0.0 }, "net_j": { "core": 11.103111635187604, "dram": 8.180287073099805, "package-0": 83.98965823434425, "psys": 124.69915985880303, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 17, "gross_j": { "core": 3.254264, "dram": 0.264526, "package-0": 4.026052, "psys": 5.968063, "uncore": 0.0 }, "logits": { "l2": 479.5725935312274, "sha256": "0c2ef8bebe7bf52d4e6a3605f2b28cbeae5302dda9b1e081f94832c5649560d8", "sum": 99441.07571207639, "top1": 546 }, "net_j": { "core": 2.992744204549545, "dram": 0.18106572542892332, "package-0": 3.469330816038266, "psys": 4.595281806781639, "uncore": 0.0 }, "npu_busy_delta_us": 47906, "npu_utilization": 0.27952758542711004, "seconds": 0.171382012, "watts": { "core": 18.988363842991877, "dram": 1.543487539404077, "package-0": 23.491683596292475, "psys": 34.82315868715557, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 1376.0189132402104, "decode_tokens_per_s": 17.851990135218756, "end_to_end_ms": 5953.406542000001, "handoff_estimate_ms": 124.313991, "handoff_plus_first_decode_ms": 171.382012, "output_tokens_per_s_e2e": 2.687536939922286, "prefill_tokens_per_s": 1599.009900579588, "steady_tpot_ms": 47.068021, "steady_tpot_p95_ms": 50.8961629, "ttft_ms": 5113.164088 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 505, "gross_j": { "core": 12.421477, "dram": 8.858436, "package-0": 88.282123, "psys": 147.004995, "uncore": 0.0 }, "logits": { "l2": 393.33541703766207, "sha256": "73521a985b1e954f610989386d69c028e3b55f95b4a5e007c6f02207bfccaece", "sum": 10675.848786275834, "top1": 13 }, "net_j": { "core": 4.619061736031843, "dram": 6.368407590767362, "package-0": 71.67240572779305, "psys": 106.04821420409061, "uncore": 0.0 }, "npu_busy_delta_us": 5012704, "npu_utilization": 0.9803526571275566, "seconds": 5.113164088, "watts": { "core": 2.429313197507539, "dram": 1.7324763781373094, "package-0": 17.265654197796593, "psys": 28.75029873283425, "uncore": 0.0 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 530.2405363191889, "sha256": "fe7a0eed3c4c1cc5e8a1c731ca976a3a6dc917e8d9768da56d0cca26eb6e651d", "sum": 99033.82702467777, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 66, "gross_j": { "core": 4.511951, "dram": 1.956538, "package-0": 11.020663, "psys": 19.41328, "uncore": 0.0 }, "net_j": { "core": 3.4913056946062158, "dram": 1.630813756903521, "package-0": 8.847921690512932, "psys": 14.055663847930784, "uncore": 0.0 }, "npu_busy_delta_us": 650694, "npu_utilization": 0.9728397123536272, "seconds": 0.668860442, "token_latencies_s": [ 0.056285127, 0.047994413, 0.047450013, 0.047105563, 0.047289358, 0.046911115, 0.047139601, 0.047166355, 0.046876048, 0.046754606, 0.046913468, 0.047030479, 0.046919979, 0.046996221 ], "watts": { "core": 6.745728580551934, "dram": 2.925181214409448, "package-0": 16.476774986193607, "psys": 29.02441044644706, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 16.814593, "dram": 11.035738, "package-0": 99.945911, "psys": 166.599305, "uncore": 0.0 }, "net_j": { "core": 7.70228147977099, "dram": 8.127675074417079, "package-0": 80.54769737937518, "psys": 118.76655926015694, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 15, "gross_j": { "core": 2.849968, "dram": 0.251098, "package-0": 3.583365, "psys": 5.322679, "uncore": 0.0 }, "logits": { "l2": 479.5725935312274, "sha256": "0c2ef8bebe7bf52d4e6a3605f2b28cbeae5302dda9b1e081f94832c5649560d8", "sum": 99441.07571207639, "top1": 546 }, "net_j": { "core": 2.623502138040201, "dram": 0.17882468322264283, "package-0": 3.1012663405849064, "psys": 4.133904453626146, "uncore": 0.0 }, "npu_busy_delta_us": 47741, "npu_utilization": 0.32168299074823653, "seconds": 0.148410085, "watts": { "core": 19.203331094379468, "dram": 1.6919200605538363, "package-0": 24.14502356763693, "psys": 35.864671865122915, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 1371.831326560963, "decode_tokens_per_s": 18.334684491609448, "end_to_end_ms": 5971.579626000001, "handoff_estimate_ms": 101.13829499999999, "handoff_plus_first_decode_ms": 148.410085, "output_tokens_per_s_e2e": 2.6793580596893807, "prefill_tokens_per_s": 1586.5075133220696, "steady_tpot_ms": 47.27179, "steady_tpot_p95_ms": 50.53907135000001, "ttft_ms": 5153.458103 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 509, "gross_j": { "core": 9.585669, "dram": 8.824562, "package-0": 85.456507, "psys": 141.989871, "uncore": 0.0 }, "logits": { "l2": 393.33541703766207, "sha256": "73521a985b1e954f610989386d69c028e3b55f95b4a5e007c6f02207bfccaece", "sum": 10675.848786275834, "top1": 13 }, "net_j": { "core": 1.7217672215621826, "dram": 6.314911055611975, "package-0": 68.71589774283157, "psys": 100.71033249255635, "uncore": 0.0 }, "npu_busy_delta_us": 5035934, "npu_utilization": 0.9771950987761819, "seconds": 5.153458103, "watts": { "core": 1.8600459746475597, "dram": 1.7123573770519116, "package-0": 16.582361841702546, "psys": 27.552347988885938, "uncore": 0.0 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 530.2405363191889, "sha256": "fe7a0eed3c4c1cc5e8a1c731ca976a3a6dc917e8d9768da56d0cca26eb6e651d", "sum": 99033.82702467777, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 66, "gross_j": { "core": 4.378956, "dram": 1.960078, "package-0": 10.906039, "psys": 19.286755, "uncore": 0.0 }, "net_j": { "core": 3.357012120168607, "dram": 1.6339393355824614, "package-0": 8.730533295958695, "psys": 13.922322313974442, "uncore": 0.0 }, "npu_busy_delta_us": 652629, "npu_utilization": 0.9744928382125079, "seconds": 0.669711438, "token_latencies_s": [ 0.052577784, 0.047581958, 0.047213935, 0.047473785, 0.047158253, 0.047083428, 0.047261864, 0.047235505, 0.047223046, 0.049441303, 0.04754857, 0.04726505, 0.047315954, 0.04727853 ], "watts": { "core": 6.538571318233928, "dram": 2.9267500729172258, "package-0": 16.28468379242464, "psys": 28.798604750722504, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 21.303413, "dram": 11.120089, "package-0": 104.836767, "psys": 173.978315, "uncore": 0.0 }, "net_j": { "core": 12.240999615609848, "dram": 8.227950345505326, "package-0": 85.54477611946963, "psys": 126.40749674575414, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 15, "gross_j": { "core": 2.893181, "dram": 0.253051, "package-0": 3.63323, "psys": 5.374987, "uncore": 0.0 }, "logits": { "l2": 479.5725935312274, "sha256": "0c2ef8bebe7bf52d4e6a3605f2b28cbeae5302dda9b1e081f94832c5649560d8", "sum": 99441.07571207639, "top1": 546 }, "net_j": { "core": 2.6607408374348425, "dram": 0.17887107126604324, "package-0": 3.138413298722609, "psys": 4.15485189256597, "uncore": 0.0 }, "npu_busy_delta_us": 47768, "npu_utilization": 0.31359217585683064, "seconds": 0.152325229, "watts": { "core": 18.99344592483757, "dram": 1.6612546828995742, "package-0": 23.85179411087575, "psys": 35.28625583093658, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 1379.3847036777745, "decode_tokens_per_s": 18.403970998811467, "end_to_end_ms": 5938.879833999999, "handoff_estimate_ms": 105.13715850000001, "handoff_plus_first_decode_ms": 152.325229, "output_tokens_per_s_e2e": 2.6941107493706533, "prefill_tokens_per_s": 1595.6787579688396, "steady_tpot_ms": 47.188070499999995, "steady_tpot_p95_ms": 47.919905050000004, "ttft_ms": 5123.838341 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 505, "gross_j": { "core": 14.846276, "dram": 8.915383, "package-0": 91.142955, "psys": 150.666728, "uncore": 0.0 }, "logits": { "l2": 393.33541703766207, "sha256": "73521a985b1e954f610989386d69c028e3b55f95b4a5e007c6f02207bfccaece", "sum": 10675.848786275834, "top1": 13 }, "net_j": { "core": 7.027572396149672, "dram": 6.42015640178694, "package-0": 74.49856324417182, "psys": 109.62444573518468, "uncore": 0.0 }, "npu_busy_delta_us": 5009495, "npu_utilization": 0.9776840459455862, "seconds": 5.123838341, "watts": { "core": 2.897491101778693, "dram": 1.7399813199922345, "package-0": 17.788023144815295, "psys": 29.40505105213662, "uncore": 0.0 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 530.2405363191889, "sha256": "fe7a0eed3c4c1cc5e8a1c731ca976a3a6dc917e8d9768da56d0cca26eb6e651d", "sum": 99033.82702467777, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 66, "gross_j": { "core": 3.563956, "dram": 1.951655, "package-0": 10.060582, "psys": 17.9366, "uncore": 0.0 }, "net_j": { "core": 2.5526863820253327, "dram": 1.6289228724523426, "package-0": 7.907799576575208, "psys": 12.6281991180035, "uncore": 0.0 }, "npu_busy_delta_us": 650144, "npu_utilization": 0.9810291904953158, "seconds": 0.662716264, "token_latencies_s": [ 0.048366847, 0.047633622, 0.047679244, 0.047137533, 0.047037577, 0.047077295, 0.047227845, 0.047531262, 0.047332575, 0.047349869, 0.047137763, 0.047148296, 0.047035432, 0.04698602 ], "watts": { "core": 5.377800717442479, "dram": 2.9449330068048547, "package-0": 15.18082857854836, "psys": 27.065278120290703, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 14.877661527134478, "compile_target": "NPU", "context_length": 1024, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 409687, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 1.5259465821328744, "dram": 0.48698386485903, "package-0": 3.248422500499841, "psys": 8.010065800945089, "uncore": 0.0023192603038971636 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_3b_ov_fp16/openvino_model.xml", "key": "llama32_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "npu_npu", "label": "NPU -> NPU", "mode": "staged", "prefill_device": "NPU" }, "prompt_length": 1008, "prompt_sha256": "2669029615323cd09063f0eb8d822f5daea575e1396736877003e47194c143f8", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "NPU" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 1024, "NPUW_LLM_MIN_RESPONSE_LEN": 1024, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.04378183372318745, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 12.625821, "dram": 6.100204, "package-0": 42.698865, "psys": 71.28888, "uncore": 0.0 }, "net_j": { "core": 9.056031138021263, "dram": 4.96095700139857, "package-0": 35.09952577968502, "psys": 52.55018231241183, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 21, "gross_j": { "core": 3.176627, "dram": 0.395995, "package-0": 4.39751, "psys": 6.947797, "uncore": 0.0 }, "logits": { "l2": 530.8437238572551, "sha256": "274c0813f3a62c60b62c0cc581677f1b44f808364130fe3aac5dcc63f5f49df1", "sum": -124148.91462222487, "top1": 546 }, "net_j": { "core": 2.853316875641328, "dram": 0.292815232273026, "package-0": 3.709250073405361, "psys": 5.250663299156468, "uncore": 0.0 }, "npu_busy_delta_us": 98585, "npu_utilization": 0.4652976583952569, "seconds": 0.211875126, "watts": { "core": 14.992920877366227, "dram": 1.869001838375308, "package-0": 20.75519709661435, "psys": 32.79194274083877, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 437.7202469945753, "decode_tokens_per_s": 9.549546023272141, "end_to_end_ms": 2339.393727, "handoff_estimate_ms": 114.9072025, "handoff_plus_first_decode_ms": 211.875126, "output_tokens_per_s_e2e": 6.839378859290239, "prefill_tokens_per_s": 1311.4098393440984, "steady_tpot_ms": 96.9679235, "steady_tpot_p95_ms": 97.5769649, "ttft_ms": 768.63843 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 76, "gross_j": { "core": 2.516473, "dram": 1.489071, "package-0": 17.759964, "psys": 27.334037, "uncore": 0.0 }, "logits": { "l2": 868.4674711122068, "sha256": "1773fbd9c1c18029e107f5353f56d0ed01120e9f0f0ea934e28c472a840595d4", "sum": -269072.11017668527, "top1": 198 }, "net_j": { "core": 1.3435718148455214, "dram": 1.114756486679423, "package-0": 15.263101629239127, "psys": 21.177192598564872, "uncore": 0.0 }, "npu_busy_delta_us": 760854, "npu_utilization": 0.9898724423653915, "seconds": 0.76863843, "watts": { "core": 3.2739359649243664, "dram": 1.9372840881765436, "package-0": 23.105745571425565, "psys": 35.56163201467822, "uncore": 0.0 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 520.1749807146045, "sha256": "631550d11dadb58c7812a3211e73e0117bf885be1946c6a0b54350ad0baa4cd2", "sum": -88494.78699321579, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 134, "gross_j": { "core": 6.932721, "dram": 4.215138, "package-0": 20.541391, "psys": 37.007046, "uncore": 0.0 }, "net_j": { "core": 4.859142447534413, "dram": 3.5533852824461203, "package-0": 16.12717407704053, "psys": 26.122326414690487, "uncore": 0.0 }, "npu_busy_delta_us": 1347943, "npu_utilization": 0.9919513351998127, "seconds": 1.358880171, "token_latencies_s": [ 0.097993169, 0.097352855, 0.097348436, 0.097027084, 0.09702352, 0.096966228, 0.096902958, 0.096771809, 0.09681007, 0.096853403, 0.096969619, 0.09691046, 0.096922908, 0.096996998 ], "watts": { "core": 5.101789803068662, "dram": 3.1019203090571845, "package-0": 15.116410878880947, "psys": 27.23348738893328, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 13.053617, "dram": 6.079024, "package-0": 42.9568, "psys": 70.633853, "uncore": 0.0 }, "net_j": { "core": 9.64327241591715, "dram": 4.990661664448, "package-0": 35.696886572422464, "psys": 52.7321224661446, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 11, "gross_j": { "core": 2.419733, "dram": 0.322448, "package-0": 3.423209, "psys": 5.009082, "uncore": 0.0 }, "logits": { "l2": 530.8437238572551, "sha256": "274c0813f3a62c60b62c0cc581677f1b44f808364130fe3aac5dcc63f5f49df1", "sum": -124148.91462222487, "top1": 546 }, "net_j": { "core": 2.252264667306225, "dram": 0.26900289461320426, "package-0": 3.0667037943700164, "psys": 4.129999874741901, "uncore": 0.0 }, "npu_busy_delta_us": 96679, "npu_utilization": 0.8809246932898385, "seconds": 0.109747179, "watts": { "core": 22.048247818743477, "dram": 2.93809829954718, "package-0": 31.19177213657583, "psys": 45.64201144523268, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 458.1851662137208, "decode_tokens_per_s": 10.202238934793973, "end_to_end_ms": 2234.904304, "handoff_estimate_ms": 12.603467499999994, "handoff_plus_first_decode_ms": 109.747179, "output_tokens_per_s_e2e": 7.159143222089387, "prefill_tokens_per_s": 1318.2694939817445, "steady_tpot_ms": 97.14371150000001, "steady_tpot_p95_ms": 97.51050240000001, "ttft_ms": 764.6387970000001 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 76, "gross_j": { "core": 2.990349, "dram": 1.542476, "package-0": 18.239273, "psys": 27.717397, "uncore": 0.0 }, "logits": { "l2": 868.4674711122068, "sha256": "1773fbd9c1c18029e107f5353f56d0ed01120e9f0f0ea934e28c472a840595d4", "sum": -269072.11017668527, "top1": 198 }, "net_j": { "core": 1.8235510411516573, "dram": 1.1701092434157807, "package-0": 15.755403127070071, "psys": 21.592589922074502, "uncore": 0.0 }, "npu_busy_delta_us": 759652, "npu_utilization": 0.9934782317879169, "seconds": 0.764638797, "watts": { "core": 3.9107994673202544, "dram": 2.0172609682529616, "package-0": 23.85344959157232, "psys": 36.24900686277889, "uncore": 0.0 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 520.1749807146045, "sha256": "631550d11dadb58c7812a3211e73e0117bf885be1946c6a0b54350ad0baa4cd2", "sum": -88494.78699321579, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 135, "gross_j": { "core": 7.643535, "dram": 4.2141, "package-0": 21.294318, "psys": 37.907374, "uncore": 0.0 }, "net_j": { "core": 5.567456707459267, "dram": 3.5515495264190147, "package-0": 16.87477965098238, "psys": 27.009532669328202, "uncore": 0.0 }, "npu_busy_delta_us": 1348127, "npu_utilization": 0.9908922006084141, "seconds": 1.360518328, "token_latencies_s": [ 0.097487604, 0.097553028, 0.097196225, 0.097057119, 0.097248958, 0.096993922, 0.097084041, 0.097016508, 0.097074958, 0.097104771, 0.097215076, 0.096961535, 0.097312637, 0.097182652 ], "watts": { "core": 5.618105131473099, "dram": 3.097422440603829, "package-0": 15.651621563454603, "psys": 27.86245008233362, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 13.000089, "dram": 6.08464, "package-0": 42.913464, "psys": 70.08771200000001, "uncore": 0.0 }, "net_j": { "core": 9.599591088274787, "dram": 4.999420088053695, "package-0": 35.674512087337945, "psys": 52.23766905103686, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 11, "gross_j": { "core": 2.392633, "dram": 0.317992, "package-0": 3.406607, "psys": 4.953052, "uncore": 0.0 }, "logits": { "l2": 530.8437238572551, "sha256": "274c0813f3a62c60b62c0cc581677f1b44f808364130fe3aac5dcc63f5f49df1", "sum": -124148.91462222487, "top1": 546 }, "net_j": { "core": 2.2288425602185056, "dram": 0.26572064114269045, "package-0": 3.0579312624723536, "psys": 4.093276031707773, "uncore": 0.0 }, "npu_busy_delta_us": 96608, "npu_utilization": 0.9000442736667493, "seconds": 0.107336942, "watts": { "core": 22.290862357528315, "dram": 2.962558780554788, "package-0": 31.73750748367696, "psys": 46.14489576198285, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 459.5119128631688, "decode_tokens_per_s": 10.213703232918563, "end_to_end_ms": 2228.451475, "handoff_estimate_ms": 10.34657800000001, "handoff_plus_first_decode_ms": 107.33694200000001, "output_tokens_per_s_e2e": 7.179873638487012, "prefill_tokens_per_s": 1326.6016075355562, "steady_tpot_ms": 96.990364, "steady_tpot_p95_ms": 98.19586815000001, "ttft_ms": 759.836257 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 76, "gross_j": { "core": 3.143608, "dram": 1.542171, "package-0": 18.388259, "psys": 27.916249, "uncore": 0.0 }, "logits": { "l2": 868.4674711122068, "sha256": "1773fbd9c1c18029e107f5353f56d0ed01120e9f0f0ea934e28c472a840595d4", "sum": -269072.11017668527, "top1": 198 }, "net_j": { "core": 1.9841384606502135, "dram": 1.1721430029061208, "package-0": 15.919989806065622, "psys": 21.829910583486175, "uncore": 0.0 }, "npu_busy_delta_us": 753285, "npu_utilization": 0.9913780673932752, "seconds": 0.759836257, "watts": { "core": 4.1372176847833675, "dram": 2.02960965049079, "package-0": 24.20029161625016, "psys": 36.73982222198697, "uncore": 0.0 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 520.1749807146045, "sha256": "631550d11dadb58c7812a3211e73e0117bf885be1946c6a0b54350ad0baa4cd2", "sum": -88494.78699321579, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 135, "gross_j": { "core": 7.463848, "dram": 4.224477, "package-0": 21.118598, "psys": 37.218411, "uncore": 0.0 }, "net_j": { "core": 5.3866100674060675, "dram": 3.561556444004883, "package-0": 16.696591018799968, "psys": 26.314482435842912, "uncore": 0.0 }, "npu_busy_delta_us": 1350546, "npu_utilization": 0.9921160308004504, "seconds": 1.361278276, "token_latencies_s": [ 0.097855952, 0.097332537, 0.097250902, 0.096996809, 0.096983919, 0.096845612, 0.096913275, 0.096857339, 0.097707471, 0.096925419, 0.096755145, 0.096981384, 0.097019536, 0.098827141 ], "watts": { "core": 5.482970037494376, "dram": 3.103316253906046, "package-0": 15.513799325480457, "psys": 27.34078083532129, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 32.750238213688135, "compile_target": "NPU", "context_length": 2048, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 409811, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 1.5259465821328744, "dram": 0.48698386485903, "package-0": 3.248422500499841, "psys": 8.010065800945089, "uncore": 0.0023192603038971636 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_3b_ov_fp16/openvino_model.xml", "key": "llama32_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "npu_npu", "label": "NPU -> NPU", "mode": "staged", "prefill_device": "NPU" }, "prompt_length": 2032, "prompt_sha256": "cd1a97358e68d07c27508cfacc4ba6882a7a2962b1281e2e2b0c7e5e320da4fa", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "NPU" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 2048, "NPUW_LLM_MIN_RESPONSE_LEN": 2048, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.04384459741413593, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 13.805689999999998, "dram": 8.016215, "package-0": 62.817343, "psys": 103.568033, "uncore": 0.0 }, "net_j": { "core": 8.931583572579628, "dram": 6.460714162977434, "package-0": 52.44138543107081, "psys": 77.9826595265222, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 20, "gross_j": { "core": 3.318778, "dram": 0.416197, "package-0": 4.59093, "psys": 7.060834, "uncore": 0.0 }, "logits": { "l2": 518.0133203759266, "sha256": "b5b42791fa53b84d4c0febbc65f92575aa94c4de03d3a76de6c941f2c254cd68", "sum": -135962.72018502932, "top1": 319 }, "net_j": { "core": 2.9977560966309924, "dram": 0.31374748503170447, "package-0": 3.907541219378606, "psys": 5.375711729247749, "uncore": 0.0 }, "npu_busy_delta_us": 101218, "npu_utilization": 0.48112997751678255, "seconds": 0.210375584, "watts": { "core": 15.775490372494938, "dram": 1.9783522026966778, "package-0": 21.822541916271046, "psys": 33.56299179661457, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 641.1715966288639, "decode_tokens_per_s": 9.262187951654772, "end_to_end_ms": 3194.152721, "handoff_estimate_ms": 109.8744865, "handoff_plus_first_decode_ms": 210.375584, "output_tokens_per_s_e2e": 5.009153098662999, "prefill_tokens_per_s": 1290.4332335741442, "steady_tpot_ms": 100.5010975, "steady_tpot_p95_ms": 101.3804163, "ttft_ms": 1574.664963 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 156, "gross_j": { "core": 4.209401, "dram": 3.225822, "package-0": 37.782068, "psys": 58.820711, "uncore": 0.0 }, "logits": { "l2": 805.6627491508196, "sha256": "29e78d35151f46938d24fab3610c72064af8b845ed8c88ce1e8e01295747b58c", "sum": -252043.32194374222, "top1": 319 }, "net_j": { "core": 1.8065463817057608, "dram": 2.4589855704601584, "package-0": 32.66689090344205, "psys": 46.207541031927235, "uncore": 0.0 }, "npu_busy_delta_us": 1535265, "npu_utilization": 0.9749788279248073, "seconds": 1.574664963, "watts": { "core": 2.673204204645785, "dram": 2.0485767295249078, "package-0": 23.993718592695963, "psys": 37.3544292799509, "uncore": 0.0 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 543.8257039866751, "sha256": "789c4fe7e50b20ef2d86f9de8d79eba5278419cb5116623fcb97983782b06aec", "sum": -117196.24898801185, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 140, "gross_j": { "core": 6.277511, "dram": 4.374196, "package-0": 20.444345, "psys": 37.686488, "uncore": 0.0 }, "net_j": { "core": 4.127281094242875, "dram": 3.6879811074855704, "package-0": 15.866953308250151, "psys": 26.39940676534721, "uncore": 0.0 }, "npu_busy_delta_us": 1398688, "npu_utilization": 0.9926023107369731, "seconds": 1.409112174, "token_latencies_s": [ 0.102201534, 0.100752478, 0.10086571, 0.100707824, 0.100495026, 0.100920141, 0.100430559, 0.100507169, 0.100217411, 0.100224728, 0.100303294, 0.100251548, 0.100938276, 0.100267664 ], "watts": { "core": 4.454940575937426, "dram": 3.1042212825279303, "package-0": 14.508671046369086, "psys": 26.744845935877915, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 14.872947, "dram": 7.976053, "package-0": 63.652914, "psys": 102.079144, "uncore": 0.0 }, "net_j": { "core": 10.051472847622168, "dram": 6.437348995197283, "package-0": 53.3889995841553, "psys": 76.77005017567933, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 18, "gross_j": { "core": 2.762261, "dram": 0.394835, "package-0": 3.963674, "psys": 6.046371, "uncore": 0.0 }, "logits": { "l2": 518.0133203759266, "sha256": "b5b42791fa53b84d4c0febbc65f92575aa94c4de03d3a76de6c941f2c254cd68", "sum": -135962.72018502932, "top1": 319 }, "net_j": { "core": 2.4938888101910712, "dram": 0.3091878825031615, "package-0": 3.392365495435437, "psys": 4.637619917436837, "uncore": 0.0 }, "npu_busy_delta_us": 100883, "npu_utilization": 0.5736140885347017, "seconds": 0.175872598, "watts": { "core": 15.706033978073151, "dram": 2.2450057853810748, "package-0": 22.537189107765386, "psys": 34.37926697369877, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 648.170767163998, "decode_tokens_per_s": 9.476649174741327, "end_to_end_ms": 3159.661163, "handoff_estimate_ms": 75.40130049999999, "handoff_plus_first_decode_ms": 175.87259799999998, "output_tokens_per_s_e2e": 5.063834118468734, "prefill_tokens_per_s": 1288.6669790821597, "steady_tpot_ms": 100.4712975, "steady_tpot_p95_ms": 100.77077915, "ttft_ms": 1576.823208 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 156, "gross_j": { "core": 4.733874, "dram": 3.217032, "package-0": 38.154017, "psys": 58.092319, "uncore": 0.0 }, "logits": { "l2": 805.6627491508196, "sha256": "29e78d35151f46938d24fab3610c72064af8b845ed8c88ce1e8e01295747b58c", "sum": -252043.32194374222, "top1": 319 }, "net_j": { "core": 2.3277260151246058, "dram": 2.4491445399687457, "package-0": 33.03182901182246, "psys": 45.46186134746268, "uncore": 0.0 }, "npu_busy_delta_us": 1536304, "npu_utilization": 0.9743032650747235, "seconds": 1.576823208, "watts": { "core": 3.002159009318691, "dram": 2.0401982820131095, "package-0": 24.196762710255598, "psys": 36.84136478031848, "uncore": 0.0 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 543.8257039866751, "sha256": "789c4fe7e50b20ef2d86f9de8d79eba5278419cb5116623fcb97983782b06aec", "sum": -117196.24898801185, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 139, "gross_j": { "core": 7.376812, "dram": 4.364186, "package-0": 21.535223, "psys": 37.940454, "uncore": 0.0 }, "net_j": { "core": 5.229858022306491, "dram": 3.6790165727253754, "package-0": 16.964805076897406, "psys": 26.670568910779807, "uncore": 0.0 }, "npu_busy_delta_us": 1396469, "npu_utilization": 0.9925397189434849, "seconds": 1.406965357, "token_latencies_s": [ 0.100761263, 0.100536781, 0.100481018, 0.100359699, 0.100771929, 0.100412653, 0.100448994, 0.100331518, 0.100461577, 0.100310847, 0.100209227, 0.100551201, 0.100525837, 0.10077016 ], "watts": { "core": 5.243065839040414, "dram": 3.1018432531313564, "package-0": 15.306150142828285, "psys": 26.96616075956446, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 15.432638, "dram": 7.9350380000000005, "package-0": 64.16756099999999, "psys": 101.470932, "uncore": 0.0 }, "net_j": { "core": 10.588772809411378, "dram": 6.389188217976919, "package-0": 53.85598072665732, "psys": 76.04430215474652, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 18, "gross_j": { "core": 2.758843, "dram": 0.392028, "package-0": 3.971426, "psys": 5.970383, "uncore": 0.0 }, "logits": { "l2": 518.0133203759266, "sha256": "b5b42791fa53b84d4c0febbc65f92575aa94c4de03d3a76de6c941f2c254cd68", "sum": -135962.72018502932, "top1": 319 }, "net_j": { "core": 2.486692702262956, "dram": 0.3051751537953345, "package-0": 3.3920746904338017, "psys": 4.541799707409552, "uncore": 0.0 }, "npu_busy_delta_us": 100967, "npu_utilization": 0.5661219180699739, "seconds": 0.178348509, "watts": { "core": 15.468831309377531, "dram": 2.198100798252258, "package-0": 22.26778357872339, "psys": 33.47593446940451, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 645.1745614803872, "decode_tokens_per_s": 9.428149969795225, "end_to_end_ms": 3174.334703, "handoff_estimate_ms": 77.75900549999999, "handoff_plus_first_decode_ms": 178.34850899999998, "output_tokens_per_s_e2e": 5.040426261565525, "prefill_tokens_per_s": 1283.351268725201, "steady_tpot_ms": 100.58950349999999, "steady_tpot_p95_ms": 102.60561775, "ttft_ms": 1583.354495 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 157, "gross_j": { "core": 5.208482, "dram": 3.173087, "package-0": 38.567406, "psys": 57.646703, "uncore": 0.0 }, "logits": { "l2": 805.6627491508196, "sha256": "29e78d35151f46938d24fab3610c72064af8b845ed8c88ce1e8e01295747b58c", "sum": -252043.32194374222, "top1": 319 }, "net_j": { "core": 2.7923676200500265, "dram": 2.4020189085829826, "package-0": 33.42400163217444, "psys": 44.96392930882782, "uncore": 0.0 }, "npu_busy_delta_us": 1537599, "npu_utilization": 0.9711021788585631, "seconds": 1.583354495, "watts": { "core": 3.2895236135986083, "dram": 2.0040281630046466, "package-0": 24.3580361326476, "psys": 36.40795739806833, "uncore": 0.0 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 543.8257039866751, "sha256": "789c4fe7e50b20ef2d86f9de8d79eba5278419cb5116623fcb97983782b06aec", "sum": -117196.24898801185, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 140, "gross_j": { "core": 7.465313, "dram": 4.369923, "package-0": 21.628729, "psys": 37.853846, "uncore": 0.0 }, "net_j": { "core": 5.309712487098395, "dram": 3.681994155598602, "package-0": 17.039904404049082, "psys": 26.53857313850914, "uncore": 0.0 }, "npu_busy_delta_us": 1400999, "npu_utilization": 0.9917652286804588, "seconds": 1.412631699, "token_latencies_s": [ 0.101521256, 0.100695076, 0.100341163, 0.100861898, 0.10039921, 0.100323068, 0.100266442, 0.102911108, 0.100601021, 0.100703905, 0.100577986, 0.100543907, 0.100402046, 0.102441123 ], "watts": { "core": 5.284684610493085, "dram": 3.0934623675041855, "package-0": 15.310946947679955, "psys": 26.79668453341142, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 31.873993958346546, "compile_target": "NPU", "context_length": 4096, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 409987, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 1.5259465821328744, "dram": 0.48698386485903, "package-0": 3.248422500499841, "psys": 8.010065800945089, "uncore": 0.0023192603038971636 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_3b_ov_fp16/openvino_model.xml", "key": "llama32_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "npu_npu", "label": "NPU -> NPU", "mode": "staged", "prefill_device": "NPU" }, "prompt_length": 4080, "prompt_sha256": "66f8de7a9ef70b33481de3e8dae7cef6e4434d9723c6203a6a3dbe07ef228a16", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "NPU" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 4096, "NPUW_LLM_MIN_RESPONSE_LEN": 4096, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.04428494721651077, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 18.219131, "dram": 12.717863, "package-0": 110.94386899999999, "psys": 182.816183, "uncore": 0.0 }, "net_j": { "core": 9.644810345639879, "dram": 9.981492107537589, "package-0": 92.69092589815648, "psys": 137.8074825356435, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 28, "gross_j": { "core": 4.115285, "dram": 0.525572, "package-0": 5.530137, "psys": 8.811318, "uncore": 0.0 }, "logits": { "l2": 609.263242568076, "sha256": "1f93e2afdd0c63c174d4669ef6edf80514b7042aecb99e298ed65730a9a55a74", "sum": -156409.51710008364, "top1": 437 }, "net_j": { "core": 3.68111987681363, "dram": 0.387014456667933, "package-0": 4.605889866054074, "psys": 6.532279371834293, "uncore": 0.0 }, "npu_busy_delta_us": 129805, "npu_utilization": 0.4562215745015789, "seconds": 0.284521836, "watts": { "core": 14.46386350466261, "dram": 1.847211473779468, "package-0": 19.436599586683393, "psys": 30.96886384495284, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 728.9530509029807, "decode_tokens_per_s": 8.336893061617248, "end_to_end_ms": 5619.017569, "handoff_estimate_ms": 176.70001949999997, "handoff_plus_first_decode_ms": 284.52183599999995, "output_tokens_per_s_e2e": 2.8474728550897686, "prefill_tokens_per_s": 1068.1226247560955, "steady_tpot_ms": 107.8218165, "steady_tpot_p95_ms": 110.00807129999998, "ttft_ms": 3819.786142 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 378, "gross_j": { "core": 8.283426, "dram": 7.514996, "package-0": 84.643277, "psys": 135.13442, "uncore": 0.0 }, "logits": { "l2": 657.7937454973483, "sha256": "01330236be5a5d092e077e2f3012d7ef6a5af180540b794e147c26e69be8d496", "sum": -175698.30940290913, "top1": 72 }, "net_j": { "core": 2.4546363921365826, "dram": 5.654821781633876, "package-0": 72.23499774922972, "psys": 104.53768165704183, "uncore": 0.0 }, "npu_busy_delta_us": 3706905, "npu_utilization": 0.970448308412131, "seconds": 3.819786142, "watts": { "core": 2.168557529679629, "dram": 1.9673865815077352, "package-0": 22.159166469901287, "psys": 35.37748318266976, "uncore": 0.0 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 474.8007585329114, "sha256": "56e5d3e7ac8fb688d7b620d385293c7321b36fd5df528c679a4978f4fa7ebb39", "sum": -66282.34790089168, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 149, "gross_j": { "core": 5.82042, "dram": 4.677295, "package-0": 20.770455, "psys": 38.870445, "uncore": 0.0 }, "net_j": { "core": 3.5090540766896665, "dram": 3.9396558692357795, "package-0": 15.850038282872688, "psys": 26.737521506767372, "uncore": 0.0 }, "npu_busy_delta_us": 1496238, "npu_utilization": 0.9878051930813978, "seconds": 1.514709591, "token_latencies_s": [ 0.112792891, 0.107809922, 0.107868182, 0.107524315, 0.107588909, 0.107484573, 0.107572654, 0.107596105, 0.107842735, 0.108508553, 0.107833711, 0.108426957, 0.108072982, 0.107749612 ], "watts": { "core": 3.8425979703194475, "dram": 3.087915352085468, "package-0": 13.712499824000917, "psys": 25.661978527737467, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 21.34748, "dram": 12.823758999999999, "package-0": 114.036085, "psys": 183.34462299999998, "uncore": 0.0 }, "net_j": { "core": 12.83386607851687, "dram": 10.106761786633276, "package-0": 95.91237389049387, "psys": 138.65458698340248, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 27, "gross_j": { "core": 3.707388, "dram": 0.512022, "package-0": 4.968127, "psys": 7.922587, "uncore": 0.0 }, "logits": { "l2": 609.263242568076, "sha256": "1f93e2afdd0c63c174d4669ef6edf80514b7042aecb99e298ed65730a9a55a74", "sum": -156409.51710008364, "top1": 437 }, "net_j": { "core": 3.2999981823430886, "dram": 0.38200940963168073, "package-0": 4.10087891566568, "psys": 5.784098481282815, "uncore": 0.0 }, "npu_busy_delta_us": 130458, "npu_utilization": 0.4886522210025914, "seconds": 0.26697515, "watts": { "core": 13.886640760385376, "dram": 1.9178638910775028, "package-0": 18.60894918497096, "psys": 29.675372408255974, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 734.1508856355811, "decode_tokens_per_s": 8.466271671821122, "end_to_end_ms": 5579.234569, "handoff_estimate_ms": 159.79702400000002, "handoff_plus_first_decode_ms": 266.97515, "output_tokens_per_s_e2e": 2.8677768970139885, "prefill_tokens_per_s": 1071.5697301983532, "steady_tpot_ms": 107.17812599999999, "steady_tpot_p95_ms": 108.85781275000001, "ttft_ms": 3807.4983690000004 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 376, "gross_j": { "core": 10.478367, "dram": 7.644756, "package-0": 86.89376, "psys": 135.780353, "uncore": 0.0 }, "logits": { "l2": 657.7937454973483, "sha256": "01330236be5a5d092e077e2f3012d7ef6a5af180540b794e147c26e69be8d496", "sum": -175698.30940290913, "top1": 72 }, "net_j": { "core": 4.668327877347957, "dram": 5.790565728819927, "package-0": 74.52539662752395, "psys": 105.28204052731888, "uncore": 0.0 }, "npu_busy_delta_us": 3701084, "npu_utilization": 0.9720513684611377, "seconds": 3.807498369, "watts": { "core": 2.7520345340954235, "dram": 2.0078159618510396, "package-0": 22.821745823313837, "psys": 35.661302997658616, "uncore": 0.0 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 474.8007585329114, "sha256": "56e5d3e7ac8fb688d7b620d385293c7321b36fd5df528c679a4978f4fa7ebb39", "sum": -66282.34790089168, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 149, "gross_j": { "core": 7.161725, "dram": 4.666981, "package-0": 22.174198, "psys": 39.641683, "uncore": 0.0 }, "net_j": { "core": 4.865540018825825, "dram": 3.934186648181668, "package-0": 17.286098347304236, "psys": 27.58844797480078, "uncore": 0.0 }, "npu_busy_delta_us": 1495318, "npu_utilization": 0.9937245518150539, "seconds": 1.50476105, "token_latencies_s": [ 0.11150901, 0.107430245, 0.10722571, 0.107223819, 0.107261423, 0.107118572, 0.107134035, 0.107060124, 0.107079946, 0.107060764, 0.107010555, 0.107189788, 0.107262411, 0.107166464 ], "watts": { "core": 4.759376912367582, "dram": 3.10147647694629, "package-0": 14.73602602885023, "psys": 26.344171388540396, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 26.849235, "dram": 12.930997, "package-0": 119.537415, "psys": 187.20520900000002, "uncore": 0.0 }, "net_j": { "core": 18.319691259784168, "dram": 10.208916014511901, "package-0": 101.37979262373914, "psys": 142.4315534794565, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 26, "gross_j": { "core": 3.89385, "dram": 0.506224, "package-0": 5.164355, "psys": 8.112955, "uncore": 0.0 }, "logits": { "l2": 609.263242568076, "sha256": "1f93e2afdd0c63c174d4669ef6edf80514b7042aecb99e298ed65730a9a55a74", "sum": -156409.51710008364, "top1": 437 }, "net_j": { "core": 3.4984456366897096, "dram": 0.38003639469220024, "package-0": 4.322621452042083, "psys": 6.037381054502323, "uncore": 0.0 }, "npu_busy_delta_us": 129917, "npu_utilization": 0.5013763643153943, "seconds": 0.259120711, "watts": { "core": 15.02716623836371, "dram": 1.9536223023099069, "package-0": 19.93030576394181, "psys": 31.30955826992926, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 732.7797817539648, "decode_tokens_per_s": 8.494204063666638, "end_to_end_ms": 5589.673872, "handoff_estimate_ms": 151.9584165, "handoff_plus_first_decode_ms": 259.12071100000003, "output_tokens_per_s_e2e": 2.862421022476425, "prefill_tokens_per_s": 1067.011496886735, "steady_tpot_ms": 107.1622945, "steady_tpot_p95_ms": 109.60956010000001, "ttft_ms": 3823.76386 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 378, "gross_j": { "core": 14.870262, "dram": 7.751933, "package-0": 91.300792, "psys": 138.592541, "uncore": 0.0 }, "logits": { "l2": 657.7937454973483, "sha256": "01330236be5a5d092e077e2f3012d7ef6a5af180540b794e147c26e69be8d496", "sum": -175698.30940290913, "top1": 72 }, "net_j": { "core": 9.035402606949793, "dram": 5.889821697148918, "package-0": 78.87959144057788, "psys": 107.96394087412423, "uncore": 0.0 }, "npu_busy_delta_us": 3709788, "npu_utilization": 0.9701927566206978, "seconds": 3.82376386, "watts": { "core": 3.8889069891465526, "dram": 2.027304322082274, "package-0": 23.87720459285893, "psys": 36.24505750729074, "uncore": 0.0 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 474.8007585329114, "sha256": "56e5d3e7ac8fb688d7b620d385293c7321b36fd5df528c679a4978f4fa7ebb39", "sum": -66282.34790089168, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 149, "gross_j": { "core": 8.085123, "dram": 4.67284, "package-0": 23.072268, "psys": 40.499713, "uncore": 0.0 }, "net_j": { "core": 5.785843016144667, "dram": 3.9390579226707834, "package-0": 18.177579731119174, "psys": 28.430231550829944, "uncore": 0.0 }, "npu_busy_delta_us": 1495355, "npu_utilization": 0.9924114798317114, "seconds": 1.506789301, "token_latencies_s": [ 0.113447554, 0.107542948, 0.107317305, 0.10721283, 0.107154704, 0.107069677, 0.107135166, 0.107200964, 0.107249605, 0.107071279, 0.107074998, 0.107169885, 0.106999901, 0.107105706 ], "watts": { "core": 5.365795333583935, "dram": 3.101190058157972, "package-0": 15.312205883521868, "psys": 26.87815275375386, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 96.2730981754139, "compile_target": "NPU", "context_length": 8192, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 410177, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 1.5259465821328744, "dram": 0.48698386485903, "package-0": 3.248422500499841, "psys": 8.010065800945089, "uncore": 0.0023192603038971636 }, "model": { "family": "Llama 3.2", "ir": "/data/models/llama32_3b_ov_fp16/openvino_model.xml", "key": "llama32_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "npu_npu", "label": "NPU -> NPU", "mode": "staged", "prefill_device": "NPU" }, "prompt_length": 8176, "prompt_sha256": "998ace73fce48442f97bcb84c31be293d0d94180b17f70a4d127bba1092ad744", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "NPU" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 8192, "NPUW_LLM_MIN_RESPONSE_LEN": 8192, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.04408681485801935, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 44.705817, "dram": 26.179865, "package-0": 249.032627, "psys": 393.142243, "uncore": 0.0 }, "net_j": { "core": 26.42580749283173, "dram": 20.346063356027738, "package-0": 210.11829350515575, "psys": 297.1860144473222, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 41, "gross_j": { "core": 5.891281, "dram": 0.733152, "package-0": 7.692608, "psys": 12.431914, "uncore": 0.0 }, "logits": { "l2": 539.8529262282318, "sha256": "21b75f19253b392d43acf6a0728dac2329dc19092f92875416d390c56bfec5b5", "sum": -136239.45887422375, "top1": 546 }, "net_j": { "core": 5.240875637733782, "dram": 0.5255845000953016, "package-0": 6.308030428301388, "psys": 9.017777614073562, "uncore": 0.0 }, "npu_busy_delta_us": 149187, "npu_utilization": 0.3500146308072364, "seconds": 0.426230754, "watts": { "core": 13.821811177895437, "dram": 1.7200823570792831, "package-0": 18.047989094658337, "psys": 29.167097595214823, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 683.8374124439363, "decode_tokens_per_s": 6.954513862528223, "end_to_end_ms": 11979.455717, "handoff_estimate_ms": 303.0441995, "handoff_plus_first_decode_ms": 426.230754, "output_tokens_per_s_e2e": 1.335619946179563, "prefill_tokens_per_s": 832.3676017976165, "steady_tpot_ms": 123.1865545, "steady_tpot_p95_ms": 125.31922225, "ttft_ms": 9822.583174 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 967, "gross_j": { "core": 31.168621, "dram": 20.148264, "package-0": 216.592219, "psys": 336.905937, "uncore": 0.0 }, "logits": { "l2": 766.5680953013031, "sha256": "5bf315884e32b1eae4ea75eee38ef4394280af5ea8aefed17fa949ce7ecfd94b", "sum": -240251.75810273737, "top1": 27 }, "net_j": { "core": 16.17988377791882, "dram": 15.364824483026203, "package-0": 184.68431880454725, "psys": 258.22639944100393, "uncore": 0.0 }, "npu_busy_delta_us": 9557622, "npu_utilization": 0.9730253061433634, "seconds": 9.822583174, "watts": { "core": 3.1731592848714323, "dram": 2.0512184669844977, "package-0": 22.05043369582365, "psys": 34.29911776077163, "uncore": 0.0 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 442.27127637103683, "sha256": "d6e71a077cdc3ab7718cb5c7946ef61d3371f158d3d849c96c27ab82187f7a4e", "sum": -55858.642204726115, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 170, "gross_j": { "core": 7.645915, "dram": 5.298449, "package-0": 24.7478, "psys": 43.804392, "uncore": 0.0 }, "net_j": { "core": 5.005048077179126, "dram": 4.455654372906234, "package-0": 19.125944272307102, "psys": 29.941837392244672, "uncore": 0.0 }, "npu_busy_delta_us": 1718228, "npu_utilization": 0.9928270604125577, "seconds": 1.730641789, "token_latencies_s": [ 0.128157366, 0.123718048, 0.123790991, 0.123224991, 0.123290654, 0.123148118, 0.123656615, 0.123530244, 0.123034049, 0.122994115, 0.12296374, 0.122956266, 0.123091253, 0.123050681 ], "watts": { "core": 4.417965085899124, "dram": 3.0615515201800085, "package-0": 14.299781825041785, "psys": 25.311067997098963, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 34.015965, "dram": 25.939203, "package-0": 237.71039, "psys": 387.635421, "uncore": 0.0 }, "net_j": { "core": 15.766608800856353, "dram": 20.11518391761104, "package-0": 198.86131101449956, "psys": 291.8400991377456, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 39, "gross_j": { "core": 5.235216, "dram": 0.71234, "package-0": 6.988263, "psys": 11.420808, "uncore": 0.0 }, "logits": { "l2": 539.8529262282318, "sha256": "21b75f19253b392d43acf6a0728dac2329dc19092f92875416d390c56bfec5b5", "sum": -136239.45887422375, "top1": 546 }, "net_j": { "core": 4.632088468458037, "dram": 0.5198605385612796, "package-0": 5.704330084475936, "psys": 8.254844479009567, "uncore": 0.0 }, "npu_busy_delta_us": 147853, "npu_utilization": 0.3740764070764267, "seconds": 0.395248129, "watts": { "core": 13.245390973122102, "dram": 1.8022602707880244, "package-0": 17.680698496108505, "psys": 28.895286687112943, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 684.9860490650676, "decode_tokens_per_s": 7.025808287533666, "end_to_end_ms": 11959.367656000002, "handoff_estimate_ms": 271.4078135, "handoff_plus_first_decode_ms": 395.248129, "output_tokens_per_s_e2e": 1.3378633770802102, "prefill_tokens_per_s": 832.2151978604437, "steady_tpot_ms": 123.84031549999999, "steady_tpot_p95_ms": 126.18708135000001, "ttft_ms": 9824.381988000001 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 969, "gross_j": { "core": 22.851504, "dram": 19.922556, "package-0": 207.619891, "psys": 332.988039, "uncore": 0.0 }, "logits": { "l2": 766.5680953013031, "sha256": "5bf315884e32b1eae4ea75eee38ef4394280af5ea8aefed17fa949ce7ecfd94b", "sum": -240251.75810273737, "top1": 27 }, "net_j": { "core": 7.860021883843624, "dram": 15.13824048963232, "package-0": 175.70614749667544, "psys": 254.2940928225003, "uncore": 0.0 }, "npu_busy_delta_us": 9529870, "npu_utilization": 0.9700223394856051, "seconds": 9.824381988, "watts": { "core": 2.3259991343895203, "dram": 2.0278686256636216, "package-0": 21.13312483712436, "psys": 33.89404436907365, "uncore": 0.0 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 442.27127637103683, "sha256": "d6e71a077cdc3ab7718cb5c7946ef61d3371f158d3d849c96c27ab82187f7a4e", "sum": -55858.642204726115, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 171, "gross_j": { "core": 5.929245, "dram": 5.304307, "package-0": 23.102236, "psys": 43.226574, "uncore": 0.0 }, "net_j": { "core": 3.2744984485546915, "dram": 4.457082889417443, "package-0": 17.45083343334818, "psys": 29.291161836235727, "uncore": 0.0 }, "npu_busy_delta_us": 1718624, "npu_utilization": 0.9878639515865502, "seconds": 1.739737539, "token_latencies_s": [ 0.128960463, 0.124082549, 0.123907004, 0.123708785, 0.123736941, 0.124693722, 0.124401387, 0.123773627, 0.123647812, 0.124332317, 0.12411079, 0.12365935, 0.123370689, 0.123311002 ], "watts": { "core": 3.4081261495386976, "dram": 3.048912195714827, "package-0": 13.27915014886622, "psys": 24.846606474242435, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 33.828406, "dram": 26.150018000000003, "package-0": 238.091126, "psys": 392.294893, "uncore": 0.0 }, "net_j": { "core": 15.554093200864596, "dram": 20.318034378314564, "package-0": 199.18891961023888, "psys": 296.3685678572622, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 39, "gross_j": { "core": 5.429125, "dram": 0.712644, "package-0": 7.181988, "psys": 11.765411, "uncore": 0.0 }, "logits": { "l2": 539.8529262282318, "sha256": "21b75f19253b392d43acf6a0728dac2329dc19092f92875416d390c56bfec5b5", "sum": -136239.45887422375, "top1": 546 }, "net_j": { "core": 4.82348091855534, "dram": 0.5193614179199184, "package-0": 5.89269787379552, "psys": 8.586237494762425, "uncore": 0.0 }, "npu_busy_delta_us": 147283, "npu_utilization": 0.3710859188455758, "seconds": 0.396897302, "watts": { "core": 13.678916366128384, "dram": 1.795537526732797, "package-0": 18.095330867227712, "psys": 29.643464293440825, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 684.050587194936, "decode_tokens_per_s": 7.025409807563141, "end_to_end_ms": 11975.722488, "handoff_estimate_ms": 273.0188295, "handoff_plus_first_decode_ms": 396.897302, "output_tokens_per_s_e2e": 1.3360363031151095, "prefill_tokens_per_s": 830.8423201669978, "steady_tpot_ms": 123.8784725, "steady_tpot_p95_ms": 125.85470939999999, "ttft_ms": 9840.615724 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 971, "gross_j": { "core": 22.428959, "dram": 20.139963, "package-0": 207.759478, "psys": 337.076224, "uncore": 0.0 }, "logits": { "l2": 766.5680953013031, "sha256": "5bf315884e32b1eae4ea75eee38ef4394280af5ea8aefed17fa949ce7ecfd94b", "sum": -240251.75810273737, "top1": 27 }, "net_j": { "core": 7.412705069879179, "dram": 15.34774192213394, "package-0": 175.79300046338588, "psys": 258.25224452894514, "uncore": 0.0 }, "npu_busy_delta_us": 9562541, "npu_utilization": 0.9717421417725102, "seconds": 9.840615724, "watts": { "core": 2.2792231328877772, "dram": 2.0466161432237637, "package-0": 21.112447008097398, "psys": 34.253570452701894, "uncore": 0.0 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 442.27127637103683, "sha256": "d6e71a077cdc3ab7718cb5c7946ef61d3371f158d3d849c96c27ab82187f7a4e", "sum": -55858.642204726115, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 171, "gross_j": { "core": 5.970322, "dram": 5.297411, "package-0": 23.14966, "psys": 43.453258, "uncore": 0.0 }, "net_j": { "core": 3.317907212430078, "dram": 4.4509310382607055, "package-0": 17.503221273057477, "psys": 29.530085833554637, "uncore": 0.0 }, "npu_busy_delta_us": 1718014, "npu_utilization": 0.9883814566417313, "seconds": 1.738209462, "token_latencies_s": [ 0.127366209, 0.123726367, 0.123921256, 0.12388101, 0.123176337, 0.123126677, 0.123637577, 0.124589797, 0.123881658, 0.123826101, 0.123875935, 0.125040825, 0.124345598, 0.123776723 ], "watts": { "core": 3.4347540561253718, "dram": 3.0476252234323646, "package-0": 13.318107228207001, "psys": 24.998861730968994, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 2.6132829897105694, "compile_target": "NPU", "context_length": 1024, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 405965, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 1.5259465821328744, "dram": 0.48698386485903, "package-0": 3.248422500499841, "psys": 8.010065800945089, "uncore": 0.0023192603038971636 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_0_5b_ov_fp16/openvino_model.xml", "key": "qwen25_0_5b_fp16", "size_b": 0.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "npu_npu", "label": "NPU -> NPU", "mode": "staged", "prefill_device": "NPU" }, "prompt_length": 1008, "prompt_sha256": "2669029615323cd09063f0eb8d822f5daea575e1396736877003e47194c143f8", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "NPU" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 1024, "NPUW_LLM_MIN_RESPONSE_LEN": 1024, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.042768518440425396, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 5.642685, "dram": 0.99963, "package-0": 9.6724, "psys": 15.646383, "uncore": 0.0 }, "net_j": { "core": 4.833673686918092, "dram": 0.741445696189947, "package-0": 7.950183388130849, "psys": 11.39968529290373, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 9, "gross_j": { "core": 2.425347, "dram": 0.091979, "package-0": 2.718865, "psys": 3.689016, "uncore": 0.0 }, "logits": { "l2": 712.0668989995816, "sha256": "436eb677d7403000d1f335cdba1e504bf62a2332f96172cb434e632ffe2586d2", "sum": 23519.531799525023, "top1": 78 }, "net_j": { "core": 2.2874452373756147, "dram": 0.047969639436519926, "package-0": 2.425300864916381, "psys": 2.9651360125736432, "uncore": 0.0 }, "npu_busy_delta_us": 19234, "npu_utilization": 0.21283307770827353, "seconds": 0.090371291, "watts": { "core": 26.83758274516627, "dram": 1.0177900413085832, "package-0": 30.085494739695594, "psys": 40.82066283638683, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 1931.4554380599402, "decode_tokens_per_s": 45.951851992589326, "end_to_end_ms": 530.1701400000001, "handoff_estimate_ms": 73.9033345, "handoff_plus_first_decode_ms": 90.371291, "output_tokens_per_s_e2e": 30.178991219686566, "prefill_tokens_per_s": 4947.445368914313, "steady_tpot_ms": 16.4679565, "steady_tpot_p95_ms": 18.03229775, "ttft_ms": 203.741512 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 21, "gross_j": { "core": 0.349059, "dram": 0.246459, "package-0": 2.067073, "psys": 3.967641, "uncore": 0.0 }, "logits": { "l2": 841.2188648681404, "sha256": "ce6ed629ef29e138c367d484774de89c27db3dc3473b93905142453d08c5944d", "sum": 177284.36455662176, "top1": 291 }, "net_j": { "core": 0.03816033612501596, "dram": 0.14724017105401757, "package-0": 1.4052344881333418, "psys": 2.3356580824959563, "uncore": 0.0 }, "npu_busy_delta_us": 200620, "npu_utilization": 0.9846790574519737, "seconds": 0.203741512, "watts": { "core": 1.7132443780038307, "dram": 1.209665117239338, "package-0": 10.145566211366882, "psys": 19.473895923575945, "uncore": 0.0 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 722.9822522062098, "sha256": "7a9d0c61379cffa419a00b2eb4ebfdb6f2d1f954050a9188f0d146a26b6a317d", "sum": 73133.63508696854, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 24, "gross_j": { "core": 2.868279, "dram": 0.661192, "package-0": 4.886462, "psys": 7.989726, "uncore": 0.0 }, "net_j": { "core": 2.508068113417462, "dram": 0.5462358856994095, "package-0": 4.119648035081126, "psys": 6.0988911978341305, "uncore": 0.0 }, "npu_busy_delta_us": 226370, "npu_utilization": 0.9589619322020903, "seconds": 0.236057337, "token_latencies_s": [ 0.018016455, 0.01806172, 0.017438221, 0.017971406, 0.016983609, 0.016698959, 0.016499473, 0.01643644, 0.016391744, 0.016305754, 0.016296229, 0.016269944, 0.016282356, 0.016375491 ], "watts": { "core": 12.15077250490206, "dram": 2.8009805092395834, "package-0": 20.70031824513889, "psys": 33.84654805285717, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 4.278676, "dram": 0.958982, "package-0": 8.308817, "psys": 13.827296, "uncore": 0.0 }, "net_j": { "core": 3.58052776258105, "dram": 0.7361780494006368, "package-0": 6.8226047893824315, "psys": 10.162545617504191, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 2, "gross_j": { "core": 0.663023, "dram": 0.049194, "package-0": 0.799192, "psys": 1.091794, "uncore": 0.0 }, "logits": { "l2": 712.0668989995816, "sha256": "436eb677d7403000d1f335cdba1e504bf62a2332f96172cb434e632ffe2586d2", "sum": 23519.531799525023, "top1": 78 }, "net_j": { "core": 0.6319698710816558, "dram": 0.03928384133934501, "package-0": 0.7330863552347182, "psys": 0.9287885521857665, "uncore": 0.0 }, "npu_busy_delta_us": 16716, "npu_utilization": 0.8214219937065591, "seconds": 0.020350076, "watts": { "core": 32.58086112307394, "dram": 2.4173865493180466, "package-0": 39.27218748470521, "psys": 53.650610444894646, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 2238.162637036616, "decode_tokens_per_s": 58.541501709724066, "end_to_end_ms": 457.518137, "handoff_estimate_ms": 3.706964, "handoff_plus_first_decode_ms": 20.350076, "output_tokens_per_s_e2e": 34.97129120369713, "prefill_tokens_per_s": 5007.708866034781, "steady_tpot_ms": 16.643112000000002, "steady_tpot_p95_ms": 17.64209375, "ttft_ms": 201.289657 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 20, "gross_j": { "core": 0.641356, "dram": 0.24707, "package-0": 2.397333, "psys": 4.464466, "uncore": 0.0 }, "logits": { "l2": 841.2188648681404, "sha256": "ce6ed629ef29e138c367d484774de89c27db3dc3473b93905142453d08c5944d", "sum": 177284.36455662176, "top1": 291 }, "net_j": { "core": 0.33419873588215143, "dram": 0.1490451848779915, "package-0": 1.743459149083305, "psys": 2.8521226023803328, "uncore": 0.0 }, "npu_busy_delta_us": 198320, "npu_utilization": 0.9852468475317636, "seconds": 0.201289657, "watts": { "core": 3.186234253456947, "dram": 1.2274351483444577, "package-0": 11.909866784660476, "psys": 22.17931147848297, "uncore": 0.0 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 722.9822522062098, "sha256": "7a9d0c61379cffa419a00b2eb4ebfdb6f2d1f954050a9188f0d146a26b6a317d", "sum": 73133.63508696854, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 23, "gross_j": { "core": 2.974297, "dram": 0.662718, "package-0": 5.112292, "psys": 8.271036, "uncore": 0.0 }, "net_j": { "core": 2.6143591556172425, "dram": 0.5478490231833003, "package-0": 4.346059285064408, "psys": 6.3816344629380914, "uncore": 0.0 }, "npu_busy_delta_us": 223247, "npu_utilization": 0.9464495104859196, "seconds": 0.235878404, "token_latencies_s": [ 0.017965628, 0.017318933, 0.017467883, 0.017039755, 0.017103635, 0.016740186, 0.016546038, 0.016509856, 0.016455249, 0.016495377, 0.016426851, 0.01649644, 0.016885393, 0.016388464 ], "watts": { "core": 12.60945024878157, "dram": 2.8095747163017095, "package-0": 21.673421192047744, "psys": 35.06482941948344, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 4.111073, "dram": 0.9642919999999999, "package-0": 8.127299, "psys": 13.704004000000001, "uncore": 0.0 }, "net_j": { "core": 3.407356510257411, "dram": 0.7397110219025407, "package-0": 6.629233139753828, "psys": 10.010024502913383, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 2, "gross_j": { "core": 0.710509, "dram": 0.051452, "package-0": 0.812437, "psys": 1.164242, "uncore": 0.0 }, "logits": { "l2": 712.0668989995816, "sha256": "436eb677d7403000d1f335cdba1e504bf62a2332f96172cb434e632ffe2586d2", "sum": 23519.531799525023, "top1": 78 }, "net_j": { "core": 0.6774841741396297, "dram": 0.04091260266978655, "package-0": 0.7421340230439073, "psys": 0.9908866340234442, "uncore": 0.0 }, "npu_busy_delta_us": 16723, "npu_utilization": 0.7727036866416938, "seconds": 0.02164219, "watts": { "core": 32.829810661490356, "dram": 2.3773934153613845, "package-0": 37.53950039252035, "psys": 53.79501797184111, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 2220.4528711209146, "decode_tokens_per_s": 58.259557701827305, "end_to_end_ms": 461.16718499999996, "handoff_estimate_ms": 5.040392499999998, "handoff_plus_first_decode_ms": 21.64219, "output_tokens_per_s_e2e": 34.69457611126429, "prefill_tokens_per_s": 4948.485164861213, "steady_tpot_ms": 16.6017975, "steady_tpot_p95_ms": 17.67764695, "ttft_ms": 203.698701 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 21, "gross_j": { "core": 0.450682, "dram": 0.249572, "package-0": 2.210688, "psys": 4.259754, "uncore": 0.0 }, "logits": { "l2": 841.2188648681404, "sha256": "ce6ed629ef29e138c367d484774de89c27db3dc3473b93905142453d08c5944d", "sum": 177284.36455662176, "top1": 291 }, "net_j": { "core": 0.1398486634241437, "dram": 0.150374019320256, "package-0": 1.548988556349011, "psys": 2.6281140014229605, "uncore": 0.0 }, "npu_busy_delta_us": 198779, "npu_utilization": 0.9758481474066936, "seconds": 0.203698701, "watts": { "core": 2.2124932451091084, "dram": 1.2252017257586731, "package-0": 10.852734892992764, "psys": 20.91203320928394, "uncore": 0.0 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 722.9822522062098, "sha256": "7a9d0c61379cffa419a00b2eb4ebfdb6f2d1f954050a9188f0d146a26b6a317d", "sum": 73133.63508696854, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 24, "gross_j": { "core": 2.949882, "dram": 0.663268, "package-0": 5.104174, "psys": 8.280008, "uncore": 0.0 }, "net_j": { "core": 2.5900236726936376, "dram": 0.5484243999124981, "package-0": 4.33811056036091, "psys": 6.391023867466979, "uncore": 0.0 }, "npu_busy_delta_us": 222688, "npu_utilization": 0.9442882565079872, "seconds": 0.235826294, "token_latencies_s": [ 0.018017478, 0.017494661, 0.017361773, 0.017047193, 0.01711373, 0.016748173, 0.016467834, 0.016490775, 0.016367011, 0.016488662, 0.016500223, 0.016572336, 0.016631259, 0.016498169 ], "watts": { "core": 12.508706938336571, "dram": 2.8125277667298625, "package-0": 21.64378667630676, "psys": 35.11062256696448, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 5.864324741996825, "compile_target": "NPU", "context_length": 2048, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 406099, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 1.5259465821328744, "dram": 0.48698386485903, "package-0": 3.248422500499841, "psys": 8.010065800945089, "uncore": 0.0023192603038971636 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_0_5b_ov_fp16/openvino_model.xml", "key": "qwen25_0_5b_fp16", "size_b": 0.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "npu_npu", "label": "NPU -> NPU", "mode": "staged", "prefill_device": "NPU" }, "prompt_length": 2032, "prompt_sha256": "cd1a97358e68d07c27508cfacc4ba6882a7a2962b1281e2e2b0c7e5e320da4fa", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "NPU" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 2048, "NPUW_LLM_MIN_RESPONSE_LEN": 2048, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.038750503212213516, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 6.904523, "dram": 1.270443, "package-0": 13.941065, "psys": 22.660097999999998, "uncore": 0.0 }, "net_j": { "core": 5.791324035398854, "dram": 0.9151815901406796, "package-0": 11.571296224025504, "psys": 16.8166451161279, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 11, "gross_j": { "core": 2.492852, "dram": 0.099609, "package-0": 2.859184, "psys": 4.017812, "uncore": 0.0 }, "logits": { "l2": 759.4074790825704, "sha256": "c51b071629174bb3f22fc600a0fbef27184fb95d6a4b9524fdf7f550639846d2", "sum": -148789.21454068646, "top1": 522 }, "net_j": { "core": 2.3351354028972557, "dram": 0.049276020517746454, "package-0": 2.5234382086796256, "psys": 3.189919113390342, "uncore": 0.0 }, "npu_busy_delta_us": 19207, "npu_utilization": 0.1858324142254534, "seconds": 0.103356565, "watts": { "core": 24.11895170858281, "dram": 0.9637413936889254, "package-0": 27.663303245420355, "psys": 38.873312014577884, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 2807.349539107639, "decode_tokens_per_s": 42.792829269503265, "end_to_end_ms": 729.5137179999999, "handoff_estimate_ms": 86.4074495, "handoff_plus_first_decode_ms": 103.356565, "output_tokens_per_s_e2e": 21.93241827427843, "prefill_tokens_per_s": 5361.651564518712, "steady_tpot_ms": 16.9491155, "steady_tpot_p95_ms": 20.958004949999996, "ttft_ms": 378.987701 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 38, "gross_j": { "core": 1.311948, "dram": 0.491637, "package-0": 5.751328, "psys": 9.919469, "uncore": 0.0 }, "logits": { "l2": 659.9256228894352, "sha256": "ae1f9856e8f12d8b4cc30ec030d87b4669b23f386612aacb1b94e80973a058b4", "sum": -92554.5169948861, "top1": 220 }, "net_j": { "core": 0.7336330129886541, "dram": 0.3070761046329815, "package-0": 4.520215824658894, "psys": 6.883752577241097, "uncore": 0.0 }, "npu_busy_delta_us": 357540, "npu_utilization": 0.9434079234143802, "seconds": 0.378987701, "watts": { "core": 3.4617165584484226, "dram": 1.297237347551814, "package-0": 15.175500378572972, "psys": 26.173590788900032, "uncore": 0.0 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 734.1197976827015, "sha256": "572b0503dc5eccc03209bbe773e7bd58c98289cecafb7deafd8ec09b9210e019", "sum": -60055.14825826511, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 25, "gross_j": { "core": 3.099723, "dram": 0.679197, "package-0": 5.330553, "psys": 8.722817, "uncore": 0.0 }, "net_j": { "core": 2.7225556195129443, "dram": 0.5588294649899516, "package-0": 4.527642190686985, "psys": 6.74297342549646, "uncore": 0.0 }, "npu_busy_delta_us": 229064, "npu_utilization": 0.9267488281682964, "seconds": 0.247169452, "token_latencies_s": [ 0.022681491, 0.020029974, 0.017266856, 0.017665278, 0.017142405, 0.016938804, 0.016902463, 0.016959192, 0.016924811, 0.016939039, 0.016970031, 0.016846037, 0.016928737, 0.016935161 ], "watts": { "core": 12.540882276989471, "dram": 2.747900254275759, "package-0": 21.56639081758372, "psys": 35.29083763959633, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 5.409472, "dram": 1.235836, "package-0": 12.384001, "psys": 19.911143, "uncore": 0.0 }, "net_j": { "core": 4.405662745547006, "dram": 0.9154847399765256, "package-0": 10.247100136919109, "psys": 14.641903385774786, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 4, "gross_j": { "core": 1.071653, "dram": 0.063049, "package-0": 1.281186, "psys": 1.72845, "uncore": 0.0 }, "logits": { "l2": 759.4074790825704, "sha256": "c51b071629174bb3f22fc600a0fbef27184fb95d6a4b9524fdf7f550639846d2", "sum": -148789.21454068646, "top1": 522 }, "net_j": { "core": 1.0231790880922187, "dram": 0.0475792494582134, "package-0": 1.1779951360653067, "psys": 1.4739986131973049, "uncore": 0.0 }, "npu_busy_delta_us": 16491, "npu_utilization": 0.5191325415169096, "seconds": 0.031766454, "watts": { "core": 33.735367504349085, "dram": 1.984766697598668, "package-0": 40.33141376119601, "psys": 54.41117223848781, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 3113.2793270680786, "decode_tokens_per_s": 54.76064976081041, "end_to_end_ms": 657.827257, "handoff_estimate_ms": 14.449682999999999, "handoff_plus_first_decode_ms": 31.766454, "output_tokens_per_s_e2e": 24.322494742719364, "prefill_tokens_per_s": 5292.935647988272, "steady_tpot_ms": 17.316771000000003, "steady_tpot_p95_ms": 17.806366, "ttft_ms": 383.907936 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 38, "gross_j": { "core": 1.25726, "dram": 0.494872, "package-0": 5.71727, "psys": 9.543555, "uncore": 0.0 }, "logits": { "l2": 659.9256228894352, "sha256": "ae1f9856e8f12d8b4cc30ec030d87b4669b23f386612aacb1b94e80973a058b4", "sum": -92554.5169948861, "top1": 220 }, "net_j": { "core": 0.6714369972071138, "dram": 0.30791502957666683, "package-0": 4.470174822577147, "psys": 6.468427171134984, "uncore": 0.0 }, "npu_busy_delta_us": 357996, "npu_utilization": 0.9325048180301226, "seconds": 0.383907936, "watts": { "core": 3.2748997405461293, "dram": 1.289038213578372, "package-0": 14.892294385912356, "psys": 24.858967750018067, "uncore": 0.0 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 734.1197976827015, "sha256": "572b0503dc5eccc03209bbe773e7bd58c98289cecafb7deafd8ec09b9210e019", "sum": -60055.14825826511, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 24, "gross_j": { "core": 3.080559, "dram": 0.677915, "package-0": 5.385545, "psys": 8.639138, "uncore": 0.0 }, "net_j": { "core": 2.7110466602476735, "dram": 0.5599904609416454, "package-0": 4.598930178276654, "psys": 6.699477601442497, "uncore": 0.0 }, "npu_busy_delta_us": 227520, "npu_utilization": 0.9395717788466242, "seconds": 0.242152867, "token_latencies_s": [ 0.017941085, 0.017693686, 0.017619532, 0.017454019, 0.017733825, 0.017389973, 0.017064191, 0.016913076, 0.016920434, 0.017038488, 0.016919895, 0.016798849, 0.017255338, 0.017378204 ], "watts": { "core": 12.721546674894418, "dram": 2.7995332386463136, "package-0": 22.24026940800168, "psys": 35.67638123400786, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 5.414354, "dram": 1.22662, "package-0": 12.274626, "psys": 19.64979, "uncore": 0.0 }, "net_j": { "core": 4.427842732563086, "dram": 0.9117891432920843, "package-0": 10.17454894903306, "psys": 14.471351738748112, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 3, "gross_j": { "core": 1.007322, "dram": 0.05957, "package-0": 1.20477, "psys": 1.613765, "uncore": 0.0 }, "logits": { "l2": 759.4074790825704, "sha256": "c51b071629174bb3f22fc600a0fbef27184fb95d6a4b9524fdf7f550639846d2", "sum": -148789.21454068646, "top1": 522 }, "net_j": { "core": 0.960338896702405, "dram": 0.044576019545616324, "package-0": 1.1047527571408875, "psys": 1.3671392312130513, "uncore": 0.0 }, "npu_busy_delta_us": 16489, "npu_utilization": 0.5355400436921948, "seconds": 0.030789481, "watts": { "core": 32.71643325199278, "dram": 1.934751677041909, "package-0": 39.12927275389929, "psys": 52.41286788822455, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 3167.8691398301476, "decode_tokens_per_s": 55.144178959330446, "end_to_end_ms": 646.491351, "handoff_estimate_ms": 13.804062499999999, "handoff_plus_first_decode_ms": 30.789481000000002, "output_tokens_per_s_e2e": 24.748977654923028, "prefill_tokens_per_s": 5426.232339422924, "steady_tpot_ms": 16.9854185, "steady_tpot_p95_ms": 17.955736849999997, "ttft_ms": 374.477146 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 38, "gross_j": { "core": 1.403622, "dram": 0.488829, "package-0": 5.802841, "psys": 9.567053, "uncore": 0.0 }, "logits": { "l2": 659.9256228894352, "sha256": "ae1f9856e8f12d8b4cc30ec030d87b4669b23f386612aacb1b94e80973a058b4", "sum": -92554.5169948861, "top1": 220 }, "net_j": { "core": 0.8321898789744265, "dram": 0.30646467213954076, "package-0": 4.5863810130106355, "psys": 6.567466419589879, "uncore": 0.0 }, "npu_busy_delta_us": 359521, "npu_utilization": 0.9600612583177506, "seconds": 0.374477146, "watts": { "core": 3.7482180554751396, "dram": 1.3053640394920123, "package-0": 15.495848176539992, "psys": 25.54776199880566, "uncore": 0.0 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 734.1197976827015, "sha256": "572b0503dc5eccc03209bbe773e7bd58c98289cecafb7deafd8ec09b9210e019", "sum": -60055.14825826511, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 24, "gross_j": { "core": 3.00341, "dram": 0.678221, "package-0": 5.267015, "psys": 8.468972, "uncore": 0.0 }, "net_j": { "core": 2.6353139568862542, "dram": 0.5607484516069272, "package-0": 4.483415178881536, "psys": 6.536746087945183, "uncore": 0.0 }, "npu_busy_delta_us": 228114, "npu_utilization": 0.9456493356792068, "seconds": 0.241224724, "token_latencies_s": [ 0.017436233, 0.017452104, 0.017352495, 0.01732255, 0.018891055, 0.016965704, 0.017005133, 0.016849093, 0.016948205, 0.016854944, 0.016932022, 0.017425609, 0.016885595, 0.016874093 ], "watts": { "core": 12.450672344846376, "dram": 2.8115733277821056, "package-0": 21.834474147848958, "psys": 35.10822547360444, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 14.634349533356726, "compile_target": "NPU", "context_length": 4096, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 406304, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 1.5259465821328744, "dram": 0.48698386485903, "package-0": 3.248422500499841, "psys": 8.010065800945089, "uncore": 0.0023192603038971636 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_0_5b_ov_fp16/openvino_model.xml", "key": "qwen25_0_5b_fp16", "size_b": 0.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "npu_npu", "label": "NPU -> NPU", "mode": "staged", "prefill_device": "NPU" }, "prompt_length": 4080, "prompt_sha256": "66f8de7a9ef70b33481de3e8dae7cef6e4434d9723c6203a6a3dbe07ef228a16", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "NPU" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 4096, "NPUW_LLM_MIN_RESPONSE_LEN": 4096, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.03937573544681072, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 7.848979, "dram": 2.249933, "package-0": 23.131471, "psys": 40.520343, "uncore": 0.0 }, "net_j": { "core": 5.858397596178624, "dram": 1.614667626559138, "package-0": 18.89393772264779, "psys": 30.0712956916333, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 12, "gross_j": { "core": 2.589349, "dram": 0.123657, "package-0": 3.01934, "psys": 4.441944, "uncore": 0.0 }, "logits": { "l2": 745.9631922969153, "sha256": "24b1cbf9fe48ed8a6823bcda5c695ed01600d1970212ae8682be6ad913cc2694", "sum": -47925.70799851045, "top1": 78 }, "net_j": { "core": 2.40352184985598, "dram": 0.06435294254971488, "package-0": 2.6237526951758303, "psys": 3.466491964074085, "uncore": 0.0 }, "npu_busy_delta_us": 22063, "npu_utilization": 0.18117352289751504, "seconds": 0.12177828, "watts": { "core": 21.262814682552584, "dram": 1.0154273816316013, "package-0": 24.79374811337457, "psys": 36.47566708940215, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 3139.9254451073525, "decode_tokens_per_s": 36.15686424147231, "end_to_end_ms": 1304.489572, "handoff_estimate_ms": 102.36964700000001, "handoff_plus_first_decode_ms": 121.77828000000001, "output_tokens_per_s_e2e": 12.265333769950596, "prefill_tokens_per_s": 4586.173249459104, "steady_tpot_ms": 19.408633000000002, "steady_tpot_p95_ms": 27.549739649999996, "ttft_ms": 889.630587 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 88, "gross_j": { "core": 2.150263, "dram": 1.352658, "package-0": 14.464135, "psys": 26.18548, "uncore": 0.0 }, "logits": { "l2": 667.2018758805161, "sha256": "d7b52e24c142f2b72de7a972f3de94ad4cd10f7c3d5413ce59a0998a819faea1", "sum": 10659.559723772109, "top1": 220 }, "net_j": { "core": 0.792734246406487, "dram": 0.9194222584459324, "package-0": 11.574238984056318, "psys": 19.059480459596593, "uncore": 0.0 }, "npu_busy_delta_us": 853218, "npu_utilization": 0.9590699920482838, "seconds": 0.889630587, "watts": { "core": 2.417029080858255, "dram": 1.5204715527614834, "package-0": 16.25858554254048, "psys": 29.434104877511366, "uncore": 0.0 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 722.5170187115344, "sha256": "d8ec2f43616339a09570ca4449bf8889855b4f8bd315d2af2e02def48909cafd", "sum": 16701.138758618385, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 29, "gross_j": { "core": 3.109367, "dram": 0.773618, "package-0": 5.647996, "psys": 9.892919, "uncore": 0.0 }, "net_j": { "core": 2.662141499916157, "dram": 0.6308924255634908, "package-0": 4.695946043415644, "psys": 7.5453232679626225, "uncore": 0.0 }, "npu_busy_delta_us": 266669, "npu_utilization": 0.9098824844167069, "seconds": 0.293080705, "token_latencies_s": [ 0.035609648, 0.023209789, 0.019711727, 0.019431841, 0.019300816, 0.019221497, 0.019385425, 0.021546063, 0.019436998, 0.019447245, 0.019148993, 0.019087076, 0.019229221, 0.019285769 ], "watts": { "core": 10.609251810009125, "dram": 2.639607407795747, "package-0": 19.271128749332032, "psys": 33.75493108630266, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 6.760175, "dram": 2.191644, "package-0": 21.855168, "psys": 37.534449, "uncore": 0.0 }, "net_j": { "core": 4.903234967827878, "dram": 1.5990283447293832, "package-0": 17.90212937255638, "psys": 27.786917847685658, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 5, "gross_j": { "core": 1.721004, "dram": 0.08258, "package-0": 1.989985, "psys": 2.685662, "uncore": 0.0 }, "logits": { "l2": 745.9631922969153, "sha256": "24b1cbf9fe48ed8a6823bcda5c695ed01600d1970212ae8682be6ad913cc2694", "sum": -47925.70799851045, "top1": 78 }, "net_j": { "core": 1.6423635326121473, "dram": 0.05748302793605765, "package-0": 1.8225758215243928, "psys": 2.2728590056033546, "uncore": 0.0 }, "npu_busy_delta_us": 19879, "npu_utilization": 0.3857338735871374, "seconds": 0.051535531, "watts": { "core": 33.39451377730055, "dram": 1.6023896212498518, "package-0": 38.613844883057475, "psys": 52.112822898826835, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 3365.9014788458758, "decode_tokens_per_s": 45.792923101217276, "end_to_end_ms": 1216.910247, "handoff_estimate_ms": 31.905805500000003, "handoff_plus_first_decode_ms": 51.535531000000006, "output_tokens_per_s_e2e": 13.148052651741702, "prefill_tokens_per_s": 4587.626795426779, "steady_tpot_ms": 19.6297255, "steady_tpot_p95_ms": 20.2453457, "ttft_ms": 889.348716 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 89, "gross_j": { "core": 1.860774, "dram": 1.351315, "package-0": 14.14187, "psys": 25.090573, "uncore": 0.0 }, "logits": { "l2": 667.2018758805161, "sha256": "d7b52e24c142f2b72de7a972f3de94ad4cd10f7c3d5413ce59a0998a819faea1", "sum": 10659.559723772109, "top1": 220 }, "net_j": { "core": 0.5036753664955396, "dram": 0.9182165250749041, "package-0": 11.252889620154958, "psys": 17.966831264853973, "uncore": 0.0 }, "npu_busy_delta_us": 853201, "npu_utilization": 0.9593548454619909, "seconds": 0.889348716, "watts": { "core": 2.0922883977042814, "dram": 1.519443358593661, "package-0": 15.90137787976522, "psys": 28.21230024691462, "uncore": 0.0 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 722.5170187115344, "sha256": "d8ec2f43616339a09570ca4449bf8889855b4f8bd315d2af2e02def48909cafd", "sum": 16701.138758618385, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 28, "gross_j": { "core": 3.178397, "dram": 0.757749, "package-0": 5.723313, "psys": 9.758214, "uncore": 0.0 }, "net_j": { "core": 2.7571960687201913, "dram": 0.6233287917184214, "package-0": 4.826663930877031, "psys": 7.547227577228332, "uncore": 0.0 }, "npu_busy_delta_us": 260737, "npu_utilization": 0.9446102903349685, "seconds": 0.276026, "token_latencies_s": [ 0.020099603, 0.020225822, 0.019960102, 0.020281604, 0.019838353, 0.019528373, 0.019682196, 0.019430199, 0.019438017, 0.019428453, 0.019577255, 0.01969393, 0.019471025, 0.019318372 ], "watts": { "core": 11.514846427510452, "dram": 2.745208784679704, "package-0": 20.734688036634232, "psys": 35.352517516465845, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 6.955915, "dram": 2.1974430000000003, "package-0": 22.093084, "psys": 37.26321, "uncore": 0.0 }, "net_j": { "core": 5.084102852864798, "dram": 1.6000811235260004, "package-0": 18.10838573686418, "psys": 27.437611488809317, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 6, "gross_j": { "core": 1.802425, "dram": 0.085083, "package-0": 2.040461, "psys": 2.784172, "uncore": 0.0 }, "logits": { "l2": 745.9631922969153, "sha256": "24b1cbf9fe48ed8a6823bcda5c695ed01600d1970212ae8682be6ad913cc2694", "sum": -47925.70799851045, "top1": 78 }, "net_j": { "core": 1.7190717946100482, "dram": 0.05848202550687543, "package-0": 1.8630193837285818, "psys": 2.346630694353518, "uncore": 0.0 }, "npu_busy_delta_us": 18781, "npu_utilization": 0.3438236433135702, "seconds": 0.054623934, "watts": { "core": 32.99698260473147, "dram": 1.5576139206670836, "package-0": 37.354706089092744, "psys": 50.969818468219444, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 3339.1583711978074, "decode_tokens_per_s": 45.60808645967518, "end_to_end_ms": 1226.656404, "handoff_estimate_ms": 35.06952, "handoff_plus_first_decode_ms": 54.623934, "output_tokens_per_s_e2e": 13.043587387491435, "prefill_tokens_per_s": 4544.607195949524, "steady_tpot_ms": 19.554413999999998, "steady_tpot_p95_ms": 20.106267550000002, "ttft_ms": 897.767359 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 89, "gross_j": { "core": 2.179926, "dram": 1.359066, "package-0": 14.515527, "psys": 25.170407, "uncore": 0.0 }, "logits": { "l2": 667.2018758805161, "sha256": "d7b52e24c142f2b72de7a972f3de94ad4cd10f7c3d5413ce59a0998a819faea1", "sum": 10659.559723772109, "top1": 220 }, "net_j": { "core": 0.8099809669834928, "dram": 0.9218677817698958, "package-0": 11.599199310810082, "psys": 17.979231380469308, "uncore": 0.0 }, "npu_busy_delta_us": 852678, "npu_utilization": 0.9497761212323158, "seconds": 0.897767359, "watts": { "core": 2.4281635750582016, "dram": 1.5138287067084157, "package-0": 16.168472661078336, "psys": 28.03666979832801, "uncore": 0.0 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 722.5170187115344, "sha256": "d8ec2f43616339a09570ca4449bf8889855b4f8bd315d2af2e02def48909cafd", "sum": 16701.138758618385, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 27, "gross_j": { "core": 2.973564, "dram": 0.753294, "package-0": 5.537096, "psys": 9.308631, "uncore": 0.0 }, "net_j": { "core": 2.555050091271257, "dram": 0.6197313162492292, "package-0": 4.646167042325514, "psys": 7.111749413986491, "uncore": 0.0 }, "npu_busy_delta_us": 260537, "npu_utilization": 0.9499458354365752, "seconds": 0.274265111, "token_latencies_s": [ 0.019712694, 0.020071207, 0.019838815, 0.02017138, 0.019883847, 0.019660559, 0.019476924, 0.019548802, 0.019560026, 0.019497376, 0.019239628, 0.019085801, 0.019191579, 0.019298435 ], "watts": { "core": 10.84193315423193, "dram": 2.7465906883066875, "package-0": 20.18884567494259, "psys": 33.940266649519266, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 50.01766771264374, "compile_target": "NPU", "context_length": 8192, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 406493, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 1.5259465821328744, "dram": 0.48698386485903, "package-0": 3.248422500499841, "psys": 8.010065800945089, "uncore": 0.0023192603038971636 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_0_5b_ov_fp16/openvino_model.xml", "key": "qwen25_0_5b_fp16", "size_b": 0.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "npu_npu", "label": "NPU -> NPU", "mode": "staged", "prefill_device": "NPU" }, "prompt_length": 8176, "prompt_sha256": "998ace73fce48442f97bcb84c31be293d0d94180b17f70a4d127bba1092ad744", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "NPU" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 8192, "NPUW_LLM_MIN_RESPONSE_LEN": 8192, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.038754516281187534, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 11.620087999999999, "dram": 5.957077, "package-0": 53.325058999999996, "psys": 96.75225400000001, "uncore": 0.0 }, "net_j": { "core": 6.3247959297586265, "dram": 4.267160833787196, "package-0": 42.05248490789287, "psys": 68.9559743565998, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 15, "gross_j": { "core": 2.706902, "dram": 0.157348, "package-0": 3.196037, "psys": 4.814135, "uncore": 0.0 }, "logits": { "l2": 772.4011392999273, "sha256": "c1b35fe6f6b041dcd0c1e661cd3228b8443fe53355bb70c15d60c562f9bd4d03", "sum": 11189.117347188294, "top1": 78 }, "net_j": { "core": 2.483511991617699, "dram": 0.08605629757300691, "package-0": 2.7204861923160086, "psys": 3.6415063592812436, "uncore": 0.0 }, "npu_busy_delta_us": 25533, "npu_utilization": 0.1744124294714231, "seconds": 0.146394383, "watts": { "core": 18.49047719269393, "dram": 1.0748226590087135, "package-0": 21.831691452260163, "psys": 32.88469749553165, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 2360.692145969335, "decode_tokens_per_s": 30.53897379764192, "end_to_end_ms": 3470.1687019999995, "handoff_estimate_ms": 123.1576245, "handoff_plus_first_decode_ms": 146.39438299999998, "output_tokens_per_s_e2e": 4.610726847596357, "prefill_tokens_per_s": 2744.551541355688, "steady_tpot_ms": 23.2367585, "steady_tpot_p95_ms": 30.86077999999999, "ttft_ms": 2978.993062 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 294, "gross_j": { "core": 5.745225, "dram": 4.937304, "package-0": 44.037546, "psys": 81.205786, "uncore": 0.0 }, "logits": { "l2": 674.7263359808221, "sha256": "dcb96fa05f02a6152ed6574236449268a41f7b136d2e8fd7a99247570c66fbf0", "sum": -598.7580670975149, "top1": 198 }, "net_j": { "core": 1.1994407188435536, "dram": 3.486582445279004, "package-0": 34.36051790856628, "psys": 57.34385555282111, "uncore": 0.0 }, "npu_busy_delta_us": 2901686, "npu_utilization": 0.9740492641671015, "seconds": 2.978993062, "watts": { "core": 1.9285795167790156, "dram": 1.6573734470818988, "package-0": 14.782695052815804, "psys": 27.259474698299922, "uncore": 0.0 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 723.3369122262443, "sha256": "1883b45b8d5aa18bcfafb26d9ce86cb59c90e61fe4ce23a8ce37ca7672e44044", "sum": 66699.60143572465, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 33, "gross_j": { "core": 3.167961, "dram": 0.862425, "package-0": 6.091476, "psys": 10.732333, "uncore": 0.0 }, "net_j": { "core": 2.6418432192973738, "dram": 0.6945220909351855, "package-0": 4.971480807010582, "psys": 7.970612444497441, "uncore": 0.0 }, "npu_busy_delta_us": 307694, "npu_utilization": 0.8924325025011438, "seconds": 0.344781257, "token_latencies_s": [ 0.042751295, 0.024458195, 0.023101537, 0.023272606, 0.023248144, 0.023309277, 0.022952439, 0.022673416, 0.023052258, 0.02340053, 0.02320157, 0.022706691, 0.023225373, 0.023402395 ], "watts": { "core": 9.188321394164417, "dram": 2.5013685706238955, "package-0": 17.667654132370657, "psys": 31.127947886099854, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 10.757297, "dram": 5.951827, "package-0": 52.352711, "psys": 91.399119, "uncore": 0.0 }, "net_j": { "core": 5.5860765730954345, "dram": 4.30150651105338, "package-0": 41.344259599085596, "psys": 64.2541216746021, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 8, "gross_j": { "core": 2.327997, "dram": 0.106506, "package-0": 2.663995, "psys": 3.500235, "uncore": 0.0 }, "logits": { "l2": 772.4011392999273, "sha256": "c1b35fe6f6b041dcd0c1e661cd3228b8443fe53355bb70c15d60c562f9bd4d03", "sum": 11189.117347188294, "top1": 78 }, "net_j": { "core": 2.2118441664203226, "dram": 0.06943749676845401, "package-0": 2.4167298057166904, "psys": 2.8905204458778, "uncore": 0.0 }, "npu_busy_delta_us": 22482, "npu_utilization": 0.2953550938210918, "seconds": 0.076118545, "watts": { "core": 30.583834727792024, "dram": 1.3992122419050443, "package-0": 34.99797585463569, "psys": 45.983997723550814, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 2417.3315714401288, "decode_tokens_per_s": 37.76794924809452, "end_to_end_ms": 3388.860716, "handoff_estimate_ms": 53.518238499999995, "handoff_plus_first_decode_ms": 76.118545, "output_tokens_per_s_e2e": 4.7213507254690015, "prefill_tokens_per_s": 2732.8956527854257, "steady_tpot_ms": 22.600306500000002, "steady_tpot_p95_ms": 24.377819849999995, "ttft_ms": 2991.698564 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 296, "gross_j": { "core": 5.476915, "dram": 4.999193, "package-0": 43.832713, "psys": 78.008528, "uncore": 0.0 }, "logits": { "l2": 674.7263359808221, "sha256": "dcb96fa05f02a6152ed6574236449268a41f7b136d2e8fd7a99247570c66fbf0", "sum": -598.7580670975149, "top1": 198 }, "net_j": { "core": 0.9117428014923714, "dram": 3.54228407081007, "package-0": 34.114412069989335, "psys": 54.04482564576706, "uncore": 0.0 }, "npu_busy_delta_us": 2903866, "npu_utilization": 0.9706412387073633, "seconds": 2.991698564, "watts": { "core": 1.8307041578003043, "dram": 1.6710216263619533, "package-0": 14.65144701657182, "psys": 26.074995970081964, "uncore": 0.0 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 723.3369122262443, "sha256": "1883b45b8d5aa18bcfafb26d9ce86cb59c90e61fe4ce23a8ce37ca7672e44044", "sum": 66699.60143572465, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 32, "gross_j": { "core": 2.952385, "dram": 0.846128, "package-0": 5.856003, "psys": 9.890356, "uncore": 0.0 }, "net_j": { "core": 2.46248960518274, "dram": 0.6897849434748564, "package-0": 4.813117723379572, "psys": 7.318775582957246, "uncore": 0.0 }, "npu_busy_delta_us": 305229, "npu_utilization": 0.950740003366583, "seconds": 0.321043607, "token_latencies_s": [ 0.025640621, 0.02369785, 0.023048045, 0.02292465, 0.022585962, 0.022614651, 0.022536019, 0.022500048, 0.022584317, 0.022388062, 0.022579137, 0.022416491, 0.022754714, 0.022728235 ], "watts": { "core": 9.196211778171307, "dram": 2.635554739453198, "package-0": 18.24052207337678, "psys": 30.806892846802587, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 10.391271, "dram": 5.9081269999999995, "package-0": 51.835072000000004, "psys": 88.98774800000001, "uncore": 0.0 }, "net_j": { "core": 5.232744156687014, "dram": 4.261857485359242, "package-0": 40.853642595159165, "psys": 61.90938239094715, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 8, "gross_j": { "core": 2.391901, "dram": 0.117187, "package-0": 2.753411, "psys": 3.73583, "uncore": 0.0 }, "logits": { "l2": 772.4011392999273, "sha256": "c1b35fe6f6b041dcd0c1e661cd3228b8443fe53355bb70c15d60c562f9bd4d03", "sum": 11189.117347188294, "top1": 78 }, "net_j": { "core": 2.25839679721496, "dram": 0.07458105729635098, "package-0": 2.469208354038404, "psys": 3.0350338767725575, "uncore": 0.0 }, "npu_busy_delta_us": 22993, "npu_utilization": 0.26280887815550386, "seconds": 0.087489434, "watts": { "core": 27.339312767756613, "dram": 1.3394417433309718, "package-0": 31.471354586657856, "psys": 42.70035625101883, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 2423.27989763919, "decode_tokens_per_s": 36.81603411132208, "end_to_end_ms": 3380.542218, "handoff_estimate_ms": 64.87825450000001, "handoff_plus_first_decode_ms": 87.489434, "output_tokens_per_s_e2e": 4.732968550076543, "prefill_tokens_per_s": 2749.981376074025, "steady_tpot_ms": 22.611179500000002, "steady_tpot_p95_ms": 23.9025259, "ttft_ms": 2973.1110439999998 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 294, "gross_j": { "core": 5.07561, "dram": 4.948535, "package-0": 43.280346, "psys": 75.518178, "uncore": 0.0 }, "logits": { "l2": 674.7263359808221, "sha256": "dcb96fa05f02a6152ed6574236449268a41f7b136d2e8fd7a99247570c66fbf0", "sum": -598.7580670975149, "top1": 198 }, "net_j": { "core": 0.538801364106698, "dram": 3.500677893137814, "package-0": 33.62242518818583, "psys": 51.70336290404346, "uncore": 0.0 }, "npu_busy_delta_us": 2893619, "npu_utilization": 0.9732630087394746, "seconds": 2.973111044, "watts": { "core": 1.7071713517875589, "dram": 1.6644299276969756, "package-0": 14.557258494378658, "psys": 25.40038931690841, "uncore": 0.0 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 723.3369122262443, "sha256": "1883b45b8d5aa18bcfafb26d9ce86cb59c90e61fe4ce23a8ce37ca7672e44044", "sum": 66699.60143572465, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 32, "gross_j": { "core": 2.92376, "dram": 0.842405, "package-0": 5.801315, "psys": 9.73374, "uncore": 0.0 }, "net_j": { "core": 2.4355459953653553, "dram": 0.686598534925077, "package-0": 4.76200905293493, "psys": 7.170985610131134, "uncore": 0.0 }, "npu_busy_delta_us": 306017, "npu_utilization": 0.9564772636418118, "seconds": 0.31994174, "token_latencies_s": [ 0.025090631, 0.022843426, 0.023262777, 0.022992873, 0.022606033, 0.022616326, 0.02258021, 0.02253286, 0.022556541, 0.022465176, 0.022628597, 0.022443708, 0.022702033, 0.022571763 ], "watts": { "core": 9.138413762455627, "dram": 2.632994994651214, "package-0": 18.13241060700614, "psys": 30.42347647418558, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 4.496121177449822, "compile_target": "NPU", "context_length": 1024, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 406788, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 1.5259465821328744, "dram": 0.48698386485903, "package-0": 3.248422500499841, "psys": 8.010065800945089, "uncore": 0.0023192603038971636 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_1_5b_ov_fp16/openvino_model.xml", "key": "qwen25_1_5b_fp16", "size_b": 1.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "npu_npu", "label": "NPU -> NPU", "mode": "staged", "prefill_device": "NPU" }, "prompt_length": 1008, "prompt_sha256": "2669029615323cd09063f0eb8d822f5daea575e1396736877003e47194c143f8", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "NPU" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 1024, "NPUW_LLM_MIN_RESPONSE_LEN": 1024, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.0495118023827672, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 7.23863, "dram": 2.913627, "package-0": 21.426334999999998, "psys": 35.955475, "uncore": 0.0 }, "net_j": { "core": 5.303336961309061, "dram": 2.2960061116598705, "package-0": 17.306499138408263, "psys": 25.796649805504725, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 12, "gross_j": { "core": 2.576898, "dram": 0.203368, "package-0": 3.225699, "psys": 4.606434, "uncore": 0.0 }, "logits": { "l2": 992.2683369200568, "sha256": "ae79c6c31d292f024b23dc06077f4b134ca9d7fc1fefa9233a47dedcd490f0e2", "sum": 292038.3226532545, "top1": 78 }, "net_j": { "core": 2.384725661830048, "dram": 0.1420389672168258, "package-0": 2.816604111231097, "psys": 3.597674524480727, "uncore": 0.0 }, "npu_busy_delta_us": 50545, "npu_utilization": 0.40135313296597036, "seconds": 0.125936478, "watts": { "core": 20.461887142818146, "dram": 1.6148458590369663, "package-0": 25.613698677518997, "psys": 36.57744025523725, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 807.407079374919, "decode_tokens_per_s": 18.812497916437472, "end_to_end_ms": 1268.257396, "handoff_estimate_ms": 78.194027, "handoff_plus_first_decode_ms": 125.936478, "output_tokens_per_s_e2e": 12.61573561523311, "prefill_tokens_per_s": 2140.5131580879497, "steady_tpot_ms": 47.742450999999996, "steady_tpot_p95_ms": 49.104597749999996, "ttft_ms": 470.915115 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 47, "gross_j": { "core": 0.816221, "dram": 0.683531, "package-0": 7.808085, "psys": 12.827421, "uncore": 0.0 }, "logits": { "l2": 1147.1172002671653, "sha256": "d036e9984fc486f880661b10e94e4628553614043ee547042d906b12c17261b9", "sum": 371321.3503918741, "top1": 291 }, "net_j": { "core": 0.09762968979104047, "dram": 0.4542029372767654, "package-0": 6.2783537446085305, "psys": 9.055359942190375, "uncore": 0.0 }, "npu_busy_delta_us": 464276, "npu_utilization": 0.9859016735956756, "seconds": 0.470915115, "watts": { "core": 1.7332656650870082, "dram": 1.451495138354181, "package-0": 16.580663374969394, "psys": 27.239348645668336, "uncore": 0.0 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 953.3553678944744, "sha256": "d7f745389327c7b4f7c74a1b3a7960947e9141430a397b0cdfdaccad2789a89f", "sum": 273382.66879007034, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 66, "gross_j": { "core": 3.845511, "dram": 2.026728, "package-0": 10.392551, "psys": 18.52162, "uncore": 0.0 }, "net_j": { "core": 2.8209816096879723, "dram": 1.6997642071662793, "package-0": 8.211541282568636, "psys": 13.143615338833623, "uncore": 0.0 }, "npu_busy_delta_us": 654689, "npu_utilization": 0.9751017895208749, "seconds": 0.671405803, "token_latencies_s": [ 0.050040757, 0.048089667, 0.047947045, 0.048600512, 0.04754948, 0.048062948, 0.047716527, 0.047455843, 0.047546194, 0.047806947, 0.047498061, 0.04775913, 0.047725772, 0.04757785 ], "watts": { "core": 5.7275510322033965, "dram": 3.0186334269738206, "package-0": 15.478792339243453, "psys": 27.586326953447557, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 6.3616779999999995, "dram": 2.8526540000000002, "package-0": 20.359262, "psys": 34.282933, "uncore": 0.0 }, "net_j": { "core": 4.552548215048158, "dram": 2.275296296248665, "package-0": 16.508001432577156, "psys": 24.786369496670208, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 5, "gross_j": { "core": 1.645992, "dram": 0.149291, "package-0": 2.125544, "psys": 2.888604, "uncore": 0.0 }, "logits": { "l2": 992.2683369200568, "sha256": "ae79c6c31d292f024b23dc06077f4b134ca9d7fc1fefa9233a47dedcd490f0e2", "sum": 292038.3226532545, "top1": 78 }, "net_j": { "core": 1.5653396651263465, "dram": 0.12355196941625678, "package-0": 1.9538519746112566, "psys": 2.465240222458822, "uncore": 0.0 }, "npu_busy_delta_us": 46679, "npu_utilization": 0.8831692302394692, "seconds": 0.05285397, "watts": { "core": 31.142258566385834, "dram": 2.8245938762972775, "package-0": 40.215408606013895, "psys": 54.65254549469037, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 863.7132134473471, "decode_tokens_per_s": 20.989015272922824, "end_to_end_ms": 1185.5787129999999, "handoff_estimate_ms": 5.7113035000000005, "handoff_plus_first_decode_ms": 52.85397, "output_tokens_per_s_e2e": 13.495518960114799, "prefill_tokens_per_s": 2140.494712879316, "steady_tpot_ms": 47.1426665, "steady_tpot_p95_ms": 47.920721449999995, "ttft_ms": 470.919173 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 47, "gross_j": { "core": 0.88751, "dram": 0.684691, "package-0": 7.899272, "psys": 12.93057, "uncore": 0.0 }, "logits": { "l2": 1147.1172002671653, "sha256": "d036e9984fc486f880661b10e94e4628553614043ee547042d906b12c17261b9", "sum": 371321.3503918741, "top1": 291 }, "net_j": { "core": 0.1689124974998102, "dram": 0.45536096109624186, "package-0": 6.369527562510022, "psys": 9.158476437343356, "uncore": 0.0 }, "npu_busy_delta_us": 466181, "npu_utilization": 0.9899384580801512, "seconds": 0.470919173, "watts": { "core": 1.8846333954638113, "dram": 1.4539458982698927, "package-0": 16.774156698011527, "psys": 27.458151507456247, "uncore": 0.0 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 953.3553678944744, "sha256": "d7f745389327c7b4f7c74a1b3a7960947e9141430a397b0cdfdaccad2789a89f", "sum": 273382.66879007034, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 66, "gross_j": { "core": 3.828176, "dram": 2.018672, "package-0": 10.334446, "psys": 18.463759, "uncore": 0.0 }, "net_j": { "core": 2.818296052422001, "dram": 1.6963833657361667, "package-0": 8.184621895455876, "psys": 13.162652836868029, "uncore": 0.0 }, "npu_busy_delta_us": 651910, "npu_utilization": 0.9850476175351622, "seconds": 0.66180557, "token_latencies_s": [ 0.048738439, 0.047480412, 0.047287563, 0.047167345, 0.047151295, 0.047140702, 0.047061307, 0.047381115, 0.047144631, 0.047137891, 0.047034206, 0.046999555, 0.047062158, 0.046994456 ], "watts": { "core": 5.784442098303887, "dram": 3.0502493353146, "package-0": 15.615531915211896, "psys": 27.899068604091685, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 5.561388, "dram": 2.8277509999999997, "package-0": 19.482433, "psys": 32.526284000000004, "uncore": 0.0 }, "net_j": { "core": 3.9679824633553276, "dram": 2.2401243293373336, "package-0": 15.562673359858774, "psys": 22.860813256404036, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 6, "gross_j": { "core": 1.637692, "dram": 0.148803, "package-0": 2.114252, "psys": 2.882805, "uncore": 0.0 }, "logits": { "l2": 992.2683369200568, "sha256": "ae79c6c31d292f024b23dc06077f4b134ca9d7fc1fefa9233a47dedcd490f0e2", "sum": 292038.3226532545, "top1": 78 }, "net_j": { "core": 1.5585142742876166, "dram": 0.12353456968295035, "package-0": 1.945699110433572, "psys": 2.467181805616026, "uncore": 0.0 }, "npu_busy_delta_us": 46682, "npu_utilization": 0.8996752269178387, "seconds": 0.051887613, "watts": { "core": 31.562292140900755, "dram": 2.8677942845434035, "package-0": 40.74675780518175, "psys": 55.55863593108436, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 848.6195445371723, "decode_tokens_per_s": 20.557439560377347, "end_to_end_ms": 1206.6655859999998, "handoff_estimate_ms": 3.149337500000002, "handoff_plus_first_decode_ms": 51.887613, "output_tokens_per_s_e2e": 13.259680383393317, "prefill_tokens_per_s": 2113.1954394902805, "steady_tpot_ms": 48.7382755, "steady_tpot_p95_ms": 49.52061395, "ttft_ms": 477.00273300000003 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 47, "gross_j": { "core": 0.479979, "dram": 0.657774, "package-0": 7.415692, "psys": 11.833588, "uncore": 0.0 }, "logits": { "l2": 1147.1172002671653, "sha256": "d036e9984fc486f880661b10e94e4628553614043ee547042d906b12c17261b9", "sum": 371321.3503918741, "top1": 291 }, "net_j": { "core": 0.0, "dram": 0.42548136553534, "package-0": 5.866185589322882, "psys": 8.012764721439359, "uncore": 0.0 }, "npu_busy_delta_us": 467645, "npu_utilization": 0.9803822235123335, "seconds": 0.477002733, "watts": { "core": 1.0062395177094299, "dram": 1.3789732311659522, "package-0": 15.546435034786267, "psys": 24.808218446832253, "uncore": 0.0 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 953.3553678944744, "sha256": "d7f745389327c7b4f7c74a1b3a7960947e9141430a397b0cdfdaccad2789a89f", "sum": 273382.66879007034, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 67, "gross_j": { "core": 3.443717, "dram": 2.021174, "package-0": 9.952489, "psys": 17.809891, "uncore": 0.0 }, "net_j": { "core": 2.409468189067711, "dram": 1.6911083941190432, "package-0": 7.75078866010232, "psys": 12.38086672934865, "uncore": 0.0 }, "npu_busy_delta_us": 656514, "npu_utilization": 0.968630839922686, "seconds": 0.67777524, "token_latencies_s": [ 0.04844395, 0.04877334, 0.048878987, 0.049748099, 0.048703211, 0.04881916, 0.049398122, 0.04914311, 0.049133506, 0.0480643, 0.047195861, 0.047228022, 0.047117439, 0.047075034 ], "watts": { "core": 5.080912958697045, "dram": 2.9820711656566266, "package-0": 14.684055144888443, "psys": 26.27698674858645, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 10.58211692981422, "compile_target": "NPU", "context_length": 2048, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 406911, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 1.5259465821328744, "dram": 0.48698386485903, "package-0": 3.248422500499841, "psys": 8.010065800945089, "uncore": 0.0023192603038971636 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_1_5b_ov_fp16/openvino_model.xml", "key": "qwen25_1_5b_fp16", "size_b": 1.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "npu_npu", "label": "NPU -> NPU", "mode": "staged", "prefill_device": "NPU" }, "prompt_length": 2032, "prompt_sha256": "cd1a97358e68d07c27508cfacc4ba6882a7a2962b1281e2e2b0c7e5e320da4fa", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "NPU" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 2048, "NPUW_LLM_MIN_RESPONSE_LEN": 2048, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.04545198939740658, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 9.380956999999999, "dram": 3.708669, "package-0": 32.590065, "psys": 53.031236, "uncore": 0.0 }, "net_j": { "core": 6.637229988590557, "dram": 2.8330480577081034, "package-0": 26.749241563943755, "psys": 38.62874369879418, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 14, "gross_j": { "core": 2.615655, "dram": 0.210998, "package-0": 3.276053, "psys": 4.890185, "uncore": 0.0 }, "logits": { "l2": 717.7345947887626, "sha256": "f0695fbadace6f6c8d0679619c9f0011a90337bca31c052537c786abd645c276", "sum": 148310.24642302655, "top1": 220 }, "net_j": { "core": 2.4041826100090113, "dram": 0.1435096337658141, "package-0": 2.8258723200701525, "psys": 3.780114845025571, "uncore": 0.0 }, "npu_busy_delta_us": 52197, "npu_utilization": 0.37664412716470347, "seconds": 0.138584399, "watts": { "core": 18.87409419006825, "dram": 1.5225234696150756, "package-0": 23.639406914771122, "psys": 35.286691974613966, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 1139.0122221389486, "decode_tokens_per_s": 17.979076763117355, "end_to_end_ms": 1798.0491869999998, "handoff_estimate_ms": 89.11109049999999, "handoff_plus_first_decode_ms": 138.584399, "output_tokens_per_s_e2e": 8.898532985460536, "prefill_tokens_per_s": 2108.4392336886735, "steady_tpot_ms": 49.4733085, "steady_tpot_p95_ms": 50.79916185, "ttft_ms": 963.7460580000001 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 96, "gross_j": { "core": 2.64721, "dram": 1.408382, "package-0": 18.469252, "psys": 28.88341, "uncore": 0.0 }, "logits": { "l2": 831.4911202378499, "sha256": "f586669e9cb29136fdd87403e7a53cdac5ca8787a455ff4d27734f7920186588", "sum": 207554.2427772805, "top1": 220 }, "net_j": { "core": 1.1765849967508688, "dram": 0.9390532199325051, "package-0": 15.338597620424776, "psys": 21.163740660018558, "uncore": 0.0 }, "npu_busy_delta_us": 927824, "npu_utilization": 0.9627266356092322, "seconds": 0.963746058, "watts": { "core": 2.746792039278048, "dram": 1.4613621381992723, "package-0": 19.164023392560534, "psys": 29.96993840881682, "uncore": 0.0 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 674.5225862907852, "sha256": "4951873d057936f5eea807776d0b62282cdeffba732c9441debea531ffc8e8bb", "sum": 113242.55733524822, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 69, "gross_j": { "core": 4.118092, "dram": 2.089289, "package-0": 10.84476, "psys": 19.257641, "uncore": 0.0 }, "net_j": { "core": 3.056462381830676, "dram": 1.750485204009784, "package-0": 8.584771623448827, "psys": 13.68488819375005, "uncore": 0.0 }, "npu_busy_delta_us": 677582, "npu_utilization": 0.9739309447655665, "seconds": 0.69571873, "token_latencies_s": [ 0.052205691, 0.049185217, 0.049444304, 0.049973793, 0.049305569, 0.049236365, 0.049572606, 0.049463416, 0.04962484, 0.049378277, 0.049483201, 0.0500418, 0.049516981, 0.049259732 ], "watts": { "core": 5.919190935106778, "dram": 3.00306562107362, "package-0": 15.587851142084391, "psys": 27.68021065064613, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 10.818759, "dram": 3.6581930000000003, "package-0": 33.903721, "psys": 54.00206, "uncore": 0.0 }, "net_j": { "core": 8.19865162213459, "dram": 2.8220234825617743, "package-0": 28.326058020417072, "psys": 40.24847730714521, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 8, "gross_j": { "core": 2.383356, "dram": 0.174499, "package-0": 2.931511, "psys": 3.992605, "uncore": 0.0 }, "logits": { "l2": 717.7345947887626, "sha256": "f0695fbadace6f6c8d0679619c9f0011a90337bca31c052537c786abd645c276", "sum": 148310.24642302655, "top1": 220 }, "net_j": { "core": 2.2663090105534835, "dram": 0.13714513980713028, "package-0": 2.6823423340809174, "psys": 3.3781968036405523, "uncore": 0.0 }, "npu_busy_delta_us": 48582, "npu_utilization": 0.6333656013173566, "seconds": 0.076704513, "watts": { "core": 31.071913591316328, "dram": 2.2749508884829237, "package-0": 38.21823365204079, "psys": 52.05176128293781, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 1192.7521087911155, "decode_tokens_per_s": 19.655706813149937, "end_to_end_ms": 1717.037417, "handoff_estimate_ms": 27.8019725, "handoff_plus_first_decode_ms": 76.704513, "output_tokens_per_s_e2e": 9.31837584993059, "prefill_tokens_per_s": 2130.201722104344, "steady_tpot_ms": 48.9025405, "steady_tpot_p95_ms": 49.7028969, "ttft_ms": 953.900271 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 95, "gross_j": { "core": 4.054189, "dram": 1.402645, "package-0": 19.880686, "psys": 30.574385, "uncore": 0.0 }, "logits": { "l2": 831.4911202378499, "sha256": "f586669e9cb29136fdd87403e7a53cdac5ca8787a455ff4d27734f7920186588", "sum": 207554.2427772805, "top1": 220 }, "net_j": { "core": 2.5985881417719274, "dram": 0.9381109593383439, "package-0": 16.782014896450704, "psys": 22.933581061750647, "uncore": 0.0 }, "npu_busy_delta_us": 926078, "npu_utilization": 0.9708331448833397, "seconds": 0.953900271, "watts": { "core": 4.250118301937247, "dram": 1.4704314933568143, "package-0": 20.841472221366, "psys": 32.05197223389823, "uncore": 0.0 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 674.5225862907852, "sha256": "4951873d057936f5eea807776d0b62282cdeffba732c9441debea531ffc8e8bb", "sum": 113242.55733524822, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 68, "gross_j": { "core": 4.381214, "dram": 2.081049, "package-0": 11.091524, "psys": 19.43507, "uncore": 0.0 }, "net_j": { "core": 3.33375446980918, "dram": 1.7467673834163, "package-0": 8.86170078988545, "psys": 13.936699441754008, "uncore": 0.0 }, "npu_busy_delta_us": 675317, "npu_utilization": 0.9838066658465521, "seconds": 0.686432633, "token_latencies_s": [ 0.049429486, 0.05021066, 0.04888858, 0.049085281, 0.048875295, 0.048997466, 0.048916501, 0.048931819, 0.04885273, 0.048776068, 0.048819037, 0.048731533, 0.049053754, 0.048833118 ], "watts": { "core": 6.382584086732951, "dram": 3.031687160479155, "package-0": 16.15821198873568, "psys": 28.313149849913966, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 9.619179, "dram": 3.66747, "package-0": 32.736976999999996, "psys": 53.363144000000005, "uncore": 0.0 }, "net_j": { "core": 6.974752667915164, "dram": 2.8235394387259185, "package-0": 27.10754403121824, "psys": 39.48190518484798, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 9, "gross_j": { "core": 2.370417, "dram": 0.178283, "package-0": 2.949578, "psys": 4.105458, "uncore": 0.0 }, "logits": { "l2": 717.7345947887626, "sha256": "f0695fbadace6f6c8d0679619c9f0011a90337bca31c052537c786abd645c276", "sum": 148310.24642302655, "top1": 220 }, "net_j": { "core": 2.2323983376598506, "dram": 0.1342363325766008, "package-0": 2.655765009765463, "psys": 3.3809643774527642, "uncore": 0.0 }, "npu_busy_delta_us": 48844, "npu_utilization": 0.5400235996637136, "seconds": 0.090447899, "watts": { "core": 26.207540763329398, "dram": 1.971112673385592, "package-0": 32.610796188864484, "psys": 45.39030807116924, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 1181.783195202228, "decode_tokens_per_s": 19.106795571571624, "end_to_end_ms": 1732.9743799999999, "handoff_estimate_ms": 41.158146499999994, "handoff_plus_first_decode_ms": 90.44789899999999, "output_tokens_per_s_e2e": 9.232681212517406, "prefill_tokens_per_s": 2143.655779721865, "steady_tpot_ms": 49.289752500000006, "steady_tpot_p95_ms": 51.320964800000006, "ttft_ms": 947.913382 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 94, "gross_j": { "core": 2.506098, "dram": 1.398739, "package-0": 18.29402, "psys": 28.803087, "uncore": 0.0 }, "logits": { "l2": 831.4911202378499, "sha256": "f586669e9cb29136fdd87403e7a53cdac5ca8787a455ff4d27734f7920186588", "sum": 207554.2427772805, "top1": 220 }, "net_j": { "core": 1.0596328145790865, "dram": 0.9371204776820459, "package-0": 15.214796841386299, "psys": 21.2102384365836, "uncore": 0.0 }, "npu_busy_delta_us": 926241, "npu_utilization": 0.9771367485557874, "seconds": 0.947913382, "watts": { "core": 2.643804853469196, "dram": 1.4755979043663294, "package-0": 19.299252808733954, "psys": 30.385779488868955, "uncore": 0.0 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 674.5225862907852, "sha256": "4951873d057936f5eea807776d0b62282cdeffba732c9441debea531ffc8e8bb", "sum": 113242.55733524822, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 69, "gross_j": { "core": 4.742664, "dram": 2.090448, "package-0": 11.493379, "psys": 20.454599, "uncore": 0.0 }, "net_j": { "core": 3.6827215156762265, "dram": 1.752182628467272, "package-0": 9.236982180066477, "psys": 14.890702370811617, "uncore": 0.0 }, "npu_busy_delta_us": 677950, "npu_utilization": 0.9760109634788215, "seconds": 0.694613099, "token_latencies_s": [ 0.053529826, 0.049208487, 0.049084497, 0.04905678, 0.049075252, 0.04901029, 0.04950561, 0.049404834, 0.049118912, 0.050131578, 0.049157262, 0.049469716, 0.049371018, 0.049445467 ], "watts": { "core": 6.8277779483683485, "dram": 3.0095142216717683, "package-0": 16.546447247462574, "psys": 29.447470871838544, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 15.732889357022941, "compile_target": "NPU", "context_length": 4096, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 407089, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 1.5259465821328744, "dram": 0.48698386485903, "package-0": 3.248422500499841, "psys": 8.010065800945089, "uncore": 0.0023192603038971636 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_1_5b_ov_fp16/openvino_model.xml", "key": "qwen25_1_5b_fp16", "size_b": 1.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "npu_npu", "label": "NPU -> NPU", "mode": "staged", "prefill_device": "NPU" }, "prompt_length": 4080, "prompt_sha256": "66f8de7a9ef70b33481de3e8dae7cef6e4434d9723c6203a6a3dbe07ef228a16", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "NPU" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 4096, "NPUW_LLM_MIN_RESPONSE_LEN": 4096, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.046084214001894, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 11.506379, "dram": 5.885849, "package-0": 55.531901000000005, "psys": 94.529482, "uncore": 0.0 }, "net_j": { "core": 6.709011966885171, "dram": 4.35483843495905, "package-0": 45.31930562084578, "psys": 69.34693232462192, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 17, "gross_j": { "core": 3.042351, "dram": 0.251098, "package-0": 3.700552, "psys": 5.670395, "uncore": 0.0 }, "logits": { "l2": 987.7546651143514, "sha256": "65b8cf3d23c292957632e47bc3a72984a0e7f6d6be836d09ac1b62daa0526b20", "sum": 285645.4547156822, "top1": 78 }, "net_j": { "core": 2.777340133277315, "dram": 0.16652360033401178, "package-0": 3.1363990523026652, "psys": 4.279288322171777, "uncore": 0.0 }, "npu_busy_delta_us": 67409, "npu_utilization": 0.38814458602044144, "seconds": 0.173669819, "watts": { "core": 17.518017911909034, "dram": 1.4458355599483868, "package-0": 21.30797407003689, "psys": 32.650434212751726, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 1302.8557450935916, "decode_tokens_per_s": 16.62785011452382, "end_to_end_ms": 3143.8630219999995, "handoff_estimate_ms": 122.5199525, "handoff_plus_first_decode_ms": 173.669819, "output_tokens_per_s_e2e": 5.089280254271842, "prefill_tokens_per_s": 1819.9969204135443, "steady_tpot_ms": 51.1498665, "steady_tpot_p95_ms": 56.3042718, "ttft_ms": 2241.7620349999997 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 222, "gross_j": { "core": 4.322498, "dram": 3.457999, "package-0": 40.939348, "psys": 68.50898, "uncore": 0.0 }, "logits": { "l2": 1191.9284956676295, "sha256": "6ade4f6edb2f1529fce4ee1de9d91b6bcdcc6c2eb70a2a074a1cca9c1c183747", "sum": 385704.7659149729, "top1": 220 }, "net_j": { "core": 0.9016888847365134, "dram": 2.366297060101456, "package-0": 33.657157764739694, "psys": 50.55231858958943, "uncore": 0.0 }, "npu_busy_delta_us": 2163402, "npu_utilization": 0.9650453376511037, "seconds": 2.241762035, "watts": { "core": 1.9281698648268886, "dram": 1.5425361595081166, "package-0": 18.26212923621039, "psys": 30.560326622713994, "uncore": 0.0 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 816.3258561155944, "sha256": "7873c6e1ae7131c1ddafce3cc8875eb4cf52e122938a93b820e945a8560dabbe", "sum": 204753.52091340534, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 72, "gross_j": { "core": 4.14153, "dram": 2.176752, "package-0": 10.892001, "psys": 20.350107, "uncore": 0.0 }, "net_j": { "core": 3.029982948871343, "dram": 1.8220177745235826, "package-0": 8.52574880380342, "psys": 14.515325412860715, "uncore": 0.0 }, "npu_busy_delta_us": 711475, "npu_utilization": 0.9767223469493277, "seconds": 0.728431168, "token_latencies_s": [ 0.060425537, 0.054085129, 0.05262576, 0.05130668, 0.051174891, 0.050962457, 0.051116808, 0.051124842, 0.051195184, 0.05118923, 0.05072121, 0.050913282, 0.050618723, 0.050939633 ], "watts": { "core": 5.685547491564777, "dram": 2.988274109654792, "package-0": 14.952683902729435, "psys": 27.936897669925077, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 13.576502999999999, "dram": 5.826462, "package-0": 57.959874, "psys": 95.660217, "uncore": 0.0 }, "net_j": { "core": 8.916219808408012, "dram": 4.339199767788111, "package-0": 48.03910157223333, "psys": 71.19725384468838, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 13, "gross_j": { "core": 2.611748, "dram": 0.21881, "package-0": 3.281608, "psys": 4.7531, "uncore": 0.0 }, "logits": { "l2": 987.7546651143514, "sha256": "65b8cf3d23c292957632e47bc3a72984a0e7f6d6be836d09ac1b62daa0526b20", "sum": 285645.4547156822, "top1": 78 }, "net_j": { "core": 2.4149477374649306, "dram": 0.1560040318378363, "package-0": 2.8626612240062856, "psys": 3.7200474206981724, "uncore": 0.0 }, "npu_busy_delta_us": 50876, "npu_utilization": 0.39448147737484807, "seconds": 0.1289693, "watts": { "core": 20.2509279340122, "dram": 1.6966053161488819, "package-0": 25.44487719170376, "psys": 36.85450723544285, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 1341.1796973396185, "decode_tokens_per_s": 17.818719397930845, "end_to_end_ms": 3054.0277400000004, "handoff_estimate_ms": 78.1047915, "handoff_plus_first_decode_ms": 128.9693, "output_tokens_per_s_e2e": 5.238983192732885, "prefill_tokens_per_s": 1844.3041984803508, "steady_tpot_ms": 50.8645085, "steady_tpot_p95_ms": 51.4627742, "ttft_ms": 2212.2164030000004 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 219, "gross_j": { "core": 6.337508, "dram": 3.447623, "package-0": 43.023693, "psys": 69.963994, "uncore": 0.0 }, "logits": { "l2": 1191.9284956676295, "sha256": "6ade4f6edb2f1529fce4ee1de9d91b6bcdcc6c2eb70a2a074a1cca9c1c183747", "sum": 385704.7659149729, "top1": 220 }, "net_j": { "core": 2.9617839409038678, "dram": 2.3703093061625182, "package-0": 35.83747946051998, "psys": 52.24399504603994, "uncore": 0.0 }, "npu_busy_delta_us": 2155902, "npu_utilization": 0.9745438995373004, "seconds": 2.212216403, "watts": { "core": 2.86477760105461, "dram": 1.55844744452878, "package-0": 19.448229812262177, "psys": 31.62619800898384, "uncore": 0.0 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 816.3258561155944, "sha256": "7873c6e1ae7131c1ddafce3cc8875eb4cf52e122938a93b820e945a8560dabbe", "sum": 204753.52091340534, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 71, "gross_j": { "core": 4.627247, "dram": 2.160029, "package-0": 11.654573, "psys": 20.943123, "uncore": 0.0 }, "net_j": { "core": 3.5394881300392136, "dram": 1.8128864297877565, "package-0": 9.33896088770706, "psys": 15.233211377950266, "uncore": 0.0 }, "npu_busy_delta_us": 701125, "npu_utilization": 0.9835629264383575, "seconds": 0.712842037, "token_latencies_s": [ 0.051834426, 0.050896155, 0.050847052, 0.050717886, 0.050742519, 0.050920484, 0.050881965, 0.050761774, 0.05097453, 0.050661685, 0.050527041, 0.050800322, 0.051262654, 0.050970986 ], "watts": { "core": 6.491265609802975, "dram": 3.0301650125608406, "package-0": 16.349446855082142, "psys": 29.37975303496306, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 13.644496, "dram": 5.817246, "package-0": 58.016515000000005, "psys": 94.99444700000001, "uncore": 0.0 }, "net_j": { "core": 9.003646134877586, "dram": 4.336185633922896, "package-0": 48.13711207736704, "psys": 70.63349411528053, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 12, "gross_j": { "core": 2.738701, "dram": 0.216369, "package-0": 3.400199, "psys": 4.85149, "uncore": 0.0 }, "logits": { "l2": 987.7546651143514, "sha256": "65b8cf3d23c292957632e47bc3a72984a0e7f6d6be836d09ac1b62daa0526b20", "sum": 285645.4547156822, "top1": 78 }, "net_j": { "core": 2.553586976575994, "dram": 0.1572925266672149, "package-0": 3.006129793456679, "psys": 3.8797813401651036, "uncore": 0.0 }, "npu_busy_delta_us": 50762, "npu_utilization": 0.41844533963159436, "seconds": 0.121310946, "watts": { "core": 22.575877035861215, "dram": 1.783590080980821, "package-0": 28.02878975158598, "psys": 39.99218669022662, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 1346.7958201770846, "decode_tokens_per_s": 17.92484973051763, "end_to_end_ms": 3041.29248, "handoff_estimate_ms": 70.3499075, "handoff_plus_first_decode_ms": 121.310946, "output_tokens_per_s_e2e": 5.260921172566737, "prefill_tokens_per_s": 1850.7888738761737, "steady_tpot_ms": 50.9610385, "steady_tpot_p95_ms": 51.880345350000006, "ttft_ms": 2204.465381 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 218, "gross_j": { "core": 6.228561, "dram": 3.43682, "package-0": 42.895276, "psys": 69.235967, "uncore": 0.0 }, "logits": { "l2": 1191.9284956676295, "sha256": "6ade4f6edb2f1529fce4ee1de9d91b6bcdcc6c2eb70a2a074a1cca9c1c183747", "sum": 385704.7659149729, "top1": 220 }, "net_j": { "core": 2.8646645864328053, "dram": 2.363280928812686, "package-0": 35.73424105478665, "psys": 51.578054242284516, "uncore": 0.0 }, "npu_busy_delta_us": 2150338, "npu_utilization": 0.9754464817336137, "seconds": 2.204465381, "watts": { "core": 2.8254292644752583, "dram": 1.5590265239007626, "package-0": 19.458357735943054, "psys": 31.407146420504393, "uncore": 0.0 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 816.3258561155944, "sha256": "7873c6e1ae7131c1ddafce3cc8875eb4cf52e122938a93b820e945a8560dabbe", "sum": 204753.52091340534, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 71, "gross_j": { "core": 4.677234, "dram": 2.164057, "package-0": 11.72104, "psys": 20.90699, "uncore": 0.0 }, "net_j": { "core": 3.5853945718687874, "dram": 1.815612178442995, "package-0": 9.396741229123712, "psys": 15.175658532830905, "uncore": 0.0 }, "npu_busy_delta_us": 701959, "npu_utilization": 0.981052624817542, "seconds": 0.715516153, "token_latencies_s": [ 0.05243529, 0.051581529, 0.051009574, 0.050858536, 0.050938977, 0.05139146, 0.050850895, 0.050913271, 0.05103691, 0.0509831, 0.050914064, 0.050915901, 0.050662856, 0.050996186 ], "watts": { "core": 6.536867099910182, "dram": 3.0244698053658055, "package-0": 16.381237447758917, "psys": 29.21945215679848, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 46.333827616646886, "compile_target": "NPU", "context_length": 8192, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 407278, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 1.5259465821328744, "dram": 0.48698386485903, "package-0": 3.248422500499841, "psys": 8.010065800945089, "uncore": 0.0023192603038971636 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_1_5b_ov_fp16/openvino_model.xml", "key": "qwen25_1_5b_fp16", "size_b": 1.5 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "npu_npu", "label": "NPU -> NPU", "mode": "staged", "prefill_device": "NPU" }, "prompt_length": 8176, "prompt_sha256": "998ace73fce48442f97bcb84c31be293d0d94180b17f70a4d127bba1092ad744", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "NPU" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 8192, "NPUW_LLM_MIN_RESPONSE_LEN": 8192, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.04594239592552185, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 23.247012, "dram": 11.962311000000001, "package-0": 121.986687, "psys": 204.995446, "uncore": 0.0 }, "net_j": { "core": 13.409986591201342, "dram": 8.822966079028113, "package-0": 101.04570842965862, "psys": 153.35850029518318, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 19, "gross_j": { "core": 3.354118, "dram": 0.301757, "package-0": 4.244923, "psys": 6.509322, "uncore": 0.0 }, "logits": { "l2": 1201.451742992223, "sha256": "b76c1e342ce2c72df4501b64f4e3fc619680eb252044cc454aad186c16b9052a", "sum": 386158.7196010724, "top1": 261 }, "net_j": { "core": 3.050137738354793, "dram": 0.20474607944082163, "package-0": 3.5978123257482753, "psys": 4.913655463257111, "uncore": 0.0 }, "npu_busy_delta_us": 57161, "npu_utilization": 0.28694176427514945, "seconds": 0.199207669, "watts": { "core": 16.837293548171584, "dram": 1.5147860597676086, "package-0": 21.309034041254705, "psys": 32.676061281556386, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 1270.7656919998878, "decode_tokens_per_s": 15.173583703614803, "end_to_end_ms": 6446.5070559999995, "handoff_estimate_ms": 143.2111745, "handoff_plus_first_decode_ms": 199.207669, "output_tokens_per_s_e2e": 2.481964242187281, "prefill_tokens_per_s": 1497.999178598827, "steady_tpot_ms": 55.9964945, "steady_tpot_p95_ms": 58.5222284, "ttft_ms": 5457.946918000001 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 539, "gross_j": { "core": 15.216941, "dram": 9.326942, "package-0": 105.422948, "psys": 176.314123, "uncore": 0.0 }, "logits": { "l2": 1277.917301221495, "sha256": "938fc863e80af83b1ea1b09a9e8a8e1594a85ece59922f15c0118b3801e46c64", "sum": 430820.31402779557, "top1": 220 }, "net_j": { "core": 6.888405555015245, "dram": 6.669009915676929, "package-0": 87.69323042503504, "psys": 132.59560904875454, "uncore": 0.0 }, "npu_busy_delta_us": 5344647, "npu_utilization": 0.9792412935299273, "seconds": 5.457946918, "watts": { "core": 2.7880338941764693, "dram": 1.708873710229807, "package-0": 19.315495292253775, "psys": 32.304110986958484, "uncore": 0.0 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 1199.9686300410292, "sha256": "545756e65cf0318c97b46cc3463d1705c40d70fcc43ef1451157bbd1a8855e07", "sum": 387763.0401600674, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 78, "gross_j": { "core": 4.675953, "dram": 2.333612, "package-0": 12.318816, "psys": 22.172001, "uncore": 0.0 }, "net_j": { "core": 3.4714432978313043, "dram": 1.9492100839103623, "package-0": 9.754665678875297, "psys": 15.849235783171533, "uncore": 0.0 }, "npu_busy_delta_us": 766469, "npu_utilization": 0.9710098214692477, "seconds": 0.789352469, "token_latencies_s": [ 0.062525594, 0.05636657, 0.056303841, 0.056092229, 0.056279785, 0.055745845, 0.056270939, 0.055863224, 0.055574286, 0.056256687, 0.05556658, 0.055216354, 0.05590076, 0.055360972 ], "watts": { "core": 5.9237833333488945, "dram": 2.9563624510560693, "package-0": 15.606229769074176, "psys": 28.088847341022255, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 22.134894, "dram": 11.922271, "package-0": 120.69359899999999, "psys": 196.684921, "uncore": 0.0 }, "net_j": { "core": 12.34335123732851, "dram": 8.797441210196116, "package-0": 99.84944351166668, "psys": 145.28672479830834, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 16, "gross_j": { "core": 2.956048, "dram": 0.272948, "package-0": 3.745107, "psys": 5.522081, "uncore": 0.0 }, "logits": { "l2": 1201.451742992223, "sha256": "b76c1e342ce2c72df4501b64f4e3fc619680eb252044cc454aad186c16b9052a", "sum": 386158.7196010724, "top1": 261 }, "net_j": { "core": 2.7212509168041037, "dram": 0.19801589400678496, "package-0": 3.2452729055199696, "psys": 4.289573841302266, "uncore": 0.0 }, "npu_busy_delta_us": 55599, "npu_utilization": 0.36133798114187354, "seconds": 0.153869792, "watts": { "core": 19.211360212926003, "dram": 1.773889445434488, "package-0": 24.33945579129658, "psys": 35.88801237867404, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 1276.6685193357519, "decode_tokens_per_s": 16.078689655995717, "end_to_end_ms": 6416.700871000001, "handoff_estimate_ms": 98.552389, "handoff_plus_first_decode_ms": 153.86979200000002, "output_tokens_per_s_e2e": 2.4934932018276403, "prefill_tokens_per_s": 1490.9399246130151, "steady_tpot_ms": 55.317403, "steady_tpot_p95_ms": 57.49918405, "ttft_ms": 5483.789028 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 538, "gross_j": { "core": 14.047267, "dram": 9.321265, "package-0": 104.173379, "psys": 169.149164, "uncore": 0.0 }, "logits": { "l2": 1277.917301221495, "sha256": "938fc863e80af83b1ea1b09a9e8a8e1594a85ece59922f15c0118b3801e46c64", "sum": 430820.31402779557, "top1": 220 }, "net_j": { "core": 5.679297875585641, "dram": 6.650748225073016, "package-0": 86.35971533345065, "psys": 125.2236530472193, "uncore": 0.0 }, "npu_busy_delta_us": 5359116, "npu_utilization": 0.9772651669560181, "seconds": 5.483789028, "watts": { "core": 2.5615987282288275, "dram": 1.6997854863500412, "package-0": 18.996605899332565, "psys": 30.845308442088374, "uncore": 0.0 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 1199.9686300410292, "sha256": "545756e65cf0318c97b46cc3463d1705c40d70fcc43ef1451157bbd1a8855e07", "sum": 387763.0401600674, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 77, "gross_j": { "core": 5.131579, "dram": 2.328058, "package-0": 12.775113, "psys": 22.013676, "uncore": 0.0 }, "net_j": { "core": 3.942802444938766, "dram": 1.9486770911163145, "package-0": 10.244455272696054, "psys": 15.77349790978678, "uncore": 0.0 }, "npu_busy_delta_us": 763048, "npu_utilization": 0.9794695922004857, "seconds": 0.779042051, "token_latencies_s": [ 0.058245568, 0.057097285, 0.055307853, 0.055386553, 0.055481285, 0.055178232, 0.055225464, 0.055286395, 0.055123775, 0.055317715, 0.055381826, 0.055428782, 0.055317091, 0.055220937 ], "watts": { "core": 6.58703723812208, "dram": 2.988359867110691, "package-0": 16.398489636857867, "psys": 28.25736553212068, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 24.699887, "dram": 11.951142, "package-0": 123.271657, "psys": 196.55986099999998, "uncore": 0.0 }, "net_j": { "core": 14.86122190819071, "dram": 8.811273797820887, "package-0": 102.32718788598704, "psys": 144.91430819912776, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 15, "gross_j": { "core": 2.917228, "dram": 0.271789, "package-0": 3.709524, "psys": 5.432542, "uncore": 0.0 }, "logits": { "l2": 1201.451742992223, "sha256": "b76c1e342ce2c72df4501b64f4e3fc619680eb252044cc454aad186c16b9052a", "sum": 386158.7196010724, "top1": 261 }, "net_j": { "core": 2.6818414329577998, "dram": 0.19666876873090283, "package-0": 3.208435017169069, "psys": 4.1969404968129, "uncore": 0.0 }, "npu_busy_delta_us": 55619, "npu_utilization": 0.36056272886817914, "seconds": 0.154256099, "watts": { "core": 18.91158935634694, "dram": 1.7619335751515406, "package-0": 24.04782711379211, "psys": 35.217680436739165, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 1270.5539099241466, "decode_tokens_per_s": 16.07177991201729, "end_to_end_ms": 6447.581591, "handoff_estimate_ms": 98.8833005, "handoff_plus_first_decode_ms": 154.256099, "output_tokens_per_s_e2e": 2.481550605320599, "prefill_tokens_per_s": 1482.6988858604689, "steady_tpot_ms": 55.3727985, "steady_tpot_p95_ms": 56.69758749999999, "ttft_ms": 5514.268661 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 545, "gross_j": { "core": 17.431168, "dram": 9.350196, "package-0": 107.573516, "psys": 170.200492, "uncore": 0.0 }, "logits": { "l2": 1277.917301221495, "sha256": "938fc863e80af83b1ea1b09a9e8a8e1594a85ece59922f15c0118b3801e46c64", "sum": 430820.31402779557, "top1": 220 }, "net_j": { "core": 9.016688583784628, "dram": 6.664836135595192, "package-0": 89.66084160780647, "psys": 126.03083718130063, "uncore": 0.0 }, "npu_busy_delta_us": 5352984, "npu_utilization": 0.9707513958939478, "seconds": 5.514268661, "watts": { "core": 3.16110241840101, "dram": 1.6956366428298697, "package-0": 19.50821090035388, "psys": 30.865469650355145, "uncore": 0.0 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 1199.9686300410292, "sha256": "545756e65cf0318c97b46cc3463d1705c40d70fcc43ef1451157bbd1a8855e07", "sum": 387763.0401600674, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 77, "gross_j": { "core": 4.351491, "dram": 2.329157, "package-0": 11.988617, "psys": 20.926827, "uncore": 0.0 }, "net_j": { "core": 3.162691891448282, "dram": 1.9497688934947917, "package-0": 9.457911261011498, "psys": 14.686530521014241, "uncore": 0.0 }, "npu_busy_delta_us": 765561, "npu_utilization": 0.9826767053917278, "seconds": 0.779056831, "token_latencies_s": [ 0.056086977, 0.056149778, 0.055354418, 0.055277658, 0.055672377, 0.055550868, 0.057714948, 0.055249627, 0.055161305, 0.055391179, 0.055353252, 0.055595278, 0.055252188, 0.055212434 ], "watts": { "core": 5.585588659064078, "dram": 2.989713853109132, "package-0": 15.388629587666115, "psys": 26.861746367255716, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 11.477834473364055, "compile_target": "NPU", "context_length": 1024, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 407521, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 1.5259465821328744, "dram": 0.48698386485903, "package-0": 3.248422500499841, "psys": 8.010065800945089, "uncore": 0.0023192603038971636 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_3b_ov_fp16/openvino_model.xml", "key": "qwen25_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "npu_npu", "label": "NPU -> NPU", "mode": "staged", "prefill_device": "NPU" }, "prompt_length": 1008, "prompt_sha256": "2669029615323cd09063f0eb8d822f5daea575e1396736877003e47194c143f8", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "NPU" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 1024, "NPUW_LLM_MIN_RESPONSE_LEN": 1024, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.06274531874805689, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 11.222139, "dram": 5.715562, "package-0": 40.526995, "psys": 69.437077, "uncore": 0.0 }, "net_j": { "core": 7.722327151526276, "dram": 4.59864747612006, "package-0": 33.076624397768775, "psys": 51.06571097353138, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 18, "gross_j": { "core": 3.173515, "dram": 0.348388, "package-0": 4.302723, "psys": 6.532759, "uncore": 0.0 }, "logits": { "l2": 656.7043815022596, "sha256": "2cabc748d175b793c08a889d7d9410edb1eb4a784b8bdcc4b1140fb6799d78e4", "sum": 94.01911521144211, "top1": 78 }, "net_j": { "core": 2.8972747869880875, "dram": 0.2602299162655138, "package-0": 3.714665112523682, "psys": 5.082706744566702, "uncore": 0.0 }, "npu_busy_delta_us": 100319, "npu_utilization": 0.5541605746097014, "seconds": 0.181028757, "watts": { "core": 17.530446833924845, "dram": 1.9244898201449836, "package-0": 23.768174025522367, "psys": 36.08685773608886, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 446.4723727321237, "decode_tokens_per_s": 9.643070544697089, "end_to_end_ms": 2293.5349699999997, "handoff_estimate_ms": 83.11460550000002, "handoff_plus_first_decode_ms": 181.028757, "output_tokens_per_s_e2e": 6.976130823939433, "prefill_tokens_per_s": 1365.8280464846407, "steady_tpot_ms": 97.91415149999999, "steady_tpot_p95_ms": 99.5388777, "ttft_ms": 738.013839 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 73, "gross_j": { "core": 2.111017, "dram": 1.308041, "package-0": 16.941301, "psys": 26.73571, "uncore": 0.0 }, "logits": { "l2": 690.669334639667, "sha256": "dffdd879fdeb67b8ccffdd2d86deb30dc4ae6c5324e8c93b70566bb7f425595c", "sum": 123939.967737481, "top1": 290 }, "net_j": { "core": 0.9848473048111885, "dram": 0.9486401683643302, "package-0": 14.543920239712133, "psys": 20.824170587601905, "uncore": 0.0 }, "npu_busy_delta_us": 729906, "npu_utilization": 0.9890139743029942, "seconds": 0.738013839, "watts": { "core": 2.860403001196296, "dram": 1.772380043404579, "package-0": 22.955261954105445, "psys": 36.22657000067448, "uncore": 0.0 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 595.7391334219876, "sha256": "a4358e6f20367655c98e237fcca564f9503536c619fe29ca9486099ea9b451f3", "sum": -67153.35384979099, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 136, "gross_j": { "core": 5.937607, "dram": 4.059133, "package-0": 19.282971, "psys": 36.168608, "uncore": 0.0 }, "net_j": { "core": 3.8402050597269994, "dram": 3.3897773914902167, "package-0": 14.818039045532958, "psys": 25.158833641362772, "uncore": 0.0 }, "npu_busy_delta_us": 1358163, "npu_utilization": 0.9881197056390508, "seconds": 1.374492374, "token_latencies_s": [ 0.099829065, 0.097970373, 0.097866055, 0.097710669, 0.099382623, 0.098194411, 0.097773425, 0.098466244, 0.098024195, 0.097899874, 0.097918433, 0.097816606, 0.097698954, 0.09790987 ], "watts": { "core": 4.319854451225933, "dram": 2.9531869923637717, "package-0": 14.029158229436637, "psys": 26.31415690924743, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 11.238434999999999, "dram": 5.659653, "package-0": 40.326923, "psys": 67.88605799999999, "uncore": 0.0 }, "net_j": { "core": 7.863117920013003, "dram": 4.58246918768118, "package-0": 33.141575834291395, "psys": 50.16819537864944, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 10, "gross_j": { "core": 2.3034, "dram": 0.296264, "package-0": 3.268119, "psys": 4.716785, "uncore": 0.0 }, "logits": { "l2": 656.7043815022596, "sha256": "2cabc748d175b793c08a889d7d9410edb1eb4a784b8bdcc4b1140fb6799d78e4", "sum": 94.01911521144211, "top1": 78 }, "net_j": { "core": 2.1453308008945755, "dram": 0.24581849266910963, "package-0": 2.931622592444013, "psys": 3.8870412194756514, "uncore": 0.0 }, "npu_busy_delta_us": 96983, "npu_utilization": 0.9362410780375372, "seconds": 0.103587636, "watts": { "core": 22.23624448771087, "dram": 2.860032446343307, "package-0": 31.549315402853676, "psys": 45.53424696360481, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 462.94000328706386, "decode_tokens_per_s": 10.186762902220105, "end_to_end_ms": 2211.949697, "handoff_estimate_ms": 5.935249500000003, "handoff_plus_first_decode_ms": 103.587636, "output_tokens_per_s_e2e": 7.233437551360373, "prefill_tokens_per_s": 1363.1743762424758, "steady_tpot_ms": 97.65238649999999, "steady_tpot_p95_ms": 98.4875902, "ttft_ms": 739.450519 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 74, "gross_j": { "core": 2.002619, "dram": 1.308102, "package-0": 16.780841, "psys": 26.173029, "uncore": 0.0 }, "logits": { "l2": 690.669334639667, "sha256": "dffdd879fdeb67b8ccffdd2d86deb30dc4ae6c5324e8c93b70566bb7f425595c", "sum": 123939.967737481, "top1": 290 }, "net_j": { "core": 0.87425700787557, "dram": 0.9480015283853644, "package-0": 14.378793296074114, "psys": 20.249981686267002, "uncore": 0.0 }, "npu_busy_delta_us": 729099, "npu_utilization": 0.9860010660158858, "seconds": 0.739450519, "watts": { "core": 2.7082528831114394, "dram": 1.7690189761027135, "package-0": 22.693663157737262, "psys": 35.39524055699527, "uncore": 0.0 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 595.7391334219876, "sha256": "a4358e6f20367655c98e237fcca564f9503536c619fe29ca9486099ea9b451f3", "sum": -67153.35384979099, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 136, "gross_j": { "core": 6.932416, "dram": 4.055287, "package-0": 20.277963, "psys": 36.996244, "uncore": 0.0 }, "net_j": { "core": 4.843530111242857, "dram": 3.3886491666267053, "package-0": 15.831159945773267, "psys": 26.03117247290679, "uncore": 0.0 }, "npu_busy_delta_us": 1355926, "npu_utilization": 0.9905139655839063, "seconds": 1.368911542, "token_latencies_s": [ 0.099556003, 0.097912291, 0.09766316, 0.097610582, 0.097817692, 0.097761904, 0.097874747, 0.097827015, 0.097549849, 0.09738773, 0.097427545, 0.097396938, 0.097447996, 0.097641613 ], "watts": { "core": 5.064181130266195, "dram": 2.9624171289221257, "package-0": 14.81320185990513, "psys": 27.02602970674638, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 11.764924, "dram": 5.662644, "package-0": 40.901323, "psys": 68.740912, "uncore": 0.0 }, "net_j": { "core": 8.394726083205628, "dram": 4.58709389485388, "package-0": 33.726873466681255, "psys": 51.049921114854904, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 10, "gross_j": { "core": 2.355524, "dram": 0.29779, "package-0": 3.322196, "psys": 4.779956, "uncore": 0.0 }, "logits": { "l2": 656.7043815022596, "sha256": "2cabc748d175b793c08a889d7d9410edb1eb4a784b8bdcc4b1140fb6799d78e4", "sum": 94.01911521144211, "top1": 78 }, "net_j": { "core": 2.1981994597747634, "dram": 0.2475821398211931, "package-0": 2.9872848161274117, "psys": 3.9541211155663816, "uncore": 0.0 }, "npu_busy_delta_us": 96896, "npu_utilization": 0.9398287121046924, "seconds": 0.103099638, "watts": { "core": 22.847063730718435, "dram": 2.888370956258838, "package-0": 32.22315872728865, "psys": 46.36249062290597, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 463.6431861516104, "decode_tokens_per_s": 10.188324121057455, "end_to_end_ms": 2208.594951, "handoff_estimate_ms": 5.349535999999988, "handoff_plus_first_decode_ms": 103.099638, "output_tokens_per_s_e2e": 7.244424783618912, "prefill_tokens_per_s": 1368.9673859858262, "steady_tpot_ms": 97.750102, "steady_tpot_p95_ms": 98.0997677, "ttft_ms": 736.321413 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 73, "gross_j": { "core": 2.143489, "dram": 1.304501, "package-0": 16.955279, "psys": 26.389703, "uncore": 0.0 }, "logits": { "l2": 690.669334639667, "sha256": "dffdd879fdeb67b8ccffdd2d86deb30dc4ae6c5324e8c93b70566bb7f425595c", "sum": 123939.967737481, "top1": 290 }, "net_j": { "core": 1.0199018564814015, "dram": 0.9459243525187979, "package-0": 14.563395954410964, "psys": 20.491720031225135, "uncore": 0.0 }, "npu_busy_delta_us": 730400, "npu_utilization": 0.9919581138135393, "seconds": 0.736321413, "watts": { "core": 2.9110779099398543, "dram": 1.771646154747913, "package-0": 23.027007908026167, "psys": 35.83992334608365, "uncore": 0.0 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 595.7391334219876, "sha256": "a4358e6f20367655c98e237fcca564f9503536c619fe29ca9486099ea9b451f3", "sum": -67153.35384979099, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 135, "gross_j": { "core": 7.265911, "dram": 4.060353, "package-0": 20.623848, "psys": 37.571253, "uncore": 0.0 }, "net_j": { "core": 5.1766247669494625, "dram": 3.393587402513889, "package-0": 16.17619269614288, "psys": 26.604079968063388, "uncore": 0.0 }, "npu_busy_delta_us": 1354569, "npu_utilization": 0.989333056962304, "seconds": 1.3691739, "token_latencies_s": [ 0.098289257, 0.097748221, 0.097670707, 0.097602527, 0.097751983, 0.097770435, 0.097692025, 0.097799931, 0.097711887, 0.097741773, 0.097947507, 0.097997735, 0.097611626, 0.097775414 ], "watts": { "core": 5.306784623925419, "dram": 2.9655495185819714, "package-0": 15.062986520558125, "psys": 27.440818876258156, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 23.044777703471482, "compile_target": "NPU", "context_length": 2048, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 407644, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 1.5259465821328744, "dram": 0.48698386485903, "package-0": 3.248422500499841, "psys": 8.010065800945089, "uncore": 0.0023192603038971636 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_3b_ov_fp16/openvino_model.xml", "key": "qwen25_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "npu_npu", "label": "NPU -> NPU", "mode": "staged", "prefill_device": "NPU" }, "prompt_length": 2032, "prompt_sha256": "cd1a97358e68d07c27508cfacc4ba6882a7a2962b1281e2e2b0c7e5e320da4fa", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "NPU" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 2048, "NPUW_LLM_MIN_RESPONSE_LEN": 2048, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.05915387440472841, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 14.601586000000001, "dram": 7.2031670000000005, "package-0": 61.518457000000005, "psys": 98.975576, "uncore": 0.0 }, "net_j": { "core": 9.927391047926399, "dram": 5.7114650298800775, "package-0": 51.568069331241816, "psys": 74.43958662072386, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 18, "gross_j": { "core": 2.977104, "dram": 0.356505, "package-0": 4.123585, "psys": 6.31169, "uncore": 0.0 }, "logits": { "l2": 598.095176963933, "sha256": "0883b6dfe42b96f14222016a116c7843ca78c62813e9c5095fbc666e8cbfe150", "sum": 14349.924920428544, "top1": 220 }, "net_j": { "core": 2.7018475037117256, "dram": 0.2686608552838339, "package-0": 3.5376212405728547, "psys": 4.866801513585938, "uncore": 0.0 }, "npu_busy_delta_us": 101607, "npu_utilization": 0.5632813628797887, "seconds": 0.180384097, "watts": { "core": 16.50424870879832, "dram": 1.9763660207806457, "package-0": 22.860025182818642, "psys": 34.9902796586331, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 668.5939786961023, "decode_tokens_per_s": 9.50839441818555, "end_to_end_ms": 3063.144547, "handoff_estimate_ms": 80.70304949999999, "handoff_plus_first_decode_ms": 180.384097, "output_tokens_per_s_e2e": 5.2233904585632995, "prefill_tokens_per_s": 1367.8056794497268, "steady_tpot_ms": 99.6810475, "steady_tpot_p95_ms": 100.4612143, "ttft_ms": 1485.591141 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 147, "gross_j": { "core": 5.465868, "dram": 2.71502, "package-0": 37.671961, "psys": 56.88053, "uncore": 0.0 }, "logits": { "l2": 599.2833854145028, "sha256": "c07e61c2839864d29446765c2cb7b6ae7bde6f2704446b2060059c8f9d6eff63", "sum": -5876.612534986809, "top1": 220 }, "net_j": { "core": 3.1989352759441734, "dram": 1.9915610845554839, "package-0": 32.846133311032375, "psys": 44.9808472072889, "uncore": 0.0 }, "npu_busy_delta_us": 1457431, "npu_utilization": 0.981044487798275, "seconds": 1.485591141, "watts": { "core": 3.6792545735839175, "dram": 1.827568787312794, "package-0": 25.358229434945184, "psys": 38.28814566147174, "uncore": 0.0 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 677.0484772729441, "sha256": "d7335174072bf9ed9b0a52e0cbb84640148dea91348cc0fb13fdba6b8c11a0d3", "sum": -131147.21031796, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 138, "gross_j": { "core": 6.158614, "dram": 4.131642, "package-0": 19.722911, "psys": 35.783356, "uncore": 0.0 }, "net_j": { "core": 4.0266082682705004, "dram": 3.45124309004076, "package-0": 15.184314779636585, "psys": 24.591937899849015, "uncore": 0.0 }, "npu_busy_delta_us": 1382592, "npu_utilization": 0.9895665407863609, "seconds": 1.397169309, "token_latencies_s": [ 0.101128282, 0.099647102, 0.099701659, 0.099521264, 0.099425763, 0.099634718, 0.099560782, 0.099660436, 0.099777722, 0.099784135, 0.100102024, 0.09989721, 0.099775444, 0.099518445 ], "watts": { "core": 4.40792247605834, "dram": 2.957151988227649, "package-0": 14.116335703162802, "psys": 25.611324103312377, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 15.065757, "dram": 7.170575, "package-0": 62.002099, "psys": 98.18059600000001, "uncore": 0.0 }, "net_j": { "core": 10.464868805558126, "dram": 5.702267825326905, "package-0": 52.20776615237241, "psys": 74.02941167167351, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 14, "gross_j": { "core": 2.486016, "dram": 0.322875, "package-0": 3.533927, "psys": 5.187365, "uncore": 0.0 }, "logits": { "l2": 598.095176963933, "sha256": "0883b6dfe42b96f14222016a116c7843ca78c62813e9c5095fbc666e8cbfe150", "sum": 14349.924920428544, "top1": 220 }, "net_j": { "core": 2.28264421951599, "dram": 0.2579718265711275, "package-0": 3.1009908307878558, "psys": 4.119816920135152, "uncore": 0.0 }, "npu_busy_delta_us": 99110, "npu_utilization": 0.7436457771833315, "seconds": 0.133275819, "watts": { "core": 18.65316618313184, "dram": 2.42260750991896, "package-0": 26.515890328162232, "psys": 38.92202680817891, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 679.2468036896585, "decode_tokens_per_s": 9.842233963179135, "end_to_end_ms": 3015.1043609999997, "handoff_estimate_ms": 34.017079499999994, "handoff_plus_first_decode_ms": 133.27581899999998, "output_tokens_per_s_e2e": 5.306615653825457, "prefill_tokens_per_s": 1362.7887796789769, "steady_tpot_ms": 99.2587395, "steady_tpot_p95_ms": 99.68005465, "ttft_ms": 1491.060119 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 148, "gross_j": { "core": 5.133959, "dram": 2.718377, "package-0": 37.439906, "psys": 55.939127, "uncore": 0.0 }, "logits": { "l2": 599.2833854145028, "sha256": "c07e61c2839864d29446765c2cb7b6ae7bde6f2704446b2060059c8f9d6eff63", "sum": -5876.612534986809, "top1": 220 }, "net_j": { "core": 2.858680907657313, "dram": 1.9922547805122148, "package-0": 32.596312759842434, "psys": 43.995637333644986, "uncore": 0.0 }, "npu_busy_delta_us": 1453357, "npu_utilization": 0.9747138840885329, "seconds": 1.491060119, "watts": { "core": 3.4431602955373526, "dram": 1.8231169658156487, "package-0": 25.109588488698627, "psys": 37.516345777872694, "uncore": 0.0 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 677.0484772729441, "sha256": "d7335174072bf9ed9b0a52e0cbb84640148dea91348cc0fb13fdba6b8c11a0d3", "sum": -131147.21031796, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 137, "gross_j": { "core": 7.445782, "dram": 4.129323, "package-0": 21.028266, "psys": 37.054104, "uncore": 0.0 }, "net_j": { "core": 5.323543678384823, "dram": 3.4520412182435622, "package-0": 16.510462561742116, "psys": 25.91395741789337, "uncore": 0.0 }, "npu_busy_delta_us": 1381352, "npu_utilization": 0.9932293379370176, "seconds": 1.390768423, "token_latencies_s": [ 0.100237533, 0.099368385, 0.099345932, 0.099266446, 0.099247309, 0.09921427, 0.099258712, 0.099111658, 0.099233088, 0.099312333, 0.099379874, 0.099253428, 0.099255143, 0.099258767 ], "watts": { "core": 5.35371804310803, "dram": 2.9690945895167196, "package-0": 15.119890308294696, "psys": 26.642899987671065, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 12.723234999999999, "dram": 7.186382999999999, "package-0": 59.613311, "psys": 94.738344, "uncore": 0.0 }, "net_j": { "core": 8.082295291720671, "dram": 5.705293961773884, "package-0": 49.73371681999548, "psys": 70.37691950663637, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 13, "gross_j": { "core": 2.485895, "dram": 0.322326, "package-0": 3.537893, "psys": 5.136217, "uncore": 0.0 }, "logits": { "l2": 598.095176963933, "sha256": "0883b6dfe42b96f14222016a116c7843ca78c62813e9c5095fbc666e8cbfe150", "sum": 14349.924920428544, "top1": 220 }, "net_j": { "core": 2.2811357678456523, "dram": 0.25698004136184305, "package-0": 3.1020032351135014, "psys": 4.061385847906301, "uncore": 0.0 }, "npu_busy_delta_us": 99166, "npu_utilization": 0.7390241561841845, "seconds": 0.134185059, "watts": { "core": 18.525870305724577, "dram": 2.4021005199990264, "package-0": 26.36577444885276, "psys": 38.2771154872019, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 673.3848738937401, "decode_tokens_per_s": 9.815708810668962, "end_to_end_ms": 3041.351357, "handoff_estimate_ms": 34.76565049999999, "handoff_plus_first_decode_ms": 134.185059, "output_tokens_per_s_e2e": 5.260819327294844, "prefill_tokens_per_s": 1342.859649295615, "steady_tpot_ms": 99.4194085, "steady_tpot_p95_ms": 100.32548745, "ttft_ms": 1513.188665 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 150, "gross_j": { "core": 4.185231, "dram": 2.726434, "package-0": 36.457182, "psys": 54.469954, "uncore": 0.0 }, "logits": { "l2": 599.2833854145028, "sha256": "c07e61c2839864d29446765c2cb7b6ae7bde6f2704446b2060059c8f9d6eff63", "sum": -5876.612534986809, "top1": 220 }, "net_j": { "core": 1.8761859285210432, "dram": 1.9895355356574238, "package-0": 31.541705893112688, "psys": 42.349213224105746, "uncore": 0.0 }, "npu_busy_delta_us": 1461733, "npu_utilization": 0.9659952085353481, "seconds": 1.513188665, "watts": { "core": 2.7658355476777245, "dram": 1.8017806127301383, "package-0": 24.09295208406812, "psys": 35.99680281771077, "uncore": 0.0 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 677.0484772729441, "sha256": "d7335174072bf9ed9b0a52e0cbb84640148dea91348cc0fb13fdba6b8c11a0d3", "sum": -131147.21031796, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 137, "gross_j": { "core": 6.052109, "dram": 4.137623, "package-0": 19.618236, "psys": 35.132173, "uncore": 0.0 }, "net_j": { "core": 3.9249735953539755, "dram": 3.458778384754617, "package-0": 15.09000769176929, "psys": 23.96632043462432, "uncore": 0.0 }, "npu_busy_delta_us": 1382596, "npu_utilization": 0.9918351394379941, "seconds": 1.393977633, "token_latencies_s": [ 0.100106336, 0.099676829, 0.09966673, 0.099440668, 0.100732483, 0.099398149, 0.0992854, 0.099314426, 0.099307562, 0.099366736, 0.099546878, 0.09923671, 0.099396817, 0.099462917 ], "watts": { "core": 4.3416112688803805, "dram": 2.9682133357443905, "package-0": 14.073565841784205, "psys": 25.20282403986033, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 21.653414119966328, "compile_target": "NPU", "context_length": 4096, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 407815, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 1.5259465821328744, "dram": 0.48698386485903, "package-0": 3.248422500499841, "psys": 8.010065800945089, "uncore": 0.0023192603038971636 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_3b_ov_fp16/openvino_model.xml", "key": "qwen25_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "npu_npu", "label": "NPU -> NPU", "mode": "staged", "prefill_device": "NPU" }, "prompt_length": 4080, "prompt_sha256": "66f8de7a9ef70b33481de3e8dae7cef6e4434d9723c6203a6a3dbe07ef228a16", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "NPU" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 4096, "NPUW_LLM_MIN_RESPONSE_LEN": 4096, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.05971842538565397, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 17.300981, "dram": 11.186616, "package-0": 105.502476, "psys": 173.102829, "uncore": 0.0 }, "net_j": { "core": 9.330816342289001, "dram": 8.643052788796465, "package-0": 88.53565520171955, "psys": 131.26549072083276, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 21, "gross_j": { "core": 3.825246, "dram": 0.401305, "package-0": 5.096728, "psys": 7.693157, "uncore": 0.0 }, "logits": { "l2": 674.5130808654125, "sha256": "68592a61579d632d5b142b4d10b5be1b44d2f298fcb42b81fe8b59148d40b24f", "sum": -59216.823038564995, "top1": 1718 }, "net_j": { "core": 3.4874076937783944, "dram": 0.29348877236294374, "package-0": 4.377540599660544, "psys": 5.919761329011002, "uncore": 0.0 }, "npu_busy_delta_us": 104775, "npu_utilization": 0.4732472611856444, "seconds": 0.221395893, "watts": { "core": 17.277854381878708, "dram": 1.8126126666676694, "package-0": 23.020878711602837, "psys": 34.7484178489255, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 784.209294141648, "decode_tokens_per_s": 9.085405482646086, "end_to_end_ms": 5223.095455000001, "handoff_estimate_ms": 119.53060350000001, "handoff_plus_first_decode_ms": 221.395893, "output_tokens_per_s_e2e": 3.0633175552408125, "prefill_tokens_per_s": 1142.1865657642632, "steady_tpot_ms": 101.8652895, "steady_tpot_p95_ms": 103.19750105, "ttft_ms": 3572.095945 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 353, "gross_j": { "core": 7.016889, "dram": 6.548994, "package-0": 79.992715, "psys": 127.320597, "uncore": 0.0 }, "logits": { "l2": 667.6443131718496, "sha256": "b618e00272bb302626d98d7f306a45e1f92c71ccf07eeff3434920f53951ad1a", "sum": -472.4365739785135, "top1": 36 }, "net_j": { "core": 1.5660614016765502, "dram": 4.809440911056631, "package-0": 68.38903815831776, "psys": 98.70787343326087, "uncore": 0.0 }, "npu_busy_delta_us": 3467063, "npu_utilization": 0.9705962699162606, "seconds": 3.572095945, "watts": { "core": 1.9643618503085867, "dram": 1.8333757269781286, "package-0": 22.39377559608075, "psys": 35.643106725119054, "uncore": 0.0 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 708.380856841874, "sha256": "fd2bd305053dee7b73acdaf34aa5d6eddb70fcf63291f691749f6eb56bcd0c8d", "sum": -134037.2010315787, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 141, "gross_j": { "core": 6.458846, "dram": 4.236317, "package-0": 20.413033, "psys": 38.089075, "uncore": 0.0 }, "net_j": { "core": 4.277347246834056, "dram": 3.540123105376891, "package-0": 15.769076443741241, "psys": 26.6378559585609, "uncore": 0.0 }, "npu_busy_delta_us": 1418238, "npu_utilization": 0.992049812364178, "seconds": 1.429603617, "token_latencies_s": [ 0.104826052, 0.102320589, 0.102107768, 0.101992299, 0.101964491, 0.10183759, 0.101881688, 0.101848891, 0.101732823, 0.101914355, 0.101827184, 0.101836468, 0.101734332, 0.101752354 ], "watts": { "core": 4.5179278530043065, "dram": 2.9632808350680047, "package-0": 14.278806207021509, "psys": 26.64310200888363, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 18.157363, "dram": 11.129732, "package-0": 106.278658, "psys": 172.197252, "uncore": 0.0 }, "net_j": { "core": 10.309671996053638, "dram": 8.625254490715564, "package-0": 89.57255810588867, "psys": 131.00280780180756, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 18, "gross_j": { "core": 2.80523, "dram": 0.374633, "package-0": 3.980703, "psys": 6.033615, "uncore": 0.0 }, "logits": { "l2": 674.5130808654125, "sha256": "68592a61579d632d5b142b4d10b5be1b44d2f298fcb42b81fe8b59148d40b24f", "sum": -59216.823038564995, "top1": 1718 }, "net_j": { "core": 2.5270395433290593, "dram": 0.28585252410211137, "package-0": 3.388493449904806, "psys": 4.573325439919474, "uncore": 0.0 }, "npu_busy_delta_us": 102371, "npu_utilization": 0.5615314034453084, "seconds": 0.182306812, "watts": { "core": 15.387411853814873, "dram": 2.05495886791109, "package-0": 21.83518518221908, "psys": 33.09593829110456, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 796.4479229971188, "decode_tokens_per_s": 9.297816477719788, "end_to_end_ms": 5142.834681, "handoff_estimate_ms": 80.20840000000001, "handoff_plus_first_decode_ms": 182.306812, "output_tokens_per_s_e2e": 3.1111246992074952, "prefill_tokens_per_s": 1155.9538483692108, "steady_tpot_ms": 102.098412, "steady_tpot_p95_ms": 102.67646659999998, "ttft_ms": 3529.5526769999997 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 349, "gross_j": { "core": 9.22721, "dram": 6.530195, "package-0": 82.207309, "psys": 128.864294, "uncore": 0.0 }, "logits": { "l2": 667.6443131718496, "sha256": "b618e00272bb302626d98d7f306a45e1f92c71ccf07eeff3434920f53951ad1a", "sum": -472.4365739785135, "top1": 36 }, "net_j": { "core": 3.8413011560739125, "dram": 4.811359796131004, "package-0": 70.74183066733374, "psys": 100.59234480932811, "uncore": 0.0 }, "npu_busy_delta_us": 3465176, "npu_utilization": 0.9817606697246638, "seconds": 3.529552677, "watts": { "core": 2.6142717914732514, "dram": 1.8501480492282791, "package-0": 23.291140981035994, "psys": 36.51009229575525, "uncore": 0.0 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 708.380856841874, "sha256": "fd2bd305053dee7b73acdaf34aa5d6eddb70fcf63291f691749f6eb56bcd0c8d", "sum": -134037.2010315787, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 141, "gross_j": { "core": 6.124923, "dram": 4.224904, "package-0": 20.090646, "psys": 37.299343, "uncore": 0.0 }, "net_j": { "core": 3.941331296650666, "dram": 3.528042170482448, "package-0": 15.44223398865012, "psys": 25.837137552559966, "uncore": 0.0 }, "npu_busy_delta_us": 1417705, "npu_utilization": 0.9907264695613256, "seconds": 1.430975192, "token_latencies_s": [ 0.102719273, 0.102653417, 0.102406064, 0.101976759, 0.102039746, 0.102068042, 0.102182677, 0.102017763, 0.102527981, 0.102028255, 0.102100749, 0.102096075, 0.101958804, 0.102174286 ], "watts": { "core": 4.280244014181345, "dram": 2.9524648810263936, "package-0": 14.03982830192908, "psys": 26.06568108834133, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 21.815129, "dram": 11.166414, "package-0": 109.662866, "psys": 173.670759, "uncore": 0.0 }, "net_j": { "core": 13.921682586974912, "dram": 8.647334311181716, "package-0": 92.85936236835866, "psys": 132.2361334994205, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 21, "gross_j": { "core": 2.947624, "dram": 0.393126, "package-0": 3.948232, "psys": 6.114304, "uncore": 0.0 }, "logits": { "l2": 674.5130808654125, "sha256": "68592a61579d632d5b142b4d10b5be1b44d2f298fcb42b81fe8b59148d40b24f", "sum": -59216.823038564995, "top1": 1718 }, "net_j": { "core": 2.634132078413583, "dram": 0.29307957022075376, "package-0": 3.280872982502247, "psys": 4.468708440251066, "uncore": 0.0 }, "npu_busy_delta_us": 126162, "npu_utilization": 0.6141034567041583, "seconds": 0.205440954, "watts": { "core": 14.347791628732407, "dram": 1.9135717214397279, "package-0": 19.218329759119012, "psys": 29.76185556459205, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 791.831206974761, "decode_tokens_per_s": 9.131076469404587, "end_to_end_ms": 5172.819616, "handoff_estimate_ms": 103.144987, "handoff_plus_first_decode_ms": 205.440954, "output_tokens_per_s_e2e": 3.09309065224516, "prefill_tokens_per_s": 1155.7818514728679, "steady_tpot_ms": 102.295967, "steady_tpot_p95_ms": 104.3798861, "ttft_ms": 3530.077925 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 349, "gross_j": { "core": 11.187166, "dram": 6.532393, "package-0": 84.230863, "psys": 129.730258, "uncore": 0.0 }, "logits": { "l2": 667.6443131718496, "sha256": "b618e00272bb302626d98d7f306a45e1f92c71ccf07eeff3434920f53951ad1a", "sum": -472.4365739785135, "top1": 36 }, "net_j": { "core": 5.80045565568354, "dram": 4.813302008829955, "package-0": 72.76367843991221, "psys": 101.4541015382863, "uncore": 0.0 }, "npu_busy_delta_us": 3459794, "npu_utilization": 0.9800899791751764, "seconds": 3.530077925, "watts": { "core": 3.1690988804446856, "dram": 1.850495410805981, "package-0": 23.860907546396444, "psys": 36.749970044924716, "uncore": 0.0 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 708.380856841874, "sha256": "fd2bd305053dee7b73acdaf34aa5d6eddb70fcf63291f691749f6eb56bcd0c8d", "sum": -134037.2010315787, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 142, "gross_j": { "core": 7.680339, "dram": 4.240895, "package-0": 21.483771, "psys": 37.826197, "uncore": 0.0 }, "net_j": { "core": 5.487094852877789, "dram": 3.540952732131008, "package-0": 16.814810945944195, "psys": 26.31332352088313, "uncore": 0.0 }, "npu_busy_delta_us": 1422136, "npu_utilization": 0.9894491552048791, "seconds": 1.437300737, "token_latencies_s": [ 0.107329655, 0.102791549, 0.102281906, 0.102314533, 0.102207795, 0.102325921, 0.102323002, 0.102677839, 0.102310028, 0.102066061, 0.102226658, 0.102143562, 0.102079772, 0.102185705 ], "watts": { "core": 5.3435852374435955, "dram": 2.950596831148776, "package-0": 14.947303961481238, "psys": 26.31752424962404, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 63.33386187534779, "compile_target": "NPU", "context_length": 8192, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 407992, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 1.5259465821328744, "dram": 0.48698386485903, "package-0": 3.248422500499841, "psys": 8.010065800945089, "uncore": 0.0023192603038971636 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_3b_ov_fp16/openvino_model.xml", "key": "qwen25_3b_fp16", "size_b": 3.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "npu_npu", "label": "NPU -> NPU", "mode": "staged", "prefill_device": "NPU" }, "prompt_length": 8176, "prompt_sha256": "998ace73fce48442f97bcb84c31be293d0d94180b17f70a4d127bba1092ad744", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "NPU" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 8192, "NPUW_LLM_MIN_RESPONSE_LEN": 8192, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.06031969655305147, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 35.801361, "dram": 22.174321, "package-0": 226.071382, "psys": 366.207558, "uncore": 0.0 }, "net_j": { "core": 19.501787158602674, "dram": 16.972546863356182, "package-0": 191.37298361901188, "psys": 280.6471193020305, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 28, "gross_j": { "core": 4.161, "dram": 0.492308, "package-0": 5.430772, "psys": 8.681374, "uncore": 0.0 }, "logits": { "l2": 648.2391451311615, "sha256": "e85afbe82578c61fa2818aef24e755aad5e884df578810e179b8993cd7997204", "sum": -34242.11205828376, "top1": 78 }, "net_j": { "core": 3.721802352290099, "dram": 0.35214439897812766, "package-0": 4.495811669348524, "psys": 6.375918423134816, "uncore": 0.0 }, "npu_busy_delta_us": 135153, "npu_utilization": 0.4695750523491599, "seconds": 0.287819805, "watts": { "core": 14.45696205652005, "dram": 1.7104729815239783, "package-0": 18.86865290593884, "psys": 30.162531727099186, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 766.9252289948744, "decode_tokens_per_s": 8.293658544272171, "end_to_end_ms": 10681.614961, "handoff_estimate_ms": 179.63187399999998, "handoff_plus_first_decode_ms": 287.819805, "output_tokens_per_s_e2e": 1.497900837880614, "prefill_tokens_per_s": 921.4466493619024, "steady_tpot_ms": 108.187931, "steady_tpot_p95_ms": 110.89314859999999, "ttft_ms": 8873.004211 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 876, "gross_j": { "core": 24.734007, "dram": 17.223833, "package-0": 199.070169, "psys": 317.773905, "uncore": 0.0 }, "logits": { "l2": 607.3833136095392, "sha256": "597bcab04865e78eb1aaf108263fece075d0b3ccf1e95f89435d49f34b5a7e76", "sum": 46667.203135058284, "top1": 198 }, "net_j": { "core": 11.194276550973948, "dram": 12.902823116416771, "package-0": 170.24690247395776, "psys": 246.70055741782716, "uncore": 0.0 }, "npu_busy_delta_us": 8685634, "npu_utilization": 0.9788831148341265, "seconds": 8.873004211, "watts": { "core": 2.787557225470137, "dram": 1.9411500987058419, "package-0": 22.43548681665333, "psys": 35.81356409208629, "uncore": 0.0 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 608.4144131591894, "sha256": "75dc1394dccec2f9374185c7ea194761337790f490c37e1fedd5be61da86a67d", "sum": -106184.73783594742, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 150, "gross_j": { "core": 6.906354, "dram": 4.45818, "package-0": 21.570441, "psys": 39.752279, "uncore": 0.0 }, "net_j": { "core": 4.585708255338626, "dram": 3.717579347961283, "package-0": 16.630269475705582, "psys": 27.57064346106854, "uncore": 0.0 }, "npu_busy_delta_us": 1506008, "npu_utilization": 0.9902794364678441, "seconds": 1.520790945, "token_latencies_s": [ 0.113353526, 0.10956833, 0.108401429, 0.108262132, 0.108255064, 0.108037037, 0.107998525, 0.108458194, 0.108389314, 0.108120798, 0.107970743, 0.107897816, 0.108096181, 0.107928707 ], "watts": { "core": 4.5412908478357625, "dram": 2.9314877331808415, "package-0": 14.183698996182542, "psys": 26.139213368343672, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 30.075605, "dram": 22.04041, "package-0": 220.912581, "psys": 362.20958299999995, "uncore": 0.0 }, "net_j": { "core": 13.829812997873127, "dram": 16.855799562435706, "package-0": 186.328672954945, "psys": 276.93145829420837, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 25, "gross_j": { "core": 3.225089, "dram": 0.466063, "package-0": 4.586292, "psys": 7.777019, "uncore": 0.0 }, "logits": { "l2": 648.2391451311615, "sha256": "e85afbe82578c61fa2818aef24e755aad5e884df578810e179b8993cd7997204", "sum": -34242.11205828376, "top1": 78 }, "net_j": { "core": 2.842729055237827, "dram": 0.3440383329193837, "package-0": 3.7723273026780038, "psys": 5.769918389105847, "uncore": 0.0 }, "npu_busy_delta_us": 132923, "npu_utilization": 0.5304776306080121, "seconds": 0.2505723, "watts": { "core": 12.870891954138585, "dram": 1.8599941015028396, "package-0": 18.303268158531488, "psys": 31.037026039989254, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 769.4641418033639, "decode_tokens_per_s": 8.465157969907148, "end_to_end_ms": 10646.370058, "handoff_estimate_ms": 142.44708900000003, "handoff_plus_first_decode_ms": 250.5723, "output_tokens_per_s_e2e": 1.502859651959695, "prefill_tokens_per_s": 921.3016425952004, "steady_tpot_ms": 108.125211, "steady_tpot_p95_ms": 111.0312522, "ttft_ms": 8874.400763000001 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 876, "gross_j": { "core": 21.24494, "dram": 17.119768, "package-0": 195.062367, "psys": 312.728508, "uncore": 0.0 }, "logits": { "l2": 607.3833136095392, "sha256": "597bcab04865e78eb1aaf108263fece075d0b3ccf1e95f89435d49f34b5a7e76", "sum": 46667.203135058284, "top1": 198 }, "net_j": { "core": 7.7030784872227755, "dram": 12.798078018126336, "package-0": 166.23456388301784, "psys": 241.64397394441266, "uncore": 0.0 }, "npu_busy_delta_us": 8671700, "npu_utilization": 0.9771589351874753, "seconds": 8.874400763, "watts": { "core": 2.3939576955524067, "dram": 1.9291181970705418, "package-0": 21.980342358807217, "psys": 35.239394337909275, "uncore": 0.0 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 608.4144131591894, "sha256": "75dc1394dccec2f9374185c7ea194761337790f490c37e1fedd5be61da86a67d", "sum": -106184.73783594742, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 151, "gross_j": { "core": 5.605576, "dram": 4.454579, "package-0": 21.263922, "psys": 41.704056, "uncore": 0.0 }, "net_j": { "core": 3.2840054554125246, "dram": 3.7136832113899856, "package-0": 16.321781769249156, "psys": 29.517565960689875, "uncore": 0.0 }, "npu_busy_delta_us": 1504336, "npu_utilization": 0.9887859677282983, "seconds": 1.521396995, "token_latencies_s": [ 0.115782487, 0.108472895, 0.108073991, 0.108176431, 0.108275055, 0.107944352, 0.108001312, 0.108023482, 0.107952397, 0.10822568, 0.107899985, 0.10805141, 0.10828673, 0.108184362 ], "watts": { "core": 3.6844926198897876, "dram": 2.9279530685546016, "package-0": 13.976576836869592, "psys": 27.411685534451845, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 24.925228999999998, "dram": 21.810308, "package-0": 216.495296, "psys": 359.33758099999994, "uncore": 0.0 }, "net_j": { "core": 8.763161781937455, "dram": 16.652417120908808, "package-0": 182.08962058483425, "psys": 274.49894810694786, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 24, "gross_j": { "core": 3.270622, "dram": 0.463927, "package-0": 4.453419, "psys": 7.162579, "uncore": 0.0 }, "logits": { "l2": 648.2391451311615, "sha256": "e85afbe82578c61fa2818aef24e755aad5e884df578810e179b8993cd7997204", "sum": -34242.11205828376, "top1": 78 }, "net_j": { "core": 2.891443116728377, "dram": 0.34291752285929433, "package-0": 3.646226120146826, "psys": 5.172176556757655, "uncore": 0.0 }, "npu_busy_delta_us": 134759, "npu_utilization": 0.5423166862231045, "seconds": 0.248487652, "watts": { "core": 13.162110767580515, "dram": 1.867002228344127, "package-0": 17.9220937706796, "psys": 28.82468783599758, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 773.4502172384254, "decode_tokens_per_s": 8.481422197261761, "end_to_end_ms": 10591.502617, "handoff_estimate_ms": 140.4493175, "handoff_plus_first_decode_ms": 248.487652, "output_tokens_per_s_e2e": 1.5106449555437995, "prefill_tokens_per_s": 926.6761486876783, "steady_tpot_ms": 108.0383345, "steady_tpot_p95_ms": 110.94088165, "ttft_ms": 8822.931303000001 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 872, "gross_j": { "core": 16.006245, "dram": 16.902179, "package-0": 191.791929, "psys": 314.443409, "uncore": 0.0 }, "logits": { "l2": 607.3833136095392, "sha256": "597bcab04865e78eb1aaf108263fece075d0b3ccf1e95f89435d49f34b5a7e76", "sum": 46667.203135058284, "top1": 198 }, "net_j": { "core": 2.5429231337940017, "dram": 12.605553814679343, "package-0": 163.13132043497043, "psys": 243.77114870575178, "uncore": 0.0 }, "npu_busy_delta_us": 8631732, "npu_utilization": 0.978329276695718, "seconds": 8.822931303, "watts": { "core": 1.8141640743091252, "dram": 1.9157101443431694, "package-0": 21.737892137365538, "psys": 35.63933552254702, "uncore": 0.0 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 608.4144131591894, "sha256": "75dc1394dccec2f9374185c7ea194761337790f490c37e1fedd5be61da86a67d", "sum": -106184.73783594742, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 150, "gross_j": { "core": 5.648362, "dram": 4.444202, "package-0": 20.249948, "psys": 37.731593, "uncore": 0.0 }, "net_j": { "core": 3.3287955314150763, "dram": 3.7039457833701723, "package-0": 15.312074029717005, "psys": 25.555622844438425, "uncore": 0.0 }, "npu_busy_delta_us": 1505722, "npu_utilization": 0.9905520581800714, "seconds": 1.520083662, "token_latencies_s": [ 0.114859211, 0.108831012, 0.107915531, 0.108096554, 0.107850659, 0.107825549, 0.107884003, 0.108296888, 0.108025608, 0.107923369, 0.108051061, 0.10799602, 0.108304892, 0.108178624 ], "watts": { "core": 3.7158231097414425, "dram": 2.92365618491859, "package-0": 13.321600979091373, "psys": 24.82205022212784, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 235.41330679878592, "compile_target": "NPU", "context_length": 1024, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 408211, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 1.5259465821328744, "dram": 0.48698386485903, "package-0": 3.248422500499841, "psys": 8.010065800945089, "uncore": 0.0023192603038971636 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_7b_ov_fp16/openvino_model.xml", "key": "qwen25_7b_fp16", "size_b": 7.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "npu_npu", "label": "NPU -> NPU", "mode": "staged", "prefill_device": "NPU" }, "prompt_length": 1008, "prompt_sha256": "2669029615323cd09063f0eb8d822f5daea575e1396736877003e47194c143f8", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "NPU" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 1024, "NPUW_LLM_MIN_RESPONSE_LEN": 1024, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.046412899158895016, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 14.223962, "dram": 13.447170999999999, "package-0": 85.998559, "psys": 140.942816, "uncore": 0.0 }, "net_j": { "core": 6.514010024104772, "dram": 10.986650916884775, "package-0": 69.58567681295473, "psys": 100.47139755576656, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 27, "gross_j": { "core": 2.994072, "dram": 0.702269, "package-0": 5.266649, "psys": 8.406717, "uncore": 0.0 }, "logits": { "l2": 827.3468858325061, "sha256": "75e87e5dc1b7ba8c4b5a6f35fc368b20c39fd2db3eff946d358b57e684212379", "sum": 6878.080895855557, "top1": 78 }, "net_j": { "core": 2.566239245362325, "dram": 0.5657323408493733, "package-0": 4.355882160001381, "psys": 6.160918518864857, "uncore": 0.0 }, "npu_busy_delta_us": 208200, "npu_utilization": 0.7425847482601501, "seconds": 0.280372039, "watts": { "core": 10.678925083538733, "dram": 2.5047754494520045, "package-0": 18.784501545819268, "psys": 29.98414902564517, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 202.66913529284705, "decode_tokens_per_s": 4.728974116777969, "end_to_end_ms": 5052.570035000001, "handoff_estimate_ms": 73.94627750000002, "handoff_plus_first_decode_ms": 280.37203900000003, "output_tokens_per_s_e2e": 3.166705238950735, "prefill_tokens_per_s": 535.9892533856925, "steady_tpot_ms": 206.4257615, "steady_tpot_p95_ms": 207.8030635, "ttft_ms": 1880.6347210000001 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 185, "gross_j": { "core": 3.985036, "dram": 3.734671, "package-0": 43.95289, "psys": 64.392231, "uncore": 0.0 }, "logits": { "l2": 852.4974085710298, "sha256": "7a1dee9f32cc78547b6aaf52f69c32ba16165afe1af665e77f2d4efbcabdb732", "sum": -209736.7540479228, "top1": 290 }, "net_j": { "core": 1.115287875249638, "dram": 2.8188322351793365, "package-0": 37.84379385708235, "psys": 49.32822313724799, "uncore": 0.0 }, "npu_busy_delta_us": 1871185, "npu_utilization": 0.9949752491036775, "seconds": 1.880634721, "watts": { "core": 2.1189845936062564, "dram": 1.9858566675904736, "package-0": 23.371306245281215, "psys": 34.2396268031042, "uncore": 0.0 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 764.6497230269619, "sha256": "76d71fd376731f9b77c6d5caa990ea4faf09a3dfc4e5c1add9c3562517997b0b", "sum": -165327.4534454923, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 284, "gross_j": { "core": 7.244854, "dram": 9.010231, "package-0": 36.77902, "psys": 68.143868, "uncore": 0.0 }, "net_j": { "core": 2.832482903492809, "dram": 7.602086340856065, "package-0": 27.38600079587099, "psys": 44.982255899653715, "uncore": 0.0 }, "npu_busy_delta_us": 2869046, "npu_utilization": 0.9922127676766817, "seconds": 2.891563275, "token_latencies_s": [ 0.209389031, 0.206206506, 0.206341476, 0.206047133, 0.206523493, 0.205477716, 0.20615996, 0.205791867, 0.206624987, 0.206949081, 0.206510047, 0.206701205, 0.206192714, 0.206595171 ], "watts": { "core": 2.5055145991920234, "dram": 3.1160414430149377, "package-0": 12.719424236012957, "psys": 23.566445385844094, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 14.958641, "dram": 13.44186, "package-0": 86.83181, "psys": 144.183043, "uncore": 0.0 }, "net_j": { "core": 7.362512600939299, "dram": 11.01766507138755, "package-0": 70.66123450468692, "psys": 104.30911227310797, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 21, "gross_j": { "core": 2.545037, "dram": 0.650328, "package-0": 4.690845, "psys": 7.408246, "uncore": 0.0 }, "logits": { "l2": 827.3468858325061, "sha256": "75e87e5dc1b7ba8c4b5a6f35fc368b20c39fd2db3eff946d358b57e684212379", "sum": 6878.080895855557, "top1": 78 }, "net_j": { "core": 2.2199965773448094, "dram": 0.5465960325029529, "package-0": 3.9989016286604397, "psys": 5.702029541363593, "uncore": 0.0 }, "npu_busy_delta_us": 204780, "npu_utilization": 0.9613676309443461, "seconds": 0.213009044, "watts": { "core": 11.948023202244878, "dram": 3.0530534656547257, "package-0": 22.021811430692118, "psys": 34.77902093208775, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 205.7060146978667, "decode_tokens_per_s": 4.8466610673536, "end_to_end_ms": 4977.977924, "handoff_estimate_ms": 7.175004000000013, "handoff_plus_first_decode_ms": 213.00904400000002, "output_tokens_per_s_e2e": 3.214156479654167, "prefill_tokens_per_s": 535.2978603368659, "steady_tpot_ms": 205.83404, "steady_tpot_p95_ms": 206.16559439999997, "ttft_ms": 1883.063757 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 187, "gross_j": { "core": 4.707264, "dram": 3.783742, "package-0": 44.781624, "psys": 67.045849, "uncore": 0.0 }, "logits": { "l2": 852.4974085710298, "sha256": "7a1dee9f32cc78547b6aaf52f69c32ba16165afe1af665e77f2d4efbcabdb732", "sum": -209736.7540479228, "top1": 290 }, "net_j": { "core": 1.8338092960675607, "dram": 2.866720333840175, "package-0": 38.66463732188544, "psys": 51.96238439905513, "uncore": 0.0 }, "npu_busy_delta_us": 1876379, "npu_utilization": 0.9964500633740359, "seconds": 1.883063757, "watts": { "core": 2.4997900270245603, "dram": 2.009354163359855, "package-0": 23.781257450009964, "psys": 35.604662216437106, "uncore": 0.0 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 764.6497230269619, "sha256": "76d71fd376731f9b77c6d5caa990ea4faf09a3dfc4e5c1add9c3562517997b0b", "sum": -165327.4534454923, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 282, "gross_j": { "core": 7.70634, "dram": 9.00779, "package-0": 37.359341, "psys": 69.728948, "uncore": 0.0 }, "net_j": { "core": 3.3087067275269293, "dram": 7.604348705044422, "package-0": 27.99769555414104, "psys": 46.64469833268925, "uncore": 0.0 }, "npu_busy_delta_us": 2865011, "npu_utilization": 0.9941378628792562, "seconds": 2.881905123, "token_latencies_s": [ 0.205722778, 0.205592499, 0.205914427, 0.206225774, 0.205656983, 0.20613319, 0.205656623, 0.205998249, 0.205715775, 0.205775709, 0.205675284, 0.205892371, 0.205952179, 0.205928122 ], "watts": { "core": 2.674043617361653, "dram": 3.125637248815148, "package-0": 12.963418088208867, "psys": 24.195434972339996, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 16.2524, "dram": 13.396022, "package-0": 88.18233000000001, "psys": 146.024651, "uncore": 0.0 }, "net_j": { "core": 8.640808787476434, "dram": 10.966892337269128, "package-0": 71.9788373948995, "psys": 106.06955219372988, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 21, "gross_j": { "core": 2.636712, "dram": 0.649351, "package-0": 4.792102, "psys": 7.419964, "uncore": 0.0 }, "logits": { "l2": 827.3468858325061, "sha256": "75e87e5dc1b7ba8c4b5a6f35fc368b20c39fd2db3eff946d358b57e684212379", "sum": 6878.080895855557, "top1": 78 }, "net_j": { "core": 2.313368344491132, "dram": 0.5461605312895796, "package-0": 4.103770692569335, "psys": 5.722654285970559, "uncore": 0.0 }, "npu_busy_delta_us": 204825, "npu_utilization": 0.9666248381879695, "seconds": 0.2118971, "watts": { "core": 12.443360480157587, "dram": 3.064463836456469, "package-0": 22.615231638375416, "psys": 35.01682656345934, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 205.28812655271398, "decode_tokens_per_s": 4.847297358948806, "end_to_end_ms": 4988.111184, "handoff_estimate_ms": 6.186430500000006, "handoff_plus_first_decode_ms": 211.8971, "output_tokens_per_s_e2e": 3.207626977386156, "prefill_tokens_per_s": 532.3184701415584, "steady_tpot_ms": 205.7106695, "steady_tpot_p95_ms": 206.5551994, "ttft_ms": 1893.603278 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 187, "gross_j": { "core": 3.676626, "dram": 3.740835, "package-0": 43.650828, "psys": 65.083696, "uncore": 0.0 }, "logits": { "l2": 852.4974085710298, "sha256": "7a1dee9f32cc78547b6aaf52f69c32ba16165afe1af665e77f2d4efbcabdb732", "sum": -209736.7540479228, "top1": 290 }, "net_j": { "core": 0.7870885500202931, "dram": 2.818680757169832, "package-0": 37.499604504724545, "psys": 49.91580914233469, "uncore": 0.0 }, "npu_busy_delta_us": 1881441, "npu_utilization": 0.993577177362702, "seconds": 1.893603278, "watts": { "core": 1.9416031027804337, "dram": 1.9755114724722187, "package-0": 23.05172815612331, "psys": 34.37029115662526, "uncore": 0.0 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 764.6497230269619, "sha256": "76d71fd376731f9b77c6d5caa990ea4faf09a3dfc4e5c1add9c3562517997b0b", "sum": -165327.4534454923, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 285, "gross_j": { "core": 9.939062, "dram": 9.005836, "package-0": 39.7394, "psys": 73.520991, "uncore": 0.0 }, "net_j": { "core": 5.54035189296501, "dram": 7.602051048809717, "package-0": 30.37546219760562, "psys": 50.43108876542463, "uncore": 0.0 }, "npu_busy_delta_us": 2864769, "npu_utilization": 0.9938105394030774, "seconds": 2.882610806, "token_latencies_s": [ 0.206436279, 0.205604898, 0.205548187, 0.205792852, 0.205689408, 0.205449851, 0.205719907, 0.205701432, 0.205562851, 0.206477729, 0.206443803, 0.206699073, 0.205846886, 0.205566251 ], "watts": { "core": 3.4479375361087157, "dram": 3.124194213542402, "package-0": 13.785905442831398, "psys": 25.50500083013981, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 383.0334553383291, "compile_target": "NPU", "context_length": 2048, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 408344, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 1.5259465821328744, "dram": 0.48698386485903, "package-0": 3.248422500499841, "psys": 8.010065800945089, "uncore": 0.0023192603038971636 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_7b_ov_fp16/openvino_model.xml", "key": "qwen25_7b_fp16", "size_b": 7.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "npu_npu", "label": "NPU -> NPU", "mode": "staged", "prefill_device": "NPU" }, "prompt_length": 2032, "prompt_sha256": "cd1a97358e68d07c27508cfacc4ba6882a7a2962b1281e2e2b0c7e5e320da4fa", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "NPU" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 2048, "NPUW_LLM_MIN_RESPONSE_LEN": 2048, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.04666558373719454, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 18.804945, "dram": 17.360122999999998, "package-0": 133.89669, "psys": 213.593874, "uncore": 0.0 }, "net_j": { "core": 8.122277777820676, "dram": 13.950903678843732, "package-0": 111.15551611082272, "psys": 157.51794815752865, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 29, "gross_j": { "core": 3.278557, "dram": 0.718626, "package-0": 5.611619, "psys": 9.057289, "uncore": 0.0 }, "logits": { "l2": 656.4427383867821, "sha256": "bbd35af41712a9004a15d466ce75c6984ad14e8615b96488be7f0c044f186394", "sum": -33059.47061268799, "top1": 220 }, "net_j": { "core": 2.8291799701694265, "dram": 0.5752137958454586, "package-0": 4.654988884633149, "psys": 6.698399363545661, "uncore": 0.0 }, "npu_busy_delta_us": 209202, "npu_utilization": 0.7103858356883966, "seconds": 0.294490669, "watts": { "core": 11.132974131686327, "dram": 2.440233513816358, "package-0": 19.055337199835016, "psys": 30.755775830710622, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 292.54291416283405, "decode_tokens_per_s": 4.67141247148557, "end_to_end_ms": 7000.682296, "handoff_estimate_ms": 86.46781199999998, "handoff_plus_first_decode_ms": 294.49066899999997, "output_tokens_per_s_e2e": 2.285491516897141, "prefill_tokens_per_s": 536.1955684990492, "steady_tpot_ms": 208.02285700000002, "steady_tpot_p95_ms": 209.51920255000002, "ttft_ms": 3789.662055 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 374, "gross_j": { "core": 7.47343, "dram": 7.555705, "package-0": 90.310254, "psys": 134.026085, "uncore": 0.0 }, "logits": { "l2": 639.6205341218227, "sha256": "0745c5d78f0230e7056bb6b8f550d6c0caf532bb86cab6ab11e739777c0918a6", "sum": -21376.843404488638, "top1": 220 }, "net_j": { "core": 1.6906081397341044, "dram": 5.7102007259464855, "package-0": 77.99983051124754, "psys": 103.67064257610521, "uncore": 0.0 }, "npu_busy_delta_us": 3753151, "npu_utilization": 0.990365617179023, "seconds": 3.789662055, "watts": { "core": 1.9720571099841777, "dram": 1.9937674891171793, "package-0": 23.83068798465725, "psys": 35.36623663399453, "uncore": 0.0 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 840.7172376253137, "sha256": "398e2c128d6fbbccb88ad985ccb32f4f56ee6a0eb59aed5f4d2565cdfaec3cbf", "sum": -213465.7986249458, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 287, "gross_j": { "core": 8.052958, "dram": 9.085792, "package-0": 37.974817, "psys": 70.5105, "uncore": 0.0 }, "net_j": { "core": 3.602489667917145, "dram": 7.665489157051788, "package-0": 28.50069671494203, "psys": 47.14890621787778, "uncore": 0.0 }, "npu_busy_delta_us": 2895714, "npu_utilization": 0.9928628969855684, "seconds": 2.916529572, "token_latencies_s": [ 0.211474853, 0.20819779, 0.207863707, 0.207756131, 0.207890492, 0.207873048, 0.208079224, 0.20796649, 0.207883153, 0.207922069, 0.208460489, 0.208225288, 0.208412243, 0.20846616 ], "watts": { "core": 2.7611439559235165, "dram": 3.115275115749795, "package-0": 13.020549273552849, "psys": 24.17616494512266, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 25.226559, "dram": 17.473343, "package-0": 140.648687, "psys": 223.515542, "uncore": 0.0 }, "net_j": { "core": 14.61191351712577, "dram": 14.085831836841587, "package-0": 118.05231722852187, "psys": 167.79667885822093, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 26, "gross_j": { "core": 2.763726, "dram": 0.699705, "package-0": 5.036608, "psys": 8.131938, "uncore": 0.0 }, "logits": { "l2": 656.4427383867821, "sha256": "bbd35af41712a9004a15d466ce75c6984ad14e8615b96488be7f0c044f186394", "sum": -33059.47061268799, "top1": 220 }, "net_j": { "core": 2.3627363837951996, "dram": 0.5717349417068658, "package-0": 4.1829846117384015, "psys": 6.027045699768429, "uncore": 0.0 }, "npu_busy_delta_us": 207401, "npu_utilization": 0.7892544701688746, "seconds": 0.2627809, "watts": { "core": 10.517225567002779, "dram": 2.6626935214850094, "package-0": 19.166568042045675, "psys": 30.945696586015195, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 294.41761434710696, "decode_tokens_per_s": 4.7291906814509685, "end_to_end_ms": 6956.105546, "handoff_estimate_ms": 55.00022299999996, "handoff_plus_first_decode_ms": 262.7809, "output_tokens_per_s_e2e": 2.300137612086773, "prefill_tokens_per_s": 536.9531182202691, "steady_tpot_ms": 207.78067700000003, "steady_tpot_p95_ms": 208.00556804999997, "ttft_ms": 3784.315485 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 375, "gross_j": { "core": 9.476355, "dram": 7.695415, "package-0": 92.425239, "psys": 137.867567, "uncore": 0.0 }, "logits": { "l2": 639.6205341218227, "sha256": "0745c5d78f0230e7056bb6b8f550d6c0caf532bb86cab6ab11e739777c0918a6", "sum": -21376.843404488638, "top1": 220 }, "net_j": { "core": 3.7016917199517394, "dram": 5.852514419268825, "package-0": 80.13218342953604, "psys": 107.55495095361458, "uncore": 0.0 }, "npu_busy_delta_us": 3750046, "npu_utilization": 0.9909443371896887, "seconds": 3.784315485, "watts": { "core": 2.5041133693957867, "dram": 2.0335025001225553, "package-0": 24.423238328397456, "psys": 36.431309056147576, "uncore": 0.0 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 840.7172376253137, "sha256": "398e2c128d6fbbccb88ad985ccb32f4f56ee6a0eb59aed5f4d2565cdfaec3cbf", "sum": -213465.7986249458, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 288, "gross_j": { "core": 12.986478, "dram": 9.078223, "package-0": 43.18684, "psys": 77.516037, "uncore": 0.0 }, "net_j": { "core": 8.54748541337883, "dram": 7.661582475865895, "package-0": 33.73714918724743, "psys": 54.21468220483793, "uncore": 0.0 }, "npu_busy_delta_us": 2895085, "npu_utilization": 0.9952134351494398, "seconds": 2.909009161, "token_latencies_s": [ 0.208025889, 0.207849307, 0.207632777, 0.207994626, 0.20796089, 0.207967161, 0.207593927, 0.207645205, 0.207481571, 0.207664276, 0.207588158, 0.207944997, 0.207712047, 0.207914855 ], "watts": { "core": 4.464227261331749, "dram": 3.1207268515026856, "package-0": 14.845893433059558, "psys": 26.64688652040996, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 23.181887, "dram": 17.159014, "package-0": 137.77161999999998, "psys": 212.96588599999998, "uncore": 0.0 }, "net_j": { "core": 12.580058526906644, "dram": 13.77559320083736, "package-0": 115.20253497324937, "psys": 157.31430246873828, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 26, "gross_j": { "core": 2.766839, "dram": 0.693419, "package-0": 5.011218, "psys": 7.827983, "uncore": 0.0 }, "logits": { "l2": 656.4427383867821, "sha256": "bbd35af41712a9004a15d466ce75c6984ad14e8615b96488be7f0c044f186394", "sum": -33059.47061268799, "top1": 220 }, "net_j": { "core": 2.3749461571720865, "dram": 0.5683520458578848, "package-0": 4.176959747887614, "psys": 5.770841882003985, "uncore": 0.0 }, "npu_busy_delta_us": 207966, "npu_utilization": 0.8097749492179338, "seconds": 0.256819503, "watts": { "core": 10.773476966038674, "dram": 2.7000246939968573, "package-0": 19.51260687549886, "psys": 30.48048496534938, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 294.7735485571657, "decode_tokens_per_s": 4.728065618031263, "end_to_end_ms": 6947.706163, "handoff_estimate_ms": 48.7699865, "handoff_plus_first_decode_ms": 256.819503, "output_tokens_per_s_e2e": 2.3029183481028572, "prefill_tokens_per_s": 538.2551381755361, "steady_tpot_ms": 208.0495165, "steady_tpot_p95_ms": 209.2143584, "ttft_ms": 3775.161361 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 371, "gross_j": { "core": 8.595437, "dram": 7.393658, "package-0": 90.820934, "psys": 131.256928, "uncore": 0.0 }, "logits": { "l2": 639.6205341218227, "sha256": "0745c5d78f0230e7056bb6b8f550d6c0caf532bb86cab6ab11e739777c0918a6", "sum": -21376.843404488638, "top1": 220 }, "net_j": { "core": 2.83474242418196, "dram": 5.555215329953745, "package-0": 78.55761489190999, "psys": 101.01763708920457, "uncore": 0.0 }, "npu_busy_delta_us": 3728764, "npu_utilization": 0.9877098336830535, "seconds": 3.775161361, "watts": { "core": 2.276839630961671, "dram": 1.9585011852424499, "package-0": 24.057497233957307, "psys": 34.768560982842715, "uncore": 0.0 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 840.7172376253137, "sha256": "398e2c128d6fbbccb88ad985ccb32f4f56ee6a0eb59aed5f4d2565cdfaec3cbf", "sum": -213465.7986249458, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 287, "gross_j": { "core": 11.819611, "dram": 9.071937, "package-0": 41.939468, "psys": 73.880975, "uncore": 0.0 }, "net_j": { "core": 7.370369945552597, "dram": 7.65202582502573, "package-0": 32.46796033345177, "psys": 50.525823497529714, "uncore": 0.0 }, "npu_busy_delta_us": 2897016, "npu_utilization": 0.9935833121843073, "seconds": 2.915725299, "token_latencies_s": [ 0.209631232, 0.208989888, 0.208259828, 0.208009894, 0.208339283, 0.207821217, 0.2079403, 0.208638078, 0.208150652, 0.2079932, 0.207863392, 0.208021276, 0.207932833, 0.208077757 ], "watts": { "core": 4.053746422563795, "dram": 3.1113826131396474, "package-0": 14.383888638063361, "psys": 25.33879821440614, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 15.589648451656103, "compile_target": "NPU", "context_length": 4096, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 408602, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 1.5259465821328744, "dram": 0.48698386485903, "package-0": 3.248422500499841, "psys": 8.010065800945089, "uncore": 0.0023192603038971636 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_7b_ov_fp16/openvino_model.xml", "key": "qwen25_7b_fp16", "size_b": 7.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "npu_npu", "label": "NPU -> NPU", "mode": "staged", "prefill_device": "NPU" }, "prompt_length": 4080, "prompt_sha256": "66f8de7a9ef70b33481de3e8dae7cef6e4434d9723c6203a6a3dbe07ef228a16", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "NPU" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 4096, "NPUW_LLM_MIN_RESPONSE_LEN": 4096, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.046240122988820076, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 39.246176, "dram": 26.771172, "package-0": 248.744115, "psys": 390.289466, "uncore": 0.0 }, "net_j": { "core": 21.47481823554713, "dram": 21.09969923073094, "package-0": 210.91259514050964, "psys": 297.0032745093148, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 33, "gross_j": { "core": 4.64397, "dram": 0.777647, "package-0": 7.254071, "psys": 11.181428, "uncore": 0.0 }, "logits": { "l2": 738.8544173581239, "sha256": "756a111ecb880182890767495a4326b190317a49fe7bcb1d25e5e6a9434fe988", "sum": -54558.6855568951, "top1": 78 }, "net_j": { "core": 4.1402585429150776, "dram": 0.6168947462729233, "package-0": 6.181774203956754, "psys": 8.537323702259368, "uncore": 0.0 }, "npu_busy_delta_us": 215530, "npu_utilization": 0.6529279058896805, "seconds": 0.3300977, "watts": { "core": 14.068471243513665, "dram": 2.3558085984846304, "package-0": 21.975527245418554, "psys": 33.873086664947984, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 351.7051022920916, "decode_tokens_per_s": 4.503460418454397, "end_to_end_ms": 11646.120495, "handoff_estimate_ms": 115.8443975, "handoff_plus_first_decode_ms": 330.0977, "output_tokens_per_s_e2e": 1.373848055828483, "prefill_tokens_per_s": 490.6589323553983, "steady_tpot_ms": 214.2533025, "steady_tpot_p95_ms": 215.05255904999999, "ttft_ms": 8315.348465 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 821, "gross_j": { "core": 21.158271, "dram": 16.673908, "package-0": 196.955917, "psys": 297.429353, "uncore": 0.0 }, "logits": { "l2": 696.8009887540125, "sha256": "ee9a1e49729de1ec15cfac4c7e2d3b98754929839e7fba889ae61a8056468138", "sum": 28229.591541170143, "top1": 49 }, "net_j": { "core": 8.469493430589406, "dram": 12.624467466864697, "package-0": 169.94415194679718, "psys": 230.82286463756225, "uncore": 0.0 }, "npu_busy_delta_us": 8232952, "npu_utilization": 0.9900910388365787, "seconds": 8.315348465, "watts": { "core": 2.544483985133869, "dram": 2.0051965434980725, "package-0": 23.685828420661384, "psys": 35.768717841700216, "uncore": 0.0 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 807.7971145138014, "sha256": "12bab974e4665b4bad98d081d32a834c49bfe99679cebc03d4d2530f6e3343b7", "sum": -188106.27967495285, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 295, "gross_j": { "core": 13.443935, "dram": 9.319617, "package-0": 44.534127, "psys": 81.678685, "uncore": 0.0 }, "net_j": { "core": 8.865066262042646, "dram": 7.858337017593319, "package-0": 34.78666898975571, "psys": 57.64308616949319, "uncore": 0.0 }, "npu_busy_delta_us": 2982914, "npu_utilization": 0.9940812204035485, "seconds": 3.00067433, "token_latencies_s": [ 0.216194819, 0.214429381, 0.214437496, 0.214042825, 0.214269138, 0.214237467, 0.214345016, 0.214175515, 0.214069859, 0.214022498, 0.21397039, 0.214380772, 0.214274325, 0.213793645 ], "watts": { "core": 4.480304598733312, "dram": 3.1058408794399224, "package-0": 14.841373005647034, "psys": 27.220109887766462, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 37.574184, "dram": 26.835807, "package-0": 247.278908, "psys": 383.122981, "uncore": 0.0 }, "net_j": { "core": 19.819443516104283, "dram": 21.169637396388193, "package-0": 209.48276287032724, "psys": 289.9240176680551, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 30, "gross_j": { "core": 3.652518, "dram": 0.760923, "package-0": 6.108078, "psys": 9.606909, "uncore": 0.0 }, "logits": { "l2": 738.8544173581239, "sha256": "756a111ecb880182890767495a4326b190317a49fe7bcb1d25e5e6a9434fe988", "sum": -54558.6855568951, "top1": 78 }, "net_j": { "core": 3.191353672194068, "dram": 0.6137490495356621, "package-0": 5.126355181905815, "psys": 7.18614496145093, "uncore": 0.0 }, "npu_busy_delta_us": 214068, "npu_utilization": 0.7083295763532781, "seconds": 0.30221525, "watts": { "core": 12.085816318005131, "dram": 2.5178180121618614, "package-0": 20.21101847110627, "psys": 31.788299895521487, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 352.03427535792554, "decode_tokens_per_s": 4.5342164871431505, "end_to_end_ms": 11635.230677, "handoff_estimate_ms": 87.79911499999999, "handoff_plus_first_decode_ms": 302.21524999999997, "output_tokens_per_s_e2e": 1.3751338881168966, "prefill_tokens_per_s": 489.9693413393484, "steady_tpot_ms": 214.416135, "steady_tpot_p95_ms": 215.8251669, "ttft_ms": 8327.051625 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 822, "gross_j": { "core": 23.016237, "dram": 16.746295, "package-0": 199.239968, "psys": 298.86276, "uncore": 0.0 }, "logits": { "l2": 696.8009887540125, "sha256": "ee9a1e49729de1ec15cfac4c7e2d3b98754929839e7fba889ae61a8056468138", "sum": 28229.591541170143, "top1": 49 }, "net_j": { "core": 10.309601033587255, "dram": 12.691155216776835, "package-0": 172.19018613852626, "psys": 232.16252855588326, "uncore": 0.0 }, "npu_busy_delta_us": 8248030, "npu_utilization": 0.9905102515801925, "seconds": 8.327051625, "watts": { "core": 2.764031981127534, "dram": 2.011071355643241, "package-0": 23.926832325841385, "psys": 35.89058570295582, "uncore": 0.0 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 807.7971145138014, "sha256": "12bab974e4665b4bad98d081d32a834c49bfe99679cebc03d4d2530f6e3343b7", "sum": -188106.27967495285, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 296, "gross_j": { "core": 10.905429, "dram": 9.328589, "package-0": 41.930862, "psys": 74.653312, "uncore": 0.0 }, "net_j": { "core": 6.318488810322959, "dram": 7.864733130075695, "package-0": 32.16622154989515, "psys": 50.57534415072092, "uncore": 0.0 }, "npu_busy_delta_us": 2983659, "npu_utilization": 0.992579816834401, "seconds": 3.005963802, "token_latencies_s": [ 0.217259245, 0.214424887, 0.214065245, 0.214083861, 0.214407383, 0.215052971, 0.21434233, 0.214384469, 0.214740951, 0.215012252, 0.214780049, 0.214695391, 0.214320501, 0.214336702 ], "watts": { "core": 3.6279309127888157, "dram": 3.103360391031082, "package-0": 13.949223863607921, "psys": 24.835066859530997, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 45.51795, "dram": 26.996146, "package-0": 255.931535, "psys": 391.566258, "uncore": 0.0 }, "net_j": { "core": 27.709570088850494, "dram": 21.312858146051827, "package-0": 218.02120269992076, "psys": 298.0857282312882, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 33, "gross_j": { "core": 3.75054, "dram": 0.785215, "package-0": 5.853562, "psys": 9.619909, "uncore": 0.0 }, "logits": { "l2": 738.8544173581239, "sha256": "756a111ecb880182890767495a4326b190317a49fe7bcb1d25e5e6a9434fe988", "sum": -54558.6855568951, "top1": 78 }, "net_j": { "core": 3.239667423621946, "dram": 0.6221773782325748, "package-0": 4.7660206722588905, "psys": 6.938214240431916, "uncore": 0.0 }, "npu_busy_delta_us": 241067, "npu_utilization": 0.7200530655276486, "seconds": 0.334790603, "watts": { "core": 11.202644179352907, "dram": 2.3453913967830213, "package-0": 17.48424820633332, "psys": 28.734106972530526, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 350.97393651755306, "decode_tokens_per_s": 4.495824289639343, "end_to_end_ms": 11670.382253, "handoff_estimate_ms": 120.38274850000002, "handoff_plus_first_decode_ms": 334.79060300000003, "output_tokens_per_s_e2e": 1.3709919395216916, "prefill_tokens_per_s": 489.5636002805126, "steady_tpot_ms": 214.40785449999998, "steady_tpot_p95_ms": 215.12930295, "ttft_ms": 8333.952928 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 824, "gross_j": { "core": 31.541179, "dram": 16.864764, "package-0": 208.53035, "psys": 305.749889, "uncore": 0.0 }, "logits": { "l2": 696.8009887540125, "sha256": "ee9a1e49729de1ec15cfac4c7e2d3b98754929839e7fba889ae61a8056468138", "sum": 28229.591541170143, "top1": 49 }, "net_j": { "core": 18.824012013862138, "dram": 12.806263393569331, "package-0": 181.45814979057826, "psys": 238.99437766474102, "uncore": 0.0 }, "npu_busy_delta_us": 8235990, "npu_utilization": 0.9882453226162498, "seconds": 8.333952928, "watts": { "core": 3.7846600853755143, "dram": 2.0236212210100932, "package-0": 25.021781596508674, "psys": 36.6872589324037, "uncore": 0.0 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 807.7971145138014, "sha256": "12bab974e4665b4bad98d081d32a834c49bfe99679cebc03d4d2530f6e3343b7", "sum": -188106.27967495285, "top1": 42 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 295, "gross_j": { "core": 10.226231, "dram": 9.346167, "package-0": 41.547623, "psys": 76.19646, "uncore": 0.0 }, "net_j": { "core": 5.645890651366411, "dram": 7.88441737424992, "package-0": 31.797032237083613, "psys": 52.15313632611527, "uncore": 0.0 }, "npu_busy_delta_us": 2982719, "npu_utilization": 0.993696869026465, "seconds": 3.001638722, "token_latencies_s": [ 0.215038941, 0.214048186, 0.213877246, 0.213981359, 0.214170559, 0.215297118, 0.214493708, 0.214116809, 0.214437614, 0.214098724, 0.214443708, 0.214378095, 0.214455961, 0.214737187 ], "watts": { "core": 3.406882688795484, "dram": 3.1136881768944606, "package-0": 13.841646796292896, "psys": 25.384953705964353, "uncore": 0.0 } } } ] }, { "available_devices": [ "CPU", "GPU", "NPU" ], "compile_s": 32.28103984426707, "compile_target": "NPU", "context_length": 8192, "devices": { "GPU": { "DEVICE_ARCHITECTURE": "GPU: vendor=0x8086 arch=v30.1.0", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) Graphics (iGPU)" }, "NPU": { "DEVICE_ARCHITECTURE": "5010", "DEVICE_TYPE": "Type.INTEGRATED", "FULL_DEVICE_NAME": "Intel(R) AI Boost" } }, "energy_domains": [ "package-0", "core", "uncore", "dram", "psys" ], "execution_devices": [ "NPU" ], "host": { "hostname": "wuklab-NUC16", "pid": 408778, "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]" }, "idle_watts": { "core": 1.5259465821328744, "dram": 0.48698386485903, "package-0": 3.248422500499841, "psys": 8.010065800945089, "uncore": 0.0023192603038971636 }, "model": { "family": "Qwen2.5", "ir": "/data/models/qwen25_7b_ov_fp16/openvino_model.xml", "key": "qwen25_7b_fp16", "size_b": 7.0 }, "ok": true, "openvino_version": "2026.3.0-22451-8a17657b995-releases/2026/3", "output_tokens": 16, "placement": { "decode_device": "NPU", "key": "npu_npu", "label": "NPU -> NPU", "mode": "staged", "prefill_device": "NPU" }, "prompt_length": 8176, "prompt_sha256": "998ace73fce48442f97bcb84c31be293d0d94180b17f70a4d127bba1092ad744", "properties": { "++NPUW_LLM_GENERATE_CONFIG": { "NPUW_DEVICES": "NPU" }, "++NPUW_LLM_PREFILL_CONFIG": { "NPUW_DEVICES": "NPU" }, "NPUW_LLM": "YES", "NPUW_LLM_GENERATE_HINT": "FAST_COMPILE", "NPUW_LLM_MAX_GENERATION_TOKEN_LEN": 1, "NPUW_LLM_MAX_PROMPT_LEN": 8192, "NPUW_LLM_MIN_RESPONSE_LEN": 8192, "NPU_USE_NPUW": "YES", "PERFORMANCE_HINT": "LATENCY" }, "read_model_s": 0.045671031810343266, "schema_version": 1, "trials": [ { "energy": { "gross_j": { "core": 63.986957000000004, "dram": 49.518855, "package-0": 489.540861, "psys": 774.901154, "uncore": 0.0 }, "net_j": { "core": 29.524571885076412, "dram": 38.520681346982556, "package-0": 416.1776197574829, "psys": 593.999688863384, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 42, "gross_j": { "core": 5.017077, "dram": 0.910642, "package-0": 7.468059, "psys": 12.535613, "uncore": 0.0 }, "logits": { "l2": 853.2973260506939, "sha256": "903a38a0133d569cf197c1de4a4fa2499328b7104c3bb75a4c1e60690bc64ff9", "sum": -24865.39645318687, "top1": 78 }, "net_j": { "core": 4.362365329500047, "dram": 0.7017002038705891, "package-0": 6.074314194844895, "psys": 9.098871751809543, "uncore": 0.0 }, "npu_busy_delta_us": 252885, "npu_utilization": 0.5894029674589407, "seconds": 0.429052811, "watts": { "core": 11.693378696917568, "dram": 2.1224473459981597, "package-0": 17.405920223652842, "psys": 29.216946442520804, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 362.7303902253494, "decode_tokens_per_s": 4.196490936579461, "end_to_end_ms": 22584.26705, "handoff_estimate_ms": 204.59441599999997, "handoff_plus_first_decode_ms": 429.05281099999996, "output_tokens_per_s_e2e": 0.7084577934088856, "prefill_tokens_per_s": 430.09277310586225, "steady_tpot_ms": 224.458395, "steady_tpot_p95_ms": 226.4199107, "ttft_ms": 19009.852086 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 1873, "gross_j": { "core": 47.952758, "dram": 38.917686, "package-0": 438.461647, "psys": 680.977322, "uncore": 0.0 }, "logits": { "l2": 1035.6859878171401, "sha256": "db3bb6aec8041a493e2031745915bf05e60ecc0d225663002c7a7306b7dc4726", "sum": -316508.9812684683, "top1": 8 }, "net_j": { "core": 18.944739182516813, "dram": 29.66019476076123, "package-0": 376.7096157526638, "psys": 528.7071559249067, "uncore": 0.0 }, "npu_busy_delta_us": 18831046, "npu_utilization": 0.9905940306536272, "seconds": 19.009852086, "watts": { "core": 2.5225213632943153, "dram": 2.047237707265557, "package-0": 23.064968891731127, "psys": 35.82233669779644, "uncore": 0.0 } }, "repeat": 0, "steady_decode": { "final_logits": { "l2": 729.8049311494284, "sha256": "ba8fd8377af1d70434f76c9576db141e6f73b12f7b59811353caa97ab51d741c", "sum": -13244.23879000172, "top1": 730 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 311, "gross_j": { "core": 11.017122, "dram": 9.690527, "package-0": 43.611155, "psys": 81.388219, "uncore": 0.0 }, "net_j": { "core": 6.217467373059551, "dram": 8.15878638235074, "package-0": 33.39368980997418, "psys": 56.19366118666769, "uncore": 0.0 }, "npu_busy_delta_us": 3128724, "npu_utilization": 0.9947102584088351, "seconds": 3.145362153, "token_latencies_s": [ 0.227211196, 0.225993834, 0.224869697, 0.224925683, 0.225050445, 0.224959263, 0.224047093, 0.223549384, 0.225517242, 0.223962014, 0.223812155, 0.22401702, 0.22373254, 0.223680659 ], "watts": { "core": 3.5026561216462886, "dram": 3.08089387759604, "package-0": 13.865225331335635, "psys": 25.875627365317257, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 67.928658, "dram": 49.421689, "package-0": 494.684279, "psys": 783.1447850000001, "uncore": 0.0 }, "net_j": { "core": 33.443595513671106, "dram": 38.41627819064884, "package-0": 421.27276235624157, "psys": 602.1242808096741, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 40, "gross_j": { "core": 4.055592, "dram": 0.882933, "package-0": 6.412703, "psys": 10.812045, "uncore": 0.0 }, "logits": { "l2": 853.2973260506939, "sha256": "903a38a0133d569cf197c1de4a4fa2499328b7104c3bb75a4c1e60690bc64ff9", "sum": -24865.39645318687, "top1": 78 }, "net_j": { "core": 3.44121692372987, "dram": 0.6868640467232092, "package-0": 5.104826404010212, "psys": 7.587040373359749, "uncore": 0.0 }, "npu_busy_delta_us": 254460, "npu_utilization": 0.6320119130594518, "seconds": 0.402618993, "watts": { "core": 10.073027031787346, "dram": 2.1929740408446157, "package-0": 15.927472651544782, "psys": 26.854284541911813, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 362.49185878054215, "decode_tokens_per_s": 4.228329211382308, "end_to_end_ms": 22599.128233000003, "handoff_estimate_ms": 178.09616499999998, "handoff_plus_first_decode_ms": 402.618993, "output_tokens_per_s_e2e": 0.7079919116807464, "prefill_tokens_per_s": 429.14968292690077, "steady_tpot_ms": 224.522828, "steady_tpot_p95_ms": 225.88628919999996, "ttft_ms": 19051.627731 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 1875, "gross_j": { "core": 51.557363, "dram": 38.867943, "package-0": 443.287439, "psys": 691.30304, "uncore": 0.0 }, "logits": { "l2": 1035.6859878171401, "sha256": "db3bb6aec8041a493e2031745915bf05e60ecc0d225663002c7a7306b7dc4726", "sum": -316508.9812684683, "top1": 8 }, "net_j": { "core": 22.485596779812663, "dram": 29.590107695702144, "package-0": 381.39970280747286, "psys": 538.6982482595798, "uncore": 0.0 }, "npu_busy_delta_us": 18869638, "npu_utilization": 0.9904475494918539, "seconds": 19.051627731, "watts": { "core": 2.7061920234830144, "dram": 2.0401376485409553, "package-0": 23.267693724599788, "psys": 36.28577304579288, "uncore": 0.0 } }, "repeat": 1, "steady_decode": { "final_logits": { "l2": 729.8049311494284, "sha256": "ba8fd8377af1d70434f76c9576db141e6f73b12f7b59811353caa97ab51d741c", "sum": -13244.23879000172, "top1": 730 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 310, "gross_j": { "core": 12.315703, "dram": 9.670813, "package-0": 44.984137, "psys": 81.0297, "uncore": 0.0 }, "net_j": { "core": 7.516781810128572, "dram": 8.139306448223483, "package-0": 34.768233144758504, "psys": 55.83899217673452, "uncore": 0.0 }, "npu_busy_delta_us": 3128244, "npu_utilization": 0.9947096547350395, "seconds": 3.144881509, "token_latencies_s": [ 0.225905459, 0.224533631, 0.224512025, 0.225521625, 0.225875967, 0.22478843, 0.225705447, 0.223566774, 0.223798248, 0.223645236, 0.223619253, 0.224820536, 0.224287232, 0.224251434 ], "watts": { "core": 3.91611034144053, "dram": 3.0750961434712676, "package-0": 14.30392110839938, "psys": 25.76558123672061, "uncore": 0.0 } } }, { "energy": { "gross_j": { "core": 59.053499, "dram": 49.319086999999996, "package-0": 485.008462, "psys": 765.983879, "uncore": 0.0 }, "net_j": { "core": 24.515326342677973, "dram": 38.29672684482219, "package-0": 411.4838848621984, "psys": 584.6845865681095, "uncore": 0.0 } }, "first_decode": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 39, "gross_j": { "core": 3.902517, "dram": 0.884458, "package-0": 6.2769, "psys": 11.154146, "uncore": 0.0 }, "logits": { "l2": 853.2973260506939, "sha256": "903a38a0133d569cf197c1de4a4fa2499328b7104c3bb75a4c1e60690bc64ff9", "sum": -24865.39645318687, "top1": 78 }, "net_j": { "core": 3.2891431155937667, "dram": 0.6887085626807496, "package-0": 4.971154732993861, "psys": 7.93439687365254, "uncore": 0.0 }, "npu_busy_delta_us": 251861, "npu_utilization": 0.6265777560689739, "seconds": 0.401962881, "watts": { "core": 9.708650187528137, "dram": 2.2003474494949695, "package-0": 15.615620985660117, "psys": 27.749194085411087, "uncore": 0.0 } }, "metrics": { "all_tokens_per_s_e2e": 361.9344464126539, "decode_tokens_per_s": 4.22430750449732, "end_to_end_ms": 22633.932971, "handoff_estimate_ms": 176.9532425, "handoff_plus_first_decode_ms": 401.962881, "output_tokens_per_s_e2e": 0.7069032156497147, "prefill_tokens_per_s": 428.442927681114, "steady_tpot_ms": 225.0096385, "steady_tpot_p95_ms": 225.54362394999998, "ttft_ms": 19083.055109 }, "prefill": { "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 1875, "gross_j": { "core": 46.56586, "dram": 38.749229, "package-0": 437.514444, "psys": 676.893287, "uncore": 0.0 }, "logits": { "l2": 1035.6859878171401, "sha256": "db3bb6aec8041a493e2031745915bf05e60ecc0d225663002c7a7306b7dc4726", "sum": -316508.9812684683, "top1": 8 }, "net_j": { "core": 17.446137279768163, "dram": 29.45608906970132, "package-0": 375.524618405646, "psys": 524.0367598938486, "uncore": 0.0 }, "npu_busy_delta_us": 18857980, "npu_utilization": 0.9882054991868756, "seconds": 19.083055109, "watts": { "core": 2.440167978031908, "dram": 2.0305568882272413, "package-0": 22.926855343705334, "psys": 35.47090773116103, "uncore": 0.0 } }, "repeat": 2, "steady_decode": { "final_logits": { "l2": 729.8049311494284, "sha256": "ba8fd8377af1d70434f76c9576db141e6f73b12f7b59811353caa97ab51d741c", "sum": -13244.23879000172, "top1": 730 }, "gpu_active_fraction": 0.0, "gpu_freq_mhz_after": 0, "gpu_freq_mhz_before": 0, "gpu_freq_mhz_max": 0, "gpu_sample_count": 309, "gross_j": { "core": 8.585122, "dram": 9.6854, "package-0": 41.217118, "psys": 77.936446, "uncore": 0.0 }, "net_j": { "core": 3.7800459473160455, "dram": 8.15192921244012, "package-0": 30.98811172355857, "psys": 52.71342980060825, "uncore": 0.0 }, "npu_busy_delta_us": 3125178, "npu_utilization": 0.9924618539581969, "seconds": 3.148914981, "token_latencies_s": [ 0.225773722, 0.225231111, 0.224771474, 0.224611573, 0.224468478, 0.225158248, 0.225276914, 0.225019186, 0.224511305, 0.224095895, 0.225376798, 0.225419725, 0.225000091, 0.224143085 ], "watts": { "core": 2.7263746566042966, "dram": 3.0757896159280267, "package-0": 13.089307983447268, "psys": 24.750254125708327, "uncore": 0.0 } } } ] } ]