{"run": {"type": "run", "run_id": 1794760960969, "mode": "wait", "pull": "heap", "evict": "lookahead", "gpu_store": "packed", "gpu_pool": "global", "streams": 600, "spec": 4, "gpu_slots": 108, "ram_slots": 1080, "warmup": 200.0, "measure": 420.0, "out": "results/engine-scale-600-spec4.json", "layers": "3-44", "pull_detail": {"ttl": 2.0, "ram_first": true, "ram_prefetch": false}, "link_max_gbs": {"ssd_to_ram": 3.12, "ram_to_gpu": 55.4, "gpu_to_ram": 49.8, "gpu_decode": 55.0}}, "samples": [{"type": "s", "t": 320.3, "phase": "measure", "dt": 1.001, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.122, "ram_to_gpu_gbs": 3.704, "gpu_to_ram_gbs": 3.316, "decode_gbs": 4.376, "ssd_reads": 145, "ram_promotions": 27, "gpu_decodes": 174, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 27, "l2": 259}, "lkw": {"l0": 2383, "arriving": 0, "ssd_inflight": 0, "l1": 4950, "l2": 16667}, "missing": 154, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 11, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 27, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 135.6, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.527, "pcie_tx_gbs": 2.562}], "pulls": {"expired": 0, "need": 18925, "idle": 0, "idle_used": 0, "ram_first": 1603, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 4.57, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 104.53, "ssd_below_cap": 11.03, "nothing_needed": 0.19}, "sd": {"all_readers": 115.56, "some_readers": 4.31, "idle_nothing": 0.38, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.07}, "readers_busy": 12, "h2d_ssd_gbs_cum": 375641780224, "h2d_ram_gbs_cum": 34553630720, "h2d_used_cum": 410216980480, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 321.3, "phase": "measure", "dt": 1.001, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.101, "ram_to_gpu_gbs": 3.101, "gpu_to_ram_gbs": 2.756, "decode_gbs": 3.621, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 144, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 10, "reads_in_flight": 10, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1066, "ram_cap": 1080, "layer": 11, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 31, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 145.6, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.354, "pcie_tx_gbs": 2.765}], "pulls": {"expired": 0, "need": 19043, "idle": 0, "idle_used": 0, "ram_first": 1603, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 4.62, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 105.35, "ssd_below_cap": 11.17, "nothing_needed": 0.19}, "sd": {"all_readers": 116.54, "some_readers": 4.33, "idle_nothing": 0.38, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.07}, "readers_busy": 10, "h2d_ssd_gbs_cum": 378745647104, "h2d_ram_gbs_cum": 34553630720, "h2d_used_cum": 413320847360, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 322.3, "phase": "measure", "dt": 1.001, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.143, "ram_to_gpu_gbs": 3.832, "gpu_to_ram_gbs": 3.488, "decode_gbs": 4.576, "ssd_reads": 146, "ram_promotions": 32, "gpu_decodes": 182, "scratch_hits": 0, "lk": {"l0": 5, "arriving": 0, "ssd_inflight": 0, "l1": 32, "l2": 249}, "lkw": {"l0": 2607, "arriving": 0, "ssd_inflight": 0, "l1": 6326, "l2": 15067}, "missing": 114, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 12, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 27, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 139.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.518, "pcie_tx_gbs": 1.284}], "pulls": {"expired": 0, "need": 19245, "idle": 0, "idle_used": 0, "ram_first": 1635, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 4.64, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 106.3, "ssd_below_cap": 11.2, "nothing_needed": 0.19}, "sd": {"all_readers": 117.48, "some_readers": 4.39, "idle_nothing": 0.39, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.07}, "readers_busy": 12, "h2d_ssd_gbs_cum": 381892956160, "h2d_ram_gbs_cum": 35243470848, "h2d_used_cum": 417136439296, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 323.3, "phase": "measure", "dt": 1.001, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.122, "ram_to_gpu_gbs": 3.768, "gpu_to_ram_gbs": 3.553, "decode_gbs": 4.552, "ssd_reads": 145, "ram_promotions": 30, "gpu_decodes": 181, "scratch_hits": 0, "lk": {"l0": 5, "arriving": 0, "ssd_inflight": 0, "l1": 30, "l2": 250}, "lkw": {"l0": 3429, "arriving": 0, "ssd_inflight": 0, "l1": 5921, "l2": 14650}, "missing": 218, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 13, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 146.5, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.352, "pcie_tx_gbs": 3.846}], "pulls": {"expired": 0, "need": 19420, "idle": 0, "idle_used": 0, "ram_first": 1665, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 4.7, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 107.06, "ssd_below_cap": 11.37, "nothing_needed": 0.2}, "sd": {"all_readers": 118.4, "some_readers": 4.46, "idle_nothing": 0.39, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.07}, "readers_busy": 12, "h2d_ssd_gbs_cum": 385018548224, "h2d_ram_gbs_cum": 35890106368, "h2d_used_cum": 420930224128, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 324.3, "phase": "measure", "dt": 1.0, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.125, "ram_to_gpu_gbs": 3.125, "gpu_to_ram_gbs": 2.586, "decode_gbs": 3.649, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 73, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 13, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 32, "mem_util_pct": 14, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 151.5, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.698, "pcie_tx_gbs": 2.563}], "pulls": {"expired": 0, "need": 19565, "idle": 0, "idle_used": 0, "ram_first": 1665, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 4.7, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 108.06, "ssd_below_cap": 11.37, "nothing_needed": 0.2}, "sd": {"all_readers": 119.4, "some_readers": 4.46, "idle_nothing": 0.39, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.07}, "readers_busy": 12, "h2d_ssd_gbs_cum": 388144001024, "h2d_ram_gbs_cum": 35890106368, "h2d_used_cum": 424055676928, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 325.3, "phase": "measure", "dt": 1.003, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.117, "ram_to_gpu_gbs": 3.698, "gpu_to_ram_gbs": 3.332, "decode_gbs": 4.418, "ssd_reads": 145, "ram_promotions": 27, "gpu_decodes": 176, "scratch_hits": 0, "lk": {"l0": 4, "arriving": 0, "ssd_inflight": 0, "l1": 27, "l2": 256}, "lkw": {"l0": 3408, "arriving": 0, "ssd_inflight": 0, "l1": 5509, "l2": 15083}, "missing": 184, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 14, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 31, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 137.9, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.697, "pcie_tx_gbs": 3.843}], "pulls": {"expired": 0, "need": 19735, "idle": 0, "idle_used": 0, "ram_first": 1692, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 4.77, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 108.83, "ssd_below_cap": 11.53, "nothing_needed": 0.2}, "sd": {"all_readers": 120.32, "some_readers": 4.53, "idle_nothing": 0.4, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.07}, "readers_busy": 12, "h2d_ssd_gbs_cum": 391269683200, "h2d_ram_gbs_cum": 36472184832, "h2d_used_cum": 427763437568, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 326.3, "phase": "measure", "dt": 1.001, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.081, "ram_to_gpu_gbs": 3.081, "gpu_to_ram_gbs": 2.672, "decode_gbs": 3.597, "ssd_reads": 143, "ram_promotions": 0, "gpu_decodes": 143, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 41, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 14, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 146.9, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 5.842, "pcie_tx_gbs": 3.847}], "pulls": {"expired": 0, "need": 19878, "idle": 0, "idle_used": 0, "ram_first": 1692, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 4.77, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 109.84, "ssd_below_cap": 11.53, "nothing_needed": 0.2}, "sd": {"all_readers": 121.33, "some_readers": 4.53, "idle_nothing": 0.4, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.07}, "readers_busy": 12, "h2d_ssd_gbs_cum": 394352558080, "h2d_ram_gbs_cum": 36472184832, "h2d_used_cum": 430846312448, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 327.3, "phase": "measure", "dt": 1.0, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.169, "ram_to_gpu_gbs": 3.751, "gpu_to_ram_gbs": 3.298, "decode_gbs": 4.455, "ssd_reads": 147, "ram_promotions": 27, "gpu_decodes": 177, "scratch_hits": 0, "lk": {"l0": 4, "arriving": 0, "ssd_inflight": 0, "l1": 27, "l2": 256}, "lkw": {"l0": 2825, "arriving": 0, "ssd_inflight": 0, "l1": 6172, "l2": 15003}, "missing": 150, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 15, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 31, "mem_util_pct": 14, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 137.7, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 5.863, "pcie_tx_gbs": 0.001}], "pulls": {"expired": 0, "need": 20052, "idle": 0, "idle_used": 0, "ram_first": 1719, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 4.84, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 110.6, "ssd_below_cap": 11.69, "nothing_needed": 0.2}, "sd": {"all_readers": 122.25, "some_readers": 4.6, "idle_nothing": 0.4, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.07}, "readers_busy": 12, "h2d_ssd_gbs_cum": 397521752064, "h2d_ram_gbs_cum": 37054398464, "h2d_used_cum": 434576158720, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 328.3, "phase": "measure", "dt": 1.001, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.123, "ram_to_gpu_gbs": 3.123, "gpu_to_ram_gbs": 2.757, "decode_gbs": 3.646, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 5, "reads_in_flight": 5, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1073, "ram_cap": 1080, "layer": 15, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 145.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.344, "pcie_tx_gbs": 1.956}], "pulls": {"expired": 0, "need": 20166, "idle": 0, "idle_used": 0, "ram_first": 1719, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 4.9, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 111.4, "ssd_below_cap": 11.83, "nothing_needed": 0.2}, "sd": {"all_readers": 123.2, "some_readers": 4.65, "idle_nothing": 0.4, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.07}, "readers_busy": 5, "h2d_ssd_gbs_cum": 400648196096, "h2d_ram_gbs_cum": 37054398464, "h2d_used_cum": 437702598656, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 329.3, "phase": "measure", "dt": 1.003, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.118, "ram_to_gpu_gbs": 4.021, "gpu_to_ram_gbs": 3.613, "decode_gbs": 4.745, "ssd_reads": 145, "ram_promotions": 42, "gpu_decodes": 189, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 42, "l2": 243}, "lkw": {"l0": 833, "arriving": 0, "ssd_inflight": 0, "l1": 9018, "l2": 14149}, "missing": 103, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 16, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 146.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 1.183, "pcie_tx_gbs": 3.299}], "pulls": {"expired": 0, "need": 20377, "idle": 0, "idle_used": 0, "ram_first": 1761, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 4.91, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 112.37, "ssd_below_cap": 11.85, "nothing_needed": 0.2}, "sd": {"all_readers": 124.17, "some_readers": 4.67, "idle_nothing": 0.41, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.07}, "readers_busy": 12, "h2d_ssd_gbs_cum": 403774525440, "h2d_ram_gbs_cum": 37960044544, "h2d_used_cum": 441756139520, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 330.3, "phase": "measure", "dt": 1.001, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.101, "ram_to_gpu_gbs": 3.919, "gpu_to_ram_gbs": 3.769, "decode_gbs": 4.626, "ssd_reads": 144, "ram_promotions": 38, "gpu_decodes": 184, "scratch_hits": 0, "lk": {"l0": 3, "arriving": 0, "ssd_inflight": 0, "l1": 38, "l2": 244}, "lkw": {"l0": 1804, "arriving": 0, "ssd_inflight": 0, "l1": 8277, "l2": 13919}, "missing": 203, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 17, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 31, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 139.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 5.853, "pcie_tx_gbs": 3.809}], "pulls": {"expired": 0, "need": 20556, "idle": 0, "idle_used": 0, "ram_first": 1799, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 4.98, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 113.14, "ssd_below_cap": 12.02, "nothing_needed": 0.21}, "sd": {"all_readers": 125.1, "some_readers": 4.74, "idle_nothing": 0.42, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.07}, "readers_busy": 12, "h2d_ssd_gbs_cum": 406879301632, "h2d_ram_gbs_cum": 38779367424, "h2d_used_cum": 445658669056, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 331.3, "phase": "measure", "dt": 1.001, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.144, "ram_to_gpu_gbs": 3.144, "gpu_to_ram_gbs": 2.649, "decode_gbs": 3.696, "ssd_reads": 146, "ram_promotions": 0, "gpu_decodes": 147, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 57, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 17, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 32, "mem_util_pct": 14, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 150.1, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.345, "pcie_tx_gbs": 3.844}], "pulls": {"expired": 0, "need": 20702, "idle": 0, "idle_used": 0, "ram_first": 1799, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 4.98, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 114.14, "ssd_below_cap": 12.02, "nothing_needed": 0.21}, "sd": {"all_readers": 126.1, "some_readers": 4.74, "idle_nothing": 0.42, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.07}, "readers_busy": 12, "h2d_ssd_gbs_cum": 410026999808, "h2d_ram_gbs_cum": 38779367424, "h2d_used_cum": 448827936768, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 332.3, "phase": "measure", "dt": 1.002, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.119, "ram_to_gpu_gbs": 3.786, "gpu_to_ram_gbs": 3.463, "decode_gbs": 4.571, "ssd_reads": 145, "ram_promotions": 31, "gpu_decodes": 182, "scratch_hits": 0, "lk": {"l0": 6, "arriving": 0, "ssd_inflight": 0, "l1": 31, "l2": 245}, "lkw": {"l0": 4971, "arriving": 0, "ssd_inflight": 0, "l1": 7325, "l2": 11704}, "missing": 157, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 18, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 37, "mem_util_pct": 15, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 146.3, "power_limit_w": 600.0, "temp_c": 33, "sm_clock_mhz": 2325, "pcie_rx_gbs": 2.335, "pcie_tx_gbs": 6.409}], "pulls": {"expired": 0, "need": 20878, "idle": 0, "idle_used": 0, "ram_first": 1830, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 5.05, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 114.91, "ssd_below_cap": 12.17, "nothing_needed": 0.21}, "sd": {"all_readers": 127.03, "some_readers": 4.8, "idle_nothing": 0.42, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.07}, "readers_busy": 12, "h2d_ssd_gbs_cum": 413153087488, "h2d_ram_gbs_cum": 39447674880, "h2d_used_cum": 452622331904, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 333.3, "phase": "measure", "dt": 1.001, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.144, "ram_to_gpu_gbs": 3.122, "gpu_to_ram_gbs": 2.67, "decode_gbs": 3.62, "ssd_reads": 146, "ram_promotions": 0, "gpu_decodes": 144, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 13, "reads_in_flight": 12, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1064, "ram_cap": 1080, "layer": 18, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 137.0, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.518, "pcie_tx_gbs": 5.202}], "pulls": {"expired": 0, "need": 20999, "idle": 0, "idle_used": 0, "ram_first": 1830, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 5.1, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 115.76, "ssd_below_cap": 12.28, "nothing_needed": 0.21}, "sd": {"all_readers": 128.04, "some_readers": 4.8, "idle_nothing": 0.42, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.07}, "readers_busy": 11, "h2d_ssd_gbs_cum": 416279060480, "h2d_ram_gbs_cum": 39447674880, "h2d_used_cum": 455726755840, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 334.3, "phase": "measure", "dt": 1.002, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.12, "ram_to_gpu_gbs": 3.744, "gpu_to_ram_gbs": 3.206, "decode_gbs": 4.497, "ssd_reads": 145, "ram_promotions": 28, "gpu_decodes": 179, "scratch_hits": 0, "lk": {"l0": 5, "arriving": 0, "ssd_inflight": 0, "l1": 28, "l2": 245}, "lkw": {"l0": 3497, "arriving": 0, "ssd_inflight": 0, "l1": 7356, "l2": 13147}, "missing": 112, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 19, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 151.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.335, "pcie_tx_gbs": 5.126}], "pulls": {"expired": 0, "need": 21196, "idle": 0, "idle_used": 0, "ram_first": 1858, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 5.12, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 116.69, "ssd_below_cap": 12.33, "nothing_needed": 0.21}, "sd": {"all_readers": 128.95, "some_readers": 4.88, "idle_nothing": 0.43, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 419427209216, "h2d_ram_gbs_cum": 40051433472, "h2d_used_cum": 459478642688, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 335.3, "phase": "measure", "dt": 1.001, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.081, "ram_to_gpu_gbs": 3.792, "gpu_to_ram_gbs": 3.641, "decode_gbs": 4.478, "ssd_reads": 143, "ram_promotions": 33, "gpu_decodes": 178, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 33, "l2": 248}, "lkw": {"l0": 786, "arriving": 0, "ssd_inflight": 0, "l1": 8291, "l2": 14923}, "missing": 216, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 20, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 27, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 138.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.068, "pcie_tx_gbs": 1.283}], "pulls": {"expired": 0, "need": 21373, "idle": 0, "idle_used": 0, "ram_first": 1891, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 5.2, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 117.45, "ssd_below_cap": 12.49, "nothing_needed": 0.22}, "sd": {"all_readers": 129.88, "some_readers": 4.95, "idle_nothing": 0.43, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 422510297088, "h2d_ram_gbs_cum": 40762757120, "h2d_used_cum": 463294623744, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 336.3, "phase": "measure", "dt": 1.001, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.144, "ram_to_gpu_gbs": 3.144, "gpu_to_ram_gbs": 2.67, "decode_gbs": 3.671, "ssd_reads": 146, "ram_promotions": 0, "gpu_decodes": 146, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 70, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 20, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 147.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.607, "pcie_tx_gbs": 2.267}], "pulls": {"expired": 0, "need": 21519, "idle": 0, "idle_used": 0, "ram_first": 1891, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 5.2, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 118.45, "ssd_below_cap": 12.49, "nothing_needed": 0.22}, "sd": {"all_readers": 130.88, "some_readers": 4.95, "idle_nothing": 0.43, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 425657483264, "h2d_ram_gbs_cum": 40762757120, "h2d_used_cum": 466441809920, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 337.4, "phase": "measure", "dt": 1.002, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.12, "ram_to_gpu_gbs": 3.874, "gpu_to_ram_gbs": 3.615, "decode_gbs": 4.474, "ssd_reads": 145, "ram_promotions": 35, "gpu_decodes": 178, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 35, "l2": 246}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 9264, "l2": 14736}, "missing": 172, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 21, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 55, "mem_util_pct": 16, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 144.0, "power_limit_w": 600.0, "temp_c": 32, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.346, "pcie_tx_gbs": 2.562}], "pulls": {"expired": 0, "need": 21697, "idle": 0, "idle_used": 0, "ram_first": 1926, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 5.26, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 119.21, "ssd_below_cap": 12.66, "nothing_needed": 0.22}, "sd": {"all_readers": 131.81, "some_readers": 5.01, "idle_nothing": 0.44, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 428782817280, "h2d_ram_gbs_cum": 41517187072, "h2d_used_cum": 470278475776, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 338.4, "phase": "measure", "dt": 1.001, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.08, "ram_to_gpu_gbs": 3.08, "gpu_to_ram_gbs": 2.606, "decode_gbs": 3.647, "ssd_reads": 143, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 28, "reads_in_flight": 28, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1050, "ram_cap": 1080, "layer": 21, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 26, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 137.8, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.67, "pcie_tx_gbs": 1.283}], "pulls": {"expired": 0, "need": 21833, "idle": 0, "idle_used": 0, "ram_first": 1926, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 5.28, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 120.15, "ssd_below_cap": 12.7, "nothing_needed": 0.22}, "sd": {"all_readers": 132.81, "some_readers": 5.01, "idle_nothing": 0.44, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 431865106432, "h2d_ram_gbs_cum": 41517187072, "h2d_used_cum": 473403863040, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 339.4, "phase": "measure", "dt": 1.0, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.168, "ram_to_gpu_gbs": 3.858, "gpu_to_ram_gbs": 3.664, "decode_gbs": 4.505, "ssd_reads": 147, "ram_promotions": 32, "gpu_decodes": 179, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 32, "l2": 246}, "lkw": {"l0": 843, "arriving": 0, "ssd_inflight": 0, "l1": 7320, "l2": 15837}, "missing": 127, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 22, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 148.7, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 6.446, "pcie_tx_gbs": 2.565}], "pulls": {"expired": 0, "need": 22018, "idle": 0, "idle_used": 0, "ram_first": 1958, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 5.33, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 120.98, "ssd_below_cap": 12.81, "nothing_needed": 0.22}, "sd": {"all_readers": 133.74, "some_readers": 5.08, "idle_nothing": 0.44, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 435033321472, "h2d_ram_gbs_cum": 42206822400, "h2d_used_cum": 477240160256, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 340.4, "phase": "measure", "dt": 1.001, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.1, "ram_to_gpu_gbs": 3.595, "gpu_to_ram_gbs": 3.272, "decode_gbs": 4.249, "ssd_reads": 144, "ram_promotions": 23, "gpu_decodes": 169, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 23, "l2": 260}, "lkw": {"l0": 667, "arriving": 0, "ssd_inflight": 0, "l1": 5672, "l2": 17661}, "missing": 243, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 23, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 139.1, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.687, "pcie_tx_gbs": 7.688}], "pulls": {"expired": 0, "need": 22185, "idle": 0, "idle_used": 0, "ram_first": 1981, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 5.41, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 121.75, "ssd_below_cap": 12.97, "nothing_needed": 0.22}, "sd": {"all_readers": 134.67, "some_readers": 5.15, "idle_nothing": 0.45, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 438136963072, "h2d_ram_gbs_cum": 42702422016, "h2d_used_cum": 480860954624, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 341.4, "phase": "measure", "dt": 1.0, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.146, "ram_to_gpu_gbs": 3.146, "gpu_to_ram_gbs": 2.607, "decode_gbs": 3.674, "ssd_reads": 146, "ram_promotions": 0, "gpu_decodes": 146, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 97, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 23, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 32, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 141.5, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 1.17, "pcie_tx_gbs": 2.87}], "pulls": {"expired": 0, "need": 22331, "idle": 0, "idle_used": 0, "ram_first": 1981, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 5.41, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 122.75, "ssd_below_cap": 12.97, "nothing_needed": 0.22}, "sd": {"all_readers": 135.67, "some_readers": 5.15, "idle_nothing": 0.45, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 441283620864, "h2d_ram_gbs_cum": 42702422016, "h2d_used_cum": 484007612416, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 342.4, "phase": "measure", "dt": 1.003, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.117, "ram_to_gpu_gbs": 3.568, "gpu_to_ram_gbs": 3.117, "decode_gbs": 4.218, "ssd_reads": 145, "ram_promotions": 21, "gpu_decodes": 168, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 21, "l2": 261}, "lkw": {"l0": 1563, "arriving": 0, "ssd_inflight": 0, "l1": 6297, "l2": 16140}, "missing": 213, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 24, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 42, "mem_util_pct": 15, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 144.8, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.687, "pcie_tx_gbs": 1.281}], "pulls": {"expired": 0, "need": 22497, "idle": 0, "idle_used": 0, "ram_first": 2002, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 5.47, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 123.51, "ssd_below_cap": 13.15, "nothing_needed": 0.23}, "sd": {"all_readers": 136.61, "some_readers": 5.2, "idle_nothing": 0.46, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 444408688640, "h2d_ram_gbs_cum": 43155091456, "h2d_used_cum": 487585349632, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 343.4, "phase": "measure", "dt": 1.001, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.121, "ram_to_gpu_gbs": 3.121, "gpu_to_ram_gbs": 2.54, "decode_gbs": 3.645, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 68, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 24, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 25, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 136.7, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.336, "pcie_tx_gbs": 5.227}], "pulls": {"expired": 0, "need": 22642, "idle": 0, "idle_used": 0, "ram_first": 2002, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 5.47, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 124.51, "ssd_below_cap": 13.15, "nothing_needed": 0.23}, "sd": {"all_readers": 137.61, "some_readers": 5.2, "idle_nothing": 0.46, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 447534485504, "h2d_ram_gbs_cum": 43155091456, "h2d_used_cum": 490711146496, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 344.4, "phase": "measure", "dt": 1.0, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.104, "ram_to_gpu_gbs": 3.686, "gpu_to_ram_gbs": 3.448, "decode_gbs": 4.379, "ssd_reads": 144, "ram_promotions": 27, "gpu_decodes": 174, "scratch_hits": 0, "lk": {"l0": 3, "arriving": 0, "ssd_inflight": 0, "l1": 27, "l2": 251}, "lkw": {"l0": 2546, "arriving": 0, "ssd_inflight": 0, "l1": 6872, "l2": 14582}, "missing": 175, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 25, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 25, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 136.7, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.336, "pcie_tx_gbs": 5.227}], "pulls": {"expired": 0, "need": 22813, "idle": 0, "idle_used": 0, "ram_first": 2029, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 5.54, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 125.29, "ssd_below_cap": 13.29, "nothing_needed": 0.23}, "sd": {"all_readers": 138.53, "some_readers": 5.27, "idle_nothing": 0.46, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 450638946304, "h2d_ram_gbs_cum": 43737296896, "h2d_used_cum": 494397812736, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 345.4, "phase": "measure", "dt": 1.001, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.124, "ram_to_gpu_gbs": 3.124, "gpu_to_ram_gbs": 2.693, "decode_gbs": 3.647, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 30, "reads_in_flight": 30, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1048, "ram_cap": 1080, "layer": 25, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 33, "mem_util_pct": 14, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 148.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.345, "pcie_tx_gbs": 3.145}], "pulls": {"expired": 0, "need": 22952, "idle": 0, "idle_used": 0, "ram_first": 2029, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 5.55, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 126.25, "ssd_below_cap": 13.32, "nothing_needed": 0.23}, "sd": {"all_readers": 139.53, "some_readers": 5.27, "idle_nothing": 0.46, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 453765189632, "h2d_ram_gbs_cum": 43737296896, "h2d_used_cum": 497524056064, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 346.4, "phase": "measure", "dt": 1.001, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.145, "ram_to_gpu_gbs": 3.64, "gpu_to_ram_gbs": 3.123, "decode_gbs": 4.25, "ssd_reads": 146, "ram_promotions": 23, "gpu_decodes": 169, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 23, "l2": 260}, "lkw": {"l0": 928, "arriving": 0, "ssd_inflight": 0, "l1": 6334, "l2": 16738}, "missing": 144, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 26, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 26, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 140.5, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.499, "pcie_tx_gbs": 3.849}], "pulls": {"expired": 0, "need": 23126, "idle": 0, "idle_used": 0, "ram_first": 2052, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 5.61, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 127.06, "ssd_below_cap": 13.45, "nothing_needed": 0.23}, "sd": {"all_readers": 140.47, "some_readers": 5.33, "idle_nothing": 0.47, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 456913010688, "h2d_ram_gbs_cum": 44233195520, "h2d_used_cum": 501146210304, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 347.4, "phase": "measure", "dt": 1.008, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.081, "ram_to_gpu_gbs": 3.552, "gpu_to_ram_gbs": 3.274, "decode_gbs": 4.122, "ssd_reads": 144, "ram_promotions": 22, "gpu_decodes": 165, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 22, "l2": 259}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 7329, "l2": 16671}, "missing": 259, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1026, "ram_cap": 1080, "layer": 27, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 25, "mem_util_pct": 10, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 144.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.517, "pcie_tx_gbs": 3.845}], "pulls": {"expired": 0, "need": 23292, "idle": 0, "idle_used": 0, "ram_first": 2074, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 5.69, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 127.83, "ssd_below_cap": 13.61, "nothing_needed": 0.23}, "sd": {"all_readers": 141.4, "some_readers": 5.39, "idle_nothing": 0.47, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 460017463296, "h2d_ram_gbs_cum": 44707520512, "h2d_used_cum": 504703438848, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 348.4, "phase": "measure", "dt": 1.001, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.146, "ram_to_gpu_gbs": 3.146, "gpu_to_ram_gbs": 2.672, "decode_gbs": 3.724, "ssd_reads": 146, "ram_promotions": 0, "gpu_decodes": 148, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 113, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 27, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 33, "mem_util_pct": 14, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 144.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 1.44, "pcie_tx_gbs": 2.769}], "pulls": {"expired": 0, "need": 23438, "idle": 0, "idle_used": 0, "ram_first": 2074, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 5.69, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 128.83, "ssd_below_cap": 13.61, "nothing_needed": 0.23}, "sd": {"all_readers": 142.41, "some_readers": 5.39, "idle_nothing": 0.47, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 463165440000, "h2d_ram_gbs_cum": 44707520512, "h2d_used_cum": 507894530048, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 349.4, "phase": "measure", "dt": 1.002, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.12, "ram_to_gpu_gbs": 3.529, "gpu_to_ram_gbs": 3.185, "decode_gbs": 4.12, "ssd_reads": 145, "ram_promotions": 19, "gpu_decodes": 164, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 19, "l2": 263}, "lkw": {"l0": 947, "arriving": 0, "ssd_inflight": 0, "l1": 6053, "l2": 17000}, "missing": 231, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 28, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 134.6, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 1.183, "pcie_tx_gbs": 3.947}], "pulls": {"expired": 0, "need": 23602, "idle": 0, "idle_used": 0, "ram_first": 2093, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 5.76, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 129.6, "ssd_below_cap": 13.77, "nothing_needed": 0.24}, "sd": {"all_readers": 143.34, "some_readers": 5.45, "idle_nothing": 0.48, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 466291544064, "h2d_ram_gbs_cum": 45117140992, "h2d_used_cum": 511408709632, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 350.4, "phase": "measure", "dt": 1.0, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.147, "ram_to_gpu_gbs": 3.126, "gpu_to_ram_gbs": 2.544, "decode_gbs": 3.624, "ssd_reads": 146, "ram_promotions": 0, "gpu_decodes": 144, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 88, "reads_in_flight": 34, "uploads_pending": 2, "gpu_experts": 106, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 28, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 27, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 147.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 3.51, "pcie_tx_gbs": 2.667}], "pulls": {"expired": 0, "need": 23745, "idle": 0, "idle_used": 0, "ram_first": 2093, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 5.76, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 130.6, "ssd_below_cap": 13.77, "nothing_needed": 0.24}, "sd": {"all_readers": 144.34, "some_readers": 5.45, "idle_nothing": 0.48, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 469417893888, "h2d_ram_gbs_cum": 45117140992, "h2d_used_cum": 514513481728, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 351.4, "phase": "measure", "dt": 1.0, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.104, "ram_to_gpu_gbs": 3.535, "gpu_to_ram_gbs": 3.017, "decode_gbs": 4.203, "ssd_reads": 144, "ram_promotions": 19, "gpu_decodes": 167, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 19, "l2": 264}, "lkw": {"l0": 706, "arriving": 0, "ssd_inflight": 0, "l1": 4825, "l2": 18469}, "missing": 205, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 29, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 31, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 139.7, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.438, "pcie_tx_gbs": 2.683}], "pulls": {"expired": 0, "need": 23911, "idle": 0, "idle_used": 0, "ram_first": 2112, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 5.83, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 131.36, "ssd_below_cap": 13.93, "nothing_needed": 0.24}, "sd": {"all_readers": 145.28, "some_readers": 5.51, "idle_nothing": 0.49, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 472543621120, "h2d_ram_gbs_cum": 45526769664, "h2d_used_cum": 518091960320, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 352.4, "phase": "measure", "dt": 1.001, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.103, "ram_to_gpu_gbs": 3.103, "gpu_to_ram_gbs": 2.629, "decode_gbs": 3.623, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 144, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 61, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 29, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 143.8, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 5.863, "pcie_tx_gbs": 2.563}], "pulls": {"expired": 0, "need": 24055, "idle": 0, "idle_used": 0, "ram_first": 2112, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 5.83, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 132.36, "ssd_below_cap": 13.93, "nothing_needed": 0.24}, "sd": {"all_readers": 146.28, "some_readers": 5.51, "idle_nothing": 0.49, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 475647827968, "h2d_ram_gbs_cum": 45526769664, "h2d_used_cum": 521196167168, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 353.4, "phase": "measure", "dt": 1.001, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.145, "ram_to_gpu_gbs": 3.597, "gpu_to_ram_gbs": 3.338, "decode_gbs": 4.2, "ssd_reads": 146, "ram_promotions": 21, "gpu_decodes": 167, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 21, "l2": 260}, "lkw": {"l0": 1520, "arriving": 0, "ssd_inflight": 0, "l1": 6123, "l2": 16357}, "missing": 176, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 30, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 136.1, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.679, "pcie_tx_gbs": 3.933}], "pulls": {"expired": 0, "need": 24221, "idle": 0, "idle_used": 0, "ram_first": 2133, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 5.89, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 133.12, "ssd_below_cap": 14.11, "nothing_needed": 0.24}, "sd": {"all_readers": 147.2, "some_readers": 5.58, "idle_nothing": 0.49, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 478795091968, "h2d_ram_gbs_cum": 45979516928, "h2d_used_cum": 524753068032, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 354.4, "phase": "measure", "dt": 1.001, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.124, "ram_to_gpu_gbs": 3.124, "gpu_to_ram_gbs": 2.779, "decode_gbs": 3.648, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 31, "reads_in_flight": 30, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1046, "ram_cap": 1080, "layer": 30, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 31, "mem_util_pct": 14, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 147.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.365, "pcie_tx_gbs": 1.384}], "pulls": {"expired": 0, "need": 24361, "idle": 0, "idle_used": 0, "ram_first": 2133, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 5.91, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 134.08, "ssd_below_cap": 14.14, "nothing_needed": 0.24}, "sd": {"all_readers": 148.21, "some_readers": 5.58, "idle_nothing": 0.49, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 481920962560, "h2d_ram_gbs_cum": 45979516928, "h2d_used_cum": 527878942720, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 355.4, "phase": "measure", "dt": 1.001, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.102, "ram_to_gpu_gbs": 3.533, "gpu_to_ram_gbs": 3.232, "decode_gbs": 4.226, "ssd_reads": 144, "ram_promotions": 20, "gpu_decodes": 168, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 20, "l2": 262}, "lkw": {"l0": 2450, "arriving": 0, "ssd_inflight": 0, "l1": 5216, "l2": 16334}, "missing": 148, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 31, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 34, "mem_util_pct": 15, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 146.5, "power_limit_w": 600.0, "temp_c": 32, "sm_clock_mhz": 2332, "pcie_rx_gbs": 1.542, "pcie_tx_gbs": 5.124}], "pulls": {"expired": 0, "need": 24531, "idle": 0, "idle_used": 0, "ram_first": 2153, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 5.96, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 134.88, "ssd_below_cap": 14.28, "nothing_needed": 0.25}, "sd": {"all_readers": 149.14, "some_readers": 5.64, "idle_nothing": 0.5, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 485025341440, "h2d_ram_gbs_cum": 46410690560, "h2d_used_cum": 531457601536, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 356.4, "phase": "measure", "dt": 1.0, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.19, "ram_to_gpu_gbs": 3.19, "gpu_to_ram_gbs": 2.91, "decode_gbs": 3.725, "ssd_reads": 148, "ram_promotions": 0, "gpu_decodes": 148, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 18, "l2": 268}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 5365, "l2": 18635}, "missing": 0, "reads_in_flight": 0, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1077, "ram_cap": 1080, "layer": 32, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 140.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.516, "pcie_tx_gbs": 3.846}], "pulls": {"expired": 0, "need": 24643, "idle": 0, "idle_used": 0, "ram_first": 2153, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 6.03, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 135.66, "ssd_below_cap": 14.43, "nothing_needed": 0.25}, "sd": {"all_readers": 150.07, "some_readers": 5.7, "idle_nothing": 0.5, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 0, "h2d_ssd_gbs_cum": 488215805952, "h2d_ram_gbs_cum": 46410690560, "h2d_used_cum": 534648066048, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 357.4, "phase": "measure", "dt": 1.0, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.081, "ram_to_gpu_gbs": 3.469, "gpu_to_ram_gbs": 2.974, "decode_gbs": 4.026, "ssd_reads": 143, "ram_promotions": 18, "gpu_decodes": 160, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 125, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 32, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 146.7, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.515, "pcie_tx_gbs": 1.341}], "pulls": {"expired": 0, "need": 24840, "idle": 0, "idle_used": 0, "ram_first": 2171, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 6.03, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 136.65, "ssd_below_cap": 14.44, "nothing_needed": 0.25}, "sd": {"all_readers": 151.07, "some_readers": 5.71, "idle_nothing": 0.5, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 491298525184, "h2d_ram_gbs_cum": 46798774272, "h2d_used_cum": 538097311744, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 358.4, "phase": "measure", "dt": 1.001, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.124, "ram_to_gpu_gbs": 3.598, "gpu_to_ram_gbs": 3.232, "decode_gbs": 4.227, "ssd_reads": 145, "ram_promotions": 22, "gpu_decodes": 168, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 22, "l2": 261}, "lkw": {"l0": 1233, "arriving": 0, "ssd_inflight": 0, "l1": 7176, "l2": 15591}, "missing": 242, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 33, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 138.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.344, "pcie_tx_gbs": 3.682}], "pulls": {"expired": 0, "need": 25006, "idle": 0, "idle_used": 0, "ram_first": 2193, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 6.09, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 137.42, "ssd_below_cap": 14.61, "nothing_needed": 0.25}, "sd": {"all_readers": 152.01, "some_readers": 5.76, "idle_nothing": 0.51, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 494424223744, "h2d_ram_gbs_cum": 47273103360, "h2d_used_cum": 541697335296, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 359.4, "phase": "measure", "dt": 1.0, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.103, "ram_to_gpu_gbs": 3.103, "gpu_to_ram_gbs": 2.651, "decode_gbs": 3.599, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 143, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 98, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 33, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 24, "mem_util_pct": 9, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 144.5, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.688, "pcie_tx_gbs": 2.564}], "pulls": {"expired": 0, "need": 25150, "idle": 0, "idle_used": 0, "ram_first": 2193, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 6.09, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 138.42, "ssd_below_cap": 14.61, "nothing_needed": 0.25}, "sd": {"all_readers": 153.01, "some_readers": 5.76, "idle_nothing": 0.51, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 497528520704, "h2d_ram_gbs_cum": 47273103360, "h2d_used_cum": 544780083200, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 360.4, "phase": "measure", "dt": 1.0, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.126, "ram_to_gpu_gbs": 3.535, "gpu_to_ram_gbs": 3.147, "decode_gbs": 4.204, "ssd_reads": 145, "ram_promotions": 19, "gpu_decodes": 167, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 19, "l2": 262}, "lkw": {"l0": 1939, "arriving": 0, "ssd_inflight": 0, "l1": 6218, "l2": 15843}, "missing": 215, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 34, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 135.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.355, "pcie_tx_gbs": 6.402}], "pulls": {"expired": 0, "need": 25314, "idle": 0, "idle_used": 0, "ram_first": 2212, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 6.16, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 139.18, "ssd_below_cap": 14.79, "nothing_needed": 0.25}, "sd": {"all_readers": 153.94, "some_readers": 5.82, "idle_nothing": 0.51, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 500654424064, "h2d_ram_gbs_cum": 47682756608, "h2d_used_cum": 548337192960, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 361.4, "phase": "measure", "dt": 1.001, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.124, "ram_to_gpu_gbs": 3.124, "gpu_to_ram_gbs": 2.693, "decode_gbs": 3.648, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 70, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 34, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 27, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 142.0, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 1.655, "pcie_tx_gbs": 2.562}], "pulls": {"expired": 0, "need": 25459, "idle": 0, "idle_used": 0, "ram_first": 2212, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 6.16, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 140.18, "ssd_below_cap": 14.79, "nothing_needed": 0.25}, "sd": {"all_readers": 154.94, "some_readers": 5.82, "idle_nothing": 0.51, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 503780315136, "h2d_ram_gbs_cum": 47682756608, "h2d_used_cum": 551463084032, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 362.4, "phase": "measure", "dt": 1.001, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.059, "ram_to_gpu_gbs": 3.382, "gpu_to_ram_gbs": 2.994, "decode_gbs": 4.024, "ssd_reads": 142, "ram_promotions": 15, "gpu_decodes": 160, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 15, "l2": 269}, "lkw": {"l0": 2187, "arriving": 0, "ssd_inflight": 0, "l1": 4992, "l2": 16821}, "missing": 196, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 35, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 49, "mem_util_pct": 17, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 146.3, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.672, "pcie_tx_gbs": 2.744}], "pulls": {"expired": 0, "need": 25617, "idle": 0, "idle_used": 0, "ram_first": 2227, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 6.22, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 140.95, "ssd_below_cap": 14.95, "nothing_needed": 0.26}, "sd": {"all_readers": 155.88, "some_readers": 5.88, "idle_nothing": 0.52, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 506841522176, "h2d_ram_gbs_cum": 48006127616, "h2d_used_cum": 554869219328, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 363.4, "phase": "measure", "dt": 1.001, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.209, "ram_to_gpu_gbs": 3.209, "gpu_to_ram_gbs": 2.649, "decode_gbs": 3.747, "ssd_reads": 149, "ram_promotions": 0, "gpu_decodes": 149, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 47, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 35, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 26, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 137.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 1.173, "pcie_tx_gbs": 6.508}], "pulls": {"expired": 0, "need": 25766, "idle": 0, "idle_used": 0, "ram_first": 2227, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 6.22, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 141.95, "ssd_below_cap": 14.95, "nothing_needed": 0.26}, "sd": {"all_readers": 156.88, "some_readers": 5.88, "idle_nothing": 0.52, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 510053658624, "h2d_ram_gbs_cum": 48006127616, "h2d_used_cum": 558081355776, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 364.4, "phase": "measure", "dt": 1.0, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.06, "ram_to_gpu_gbs": 3.34, "gpu_to_ram_gbs": 3.017, "decode_gbs": 3.95, "ssd_reads": 142, "ram_promotions": 13, "gpu_decodes": 157, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 13, "l2": 272}, "lkw": {"l0": 1838, "arriving": 0, "ssd_inflight": 0, "l1": 4889, "l2": 17273}, "missing": 177, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 36, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 31, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 149.3, "power_limit_w": 600.0, "temp_c": 32, "sm_clock_mhz": 2340, "pcie_rx_gbs": 3.887, "pcie_tx_gbs": 5.124}], "pulls": {"expired": 0, "need": 25921, "idle": 0, "idle_used": 0, "ram_first": 2240, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 6.3, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 142.71, "ssd_below_cap": 15.11, "nothing_needed": 0.26}, "sd": {"all_readers": 157.81, "some_readers": 5.94, "idle_nothing": 0.53, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 513114841088, "h2d_ram_gbs_cum": 48286396416, "h2d_used_cum": 561422807040, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 365.4, "phase": "measure", "dt": 1.0, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.168, "ram_to_gpu_gbs": 3.168, "gpu_to_ram_gbs": 2.564, "decode_gbs": 3.699, "ssd_reads": 147, "ram_promotions": 0, "gpu_decodes": 147, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 30, "reads_in_flight": 30, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1048, "ram_cap": 1080, "layer": 36, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 27, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 137.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.506, "pcie_tx_gbs": 3.949}], "pulls": {"expired": 0, "need": 26062, "idle": 0, "idle_used": 0, "ram_first": 2240, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 6.31, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 143.69, "ssd_below_cap": 15.12, "nothing_needed": 0.26}, "sd": {"all_readers": 158.81, "some_readers": 5.94, "idle_nothing": 0.53, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 516283691008, "h2d_ram_gbs_cum": 48286396416, "h2d_used_cum": 564591656960, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 366.4, "phase": "measure", "dt": 1.0, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.104, "ram_to_gpu_gbs": 3.47, "gpu_to_ram_gbs": 3.147, "decode_gbs": 4.052, "ssd_reads": 144, "ram_promotions": 17, "gpu_decodes": 161, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 17, "l2": 270}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 6183, "l2": 17817}, "missing": 156, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 37, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 146.1, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.637, "pcie_tx_gbs": 2.565}], "pulls": {"expired": 0, "need": 26229, "idle": 0, "idle_used": 0, "ram_first": 2257, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 6.36, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 144.49, "ssd_below_cap": 15.26, "nothing_needed": 0.26}, "sd": {"all_readers": 159.74, "some_readers": 6.01, "idle_nothing": 0.53, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 519387992064, "h2d_ram_gbs_cum": 48652910592, "h2d_used_cum": 568062472192, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 367.4, "phase": "measure", "dt": 1.001, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.122, "ram_to_gpu_gbs": 3.122, "gpu_to_ram_gbs": 2.713, "decode_gbs": 3.646, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 11, "reads_in_flight": 11, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1067, "ram_cap": 1080, "layer": 37, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 27, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 139.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.406, "pcie_tx_gbs": 3.844}], "pulls": {"expired": 0, "need": 26349, "idle": 0, "idle_used": 0, "ram_first": 2257, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 6.42, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 145.32, "ssd_below_cap": 15.38, "nothing_needed": 0.26}, "sd": {"all_readers": 160.73, "some_readers": 6.02, "idle_nothing": 0.53, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 11, "h2d_ssd_gbs_cum": 522513522688, "h2d_ram_gbs_cum": 48652910592, "h2d_used_cum": 571188002816, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 368.4, "phase": "measure", "dt": 1.001, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.101, "ram_to_gpu_gbs": 3.316, "gpu_to_ram_gbs": 2.864, "decode_gbs": 3.948, "ssd_reads": 144, "ram_promotions": 10, "gpu_decodes": 157, "scratch_hits": 0, "lk": {"l0": 3, "arriving": 0, "ssd_inflight": 0, "l1": 10, "l2": 270}, "lkw": {"l0": 3455, "arriving": 0, "ssd_inflight": 0, "l1": 3682, "l2": 16863}, "missing": 137, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 38, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 31, "mem_util_pct": 14, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 144.0, "power_limit_w": 600.0, "temp_c": 30, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.508, "pcie_tx_gbs": 3.844}], "pulls": {"expired": 0, "need": 26528, "idle": 0, "idle_used": 0, "ram_first": 2267, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 6.43, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 146.26, "ssd_below_cap": 15.42, "nothing_needed": 0.27}, "sd": {"all_readers": 161.67, "some_readers": 6.07, "idle_nothing": 0.54, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.08}, "readers_busy": 12, "h2d_ssd_gbs_cum": 525617364992, "h2d_ram_gbs_cum": 48868458496, "h2d_used_cum": 574507393024, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 369.4, "phase": "measure", "dt": 1.0, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.146, "ram_to_gpu_gbs": 3.491, "gpu_to_ram_gbs": 2.952, "decode_gbs": 4.102, "ssd_reads": 146, "ram_promotions": 16, "gpu_decodes": 163, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 16, "l2": 267}, "lkw": {"l0": 1025, "arriving": 0, "ssd_inflight": 0, "l1": 5536, "l2": 17439}, "missing": 258, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1041, "ram_cap": 1080, "layer": 39, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 31, "mem_util_pct": 14, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 144.0, "power_limit_w": 600.0, "temp_c": 30, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.508, "pcie_tx_gbs": 3.844}], "pulls": {"expired": 0, "need": 26690, "idle": 0, "idle_used": 0, "ram_first": 2283, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 6.51, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 147.03, "ssd_below_cap": 15.58, "nothing_needed": 0.27}, "sd": {"all_readers": 162.6, "some_readers": 6.14, "idle_nothing": 0.54, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.09}, "readers_busy": 12, "h2d_ssd_gbs_cum": 528764452864, "h2d_ram_gbs_cum": 49213374464, "h2d_used_cum": 577999396864, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 370.4, "phase": "measure", "dt": 1.001, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.144, "ram_to_gpu_gbs": 3.144, "gpu_to_ram_gbs": 2.541, "decode_gbs": 3.646, "ssd_reads": 146, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 112, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 39, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 27, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 140.5, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 9.33, "pcie_tx_gbs": 1.283}], "pulls": {"expired": 0, "need": 26836, "idle": 0, "idle_used": 0, "ram_first": 2283, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 6.51, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 148.03, "ssd_below_cap": 15.58, "nothing_needed": 0.27}, "sd": {"all_readers": 163.6, "some_readers": 6.14, "idle_nothing": 0.54, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.09}, "readers_busy": 12, "h2d_ssd_gbs_cum": 531911548928, "h2d_ram_gbs_cum": 49213374464, "h2d_used_cum": 581124931584, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 371.4, "phase": "measure", "dt": 1.003, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.095, "ram_to_gpu_gbs": 3.417, "gpu_to_ram_gbs": 3.052, "decode_gbs": 4.016, "ssd_reads": 144, "ram_promotions": 15, "gpu_decodes": 160, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 15, "l2": 271}, "lkw": {"l0": 1123, "arriving": 0, "ssd_inflight": 0, "l1": 5651, "l2": 17226}, "missing": 239, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 40, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 138.8, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.342, "pcie_tx_gbs": 4.224}], "pulls": {"expired": 0, "need": 26995, "idle": 0, "idle_used": 0, "ram_first": 2298, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 6.57, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 148.79, "ssd_below_cap": 15.75, "nothing_needed": 0.27}, "sd": {"all_readers": 164.54, "some_readers": 6.19, "idle_nothing": 0.55, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.09}, "readers_busy": 12, "h2d_ssd_gbs_cum": 535015116800, "h2d_ram_gbs_cum": 49536618496, "h2d_used_cum": 584551739392, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 372.4, "phase": "measure", "dt": 1.001, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.142, "ram_to_gpu_gbs": 3.142, "gpu_to_ram_gbs": 2.712, "decode_gbs": 3.696, "ssd_reads": 146, "ram_promotions": 0, "gpu_decodes": 147, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 93, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 40, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 142.6, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 3.979, "pcie_tx_gbs": 1.383}], "pulls": {"expired": 0, "need": 27141, "idle": 0, "idle_used": 0, "ram_first": 2298, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 6.57, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 149.79, "ssd_below_cap": 15.75, "nothing_needed": 0.27}, "sd": {"all_readers": 165.54, "some_readers": 6.19, "idle_nothing": 0.55, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.09}, "readers_busy": 12, "h2d_ssd_gbs_cum": 538161598464, "h2d_ram_gbs_cum": 49536618496, "h2d_used_cum": 587719786496, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 373.4, "phase": "measure", "dt": 1.001, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.079, "ram_to_gpu_gbs": 3.424, "gpu_to_ram_gbs": 3.058, "decode_gbs": 4.0, "ssd_reads": 143, "ram_promotions": 16, "gpu_decodes": 159, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 16, "l2": 270}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 5919, "l2": 18081}, "missing": 220, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 41, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 139.3, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.516, "pcie_tx_gbs": 4.107}], "pulls": {"expired": 0, "need": 27300, "idle": 0, "idle_used": 0, "ram_first": 2314, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 6.65, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 150.55, "ssd_below_cap": 15.91, "nothing_needed": 0.27}, "sd": {"all_readers": 166.49, "some_readers": 6.24, "idle_nothing": 0.55, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.09}, "readers_busy": 12, "h2d_ssd_gbs_cum": 541242884096, "h2d_ram_gbs_cum": 49881341952, "h2d_used_cum": 591145795584, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 374.4, "phase": "measure", "dt": 1.001, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.185, "ram_to_gpu_gbs": 3.185, "gpu_to_ram_gbs": 2.734, "decode_gbs": 3.722, "ssd_reads": 148, "ram_promotions": 0, "gpu_decodes": 148, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 72, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 41, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 27, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 144.7, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 4.658, "pcie_tx_gbs": 1.282}], "pulls": {"expired": 0, "need": 27448, "idle": 0, "idle_used": 0, "ram_first": 2314, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 6.65, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 151.55, "ssd_below_cap": 15.91, "nothing_needed": 0.27}, "sd": {"all_readers": 167.49, "some_readers": 6.24, "idle_nothing": 0.55, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.09}, "readers_busy": 12, "h2d_ssd_gbs_cum": 544431398912, "h2d_ram_gbs_cum": 49881341952, "h2d_used_cum": 594334310400, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 375.4, "phase": "measure", "dt": 1.0, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.08, "ram_to_gpu_gbs": 3.511, "gpu_to_ram_gbs": 3.209, "decode_gbs": 4.127, "ssd_reads": 143, "ram_promotions": 20, "gpu_decodes": 164, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 20, "l2": 260}, "lkw": {"l0": 960, "arriving": 0, "ssd_inflight": 0, "l1": 7012, "l2": 16028}, "missing": 189, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 42, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 139.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.687, "pcie_tx_gbs": 1.383}], "pulls": {"expired": 0, "need": 27611, "idle": 0, "idle_used": 0, "ram_first": 2334, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 6.71, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 152.31, "ssd_below_cap": 16.09, "nothing_needed": 0.28}, "sd": {"all_readers": 168.42, "some_readers": 6.3, "idle_nothing": 0.56, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.09}, "readers_busy": 12, "h2d_ssd_gbs_cum": 547511795712, "h2d_ram_gbs_cum": 50312105984, "h2d_used_cum": 597845471232, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 376.4, "phase": "measure", "dt": 1.003, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.155, "ram_to_gpu_gbs": 3.155, "gpu_to_ram_gbs": 2.49, "decode_gbs": 3.663, "ssd_reads": 147, "ram_promotions": 0, "gpu_decodes": 146, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 42, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 42, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 27, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 143.1, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.492, "pcie_tx_gbs": 3.844}], "pulls": {"expired": 0, "need": 27758, "idle": 0, "idle_used": 0, "ram_first": 2334, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 6.71, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 153.31, "ssd_below_cap": 16.09, "nothing_needed": 0.28}, "sd": {"all_readers": 169.42, "some_readers": 6.3, "idle_nothing": 0.56, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.09}, "readers_busy": 12, "h2d_ssd_gbs_cum": 550678028288, "h2d_ram_gbs_cum": 50312105984, "h2d_used_cum": 600990175232, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 377.4, "phase": "measure", "dt": 1.002, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.096, "ram_to_gpu_gbs": 3.376, "gpu_to_ram_gbs": 2.924, "decode_gbs": 3.945, "ssd_reads": 144, "ram_promotions": 13, "gpu_decodes": 157, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 13, "l2": 273}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 5579, "l2": 18421}, "missing": 171, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 43, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 140.5, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.685, "pcie_tx_gbs": 12.745}], "pulls": {"expired": 0, "need": 27915, "idle": 0, "idle_used": 0, "ram_first": 2347, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 6.78, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 154.08, "ssd_below_cap": 16.24, "nothing_needed": 0.28}, "sd": {"all_readers": 170.35, "some_readers": 6.37, "idle_nothing": 0.56, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.09}, "readers_busy": 12, "h2d_ssd_gbs_cum": 553779478528, "h2d_ram_gbs_cum": 50592124928, "h2d_used_cum": 604371640320, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 378.4, "phase": "measure", "dt": 1.0, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.101, "ram_to_gpu_gbs": 3.101, "gpu_to_ram_gbs": 2.606, "decode_gbs": 3.649, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 27, "reads_in_flight": 27, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1051, "ram_cap": 1080, "layer": 43, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 145.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.666, "pcie_tx_gbs": 5.136}], "pulls": {"expired": 0, "need": 28050, "idle": 0, "idle_used": 0, "ram_first": 2347, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 6.8, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 155.02, "ssd_below_cap": 16.29, "nothing_needed": 0.28}, "sd": {"all_readers": 171.35, "some_readers": 6.37, "idle_nothing": 0.56, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.09}, "readers_busy": 12, "h2d_ssd_gbs_cum": 556881596416, "h2d_ram_gbs_cum": 50592124928, "h2d_used_cum": 607495290880, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 379.4, "phase": "measure", "dt": 1.0, "tokens": 6578, "tok_s": 0.0, "ssd_gbs": 3.168, "ram_to_gpu_gbs": 3.534, "gpu_to_ram_gbs": 3.189, "decode_gbs": 4.126, "ssd_reads": 147, "ram_promotions": 17, "gpu_decodes": 164, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 17, "l2": 262}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 7040, "l2": 16960}, "missing": 142, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 44, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 52, "mem_util_pct": 18, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 141.6, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.517, "pcie_tx_gbs": 2.563}], "pulls": {"expired": 0, "need": 28223, "idle": 0, "idle_used": 0, "ram_first": 2364, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 6.85, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 155.85, "ssd_below_cap": 16.4, "nothing_needed": 0.28}, "sd": {"all_readers": 172.28, "some_readers": 6.44, "idle_nothing": 0.57, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.09}, "readers_busy": 12, "h2d_ssd_gbs_cum": 560050860032, "h2d_ram_gbs_cum": 50958364672, "h2d_used_cum": 611030794240, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 380.4, "phase": "measure", "dt": 1.004, "tokens": 8197, "tok_s": 1612.851, "ssd_gbs": 3.052, "ram_to_gpu_gbs": 3.46, "gpu_to_ram_gbs": 3.116, "decode_gbs": 3.837, "ssd_reads": 142, "ram_promotions": 19, "gpu_decodes": 153, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 19, "l2": 269}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 2885, "l2": 21115}, "missing": 269, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1029, "ram_cap": 1080, "layer": 3, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 31, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 139.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.519, "pcie_tx_gbs": 3.964}], "pulls": {"expired": 0, "need": 28382, "idle": 0, "idle_used": 0, "ram_first": 2383, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 6.93, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 156.6, "ssd_below_cap": 16.58, "nothing_needed": 0.28}, "sd": {"all_readers": 173.19, "some_readers": 6.52, "idle_nothing": 0.58, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.09}, "readers_busy": 12, "h2d_ssd_gbs_cum": 563114217472, "h2d_ram_gbs_cum": 51367772160, "h2d_used_cum": 614331179008, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 381.4, "phase": "measure", "dt": 1.0, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.167, "ram_to_gpu_gbs": 3.167, "gpu_to_ram_gbs": 2.629, "decode_gbs": 3.875, "ssd_reads": 147, "ram_promotions": 0, "gpu_decodes": 154, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 122, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 3, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 26, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 152.5, "power_limit_w": 600.0, "temp_c": 32, "sm_clock_mhz": 2325, "pcie_rx_gbs": 2.327, "pcie_tx_gbs": 2.566}], "pulls": {"expired": 0, "need": 28529, "idle": 0, "idle_used": 0, "ram_first": 2383, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 6.93, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 157.61, "ssd_below_cap": 16.58, "nothing_needed": 0.28}, "sd": {"all_readers": 174.2, "some_readers": 6.52, "idle_nothing": 0.58, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.09}, "readers_busy": 12, "h2d_ssd_gbs_cum": 566282227712, "h2d_ram_gbs_cum": 51367772160, "h2d_used_cum": 617650020352, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 382.4, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.165, "ram_to_gpu_gbs": 3.876, "gpu_to_ram_gbs": 3.66, "decode_gbs": 4.526, "ssd_reads": 147, "ram_promotions": 33, "gpu_decodes": 180, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 33, "l2": 255}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 4406, "l2": 19594}, "missing": 231, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1041, "ram_cap": 1080, "layer": 4, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 139.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 3.526, "pcie_tx_gbs": 2.663}], "pulls": {"expired": 0, "need": 28700, "idle": 0, "idle_used": 0, "ram_first": 2416, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 7.0, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 158.37, "ssd_below_cap": 16.74, "nothing_needed": 0.29}, "sd": {"all_readers": 175.13, "some_readers": 6.58, "idle_nothing": 0.58, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.09}, "readers_busy": 12, "h2d_ssd_gbs_cum": 569450242048, "h2d_ram_gbs_cum": 52079112192, "h2d_used_cum": 621529370624, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 383.4, "phase": "measure", "dt": 1.0, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.103, "ram_to_gpu_gbs": 3.103, "gpu_to_ram_gbs": 2.672, "decode_gbs": 3.624, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 144, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 87, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 4, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 150.1, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.679, "pcie_tx_gbs": 2.564}], "pulls": {"expired": 0, "need": 28844, "idle": 0, "idle_used": 0, "ram_first": 2416, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 7.0, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 159.37, "ssd_below_cap": 16.74, "nothing_needed": 0.29}, "sd": {"all_readers": 176.13, "some_readers": 6.58, "idle_nothing": 0.58, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.09}, "readers_busy": 12, "h2d_ssd_gbs_cum": 572554186752, "h2d_ram_gbs_cum": 52079112192, "h2d_used_cum": 624633311232, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 384.4, "phase": "measure", "dt": 1.002, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.121, "ram_to_gpu_gbs": 3.766, "gpu_to_ram_gbs": 3.487, "decode_gbs": 4.373, "ssd_reads": 145, "ram_promotions": 30, "gpu_decodes": 174, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 30, "l2": 258}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 4133, "l2": 19867}, "missing": 200, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 5, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 31, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 139.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.344, "pcie_tx_gbs": 5.123}], "pulls": {"expired": 0, "need": 29015, "idle": 0, "idle_used": 0, "ram_first": 2446, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 7.06, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 160.14, "ssd_below_cap": 16.91, "nothing_needed": 0.29}, "sd": {"all_readers": 177.06, "some_readers": 6.65, "idle_nothing": 0.59, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.09}, "readers_busy": 12, "h2d_ssd_gbs_cum": 575679635456, "h2d_ram_gbs_cum": 52725772288, "h2d_used_cum": 628383870976, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 385.4, "phase": "measure", "dt": 1.0, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.104, "ram_to_gpu_gbs": 3.104, "gpu_to_ram_gbs": 2.586, "decode_gbs": 3.675, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 146, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 55, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 5, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 145.7, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.509, "pcie_tx_gbs": 1.432}], "pulls": {"expired": 0, "need": 29160, "idle": 0, "idle_used": 0, "ram_first": 2446, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 7.06, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 161.14, "ssd_below_cap": 16.91, "nothing_needed": 0.29}, "sd": {"all_readers": 178.05, "some_readers": 6.65, "idle_nothing": 0.59, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.09}, "readers_busy": 12, "h2d_ssd_gbs_cum": 578783645696, "h2d_ram_gbs_cum": 52725772288, "h2d_used_cum": 631530987520, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 386.4, "phase": "measure", "dt": 1.003, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.137, "ram_to_gpu_gbs": 3.847, "gpu_to_ram_gbs": 3.632, "decode_gbs": 4.467, "ssd_reads": 146, "ram_promotions": 33, "gpu_decodes": 178, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 33, "l2": 255}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 4760, "l2": 19240}, "missing": 164, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 6, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 64, "mem_util_pct": 21, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 145.1, "power_limit_w": 600.0, "temp_c": 32, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.345, "pcie_tx_gbs": 3.844}], "pulls": {"expired": 0, "need": 29335, "idle": 0, "idle_used": 0, "ram_first": 2479, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 7.13, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 161.9, "ssd_below_cap": 17.08, "nothing_needed": 0.29}, "sd": {"all_readers": 178.99, "some_readers": 6.71, "idle_nothing": 0.59, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.1}, "readers_busy": 12, "h2d_ssd_gbs_cum": 581930721280, "h2d_ram_gbs_cum": 53437112320, "h2d_used_cum": 635367849984, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 387.4, "phase": "measure", "dt": 1.003, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.117, "ram_to_gpu_gbs": 3.117, "gpu_to_ram_gbs": 2.515, "decode_gbs": 3.666, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 146, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 19, "reads_in_flight": 19, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1059, "ram_cap": 1080, "layer": 6, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 32, "mem_util_pct": 14, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 137.5, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 1.173, "pcie_tx_gbs": 4.15}], "pulls": {"expired": 0, "need": 29463, "idle": 0, "idle_used": 0, "ram_first": 2479, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 7.17, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 162.8, "ssd_below_cap": 17.14, "nothing_needed": 0.29}, "sd": {"all_readers": 180.0, "some_readers": 6.71, "idle_nothing": 0.59, "idle_no_ram_slot": 0.01, "idle_upload_backlog": 0.1}, "readers_busy": 12, "h2d_ssd_gbs_cum": 585056219136, "h2d_ram_gbs_cum": 53437112320, "h2d_used_cum": 638514900992, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 388.4, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.122, "ram_to_gpu_gbs": 3.962, "gpu_to_ram_gbs": 3.639, "decode_gbs": 4.627, "ssd_reads": 145, "ram_promotions": 39, "gpu_decodes": 184, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 39, "l2": 247}, "lkw": {"l0": 804, "arriving": 0, "ssd_inflight": 0, "l1": 7358, "l2": 15838}, "missing": 122, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 7, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 27, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 150.5, "power_limit_w": 600.0, "temp_c": 32, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.689, "pcie_tx_gbs": 2.62}], "pulls": {"expired": 0, "need": 29660, "idle": 0, "idle_used": 0, "ram_first": 2518, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 7.2, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 163.67, "ssd_below_cap": 17.24, "nothing_needed": 0.3}, "sd": {"all_readers": 180.92, "some_readers": 6.76, "idle_nothing": 0.6, "idle_no_ram_slot": 0.02, "idle_upload_backlog": 0.1}, "readers_busy": 12, "h2d_ssd_gbs_cum": 588181704704, "h2d_ram_gbs_cum": 54277750784, "h2d_used_cum": 642459471872, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 389.4, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.078, "ram_to_gpu_gbs": 3.917, "gpu_to_ram_gbs": 3.638, "decode_gbs": 4.6, "ssd_reads": 143, "ram_promotions": 39, "gpu_decodes": 183, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 39, "l2": 249}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 7696, "l2": 16304}, "missing": 227, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1032, "ram_cap": 1080, "layer": 8, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 139.9, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 5.841, "pcie_tx_gbs": 1.284}], "pulls": {"expired": 0, "need": 29839, "idle": 0, "idle_used": 0, "ram_first": 2557, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 7.27, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 164.43, "ssd_below_cap": 17.4, "nothing_needed": 0.3}, "sd": {"all_readers": 181.85, "some_readers": 6.83, "idle_nothing": 0.6, "idle_no_ram_slot": 0.02, "idle_upload_backlog": 0.1}, "readers_busy": 12, "h2d_ssd_gbs_cum": 591263903744, "h2d_ram_gbs_cum": 55118364672, "h2d_used_cum": 646403837952, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 390.4, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.144, "ram_to_gpu_gbs": 3.144, "gpu_to_ram_gbs": 2.584, "decode_gbs": 3.671, "ssd_reads": 146, "ram_promotions": 0, "gpu_decodes": 146, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 81, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 8, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 150.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.506, "pcie_tx_gbs": 0.03}], "pulls": {"expired": 0, "need": 29985, "idle": 0, "idle_used": 0, "ram_first": 2557, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 7.27, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 165.43, "ssd_below_cap": 17.4, "nothing_needed": 0.3}, "sd": {"all_readers": 182.85, "some_readers": 6.83, "idle_nothing": 0.6, "idle_no_ram_slot": 0.02, "idle_upload_backlog": 0.1}, "readers_busy": 12, "h2d_ssd_gbs_cum": 594410889216, "h2d_ram_gbs_cum": 55118364672, "h2d_used_cum": 649550823424, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 391.4, "phase": "measure", "dt": 1.002, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.119, "ram_to_gpu_gbs": 3.872, "gpu_to_ram_gbs": 3.527, "decode_gbs": 4.521, "ssd_reads": 145, "ram_promotions": 35, "gpu_decodes": 180, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 35, "l2": 250}, "lkw": {"l0": 1213, "arriving": 0, "ssd_inflight": 0, "l1": 6060, "l2": 16727}, "missing": 187, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 9, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 66, "mem_util_pct": 21, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 147.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 5.861, "pcie_tx_gbs": 1.282}], "pulls": {"expired": 0, "need": 30161, "idle": 0, "idle_used": 0, "ram_first": 2592, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 7.33, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 166.2, "ssd_below_cap": 17.58, "nothing_needed": 0.3}, "sd": {"all_readers": 183.77, "some_readers": 6.9, "idle_nothing": 0.61, "idle_no_ram_slot": 0.02, "idle_upload_backlog": 0.1}, "readers_busy": 12, "h2d_ssd_gbs_cum": 597536305152, "h2d_ram_gbs_cum": 55872802816, "h2d_used_cum": 653387563008, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 392.4, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.144, "ram_to_gpu_gbs": 3.144, "gpu_to_ram_gbs": 2.713, "decode_gbs": 3.697, "ssd_reads": 146, "ram_promotions": 0, "gpu_decodes": 147, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 41, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 9, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 134.5, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.507, "pcie_tx_gbs": 2.564}], "pulls": {"expired": 0, "need": 30307, "idle": 0, "idle_used": 0, "ram_first": 2592, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 7.33, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 167.2, "ssd_below_cap": 17.58, "nothing_needed": 0.3}, "sd": {"all_readers": 184.78, "some_readers": 6.9, "idle_nothing": 0.61, "idle_no_ram_slot": 0.02, "idle_upload_backlog": 0.1}, "readers_busy": 12, "h2d_ssd_gbs_cum": 600683286528, "h2d_ram_gbs_cum": 55872802816, "h2d_used_cum": 656556105728, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 393.4, "phase": "measure", "dt": 1.0, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.103, "ram_to_gpu_gbs": 3.749, "gpu_to_ram_gbs": 3.405, "decode_gbs": 4.429, "ssd_reads": 144, "ram_promotions": 30, "gpu_decodes": 176, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 30, "l2": 257}, "lkw": {"l0": 461, "arriving": 0, "ssd_inflight": 0, "l1": 5049, "l2": 18490}, "missing": 153, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 10, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 26, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 148.3, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.67, "pcie_tx_gbs": 2.566}], "pulls": {"expired": 0, "need": 30480, "idle": 0, "idle_used": 0, "ram_first": 2622, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 7.4, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 167.97, "ssd_below_cap": 17.74, "nothing_needed": 0.31}, "sd": {"all_readers": 185.7, "some_readers": 6.97, "idle_nothing": 0.61, "idle_no_ram_slot": 0.02, "idle_upload_backlog": 0.1}, "readers_busy": 12, "h2d_ssd_gbs_cum": 603787112448, "h2d_ram_gbs_cum": 56519458816, "h2d_used_cum": 660328140800, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 394.4, "phase": "measure", "dt": 1.0, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.125, "ram_to_gpu_gbs": 3.125, "gpu_to_ram_gbs": 2.78, "decode_gbs": 3.624, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 144, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 8, "reads_in_flight": 8, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1069, "ram_cap": 1080, "layer": 10, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 137.6, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.669, "pcie_tx_gbs": 3.829}], "pulls": {"expired": 0, "need": 30597, "idle": 0, "idle_used": 0, "ram_first": 2622, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 7.46, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 168.78, "ssd_below_cap": 17.87, "nothing_needed": 0.31}, "sd": {"all_readers": 186.68, "some_readers": 7.0, "idle_nothing": 0.61, "idle_no_ram_slot": 0.02, "idle_upload_backlog": 0.1}, "readers_busy": 8, "h2d_ssd_gbs_cum": 606912487424, "h2d_ram_gbs_cum": 56519458816, "h2d_used_cum": 663431966720, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 395.4, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.101, "ram_to_gpu_gbs": 3.683, "gpu_to_ram_gbs": 3.317, "decode_gbs": 4.376, "ssd_reads": 144, "ram_promotions": 27, "gpu_decodes": 174, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 27, "l2": 259}, "lkw": {"l0": 2376, "arriving": 0, "ssd_inflight": 0, "l1": 4888, "l2": 16736}, "missing": 123, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 11, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 137.6, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.669, "pcie_tx_gbs": 3.829}], "pulls": {"expired": 0, "need": 30795, "idle": 0, "idle_used": 0, "ram_first": 2649, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 7.47, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 169.73, "ssd_below_cap": 17.9, "nothing_needed": 0.31}, "sd": {"all_readers": 187.63, "some_readers": 7.04, "idle_nothing": 0.62, "idle_no_ram_slot": 0.02, "idle_upload_backlog": 0.1}, "readers_busy": 12, "h2d_ssd_gbs_cum": 610016444416, "h2d_ram_gbs_cum": 57101459456, "h2d_used_cum": 667139473408, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 396.4, "phase": "measure", "dt": 1.0, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.147, "ram_to_gpu_gbs": 3.858, "gpu_to_ram_gbs": 3.535, "decode_gbs": 4.581, "ssd_reads": 146, "ram_promotions": 33, "gpu_decodes": 182, "scratch_hits": 0, "lk": {"l0": 4, "arriving": 0, "ssd_inflight": 0, "l1": 33, "l2": 250}, "lkw": {"l0": 2140, "arriving": 0, "ssd_inflight": 0, "l1": 6898, "l2": 14962}, "missing": 227, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 12, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 27, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 147.0, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.517, "pcie_tx_gbs": 1.282}], "pulls": {"expired": 0, "need": 30972, "idle": 0, "idle_used": 0, "ram_first": 2682, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 7.54, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 170.49, "ssd_below_cap": 18.07, "nothing_needed": 0.31}, "sd": {"all_readers": 188.56, "some_readers": 7.1, "idle_nothing": 0.63, "idle_no_ram_slot": 0.02, "idle_upload_backlog": 0.1}, "readers_busy": 12, "h2d_ssd_gbs_cum": 613163446272, "h2d_ram_gbs_cum": 57812856832, "h2d_used_cum": 670976315392, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 397.4, "phase": "measure", "dt": 1.002, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.118, "ram_to_gpu_gbs": 3.118, "gpu_to_ram_gbs": 2.688, "decode_gbs": 3.641, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 82, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 12, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 31, "mem_util_pct": 14, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 146.7, "power_limit_w": 600.0, "temp_c": 32, "sm_clock_mhz": 2325, "pcie_rx_gbs": 2.345, "pcie_tx_gbs": 6.404}], "pulls": {"expired": 0, "need": 31117, "idle": 0, "idle_used": 0, "ram_first": 2682, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 7.54, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 171.49, "ssd_below_cap": 18.07, "nothing_needed": 0.31}, "sd": {"all_readers": 189.57, "some_readers": 7.1, "idle_nothing": 0.63, "idle_no_ram_slot": 0.02, "idle_upload_backlog": 0.1}, "readers_busy": 12, "h2d_ssd_gbs_cum": 616289202176, "h2d_ram_gbs_cum": 57812856832, "h2d_used_cum": 674102071296, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 398.4, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.145, "ram_to_gpu_gbs": 3.791, "gpu_to_ram_gbs": 3.619, "decode_gbs": 4.528, "ssd_reads": 146, "ram_promotions": 30, "gpu_decodes": 180, "scratch_hits": 0, "lk": {"l0": 4, "arriving": 0, "ssd_inflight": 0, "l1": 30, "l2": 252}, "lkw": {"l0": 3010, "arriving": 0, "ssd_inflight": 0, "l1": 6220, "l2": 14770}, "missing": 188, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 13, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 33, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 140.3, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 3.515, "pcie_tx_gbs": 2.562}], "pulls": {"expired": 0, "need": 31293, "idle": 0, "idle_used": 0, "ram_first": 2712, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 7.6, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 172.25, "ssd_below_cap": 18.25, "nothing_needed": 0.31}, "sd": {"all_readers": 190.48, "some_readers": 7.18, "idle_nothing": 0.63, "idle_no_ram_slot": 0.02, "idle_upload_backlog": 0.1}, "readers_busy": 12, "h2d_ssd_gbs_cum": 619436335104, "h2d_ram_gbs_cum": 58459492352, "h2d_used_cum": 677895843840, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 399.4, "phase": "measure", "dt": 1.0, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.104, "ram_to_gpu_gbs": 3.104, "gpu_to_ram_gbs": 2.694, "decode_gbs": 3.599, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 143, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 46, "reads_in_flight": 34, "uploads_pending": 2, "gpu_experts": 106, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 13, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 146.0, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.345, "pcie_tx_gbs": 5.228}], "pulls": {"expired": 0, "need": 31435, "idle": 0, "idle_used": 0, "ram_first": 2712, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 7.6, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 173.25, "ssd_below_cap": 18.25, "nothing_needed": 0.31}, "sd": {"all_readers": 191.48, "some_readers": 7.18, "idle_nothing": 0.63, "idle_no_ram_slot": 0.02, "idle_upload_backlog": 0.1}, "readers_busy": 12, "h2d_ssd_gbs_cum": 622540201984, "h2d_ram_gbs_cum": 58459492352, "h2d_used_cum": 680978145280, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 400.4, "phase": "measure", "dt": 1.0, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.081, "ram_to_gpu_gbs": 3.685, "gpu_to_ram_gbs": 3.383, "decode_gbs": 4.428, "ssd_reads": 143, "ram_promotions": 28, "gpu_decodes": 176, "scratch_hits": 0, "lk": {"l0": 4, "arriving": 0, "ssd_inflight": 0, "l1": 28, "l2": 254}, "lkw": {"l0": 3476, "arriving": 0, "ssd_inflight": 0, "l1": 5609, "l2": 14915}, "missing": 155, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 14, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 138.8, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 2.353, "pcie_tx_gbs": 3.842}], "pulls": {"expired": 0, "need": 31607, "idle": 0, "idle_used": 0, "ram_first": 2740, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 7.67, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 174.01, "ssd_below_cap": 18.42, "nothing_needed": 0.32}, "sd": {"all_readers": 192.41, "some_readers": 7.24, "idle_nothing": 0.64, "idle_no_ram_slot": 0.02, "idle_upload_backlog": 0.11}, "readers_busy": 12, "h2d_ssd_gbs_cum": 625622900736, "h2d_ram_gbs_cum": 59063132160, "h2d_used_cum": 684686028800, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 401.4, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.166, "ram_to_gpu_gbs": 3.166, "gpu_to_ram_gbs": 2.821, "decode_gbs": 3.722, "ssd_reads": 147, "ram_promotions": 0, "gpu_decodes": 148, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 8, "reads_in_flight": 8, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1069, "ram_cap": 1080, "layer": 14, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 145.9, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 5.677, "pcie_tx_gbs": 5.125}], "pulls": {"expired": 0, "need": 31726, "idle": 0, "idle_used": 0, "ram_first": 2740, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 7.73, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 174.82, "ssd_below_cap": 18.56, "nothing_needed": 0.32}, "sd": {"all_readers": 193.39, "some_readers": 7.26, "idle_nothing": 0.64, "idle_no_ram_slot": 0.02, "idle_upload_backlog": 0.11}, "readers_busy": 8, "h2d_ssd_gbs_cum": 628791889920, "h2d_ram_gbs_cum": 59063132160, "h2d_used_cum": 687876591616, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 402.4, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.102, "ram_to_gpu_gbs": 3.705, "gpu_to_ram_gbs": 3.382, "decode_gbs": 4.426, "ssd_reads": 144, "ram_promotions": 28, "gpu_decodes": 176, "scratch_hits": 0, "lk": {"l0": 4, "arriving": 0, "ssd_inflight": 0, "l1": 28, "l2": 255}, "lkw": {"l0": 2876, "arriving": 0, "ssd_inflight": 0, "l1": 6292, "l2": 14832}, "missing": 119, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 15, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 32, "mem_util_pct": 14, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 147.0, "power_limit_w": 600.0, "temp_c": 32, "sm_clock_mhz": 2325, "pcie_rx_gbs": 4.68, "pcie_tx_gbs": 2.743}], "pulls": {"expired": 0, "need": 31926, "idle": 0, "idle_used": 0, "ram_first": 2768, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 7.74, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 175.77, "ssd_below_cap": 18.58, "nothing_needed": 0.32}, "sd": {"all_readers": 194.34, "some_readers": 7.31, "idle_nothing": 0.64, "idle_no_ram_slot": 0.02, "idle_upload_backlog": 0.11}, "readers_busy": 12, "h2d_ssd_gbs_cum": 631896592384, "h2d_ram_gbs_cum": 59666915328, "h2d_used_cum": 691585077248, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 403.4, "phase": "measure", "dt": 1.003, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.139, "ram_to_gpu_gbs": 4.042, "gpu_to_ram_gbs": 3.827, "decode_gbs": 4.743, "ssd_reads": 146, "ram_promotions": 42, "gpu_decodes": 189, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 42, "l2": 243}, "lkw": {"l0": 860, "arriving": 0, "ssd_inflight": 0, "l1": 9018, "l2": 14122}, "missing": 216, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1037, "ram_cap": 1080, "layer": 16, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 136.9, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.509, "pcie_tx_gbs": 1.283}], "pulls": {"expired": 0, "need": 32108, "idle": 0, "idle_used": 0, "ram_first": 2810, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 7.82, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 176.53, "ssd_below_cap": 18.74, "nothing_needed": 0.32}, "sd": {"all_readers": 195.27, "some_readers": 7.36, "idle_nothing": 0.65, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.11}, "readers_busy": 12, "h2d_ssd_gbs_cum": 635044605952, "h2d_ram_gbs_cum": 60572557312, "h2d_used_cum": 695638732800, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 404.4, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.122, "ram_to_gpu_gbs": 3.122, "gpu_to_ram_gbs": 2.649, "decode_gbs": 3.645, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 71, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 16, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 31, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 150.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 1.183, "pcie_tx_gbs": 4.036}], "pulls": {"expired": 0, "need": 32253, "idle": 0, "idle_used": 0, "ram_first": 2810, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 7.82, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 177.54, "ssd_below_cap": 18.74, "nothing_needed": 0.32}, "sd": {"all_readers": 196.27, "some_readers": 7.36, "idle_nothing": 0.65, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.11}, "readers_busy": 12, "h2d_ssd_gbs_cum": 638170882048, "h2d_ram_gbs_cum": 60572557312, "h2d_used_cum": 698765008896, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 405.4, "phase": "measure", "dt": 1.0, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.126, "ram_to_gpu_gbs": 3.924, "gpu_to_ram_gbs": 3.622, "decode_gbs": 4.657, "ssd_reads": 145, "ram_promotions": 37, "gpu_decodes": 185, "scratch_hits": 0, "lk": {"l0": 3, "arriving": 0, "ssd_inflight": 0, "l1": 37, "l2": 244}, "lkw": {"l0": 1801, "arriving": 0, "ssd_inflight": 0, "l1": 8169, "l2": 14030}, "missing": 170, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 17, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 138.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 2.352, "pcie_tx_gbs": 2.564}], "pulls": {"expired": 0, "need": 32432, "idle": 0, "idle_used": 0, "ram_first": 2847, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 7.88, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 178.31, "ssd_below_cap": 18.9, "nothing_needed": 0.32}, "sd": {"all_readers": 197.19, "some_readers": 7.44, "idle_nothing": 0.65, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.11}, "readers_busy": 12, "h2d_ssd_gbs_cum": 641297219584, "h2d_ram_gbs_cum": 61370322944, "h2d_used_cum": 702689112064, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 406.4, "phase": "measure", "dt": 1.0, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.082, "ram_to_gpu_gbs": 3.082, "gpu_to_ram_gbs": 2.543, "decode_gbs": 3.599, "ssd_reads": 143, "ram_promotions": 0, "gpu_decodes": 143, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 27, "reads_in_flight": 27, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1051, "ram_cap": 1080, "layer": 17, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 147.6, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.343, "pcie_tx_gbs": 2.567}], "pulls": {"expired": 0, "need": 32566, "idle": 0, "idle_used": 0, "ram_first": 2847, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 7.9, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 179.23, "ssd_below_cap": 18.96, "nothing_needed": 0.32}, "sd": {"all_readers": 198.19, "some_readers": 7.44, "idle_nothing": 0.65, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.11}, "readers_busy": 12, "h2d_ssd_gbs_cum": 644380246016, "h2d_ram_gbs_cum": 61370322944, "h2d_used_cum": 705772138496, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 407.4, "phase": "measure", "dt": 1.003, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.16, "ram_to_gpu_gbs": 3.826, "gpu_to_ram_gbs": 3.611, "decode_gbs": 4.593, "ssd_reads": 147, "ram_promotions": 31, "gpu_decodes": 183, "scratch_hits": 0, "lk": {"l0": 6, "arriving": 0, "ssd_inflight": 0, "l1": 31, "l2": 246}, "lkw": {"l0": 4919, "arriving": 0, "ssd_inflight": 0, "l1": 7417, "l2": 11664}, "missing": 126, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 18, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 148.9, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 4.673, "pcie_tx_gbs": 2.666}], "pulls": {"expired": 0, "need": 32753, "idle": 0, "idle_used": 0, "ram_first": 2878, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 7.95, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 180.07, "ssd_below_cap": 19.07, "nothing_needed": 0.33}, "sd": {"all_readers": 199.12, "some_readers": 7.51, "idle_nothing": 0.66, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.11}, "readers_busy": 12, "h2d_ssd_gbs_cum": 647549435904, "h2d_ram_gbs_cum": 62038630400, "h2d_used_cum": 709588082688, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 408.4, "phase": "measure", "dt": 1.002, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.119, "ram_to_gpu_gbs": 3.7, "gpu_to_ram_gbs": 3.378, "decode_gbs": 4.471, "ssd_reads": 145, "ram_promotions": 27, "gpu_decodes": 178, "scratch_hits": 0, "lk": {"l0": 6, "arriving": 0, "ssd_inflight": 0, "l1": 27, "l2": 243}, "lkw": {"l0": 4022, "arriving": 0, "ssd_inflight": 0, "l1": 6818, "l2": 13160}, "missing": 224, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 19, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 14, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 138.0, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.338, "pcie_tx_gbs": 3.947}], "pulls": {"expired": 0, "need": 32925, "idle": 0, "idle_used": 0, "ram_first": 2905, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 8.02, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 180.84, "ssd_below_cap": 19.23, "nothing_needed": 0.33}, "sd": {"all_readers": 200.03, "some_readers": 7.59, "idle_nothing": 0.66, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.11}, "readers_busy": 12, "h2d_ssd_gbs_cum": 650675478528, "h2d_ram_gbs_cum": 62620819456, "h2d_used_cum": 713296302080, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 409.4, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.102, "ram_to_gpu_gbs": 3.102, "gpu_to_ram_gbs": 2.714, "decode_gbs": 3.646, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 80, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 19, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 32, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 147.3, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.672, "pcie_tx_gbs": 3.948}], "pulls": {"expired": 0, "need": 33069, "idle": 0, "idle_used": 0, "ram_first": 2905, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 8.02, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 181.84, "ssd_below_cap": 19.23, "nothing_needed": 0.33}, "sd": {"all_readers": 201.03, "some_readers": 7.59, "idle_nothing": 0.66, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.11}, "readers_busy": 12, "h2d_ssd_gbs_cum": 653780508672, "h2d_ram_gbs_cum": 62620819456, "h2d_used_cum": 716422897664, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 410.4, "phase": "measure", "dt": 1.0, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.147, "ram_to_gpu_gbs": 3.858, "gpu_to_ram_gbs": 3.6, "decode_gbs": 4.53, "ssd_reads": 146, "ram_promotions": 33, "gpu_decodes": 180, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 33, "l2": 249}, "lkw": {"l0": 1357, "arriving": 0, "ssd_inflight": 0, "l1": 7836, "l2": 14807}, "missing": 184, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 20, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 138.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 24.202, "pcie_tx_gbs": 3.847}], "pulls": {"expired": 0, "need": 33245, "idle": 0, "idle_used": 0, "ram_first": 2938, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 8.1, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 182.59, "ssd_below_cap": 19.4, "nothing_needed": 0.33}, "sd": {"all_readers": 201.96, "some_readers": 7.66, "idle_nothing": 0.67, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.11}, "readers_busy": 12, "h2d_ssd_gbs_cum": 656928083968, "h2d_ram_gbs_cum": 63332139008, "h2d_used_cum": 720260231168, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 411.4, "phase": "measure", "dt": 1.0, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.125, "ram_to_gpu_gbs": 3.125, "gpu_to_ram_gbs": 2.866, "decode_gbs": 3.649, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 39, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 20, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 144.7, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.557, "pcie_tx_gbs": 3.846}], "pulls": {"expired": 0, "need": 33390, "idle": 0, "idle_used": 0, "ram_first": 2938, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 8.1, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 183.59, "ssd_below_cap": 19.4, "nothing_needed": 0.33}, "sd": {"all_readers": 202.96, "some_readers": 7.66, "idle_nothing": 0.67, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.11}, "readers_busy": 12, "h2d_ssd_gbs_cum": 660053692416, "h2d_ram_gbs_cum": 63332139008, "h2d_used_cum": 723385847808, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 412.4, "phase": "measure", "dt": 1.0, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.125, "ram_to_gpu_gbs": 3.857, "gpu_to_ram_gbs": 3.642, "decode_gbs": 4.555, "ssd_reads": 145, "ram_promotions": 34, "gpu_decodes": 181, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 34, "l2": 246}, "lkw": {"l0": 573, "arriving": 0, "ssd_inflight": 0, "l1": 8651, "l2": 14776}, "missing": 139, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 21, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 27, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 145.3, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 1.179, "pcie_tx_gbs": 2.562}], "pulls": {"expired": 0, "need": 33569, "idle": 0, "idle_used": 0, "ram_first": 2972, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 8.17, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 184.37, "ssd_below_cap": 19.55, "nothing_needed": 0.34}, "sd": {"all_readers": 203.87, "some_readers": 7.73, "idle_nothing": 0.67, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.11}, "readers_busy": 12, "h2d_ssd_gbs_cum": 663179030528, "h2d_ram_gbs_cum": 64065028096, "h2d_used_cum": 727265628160, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 413.4, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.122, "ram_to_gpu_gbs": 3.832, "gpu_to_ram_gbs": 3.445, "decode_gbs": 4.476, "ssd_reads": 145, "ram_promotions": 33, "gpu_decodes": 178, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 33, "l2": 244}, "lkw": {"l0": 878, "arriving": 0, "ssd_inflight": 0, "l1": 7461, "l2": 15661}, "missing": 239, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1023, "ram_cap": 1080, "layer": 22, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 27, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 140.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 5.039, "pcie_tx_gbs": 1.282}], "pulls": {"expired": 0, "need": 33744, "idle": 0, "idle_used": 0, "ram_first": 3005, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 8.24, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 185.14, "ssd_below_cap": 19.72, "nothing_needed": 0.34}, "sd": {"all_readers": 204.8, "some_readers": 7.8, "idle_nothing": 0.68, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.11}, "readers_busy": 12, "h2d_ssd_gbs_cum": 666304253952, "h2d_ram_gbs_cum": 64776208384, "h2d_used_cum": 731080482816, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 414.4, "phase": "measure", "dt": 1.0, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.103, "ram_to_gpu_gbs": 3.103, "gpu_to_ram_gbs": 2.65, "decode_gbs": 3.649, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 94, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 22, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 149.0, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 1.173, "pcie_tx_gbs": 5.223}], "pulls": {"expired": 0, "need": 33889, "idle": 0, "idle_used": 0, "ram_first": 3005, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 8.24, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 186.14, "ssd_below_cap": 19.72, "nothing_needed": 0.34}, "sd": {"all_readers": 205.8, "some_readers": 7.8, "idle_nothing": 0.68, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.11}, "readers_busy": 12, "h2d_ssd_gbs_cum": 669407981568, "h2d_ram_gbs_cum": 64776208384, "h2d_used_cum": 734205759488, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 415.4, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.122, "ram_to_gpu_gbs": 3.639, "gpu_to_ram_gbs": 3.273, "decode_gbs": 4.275, "ssd_reads": 145, "ram_promotions": 24, "gpu_decodes": 170, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 24, "l2": 260}, "lkw": {"l0": 671, "arriving": 0, "ssd_inflight": 0, "l1": 5749, "l2": 17580}, "missing": 209, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 23, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 145.2, "power_limit_w": 600.0, "temp_c": 32, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.524, "pcie_tx_gbs": 5.121}], "pulls": {"expired": 0, "need": 34058, "idle": 0, "idle_used": 0, "ram_first": 3029, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 8.31, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 186.9, "ssd_below_cap": 19.88, "nothing_needed": 0.34}, "sd": {"all_readers": 206.73, "some_readers": 7.87, "idle_nothing": 0.69, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.11}, "readers_busy": 12, "h2d_ssd_gbs_cum": 672533061632, "h2d_ram_gbs_cum": 65293365248, "h2d_used_cum": 737847996416, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 416.4, "phase": "measure", "dt": 1.002, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.141, "ram_to_gpu_gbs": 3.141, "gpu_to_ram_gbs": 2.732, "decode_gbs": 3.643, "ssd_reads": 146, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 64, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 23, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 137.9, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 1.184, "pcie_tx_gbs": 3.946}], "pulls": {"expired": 0, "need": 34203, "idle": 0, "idle_used": 0, "ram_first": 3029, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 8.31, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 187.9, "ssd_below_cap": 19.88, "nothing_needed": 0.34}, "sd": {"all_readers": 207.73, "some_readers": 7.87, "idle_nothing": 0.69, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.11}, "readers_busy": 12, "h2d_ssd_gbs_cum": 675679678464, "h2d_ram_gbs_cum": 65293365248, "h2d_used_cum": 740973072384, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 417.4, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.144, "ram_to_gpu_gbs": 3.618, "gpu_to_ram_gbs": 3.316, "decode_gbs": 4.251, "ssd_reads": 146, "ram_promotions": 22, "gpu_decodes": 169, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 22, "l2": 259}, "lkw": {"l0": 1590, "arriving": 0, "ssd_inflight": 0, "l1": 6398, "l2": 16012}, "missing": 177, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 24, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 146.7, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 1.182, "pcie_tx_gbs": 2.765}], "pulls": {"expired": 0, "need": 34371, "idle": 0, "idle_used": 0, "ram_first": 3051, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 8.37, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 188.67, "ssd_below_cap": 20.05, "nothing_needed": 0.34}, "sd": {"all_readers": 208.66, "some_readers": 7.93, "idle_nothing": 0.69, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.11}, "readers_busy": 12, "h2d_ssd_gbs_cum": 678826549248, "h2d_ram_gbs_cum": 65767583744, "h2d_used_cum": 744572604416, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 418.4, "phase": "measure", "dt": 1.0, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.082, "ram_to_gpu_gbs": 3.082, "gpu_to_ram_gbs": 2.737, "decode_gbs": 3.65, "ssd_reads": 143, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 33, "reads_in_flight": 33, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1046, "ram_cap": 1080, "layer": 24, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 31, "mem_util_pct": 14, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 135.5, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 5.839, "pcie_tx_gbs": 2.186}], "pulls": {"expired": 0, "need": 34512, "idle": 0, "idle_used": 0, "ram_first": 3051, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 8.38, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 189.65, "ssd_below_cap": 20.06, "nothing_needed": 0.34}, "sd": {"all_readers": 209.66, "some_readers": 7.93, "idle_nothing": 0.69, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.11}, "readers_busy": 12, "h2d_ssd_gbs_cum": 681909149696, "h2d_ram_gbs_cum": 65767583744, "h2d_used_cum": 747698302976, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 419.4, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.146, "ram_to_gpu_gbs": 3.727, "gpu_to_ram_gbs": 3.404, "decode_gbs": 4.377, "ssd_reads": 146, "ram_promotions": 27, "gpu_decodes": 174, "scratch_hits": 0, "lk": {"l0": 3, "arriving": 0, "ssd_inflight": 0, "l1": 27, "l2": 254}, "lkw": {"l0": 2565, "arriving": 0, "ssd_inflight": 0, "l1": 6816, "l2": 14619}, "missing": 142, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 25, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 143.6, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 1.181, "pcie_tx_gbs": 5.139}], "pulls": {"expired": 0, "need": 34687, "idle": 0, "idle_used": 0, "ram_first": 3078, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 8.44, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 190.44, "ssd_below_cap": 20.21, "nothing_needed": 0.35}, "sd": {"all_readers": 210.59, "some_readers": 8.0, "idle_nothing": 0.7, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.11}, "readers_busy": 12, "h2d_ssd_gbs_cum": 685056913408, "h2d_ram_gbs_cum": 66349789184, "h2d_used_cum": 751385153536, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 420.4, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.057, "ram_to_gpu_gbs": 3.595, "gpu_to_ram_gbs": 3.229, "decode_gbs": 4.273, "ssd_reads": 142, "ram_promotions": 25, "gpu_decodes": 170, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 25, "l2": 257}, "lkw": {"l0": 921, "arriving": 0, "ssd_inflight": 0, "l1": 6709, "l2": 16370}, "missing": 256, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1027, "ram_cap": 1080, "layer": 26, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 25, "mem_util_pct": 10, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 139.1, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 15.15, "pcie_tx_gbs": 16.737}], "pulls": {"expired": 0, "need": 34855, "idle": 0, "idle_used": 0, "ram_first": 3103, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 8.51, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 191.21, "ssd_below_cap": 20.37, "nothing_needed": 0.35}, "sd": {"all_readers": 211.52, "some_readers": 8.06, "idle_nothing": 0.7, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.11}, "readers_busy": 12, "h2d_ssd_gbs_cum": 688118321152, "h2d_ram_gbs_cum": 66888822784, "h2d_used_cum": 755028713472, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 421.4, "phase": "measure", "dt": 1.002, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.163, "ram_to_gpu_gbs": 3.163, "gpu_to_ram_gbs": 2.732, "decode_gbs": 3.692, "ssd_reads": 147, "ram_promotions": 0, "gpu_decodes": 147, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 109, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 26, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 25, "mem_util_pct": 10, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 139.1, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 15.15, "pcie_tx_gbs": 16.737}], "pulls": {"expired": 0, "need": 35002, "idle": 0, "idle_used": 0, "ram_first": 3103, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 8.51, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 192.21, "ssd_below_cap": 20.37, "nothing_needed": 0.35}, "sd": {"all_readers": 212.52, "some_readers": 8.06, "idle_nothing": 0.7, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.11}, "readers_busy": 12, "h2d_ssd_gbs_cum": 691287838720, "h2d_ram_gbs_cum": 66888822784, "h2d_used_cum": 758198231040, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 422.4, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.101, "ram_to_gpu_gbs": 3.597, "gpu_to_ram_gbs": 3.446, "decode_gbs": 4.2, "ssd_reads": 144, "ram_promotions": 23, "gpu_decodes": 167, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 23, "l2": 259}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 7513, "l2": 16487}, "missing": 224, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 27, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 27, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 146.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.687, "pcie_tx_gbs": 1.283}], "pulls": {"expired": 0, "need": 35169, "idle": 0, "idle_used": 0, "ram_first": 3126, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 8.58, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 192.99, "ssd_below_cap": 20.52, "nothing_needed": 0.35}, "sd": {"all_readers": 213.46, "some_readers": 8.12, "idle_nothing": 0.71, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.11}, "readers_busy": 12, "h2d_ssd_gbs_cum": 694392340480, "h2d_ram_gbs_cum": 67384705024, "h2d_used_cum": 761798615040, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 423.4, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.166, "ram_to_gpu_gbs": 3.166, "gpu_to_ram_gbs": 2.477, "decode_gbs": 3.697, "ssd_reads": 147, "ram_promotions": 0, "gpu_decodes": 147, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 77, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 27, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 31, "mem_util_pct": 14, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 145.6, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 1.173, "pcie_tx_gbs": 2.675}], "pulls": {"expired": 0, "need": 35316, "idle": 0, "idle_used": 0, "ram_first": 3126, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 8.58, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 193.99, "ssd_below_cap": 20.52, "nothing_needed": 0.35}, "sd": {"all_readers": 214.46, "some_readers": 8.12, "idle_nothing": 0.71, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.11}, "readers_busy": 12, "h2d_ssd_gbs_cum": 697561796608, "h2d_ram_gbs_cum": 67384705024, "h2d_used_cum": 764968071168, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 424.4, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.123, "ram_to_gpu_gbs": 3.532, "gpu_to_ram_gbs": 3.209, "decode_gbs": 4.124, "ssd_reads": 145, "ram_promotions": 19, "gpu_decodes": 164, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 19, "l2": 264}, "lkw": {"l0": 982, "arriving": 0, "ssd_inflight": 0, "l1": 6027, "l2": 16991}, "missing": 196, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 28, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 33, "mem_util_pct": 15, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 138.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.354, "pcie_tx_gbs": 2.665}], "pulls": {"expired": 0, "need": 35480, "idle": 0, "idle_used": 0, "ram_first": 3145, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 8.65, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 194.76, "ssd_below_cap": 20.68, "nothing_needed": 0.35}, "sd": {"all_readers": 215.39, "some_readers": 8.18, "idle_nothing": 0.72, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.12}, "readers_busy": 12, "h2d_ssd_gbs_cum": 700687974400, "h2d_ram_gbs_cum": 67794325504, "h2d_used_cum": 768482320384, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 425.4, "phase": "measure", "dt": 1.0, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.104, "ram_to_gpu_gbs": 3.104, "gpu_to_ram_gbs": 2.694, "decode_gbs": 3.624, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 144, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 53, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 28, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 142.9, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 3.527, "pcie_tx_gbs": 5.126}], "pulls": {"expired": 0, "need": 35623, "idle": 0, "idle_used": 0, "ram_first": 3145, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 8.65, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 195.76, "ssd_below_cap": 20.68, "nothing_needed": 0.35}, "sd": {"all_readers": 216.39, "some_readers": 8.18, "idle_nothing": 0.72, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.12}, "readers_busy": 12, "h2d_ssd_gbs_cum": 703792627712, "h2d_ram_gbs_cum": 67794325504, "h2d_used_cum": 771586961408, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 426.4, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.1, "ram_to_gpu_gbs": 3.53, "gpu_to_ram_gbs": 3.121, "decode_gbs": 4.173, "ssd_reads": 144, "ram_promotions": 20, "gpu_decodes": 166, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 20, "l2": 262}, "lkw": {"l0": 710, "arriving": 0, "ssd_inflight": 0, "l1": 4982, "l2": 18308}, "missing": 170, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 29, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 137.6, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 3.518, "pcie_tx_gbs": 3.843}], "pulls": {"expired": 0, "need": 35788, "idle": 0, "idle_used": 0, "ram_first": 3165, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 8.72, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 196.52, "ssd_below_cap": 20.84, "nothing_needed": 0.36}, "sd": {"all_readers": 217.33, "some_readers": 8.24, "idle_nothing": 0.72, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.12}, "readers_busy": 12, "h2d_ssd_gbs_cum": 706896707584, "h2d_ram_gbs_cum": 68225515520, "h2d_used_cum": 775143792640, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 427.4, "phase": "measure", "dt": 1.0, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.169, "ram_to_gpu_gbs": 3.169, "gpu_to_ram_gbs": 2.802, "decode_gbs": 3.65, "ssd_reads": 147, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 24, "reads_in_flight": 23, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1054, "ram_cap": 1080, "layer": 29, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 32, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 143.9, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.68, "pcie_tx_gbs": 2.562}], "pulls": {"expired": 0, "need": 35922, "idle": 0, "idle_used": 0, "ram_first": 3165, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 8.75, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 197.44, "ssd_below_cap": 20.89, "nothing_needed": 0.36}, "sd": {"all_readers": 218.32, "some_readers": 8.24, "idle_nothing": 0.72, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.12}, "readers_busy": 12, "h2d_ssd_gbs_cum": 710065516544, "h2d_ram_gbs_cum": 68225515520, "h2d_used_cum": 778269499392, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 428.4, "phase": "measure", "dt": 1.002, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.142, "ram_to_gpu_gbs": 3.616, "gpu_to_ram_gbs": 3.185, "decode_gbs": 4.247, "ssd_reads": 146, "ram_promotions": 22, "gpu_decodes": 169, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 22, "l2": 259}, "lkw": {"l0": 1564, "arriving": 0, "ssd_inflight": 0, "l1": 6366, "l2": 16070}, "missing": 138, "reads_in_flight": 34, "uploads_pending": 2, "gpu_experts": 106, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 30, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 52, "mem_util_pct": 18, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 144.3, "power_limit_w": 600.0, "temp_c": 33, "sm_clock_mhz": 2332, "pcie_rx_gbs": 5.713, "pcie_tx_gbs": 1.288}], "pulls": {"expired": 0, "need": 36101, "idle": 0, "idle_used": 0, "ram_first": 3187, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 8.8, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 198.29, "ssd_below_cap": 21.0, "nothing_needed": 0.36}, "sd": {"all_readers": 219.25, "some_readers": 8.31, "idle_nothing": 0.73, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.12}, "readers_busy": 12, "h2d_ssd_gbs_cum": 713212985344, "h2d_ram_gbs_cum": 68699811840, "h2d_used_cum": 781869690880, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 429.4, "phase": "measure", "dt": 1.004, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.114, "ram_to_gpu_gbs": 3.586, "gpu_to_ram_gbs": 3.35, "decode_gbs": 4.288, "ssd_reads": 145, "ram_promotions": 22, "gpu_decodes": 171, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 22, "l2": 260}, "lkw": {"l0": 2482, "arriving": 0, "ssd_inflight": 0, "l1": 5529, "l2": 15989}, "missing": 251, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1039, "ram_cap": 1080, "layer": 31, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 136.8, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 1.644, "pcie_tx_gbs": 3.847}], "pulls": {"expired": 0, "need": 36270, "idle": 0, "idle_used": 0, "ram_first": 3209, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 8.87, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 199.05, "ssd_below_cap": 21.17, "nothing_needed": 0.36}, "sd": {"all_readers": 220.18, "some_readers": 8.38, "idle_nothing": 0.73, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.12}, "readers_busy": 12, "h2d_ssd_gbs_cum": 716338737152, "h2d_ram_gbs_cum": 69174104064, "h2d_used_cum": 785512853504, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 430.4, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.08, "ram_to_gpu_gbs": 3.08, "gpu_to_ram_gbs": 2.671, "decode_gbs": 3.621, "ssd_reads": 143, "ram_promotions": 0, "gpu_decodes": 144, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 108, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 31, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 32, "mem_util_pct": 14, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 145.1, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 1.183, "pcie_tx_gbs": 3.265}], "pulls": {"expired": 0, "need": 36413, "idle": 0, "idle_used": 0, "ram_first": 3209, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 8.87, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 200.05, "ssd_below_cap": 21.17, "nothing_needed": 0.36}, "sd": {"all_readers": 221.18, "some_readers": 8.38, "idle_nothing": 0.73, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.12}, "readers_busy": 12, "h2d_ssd_gbs_cum": 719421669376, "h2d_ram_gbs_cum": 69174104064, "h2d_used_cum": 788617342976, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 431.5, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.122, "ram_to_gpu_gbs": 3.553, "gpu_to_ram_gbs": 3.23, "decode_gbs": 4.149, "ssd_reads": 145, "ram_promotions": 20, "gpu_decodes": 165, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 20, "l2": 267}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 5823, "l2": 18177}, "missing": 230, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 32, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 139.9, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.346, "pcie_tx_gbs": 2.564}], "pulls": {"expired": 0, "need": 36578, "idle": 0, "idle_used": 0, "ram_first": 3229, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 8.94, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 200.83, "ssd_below_cap": 21.32, "nothing_needed": 0.36}, "sd": {"all_readers": 222.11, "some_readers": 8.45, "idle_nothing": 0.74, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.12}, "readers_busy": 12, "h2d_ssd_gbs_cum": 722547425280, "h2d_ram_gbs_cum": 69605302272, "h2d_used_cum": 792174297088, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 432.5, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.102, "ram_to_gpu_gbs": 3.102, "gpu_to_ram_gbs": 2.65, "decode_gbs": 3.597, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 143, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 86, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 32, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 33, "mem_util_pct": 15, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 144.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 6.659, "pcie_tx_gbs": 2.664}], "pulls": {"expired": 0, "need": 36722, "idle": 0, "idle_used": 0, "ram_first": 3229, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 8.94, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 201.83, "ssd_below_cap": 21.32, "nothing_needed": 0.36}, "sd": {"all_readers": 223.11, "some_readers": 8.45, "idle_nothing": 0.74, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.12}, "readers_busy": 12, "h2d_ssd_gbs_cum": 725651652608, "h2d_ram_gbs_cum": 69605302272, "h2d_used_cum": 795256967168, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 433.5, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.144, "ram_to_gpu_gbs": 3.639, "gpu_to_ram_gbs": 3.359, "decode_gbs": 4.25, "ssd_reads": 146, "ram_promotions": 23, "gpu_decodes": 169, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 23, "l2": 259}, "lkw": {"l0": 1232, "arriving": 0, "ssd_inflight": 0, "l1": 7371, "l2": 15397}, "missing": 200, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 33, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 27, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 136.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.721, "pcie_tx_gbs": 1.292}], "pulls": {"expired": 0, "need": 36890, "idle": 0, "idle_used": 0, "ram_first": 3252, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 9.0, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 202.59, "ssd_below_cap": 21.49, "nothing_needed": 0.37}, "sd": {"all_readers": 224.04, "some_readers": 8.51, "idle_nothing": 0.75, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.12}, "readers_busy": 12, "h2d_ssd_gbs_cum": 728798978048, "h2d_ram_gbs_cum": 70101180416, "h2d_used_cum": 798878613504, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 434.5, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.124, "ram_to_gpu_gbs": 3.124, "gpu_to_ram_gbs": 2.607, "decode_gbs": 3.698, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 147, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 54, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 33, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 143.1, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.522, "pcie_tx_gbs": 1.388}], "pulls": {"expired": 0, "need": 37036, "idle": 0, "idle_used": 0, "ram_first": 3252, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 9.0, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 203.59, "ssd_below_cap": 21.49, "nothing_needed": 0.37}, "sd": {"all_readers": 225.04, "some_readers": 8.51, "idle_nothing": 0.75, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.12}, "readers_busy": 12, "h2d_ssd_gbs_cum": 731924795392, "h2d_ram_gbs_cum": 70101180416, "h2d_used_cum": 802047545344, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 435.5, "phase": "measure", "dt": 1.0, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.125, "ram_to_gpu_gbs": 3.513, "gpu_to_ram_gbs": 3.276, "decode_gbs": 4.177, "ssd_reads": 145, "ram_promotions": 18, "gpu_decodes": 166, "scratch_hits": 0, "lk": {"l0": 3, "arriving": 0, "ssd_inflight": 0, "l1": 18, "l2": 261}, "lkw": {"l0": 2685, "arriving": 0, "ssd_inflight": 0, "l1": 5356, "l2": 15959}, "missing": 170, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 34, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 42, "mem_util_pct": 17, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 142.7, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 1.181, "pcie_tx_gbs": 3.844}], "pulls": {"expired": 0, "need": 37199, "idle": 0, "idle_used": 0, "ram_first": 3270, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 9.07, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 204.35, "ssd_below_cap": 21.65, "nothing_needed": 0.37}, "sd": {"all_readers": 225.96, "some_readers": 8.58, "idle_nothing": 0.75, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.12}, "readers_busy": 12, "h2d_ssd_gbs_cum": 735050690560, "h2d_ram_gbs_cum": 70489268224, "h2d_used_cum": 805561528320, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 436.5, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.123, "ram_to_gpu_gbs": 3.123, "gpu_to_ram_gbs": 2.8, "decode_gbs": 3.597, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 143, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 26, "reads_in_flight": 25, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1053, "ram_cap": 1080, "layer": 34, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 24, "mem_util_pct": 10, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 140.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.507, "pcie_tx_gbs": 2.801}], "pulls": {"expired": 0, "need": 37333, "idle": 0, "idle_used": 0, "ram_first": 3270, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 9.09, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 205.29, "ssd_below_cap": 21.7, "nothing_needed": 0.37}, "sd": {"all_readers": 226.97, "some_readers": 8.58, "idle_nothing": 0.75, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.12}, "readers_busy": 12, "h2d_ssd_gbs_cum": 738176618496, "h2d_ram_gbs_cum": 70489268224, "h2d_used_cum": 808644329472, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 437.5, "phase": "measure", "dt": 1.0, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.125, "ram_to_gpu_gbs": 3.449, "gpu_to_ram_gbs": 3.147, "decode_gbs": 4.077, "ssd_reads": 145, "ram_promotions": 15, "gpu_decodes": 162, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 15, "l2": 267}, "lkw": {"l0": 2199, "arriving": 0, "ssd_inflight": 0, "l1": 4880, "l2": 16921}, "missing": 149, "reads_in_flight": 34, "uploads_pending": 2, "gpu_experts": 106, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 35, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 148.3, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 4.671, "pcie_tx_gbs": 2.566}], "pulls": {"expired": 0, "need": 37502, "idle": 0, "idle_used": 0, "ram_first": 3285, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 9.14, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 206.12, "ssd_below_cap": 21.81, "nothing_needed": 0.37}, "sd": {"all_readers": 227.9, "some_readers": 8.64, "idle_nothing": 0.76, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.12}, "readers_busy": 12, "h2d_ssd_gbs_cum": 741302505472, "h2d_ram_gbs_cum": 70812639232, "h2d_used_cum": 812093591552, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 438.5, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.167, "ram_to_gpu_gbs": 3.167, "gpu_to_ram_gbs": 2.844, "decode_gbs": 3.748, "ssd_reads": 147, "ram_promotions": 0, "gpu_decodes": 149, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 15, "l2": 270}, "lkw": {"l0": 1820, "arriving": 0, "ssd_inflight": 0, "l1": 5315, "l2": 16865}, "missing": 0, "reads_in_flight": 0, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1077, "ram_cap": 1080, "layer": 36, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 135.6, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 3.081, "pcie_tx_gbs": 2.564}], "pulls": {"expired": 0, "need": 37615, "idle": 0, "idle_used": 0, "ram_first": 3285, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 9.21, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 206.89, "ssd_below_cap": 21.97, "nothing_needed": 0.38}, "sd": {"all_readers": 228.84, "some_readers": 8.69, "idle_nothing": 0.77, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.12}, "readers_busy": 0, "h2d_ssd_gbs_cum": 744471371776, "h2d_ram_gbs_cum": 70812639232, "h2d_used_cum": 815305580544, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 439.5, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.037, "ram_to_gpu_gbs": 3.36, "gpu_to_ram_gbs": 2.929, "decode_gbs": 3.974, "ssd_reads": 141, "ram_promotions": 15, "gpu_decodes": 158, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 129, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 36, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 146.8, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.69, "pcie_tx_gbs": 2.563}], "pulls": {"expired": 0, "need": 37807, "idle": 0, "idle_used": 0, "ram_first": 3300, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 9.21, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 207.89, "ssd_below_cap": 21.98, "nothing_needed": 0.38}, "sd": {"all_readers": 229.83, "some_readers": 8.7, "idle_nothing": 0.77, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.12}, "readers_busy": 12, "h2d_ssd_gbs_cum": 747511107584, "h2d_ram_gbs_cum": 71136026624, "h2d_used_cum": 818668703744, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 440.5, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.165, "ram_to_gpu_gbs": 3.532, "gpu_to_ram_gbs": 3.165, "decode_gbs": 4.099, "ssd_reads": 147, "ram_promotions": 17, "gpu_decodes": 163, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 17, "l2": 267}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 6174, "l2": 17826}, "missing": 249, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 37, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 14, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 136.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 1.173, "pcie_tx_gbs": 6.405}], "pulls": {"expired": 0, "need": 37971, "idle": 0, "idle_used": 0, "ram_first": 3317, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 9.28, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 208.65, "ssd_below_cap": 22.15, "nothing_needed": 0.38}, "sd": {"all_readers": 230.76, "some_readers": 8.76, "idle_nothing": 0.77, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.12}, "readers_busy": 12, "h2d_ssd_gbs_cum": 750679912448, "h2d_ram_gbs_cum": 71502540800, "h2d_used_cum": 822182473728, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 441.5, "phase": "measure", "dt": 1.002, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.099, "ram_to_gpu_gbs": 3.099, "gpu_to_ram_gbs": 2.754, "decode_gbs": 3.618, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 144, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 106, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 37, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 33, "mem_util_pct": 14, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 143.6, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.517, "pcie_tx_gbs": 3.844}], "pulls": {"expired": 0, "need": 38114, "idle": 0, "idle_used": 0, "ram_first": 3317, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 9.28, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 209.65, "ssd_below_cap": 22.15, "nothing_needed": 0.38}, "sd": {"all_readers": 231.77, "some_readers": 8.76, "idle_nothing": 0.77, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.12}, "readers_busy": 12, "h2d_ssd_gbs_cum": 753784164352, "h2d_ram_gbs_cum": 71502540800, "h2d_used_cum": 825286717440, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 442.5, "phase": "measure", "dt": 1.0, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.168, "ram_to_gpu_gbs": 3.405, "gpu_to_ram_gbs": 3.082, "decode_gbs": 4.027, "ssd_reads": 147, "ram_promotions": 11, "gpu_decodes": 160, "scratch_hits": 0, "lk": {"l0": 3, "arriving": 0, "ssd_inflight": 0, "l1": 11, "l2": 270}, "lkw": {"l0": 3453, "arriving": 0, "ssd_inflight": 0, "l1": 3879, "l2": 16668}, "missing": 229, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 38, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 43, "mem_util_pct": 15, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 142.2, "power_limit_w": 600.0, "temp_c": 32, "sm_clock_mhz": 2325, "pcie_rx_gbs": 2.363, "pcie_tx_gbs": 2.598}], "pulls": {"expired": 0, "need": 38272, "idle": 0, "idle_used": 0, "ram_first": 3328, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 9.34, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 210.42, "ssd_below_cap": 22.31, "nothing_needed": 0.38}, "sd": {"all_readers": 232.7, "some_readers": 8.82, "idle_nothing": 0.78, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.12}, "readers_busy": 12, "h2d_ssd_gbs_cum": 756952707072, "h2d_ram_gbs_cum": 71739650048, "h2d_used_cum": 828670820352, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 443.5, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.1, "ram_to_gpu_gbs": 3.1, "gpu_to_ram_gbs": 2.583, "decode_gbs": 3.645, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 84, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 38, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 137.8, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.517, "pcie_tx_gbs": 5.225}], "pulls": {"expired": 0, "need": 38417, "idle": 0, "idle_used": 0, "ram_first": 3328, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 9.34, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 211.42, "ssd_below_cap": 22.31, "nothing_needed": 0.38}, "sd": {"all_readers": 233.7, "some_readers": 8.82, "idle_nothing": 0.78, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.12}, "readers_busy": 12, "h2d_ssd_gbs_cum": 760056700928, "h2d_ram_gbs_cum": 71739650048, "h2d_used_cum": 831796363264, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 444.5, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.1, "ram_to_gpu_gbs": 3.444, "gpu_to_ram_gbs": 3.143, "decode_gbs": 4.072, "ssd_reads": 144, "ram_promotions": 16, "gpu_decodes": 162, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 16, "l2": 266}, "lkw": {"l0": 1030, "arriving": 0, "ssd_inflight": 0, "l1": 5486, "l2": 17484}, "missing": 206, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 39, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 23, "mem_util_pct": 10, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 147.0, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.355, "pcie_tx_gbs": 2.562}], "pulls": {"expired": 0, "need": 38577, "idle": 0, "idle_used": 0, "ram_first": 3344, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 9.41, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 212.18, "ssd_below_cap": 22.48, "nothing_needed": 0.39}, "sd": {"all_readers": 234.63, "some_readers": 8.88, "idle_nothing": 0.78, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.13}, "readers_busy": 12, "h2d_ssd_gbs_cum": 763160727552, "h2d_ram_gbs_cum": 72084566016, "h2d_used_cum": 835266863104, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 445.5, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.145, "ram_to_gpu_gbs": 3.145, "gpu_to_ram_gbs": 2.671, "decode_gbs": 3.673, "ssd_reads": 146, "ram_promotions": 0, "gpu_decodes": 146, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 60, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 39, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 26, "mem_util_pct": 10, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 138.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.517, "pcie_tx_gbs": 3.565}], "pulls": {"expired": 0, "need": 38723, "idle": 0, "idle_used": 0, "ram_first": 3344, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 9.41, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 213.18, "ssd_below_cap": 22.48, "nothing_needed": 0.39}, "sd": {"all_readers": 235.64, "some_readers": 8.88, "idle_nothing": 0.78, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.13}, "readers_busy": 12, "h2d_ssd_gbs_cum": 766307631104, "h2d_ram_gbs_cum": 72084566016, "h2d_used_cum": 838413766656, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 446.5, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.123, "ram_to_gpu_gbs": 3.446, "gpu_to_ram_gbs": 3.188, "decode_gbs": 4.025, "ssd_reads": 145, "ram_promotions": 15, "gpu_decodes": 160, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 15, "l2": 270}, "lkw": {"l0": 1113, "arriving": 0, "ssd_inflight": 0, "l1": 5712, "l2": 17175}, "missing": 185, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 40, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 26, "mem_util_pct": 10, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 138.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.517, "pcie_tx_gbs": 3.565}], "pulls": {"expired": 0, "need": 38883, "idle": 0, "idle_used": 0, "ram_first": 3359, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 9.48, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 213.94, "ssd_below_cap": 22.64, "nothing_needed": 0.39}, "sd": {"all_readers": 236.58, "some_readers": 8.93, "idle_nothing": 0.79, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.13}, "readers_busy": 12, "h2d_ssd_gbs_cum": 769432657920, "h2d_ram_gbs_cum": 72407810048, "h2d_used_cum": 841840480256, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 447.5, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.1, "ram_to_gpu_gbs": 3.1, "gpu_to_ram_gbs": 2.627, "decode_gbs": 3.621, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 144, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 41, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 40, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 26, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 140.1, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 5.843, "pcie_tx_gbs": 2.567}], "pulls": {"expired": 0, "need": 39027, "idle": 0, "idle_used": 0, "ram_first": 3359, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 9.48, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 214.95, "ssd_below_cap": 22.64, "nothing_needed": 0.39}, "sd": {"all_readers": 237.58, "some_readers": 8.93, "idle_nothing": 0.79, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.13}, "readers_busy": 12, "h2d_ssd_gbs_cum": 772535967744, "h2d_ram_gbs_cum": 72407810048, "h2d_used_cum": 844943794176, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 448.5, "phase": "measure", "dt": 1.0, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.124, "ram_to_gpu_gbs": 3.469, "gpu_to_ram_gbs": 3.016, "decode_gbs": 4.078, "ssd_reads": 145, "ram_promotions": 16, "gpu_decodes": 162, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 16, "l2": 271}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 5960, "l2": 18040}, "missing": 167, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 41, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 23, "mem_util_pct": 10, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 135.9, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.526, "pcie_tx_gbs": 3.842}], "pulls": {"expired": 0, "need": 39188, "idle": 0, "idle_used": 0, "ram_first": 3375, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 9.55, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 215.71, "ssd_below_cap": 22.81, "nothing_needed": 0.39}, "sd": {"all_readers": 238.51, "some_readers": 9.0, "idle_nothing": 0.79, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.13}, "readers_busy": 12, "h2d_ssd_gbs_cum": 775659974656, "h2d_ram_gbs_cum": 72752533504, "h2d_used_cum": 848434077696, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 449.5, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.121, "ram_to_gpu_gbs": 3.121, "gpu_to_ram_gbs": 2.863, "decode_gbs": 3.647, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 22, "reads_in_flight": 22, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1056, "ram_cap": 1080, "layer": 41, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 145.5, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.355, "pcie_tx_gbs": 3.842}], "pulls": {"expired": 0, "need": 39319, "idle": 0, "idle_used": 0, "ram_first": 3375, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 9.58, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 216.62, "ssd_below_cap": 22.87, "nothing_needed": 0.39}, "sd": {"all_readers": 239.51, "some_readers": 9.0, "idle_nothing": 0.79, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.13}, "readers_busy": 12, "h2d_ssd_gbs_cum": 778783854592, "h2d_ram_gbs_cum": 72752533504, "h2d_used_cum": 851557957632, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 450.5, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.098, "ram_to_gpu_gbs": 3.55, "gpu_to_ram_gbs": 3.206, "decode_gbs": 4.174, "ssd_reads": 144, "ram_promotions": 21, "gpu_decodes": 166, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 21, "l2": 261}, "lkw": {"l0": 956, "arriving": 0, "ssd_inflight": 0, "l1": 7364, "l2": 15680}, "missing": 139, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 42, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 137.1, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 6.92, "pcie_tx_gbs": 7.963}], "pulls": {"expired": 0, "need": 39498, "idle": 0, "idle_used": 0, "ram_first": 3396, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 9.62, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 217.48, "ssd_below_cap": 22.97, "nothing_needed": 0.39}, "sd": {"all_readers": 240.45, "some_readers": 9.05, "idle_nothing": 0.8, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.13}, "readers_busy": 12, "h2d_ssd_gbs_cum": 781885616128, "h2d_ram_gbs_cum": 73204842496, "h2d_used_cum": 855112028160, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 451.5, "phase": "measure", "dt": 1.002, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.161, "ram_to_gpu_gbs": 3.441, "gpu_to_ram_gbs": 3.14, "decode_gbs": 4.021, "ssd_reads": 147, "ram_promotions": 13, "gpu_decodes": 160, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 13, "l2": 274}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 5522, "l2": 18478}, "missing": 266, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 43, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 147.8, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.764, "pcie_tx_gbs": 3.941}], "pulls": {"expired": 0, "need": 39658, "idle": 0, "idle_used": 0, "ram_first": 3409, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 9.69, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 218.26, "ssd_below_cap": 23.13, "nothing_needed": 0.4}, "sd": {"all_readers": 241.39, "some_readers": 9.11, "idle_nothing": 0.8, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.13}, "readers_busy": 12, "h2d_ssd_gbs_cum": 785051676672, "h2d_ram_gbs_cum": 73484861440, "h2d_used_cum": 858558107648, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 452.5, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.098, "ram_to_gpu_gbs": 3.098, "gpu_to_ram_gbs": 2.646, "decode_gbs": 3.595, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 143, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 122, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 43, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 50, "mem_util_pct": 19, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 140.1, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 4.676, "pcie_tx_gbs": 3.846}], "pulls": {"expired": 0, "need": 39802, "idle": 0, "idle_used": 0, "ram_first": 3409, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 9.69, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 219.26, "ssd_below_cap": 23.13, "nothing_needed": 0.4}, "sd": {"all_readers": 242.4, "some_readers": 9.11, "idle_nothing": 0.8, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.13}, "readers_busy": 12, "h2d_ssd_gbs_cum": 788153741312, "h2d_ram_gbs_cum": 73484861440, "h2d_used_cum": 861638631424, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 453.5, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.12, "ram_to_gpu_gbs": 3.508, "gpu_to_ram_gbs": 3.206, "decode_gbs": 4.124, "ssd_reads": 145, "ram_promotions": 18, "gpu_decodes": 164, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 18, "l2": 263}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 7251, "l2": 16749}, "missing": 240, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 44, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 27, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 136.2, "power_limit_w": 600.0, "temp_c": 30, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.336, "pcie_tx_gbs": 2.664}], "pulls": {"expired": 0, "need": 39965, "idle": 0, "idle_used": 0, "ram_first": 3427, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 9.76, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 220.03, "ssd_below_cap": 23.28, "nothing_needed": 0.4}, "sd": {"all_readers": 243.32, "some_readers": 9.17, "idle_nothing": 0.81, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.13}, "readers_busy": 12, "h2d_ssd_gbs_cum": 791277142016, "h2d_ram_gbs_cum": 73872666624, "h2d_used_cum": 865171378176, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 454.5, "phase": "measure", "dt": 1.001, "tokens": 8197, "tok_s": 0.0, "ssd_gbs": 3.148, "ram_to_gpu_gbs": 3.148, "gpu_to_ram_gbs": 2.673, "decode_gbs": 3.648, "ssd_reads": 146, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 95, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 44, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 31, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 145.3, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 5.866, "pcie_tx_gbs": 6.417}], "pulls": {"expired": 0, "need": 40110, "idle": 0, "idle_used": 0, "ram_first": 3427, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 9.76, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 221.03, "ssd_below_cap": 23.28, "nothing_needed": 0.4}, "sd": {"all_readers": 244.33, "some_readers": 9.17, "idle_nothing": 0.81, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.13}, "readers_busy": 12, "h2d_ssd_gbs_cum": 794426933248, "h2d_ram_gbs_cum": 73872666624, "h2d_used_cum": 868299313152, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 455.5, "phase": "measure", "dt": 1.0, "tokens": 9869, "tok_s": 1671.514, "ssd_gbs": 3.104, "ram_to_gpu_gbs": 3.362, "gpu_to_ram_gbs": 3.147, "decode_gbs": 3.951, "ssd_reads": 144, "ram_promotions": 12, "gpu_decodes": 157, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 12, "l2": 276}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 2037, "l2": 21963}, "missing": 226, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 3, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 135.7, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.445, "pcie_tx_gbs": 1.297}], "pulls": {"expired": 0, "need": 40267, "idle": 0, "idle_used": 0, "ram_first": 3439, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 9.82, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 221.78, "ssd_below_cap": 23.46, "nothing_needed": 0.4}, "sd": {"all_readers": 245.24, "some_readers": 9.25, "idle_nothing": 0.81, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.13}, "readers_busy": 12, "h2d_ssd_gbs_cum": 797531676672, "h2d_ram_gbs_cum": 74131226624, "h2d_used_cum": 871684472832, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 456.5, "phase": "measure", "dt": 1.002, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.141, "ram_to_gpu_gbs": 3.141, "gpu_to_ram_gbs": 2.582, "decode_gbs": 3.669, "ssd_reads": 146, "ram_promotions": 0, "gpu_decodes": 146, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 80, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 3, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 151.3, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.677, "pcie_tx_gbs": 7.683}], "pulls": {"expired": 0, "need": 40413, "idle": 0, "idle_used": 0, "ram_first": 3439, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 9.82, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 222.79, "ssd_below_cap": 23.46, "nothing_needed": 0.4}, "sd": {"all_readers": 246.25, "some_readers": 9.25, "idle_nothing": 0.81, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.13}, "readers_busy": 12, "h2d_ssd_gbs_cum": 800678121472, "h2d_ram_gbs_cum": 74131226624, "h2d_used_cum": 874830917632, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 457.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.102, "ram_to_gpu_gbs": 3.727, "gpu_to_ram_gbs": 3.425, "decode_gbs": 4.353, "ssd_reads": 144, "ram_promotions": 29, "gpu_decodes": 173, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 29, "l2": 259}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 4037, "l2": 19963}, "missing": 195, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 4, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 33, "mem_util_pct": 15, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 138.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.507, "pcie_tx_gbs": 5.123}], "pulls": {"expired": 0, "need": 40584, "idle": 0, "idle_used": 0, "ram_first": 3468, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 9.9, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 223.55, "ssd_below_cap": 23.62, "nothing_needed": 0.4}, "sd": {"all_readers": 247.18, "some_readers": 9.31, "idle_nothing": 0.82, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.13}, "readers_busy": 12, "h2d_ssd_gbs_cum": 803781636096, "h2d_ram_gbs_cum": 74756345856, "h2d_used_cum": 878559551488, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 458.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.1, "ram_to_gpu_gbs": 3.1, "gpu_to_ram_gbs": 2.605, "decode_gbs": 3.595, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 143, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 52, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 4, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 25, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 147.6, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.518, "pcie_tx_gbs": 3.943}], "pulls": {"expired": 0, "need": 40727, "idle": 0, "idle_used": 0, "ram_first": 3468, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 9.9, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 224.55, "ssd_below_cap": 23.62, "nothing_needed": 0.4}, "sd": {"all_readers": 248.18, "some_readers": 9.31, "idle_nothing": 0.82, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.13}, "readers_busy": 12, "h2d_ssd_gbs_cum": 806885597184, "h2d_ram_gbs_cum": 74756345856, "h2d_used_cum": 881641955328, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 459.5, "phase": "measure", "dt": 1.0, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.168, "ram_to_gpu_gbs": 3.771, "gpu_to_ram_gbs": 3.297, "decode_gbs": 4.404, "ssd_reads": 147, "ram_promotions": 28, "gpu_decodes": 175, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 28, "l2": 260}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 3944, "l2": 20056}, "missing": 165, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 5, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 53, "mem_util_pct": 19, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 146.5, "power_limit_w": 600.0, "temp_c": 33, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.199, "pcie_tx_gbs": 2.566}], "pulls": {"expired": 0, "need": 40898, "idle": 0, "idle_used": 0, "ram_first": 3496, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 9.97, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 225.31, "ssd_below_cap": 23.79, "nothing_needed": 0.41}, "sd": {"all_readers": 249.11, "some_readers": 9.37, "idle_nothing": 0.83, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.13}, "readers_busy": 12, "h2d_ssd_gbs_cum": 810054193152, "h2d_ram_gbs_cum": 75359895552, "h2d_used_cum": 885414105088, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 460.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.144, "ram_to_gpu_gbs": 3.144, "gpu_to_ram_gbs": 2.886, "decode_gbs": 3.671, "ssd_reads": 146, "ram_promotions": 0, "gpu_decodes": 146, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 18, "reads_in_flight": 18, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1059, "ram_cap": 1080, "layer": 5, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 27, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 138.6, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.689, "pcie_tx_gbs": 3.935}], "pulls": {"expired": 0, "need": 41027, "idle": 0, "idle_used": 0, "ram_first": 3496, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 10.0, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 226.21, "ssd_below_cap": 23.86, "nothing_needed": 0.41}, "sd": {"all_readers": 250.11, "some_readers": 9.37, "idle_nothing": 0.83, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.13}, "readers_busy": 12, "h2d_ssd_gbs_cum": 813201264640, "h2d_ram_gbs_cum": 75359895552, "h2d_used_cum": 888561172480, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 461.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.078, "ram_to_gpu_gbs": 3.767, "gpu_to_ram_gbs": 3.379, "decode_gbs": 4.424, "ssd_reads": 143, "ram_promotions": 32, "gpu_decodes": 176, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 32, "l2": 256}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 4500, "l2": 19500}, "missing": 131, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 6, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 31, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 146.8, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 1.183, "pcie_tx_gbs": 6.405}], "pulls": {"expired": 0, "need": 41212, "idle": 0, "idle_used": 0, "ram_first": 3528, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 10.03, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 227.08, "ssd_below_cap": 23.95, "nothing_needed": 0.41}, "sd": {"all_readers": 251.05, "some_readers": 9.43, "idle_nothing": 0.83, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.13}, "readers_busy": 12, "h2d_ssd_gbs_cum": 816283697152, "h2d_ram_gbs_cum": 76049678336, "h2d_used_cum": 892354945024, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 462.5, "phase": "measure", "dt": 1.0, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.147, "ram_to_gpu_gbs": 4.009, "gpu_to_ram_gbs": 3.643, "decode_gbs": 4.631, "ssd_reads": 146, "ram_promotions": 40, "gpu_decodes": 184, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 40, "l2": 247}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 8157, "l2": 15843}, "missing": 232, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1024, "ram_cap": 1080, "layer": 7, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 135.7, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.502, "pcie_tx_gbs": 3.306}], "pulls": {"expired": 0, "need": 41393, "idle": 0, "idle_used": 0, "ram_first": 3568, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 10.11, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 227.84, "ssd_below_cap": 24.11, "nothing_needed": 0.41}, "sd": {"all_readers": 251.98, "some_readers": 9.5, "idle_nothing": 0.84, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.13}, "readers_busy": 12, "h2d_ssd_gbs_cum": 819430719488, "h2d_ram_gbs_cum": 76911869952, "h2d_used_cum": 896321048576, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 463.5, "phase": "measure", "dt": 1.0, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.125, "ram_to_gpu_gbs": 3.125, "gpu_to_ram_gbs": 2.866, "decode_gbs": 3.674, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 146, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 87, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 7, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 147.5, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 7.015, "pcie_tx_gbs": 2.563}], "pulls": {"expired": 0, "need": 41538, "idle": 0, "idle_used": 0, "ram_first": 3568, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 10.11, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 228.84, "ssd_below_cap": 24.11, "nothing_needed": 0.41}, "sd": {"all_readers": 252.97, "some_readers": 9.5, "idle_nothing": 0.84, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.13}, "readers_busy": 12, "h2d_ssd_gbs_cum": 822556213248, "h2d_ram_gbs_cum": 76911869952, "h2d_used_cum": 899468099584, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 464.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.122, "ram_to_gpu_gbs": 3.94, "gpu_to_ram_gbs": 3.746, "decode_gbs": 4.576, "ssd_reads": 145, "ram_promotions": 38, "gpu_decodes": 182, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 38, "l2": 250}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 7645, "l2": 16355}, "missing": 192, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 8, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 34, "mem_util_pct": 15, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 143.0, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.352, "pcie_tx_gbs": 3.845}], "pulls": {"expired": 0, "need": 41718, "idle": 0, "idle_used": 0, "ram_first": 3606, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 10.18, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 229.62, "ssd_below_cap": 24.26, "nothing_needed": 0.41}, "sd": {"all_readers": 253.9, "some_readers": 9.57, "idle_nothing": 0.84, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.13}, "readers_busy": 12, "h2d_ssd_gbs_cum": 825681539072, "h2d_ram_gbs_cum": 77730926592, "h2d_used_cum": 903390928896, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 465.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.144, "ram_to_gpu_gbs": 3.144, "gpu_to_ram_gbs": 2.606, "decode_gbs": 3.672, "ssd_reads": 146, "ram_promotions": 0, "gpu_decodes": 146, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 48, "reads_in_flight": 34, "uploads_pending": 2, "gpu_experts": 106, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 8, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 27, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 134.8, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.429, "pcie_tx_gbs": 3.843}], "pulls": {"expired": 0, "need": 41862, "idle": 0, "idle_used": 0, "ram_first": 3606, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 10.18, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 230.61, "ssd_below_cap": 24.26, "nothing_needed": 0.41}, "sd": {"all_readers": 254.9, "some_readers": 9.57, "idle_nothing": 0.84, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.13}, "readers_busy": 12, "h2d_ssd_gbs_cum": 828828487680, "h2d_ram_gbs_cum": 77730926592, "h2d_used_cum": 906537877504, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 466.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.079, "ram_to_gpu_gbs": 3.811, "gpu_to_ram_gbs": 3.596, "decode_gbs": 4.526, "ssd_reads": 143, "ram_promotions": 34, "gpu_decodes": 180, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 34, "l2": 251}, "lkw": {"l0": 1196, "arriving": 0, "ssd_inflight": 0, "l1": 5947, "l2": 16857}, "missing": 154, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 9, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 31, "mem_util_pct": 14, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 148.7, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 2.335, "pcie_tx_gbs": 6.507}], "pulls": {"expired": 0, "need": 42040, "idle": 0, "idle_used": 0, "ram_first": 3640, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 10.26, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 231.37, "ssd_below_cap": 24.43, "nothing_needed": 0.42}, "sd": {"all_readers": 255.83, "some_readers": 9.63, "idle_nothing": 0.85, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.13}, "readers_busy": 12, "h2d_ssd_gbs_cum": 831910809600, "h2d_ram_gbs_cum": 78463811584, "h2d_used_cum": 910374641664, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 467.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.101, "ram_to_gpu_gbs": 3.101, "gpu_to_ram_gbs": 2.671, "decode_gbs": 3.647, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 10, "reads_in_flight": 10, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1067, "ram_cap": 1080, "layer": 9, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 32, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 134.3, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.529, "pcie_tx_gbs": 3.843}], "pulls": {"expired": 0, "need": 42158, "idle": 0, "idle_used": 0, "ram_first": 3640, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 10.3, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 232.18, "ssd_below_cap": 24.57, "nothing_needed": 0.42}, "sd": {"all_readers": 256.82, "some_readers": 9.65, "idle_nothing": 0.85, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.13}, "readers_busy": 10, "h2d_ssd_gbs_cum": 835014631424, "h2d_ram_gbs_cum": 78463811584, "h2d_used_cum": 913500012544, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 468.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.144, "ram_to_gpu_gbs": 3.79, "gpu_to_ram_gbs": 3.424, "decode_gbs": 4.401, "ssd_reads": 146, "ram_promotions": 30, "gpu_decodes": 175, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 30, "l2": 258}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 5374, "l2": 18626}, "missing": 123, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 10, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 149.6, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.688, "pcie_tx_gbs": 3.843}], "pulls": {"expired": 0, "need": 42356, "idle": 0, "idle_used": 0, "ram_first": 3670, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 10.32, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 233.13, "ssd_below_cap": 24.6, "nothing_needed": 0.42}, "sd": {"all_readers": 257.76, "some_readers": 9.7, "idle_nothing": 0.85, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.13}, "readers_busy": 12, "h2d_ssd_gbs_cum": 838161637376, "h2d_ram_gbs_cum": 79110471680, "h2d_used_cum": 917272121344, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 469.5, "phase": "measure", "dt": 1.002, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.119, "ram_to_gpu_gbs": 3.7, "gpu_to_ram_gbs": 3.592, "decode_gbs": 4.346, "ssd_reads": 145, "ram_promotions": 27, "gpu_decodes": 173, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 27, "l2": 259}, "lkw": {"l0": 2396, "arriving": 0, "ssd_inflight": 0, "l1": 4881, "l2": 16723}, "missing": 237, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 11, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 14, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 137.8, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 8.178, "pcie_tx_gbs": 0.004}], "pulls": {"expired": 0, "need": 42527, "idle": 0, "idle_used": 0, "ram_first": 3697, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 10.4, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 233.9, "ssd_below_cap": 24.76, "nothing_needed": 0.42}, "sd": {"all_readers": 258.68, "some_readers": 9.77, "idle_nothing": 0.86, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.13}, "readers_busy": 12, "h2d_ssd_gbs_cum": 841286983680, "h2d_ram_gbs_cum": 79692472320, "h2d_used_cum": 920957919232, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 470.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.123, "ram_to_gpu_gbs": 3.123, "gpu_to_ram_gbs": 2.649, "decode_gbs": 3.673, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 146, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 91, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 11, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 33, "mem_util_pct": 14, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 143.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.689, "pcie_tx_gbs": 3.844}], "pulls": {"expired": 0, "need": 42673, "idle": 0, "idle_used": 0, "ram_first": 3697, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 10.4, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 234.9, "ssd_below_cap": 24.76, "nothing_needed": 0.42}, "sd": {"all_readers": 259.68, "some_readers": 9.77, "idle_nothing": 0.86, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.13}, "readers_busy": 12, "h2d_ssd_gbs_cum": 844412497920, "h2d_ram_gbs_cum": 79692472320, "h2d_used_cum": 924104982528, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 471.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.122, "ram_to_gpu_gbs": 3.875, "gpu_to_ram_gbs": 3.552, "decode_gbs": 4.601, "ssd_reads": 145, "ram_promotions": 35, "gpu_decodes": 183, "scratch_hits": 0, "lk": {"l0": 3, "arriving": 0, "ssd_inflight": 0, "l1": 35, "l2": 249}, "lkw": {"l0": 1754, "arriving": 0, "ssd_inflight": 0, "l1": 7466, "l2": 14780}, "missing": 195, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 12, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 40, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 148.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 1.173, "pcie_tx_gbs": 5.123}], "pulls": {"expired": 0, "need": 42851, "idle": 0, "idle_used": 0, "ram_first": 3732, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 10.47, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 235.66, "ssd_below_cap": 24.92, "nothing_needed": 0.43}, "sd": {"all_readers": 260.61, "some_readers": 9.84, "idle_nothing": 0.86, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.14}, "readers_busy": 12, "h2d_ssd_gbs_cum": 847538036736, "h2d_ram_gbs_cum": 80446980096, "h2d_used_cum": 927985029120, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 472.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.102, "ram_to_gpu_gbs": 3.102, "gpu_to_ram_gbs": 2.585, "decode_gbs": 3.647, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 51, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 12, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 40, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 148.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 1.173, "pcie_tx_gbs": 5.123}], "pulls": {"expired": 0, "need": 42995, "idle": 0, "idle_used": 0, "ram_first": 3732, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 10.47, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 236.66, "ssd_below_cap": 24.92, "nothing_needed": 0.43}, "sd": {"all_readers": 261.62, "some_readers": 9.84, "idle_nothing": 0.86, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.14}, "readers_busy": 12, "h2d_ssd_gbs_cum": 850642153472, "h2d_ram_gbs_cum": 80446980096, "h2d_used_cum": 931110703104, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 473.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.122, "ram_to_gpu_gbs": 3.768, "gpu_to_ram_gbs": 3.531, "decode_gbs": 4.5, "ssd_reads": 145, "ram_promotions": 30, "gpu_decodes": 179, "scratch_hits": 0, "lk": {"l0": 4, "arriving": 0, "ssd_inflight": 0, "l1": 30, "l2": 252}, "lkw": {"l0": 3123, "arriving": 0, "ssd_inflight": 0, "l1": 6081, "l2": 14796}, "missing": 158, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 13, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 32, "mem_util_pct": 15, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 139.0, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 1.173, "pcie_tx_gbs": 4.046}], "pulls": {"expired": 0, "need": 43168, "idle": 0, "idle_used": 0, "ram_first": 3762, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 10.55, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 237.42, "ssd_below_cap": 25.09, "nothing_needed": 0.43}, "sd": {"all_readers": 262.53, "some_readers": 9.91, "idle_nothing": 0.87, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.14}, "readers_busy": 12, "h2d_ssd_gbs_cum": 853767782400, "h2d_ram_gbs_cum": 81093615616, "h2d_used_cum": 934882967552, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 474.5, "phase": "measure", "dt": 1.0, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.147, "ram_to_gpu_gbs": 3.147, "gpu_to_ram_gbs": 2.715, "decode_gbs": 3.675, "ssd_reads": 146, "ram_promotions": 0, "gpu_decodes": 146, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 12, "reads_in_flight": 12, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1066, "ram_cap": 1080, "layer": 13, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 26, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 148.1, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 1.165, "pcie_tx_gbs": 3.842}], "pulls": {"expired": 0, "need": 43290, "idle": 0, "idle_used": 0, "ram_first": 3762, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 10.6, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 238.26, "ssd_below_cap": 25.2, "nothing_needed": 0.43}, "sd": {"all_readers": 263.53, "some_readers": 9.91, "idle_nothing": 0.87, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.14}, "readers_busy": 12, "h2d_ssd_gbs_cum": 856914612224, "h2d_ram_gbs_cum": 81093615616, "h2d_used_cum": 938029797376, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 475.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.146, "ram_to_gpu_gbs": 3.771, "gpu_to_ram_gbs": 3.404, "decode_gbs": 4.478, "ssd_reads": 146, "ram_promotions": 29, "gpu_decodes": 178, "scratch_hits": 0, "lk": {"l0": 4, "arriving": 0, "ssd_inflight": 0, "l1": 29, "l2": 254}, "lkw": {"l0": 3464, "arriving": 0, "ssd_inflight": 0, "l1": 5828, "l2": 14708}, "missing": 121, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 14, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 32, "mem_util_pct": 15, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 137.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 25.599, "pcie_tx_gbs": 5.128}], "pulls": {"expired": 0, "need": 43485, "idle": 0, "idle_used": 0, "ram_first": 3791, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 10.62, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 239.18, "ssd_below_cap": 25.26, "nothing_needed": 0.43}, "sd": {"all_readers": 264.45, "some_readers": 9.98, "idle_nothing": 0.88, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.14}, "readers_busy": 12, "h2d_ssd_gbs_cum": 860062269440, "h2d_ram_gbs_cum": 81718808576, "h2d_used_cum": 941781090304, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 476.5, "phase": "measure", "dt": 1.002, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.099, "ram_to_gpu_gbs": 3.702, "gpu_to_ram_gbs": 3.422, "decode_gbs": 4.423, "ssd_reads": 144, "ram_promotions": 28, "gpu_decodes": 176, "scratch_hits": 0, "lk": {"l0": 4, "arriving": 0, "ssd_inflight": 0, "l1": 28, "l2": 255}, "lkw": {"l0": 2973, "arriving": 0, "ssd_inflight": 0, "l1": 6288, "l2": 14739}, "missing": 231, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 15, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 146.9, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 3.527, "pcie_tx_gbs": 2.562}], "pulls": {"expired": 0, "need": 43658, "idle": 0, "idle_used": 0, "ram_first": 3819, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 10.68, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 239.94, "ssd_below_cap": 25.43, "nothing_needed": 0.43}, "sd": {"all_readers": 265.39, "some_readers": 10.04, "idle_nothing": 0.88, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.14}, "readers_busy": 12, "h2d_ssd_gbs_cum": 863166509056, "h2d_ram_gbs_cum": 82322591744, "h2d_used_cum": 945489113088, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 477.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.102, "ram_to_gpu_gbs": 3.102, "gpu_to_ram_gbs": 2.564, "decode_gbs": 3.622, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 144, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 87, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 15, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 48, "mem_util_pct": 16, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 145.8, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 3.516, "pcie_tx_gbs": 2.566}], "pulls": {"expired": 0, "need": 43802, "idle": 0, "idle_used": 0, "ram_first": 3819, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 10.68, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 240.94, "ssd_below_cap": 25.43, "nothing_needed": 0.43}, "sd": {"all_readers": 266.39, "some_readers": 10.04, "idle_nothing": 0.88, "idle_no_ram_slot": 0.03, "idle_upload_backlog": 0.14}, "readers_busy": 12, "h2d_ssd_gbs_cum": 866271330304, "h2d_ram_gbs_cum": 82322591744, "h2d_used_cum": 948593930240, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 478.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.124, "ram_to_gpu_gbs": 4.029, "gpu_to_ram_gbs": 3.921, "decode_gbs": 4.729, "ssd_reads": 145, "ram_promotions": 42, "gpu_decodes": 188, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 42, "l2": 242}, "lkw": {"l0": 869, "arriving": 0, "ssd_inflight": 0, "l1": 8999, "l2": 14132}, "missing": 184, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 16, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 138.3, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.355, "pcie_tx_gbs": 3.844}], "pulls": {"expired": 0, "need": 43983, "idle": 0, "idle_used": 0, "ram_first": 3861, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 10.75, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 241.7, "ssd_below_cap": 25.6, "nothing_needed": 0.44}, "sd": {"all_readers": 267.33, "some_readers": 10.08, "idle_nothing": 0.89, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.14}, "readers_busy": 12, "h2d_ssd_gbs_cum": 869397696512, "h2d_ram_gbs_cum": 83228233728, "h2d_used_cum": 952625946624, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 479.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.123, "ram_to_gpu_gbs": 3.123, "gpu_to_ram_gbs": 2.649, "decode_gbs": 3.672, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 146, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 39, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 16, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 150.9, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 1.173, "pcie_tx_gbs": 2.671}], "pulls": {"expired": 0, "need": 44128, "idle": 0, "idle_used": 0, "ram_first": 3861, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 10.75, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 242.7, "ssd_below_cap": 25.6, "nothing_needed": 0.44}, "sd": {"all_readers": 268.33, "some_readers": 10.08, "idle_nothing": 0.89, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.14}, "readers_busy": 12, "h2d_ssd_gbs_cum": 872523997184, "h2d_ram_gbs_cum": 83228233728, "h2d_used_cum": 955773800448, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 480.5, "phase": "measure", "dt": 1.003, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.14, "ram_to_gpu_gbs": 3.957, "gpu_to_ram_gbs": 3.742, "decode_gbs": 4.695, "ssd_reads": 146, "ram_promotions": 38, "gpu_decodes": 187, "scratch_hits": 0, "lk": {"l0": 3, "arriving": 0, "ssd_inflight": 0, "l1": 38, "l2": 243}, "lkw": {"l0": 1891, "arriving": 0, "ssd_inflight": 0, "l1": 8361, "l2": 13748}, "missing": 136, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 17, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 140.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 1.464, "pcie_tx_gbs": 8.966}], "pulls": {"expired": 0, "need": 44308, "idle": 0, "idle_used": 0, "ram_first": 3899, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 10.83, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 243.47, "ssd_below_cap": 25.75, "nothing_needed": 0.44}, "sd": {"all_readers": 269.26, "some_readers": 10.15, "idle_nothing": 0.9, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.14}, "readers_busy": 12, "h2d_ssd_gbs_cum": 875671871488, "h2d_ram_gbs_cum": 84047556608, "h2d_used_cum": 959740997632, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 481.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.103, "ram_to_gpu_gbs": 3.749, "gpu_to_ram_gbs": 3.318, "decode_gbs": 4.528, "ssd_reads": 144, "ram_promotions": 30, "gpu_decodes": 180, "scratch_hits": 0, "lk": {"l0": 7, "arriving": 0, "ssd_inflight": 0, "l1": 30, "l2": 244}, "lkw": {"l0": 5457, "arriving": 0, "ssd_inflight": 0, "l1": 6869, "l2": 11674}, "missing": 237, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1032, "ram_cap": 1080, "layer": 18, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 26, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 146.8, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 0.009, "pcie_tx_gbs": 5.206}], "pulls": {"expired": 0, "need": 44481, "idle": 0, "idle_used": 0, "ram_first": 3929, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 10.9, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 244.24, "ssd_below_cap": 25.91, "nothing_needed": 0.44}, "sd": {"all_readers": 270.19, "some_readers": 10.22, "idle_nothing": 0.9, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.14}, "readers_busy": 12, "h2d_ssd_gbs_cum": 878776438784, "h2d_ram_gbs_cum": 84694306816, "h2d_used_cum": 963470757888, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 482.5, "phase": "measure", "dt": 1.0, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.147, "ram_to_gpu_gbs": 3.147, "gpu_to_ram_gbs": 2.695, "decode_gbs": 3.7, "ssd_reads": 146, "ram_promotions": 0, "gpu_decodes": 147, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 90, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 18, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 147.7, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.682, "pcie_tx_gbs": 2.565}], "pulls": {"expired": 0, "need": 44628, "idle": 0, "idle_used": 0, "ram_first": 3929, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 10.9, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 245.24, "ssd_below_cap": 25.91, "nothing_needed": 0.44}, "sd": {"all_readers": 271.19, "some_readers": 10.22, "idle_nothing": 0.9, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.14}, "readers_busy": 12, "h2d_ssd_gbs_cum": 881924005888, "h2d_ram_gbs_cum": 84694306816, "h2d_used_cum": 966639882240, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 483.5, "phase": "measure", "dt": 1.002, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.099, "ram_to_gpu_gbs": 3.702, "gpu_to_ram_gbs": 3.379, "decode_gbs": 4.472, "ssd_reads": 144, "ram_promotions": 28, "gpu_decodes": 178, "scratch_hits": 0, "lk": {"l0": 6, "arriving": 0, "ssd_inflight": 0, "l1": 28, "l2": 242}, "lkw": {"l0": 4064, "arriving": 0, "ssd_inflight": 0, "l1": 6950, "l2": 12986}, "missing": 188, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 19, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 137.5, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.507, "pcie_tx_gbs": 5.946}], "pulls": {"expired": 0, "need": 44800, "idle": 0, "idle_used": 0, "ram_first": 3957, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 10.97, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 246.01, "ssd_below_cap": 26.08, "nothing_needed": 0.44}, "sd": {"all_readers": 272.1, "some_readers": 10.3, "idle_nothing": 0.91, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.14}, "readers_busy": 12, "h2d_ssd_gbs_cum": 885028691968, "h2d_ram_gbs_cum": 85298053120, "h2d_used_cum": 970348314624, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 484.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.124, "ram_to_gpu_gbs": 3.124, "gpu_to_ram_gbs": 2.628, "decode_gbs": 3.621, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 144, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 43, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 19, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 147.6, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 1.182, "pcie_tx_gbs": 5.124}], "pulls": {"expired": 0, "need": 44945, "idle": 0, "idle_used": 0, "ram_first": 3957, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 10.97, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 247.01, "ssd_below_cap": 26.08, "nothing_needed": 0.44}, "sd": {"all_readers": 273.1, "some_readers": 10.3, "idle_nothing": 0.91, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.14}, "readers_busy": 12, "h2d_ssd_gbs_cum": 888155254784, "h2d_ram_gbs_cum": 85298053120, "h2d_used_cum": 973453332480, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 485.5, "phase": "measure", "dt": 1.0, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.104, "ram_to_gpu_gbs": 3.836, "gpu_to_ram_gbs": 3.535, "decode_gbs": 4.53, "ssd_reads": 144, "ram_promotions": 34, "gpu_decodes": 180, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 34, "l2": 247}, "lkw": {"l0": 1383, "arriving": 0, "ssd_inflight": 0, "l1": 8003, "l2": 14614}, "missing": 147, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 20, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 67, "mem_util_pct": 22, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 148.4, "power_limit_w": 600.0, "temp_c": 34, "sm_clock_mhz": 2325, "pcie_rx_gbs": 5.832, "pcie_tx_gbs": 5.124}], "pulls": {"expired": 0, "need": 45120, "idle": 0, "idle_used": 0, "ram_first": 3991, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 11.04, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 247.77, "ssd_below_cap": 26.23, "nothing_needed": 0.45}, "sd": {"all_readers": 274.04, "some_readers": 10.35, "idle_nothing": 0.91, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.15}, "readers_busy": 12, "h2d_ssd_gbs_cum": 891259527168, "h2d_ram_gbs_cum": 86030934016, "h2d_used_cum": 977290489856, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 486.5, "phase": "measure", "dt": 1.002, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.163, "ram_to_gpu_gbs": 3.917, "gpu_to_ram_gbs": 3.594, "decode_gbs": 3.945, "ssd_reads": 147, "ram_promotions": 35, "gpu_decodes": 157, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 35, "l2": 246}, "lkw": {"l0": 1204, "arriving": 0, "ssd_inflight": 0, "l1": 8338, "l2": 14458}, "missing": 261, "reads_in_flight": 35, "uploads_pending": 16, "gpu_experts": 92, "gpu_cap": 108, "ram_victims": 1001, "ram_cap": 1080, "layer": 21, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 32, "mem_util_pct": 14, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 140.0, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 2.345, "pcie_tx_gbs": 5.125}], "pulls": {"expired": 0, "need": 45302, "idle": 0, "idle_used": 0, "ram_first": 4026, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 11.12, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 248.54, "ssd_below_cap": 26.39, "nothing_needed": 0.45}, "sd": {"all_readers": 274.95, "some_readers": 10.43, "idle_nothing": 0.92, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.15}, "readers_busy": 12, "h2d_ssd_gbs_cum": 894428151808, "h2d_ram_gbs_cum": 86785380352, "h2d_used_cum": 980631527424, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 487.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.121, "ram_to_gpu_gbs": 3.121, "gpu_to_ram_gbs": 2.583, "decode_gbs": 4.349, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 173, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 100, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 21, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 146.8, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 1.183, "pcie_tx_gbs": 3.948}], "pulls": {"expired": 0, "need": 45448, "idle": 0, "idle_used": 0, "ram_first": 4026, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 11.12, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 249.54, "ssd_below_cap": 26.39, "nothing_needed": 0.45}, "sd": {"all_readers": 275.96, "some_readers": 10.43, "idle_nothing": 0.92, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.15}, "readers_busy": 12, "h2d_ssd_gbs_cum": 897553555456, "h2d_ram_gbs_cum": 86785380352, "h2d_used_cum": 984360505344, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 488.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.078, "ram_to_gpu_gbs": 3.788, "gpu_to_ram_gbs": 3.53, "decode_gbs": 4.45, "ssd_reads": 143, "ram_promotions": 33, "gpu_decodes": 177, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 33, "l2": 244}, "lkw": {"l0": 834, "arriving": 0, "ssd_inflight": 0, "l1": 7490, "l2": 15676}, "missing": 201, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 22, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 33, "mem_util_pct": 15, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 141.7, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.688, "pcie_tx_gbs": 5.124}], "pulls": {"expired": 0, "need": 45622, "idle": 0, "idle_used": 0, "ram_first": 4059, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 11.19, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 250.31, "ssd_below_cap": 26.55, "nothing_needed": 0.45}, "sd": {"all_readers": 276.9, "some_readers": 10.49, "idle_nothing": 0.92, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.15}, "readers_busy": 12, "h2d_ssd_gbs_cum": 900635516928, "h2d_ram_gbs_cum": 87496560640, "h2d_used_cum": 988153647104, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 489.5, "phase": "measure", "dt": 1.0, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.168, "ram_to_gpu_gbs": 3.168, "gpu_to_ram_gbs": 2.715, "decode_gbs": 3.699, "ssd_reads": 147, "ram_promotions": 0, "gpu_decodes": 147, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 54, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 22, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 27, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 146.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.689, "pcie_tx_gbs": 3.846}], "pulls": {"expired": 0, "need": 45769, "idle": 0, "idle_used": 0, "ram_first": 4059, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 11.19, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 251.31, "ssd_below_cap": 26.55, "nothing_needed": 0.45}, "sd": {"all_readers": 277.89, "some_readers": 10.49, "idle_nothing": 0.92, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.15}, "readers_busy": 12, "h2d_ssd_gbs_cum": 903803994112, "h2d_ram_gbs_cum": 87496560640, "h2d_used_cum": 991322124288, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 490.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.101, "ram_to_gpu_gbs": 3.617, "gpu_to_ram_gbs": 3.445, "decode_gbs": 4.225, "ssd_reads": 144, "ram_promotions": 24, "gpu_decodes": 168, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 24, "l2": 259}, "lkw": {"l0": 694, "arriving": 0, "ssd_inflight": 0, "l1": 5740, "l2": 17566}, "missing": 170, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 23, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 145.0, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 3.505, "pcie_tx_gbs": 5.122}], "pulls": {"expired": 0, "need": 45935, "idle": 0, "idle_used": 0, "ram_first": 4083, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 11.26, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 252.08, "ssd_below_cap": 26.72, "nothing_needed": 0.45}, "sd": {"all_readers": 278.83, "some_readers": 10.55, "idle_nothing": 0.93, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.15}, "readers_busy": 12, "h2d_ssd_gbs_cum": 906907406336, "h2d_ram_gbs_cum": 88013717504, "h2d_used_cum": 994921148416, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 491.5, "phase": "measure", "dt": 1.0, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.124, "ram_to_gpu_gbs": 3.124, "gpu_to_ram_gbs": 2.564, "decode_gbs": 3.649, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 25, "reads_in_flight": 24, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1052, "ram_cap": 1080, "layer": 23, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 31, "mem_util_pct": 14, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 136.6, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 0.021, "pcie_tx_gbs": 3.844}], "pulls": {"expired": 0, "need": 46069, "idle": 0, "idle_used": 0, "ram_first": 4083, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 11.28, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 253.0, "ssd_below_cap": 26.77, "nothing_needed": 0.45}, "sd": {"all_readers": 279.83, "some_readers": 10.55, "idle_nothing": 0.93, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.15}, "readers_busy": 12, "h2d_ssd_gbs_cum": 910032535552, "h2d_ram_gbs_cum": 88013717504, "h2d_used_cum": 998046277632, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 492.5, "phase": "measure", "dt": 1.0, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.103, "ram_to_gpu_gbs": 3.599, "gpu_to_ram_gbs": 3.275, "decode_gbs": 4.278, "ssd_reads": 144, "ram_promotions": 23, "gpu_decodes": 170, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 23, "l2": 259}, "lkw": {"l0": 1569, "arriving": 0, "ssd_inflight": 0, "l1": 6629, "l2": 15802}, "missing": 139, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 24, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 147.0, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.517, "pcie_tx_gbs": 3.943}], "pulls": {"expired": 0, "need": 46248, "idle": 0, "idle_used": 0, "ram_first": 4106, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 11.32, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 253.85, "ssd_below_cap": 26.88, "nothing_needed": 0.46}, "sd": {"all_readers": 280.77, "some_readers": 10.6, "idle_nothing": 0.94, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.15}, "readers_busy": 12, "h2d_ssd_gbs_cum": 913136447488, "h2d_ram_gbs_cum": 88509489152, "h2d_used_cum": 1001667506176, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 493.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.167, "ram_to_gpu_gbs": 3.749, "gpu_to_ram_gbs": 3.382, "decode_gbs": 4.453, "ssd_reads": 147, "ram_promotions": 27, "gpu_decodes": 177, "scratch_hits": 0, "lk": {"l0": 3, "arriving": 0, "ssd_inflight": 0, "l1": 27, "l2": 254}, "lkw": {"l0": 2578, "arriving": 0, "ssd_inflight": 0, "l1": 6996, "l2": 14426}, "missing": 246, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1035, "ram_cap": 1080, "layer": 25, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 26, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 140.7, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.508, "pcie_tx_gbs": 3.843}], "pulls": {"expired": 0, "need": 46422, "idle": 0, "idle_used": 0, "ram_first": 4133, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 11.39, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 254.61, "ssd_below_cap": 27.04, "nothing_needed": 0.46}, "sd": {"all_readers": 281.7, "some_readers": 10.66, "idle_nothing": 0.94, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.16}, "readers_busy": 12, "h2d_ssd_gbs_cum": 916305244160, "h2d_ram_gbs_cum": 89091694592, "h2d_used_cum": 1005418508288, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 494.5, "phase": "measure", "dt": 1.0, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.103, "ram_to_gpu_gbs": 3.082, "gpu_to_ram_gbs": 2.672, "decode_gbs": 3.573, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 142, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 104, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 25, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 145.8, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.338, "pcie_tx_gbs": 2.669}], "pulls": {"expired": 0, "need": 46564, "idle": 0, "idle_used": 0, "ram_first": 4133, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 11.39, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 255.61, "ssd_below_cap": 27.04, "nothing_needed": 0.46}, "sd": {"all_readers": 282.7, "some_readers": 10.66, "idle_nothing": 0.94, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.16}, "readers_busy": 12, "h2d_ssd_gbs_cum": 919388504064, "h2d_ram_gbs_cum": 89091694592, "h2d_used_cum": 1008480202752, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 495.5, "phase": "measure", "dt": 1.0, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.147, "ram_to_gpu_gbs": 3.707, "gpu_to_ram_gbs": 3.362, "decode_gbs": 4.353, "ssd_reads": 146, "ram_promotions": 25, "gpu_decodes": 173, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 25, "l2": 257}, "lkw": {"l0": 912, "arriving": 0, "ssd_inflight": 0, "l1": 6707, "l2": 16381}, "missing": 213, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 26, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 52, "mem_util_pct": 19, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 143.8, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 3.525, "pcie_tx_gbs": 0.001}], "pulls": {"expired": 0, "need": 46737, "idle": 0, "idle_used": 0, "ram_first": 4158, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 11.46, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 256.39, "ssd_below_cap": 27.19, "nothing_needed": 0.46}, "sd": {"all_readers": 283.63, "some_readers": 10.73, "idle_nothing": 0.95, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.16}, "readers_busy": 12, "h2d_ssd_gbs_cum": 922557743104, "h2d_ram_gbs_cum": 89630728192, "h2d_used_cum": 1012188475392, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 496.5, "phase": "measure", "dt": 1.002, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.122, "ram_to_gpu_gbs": 3.122, "gpu_to_ram_gbs": 2.756, "decode_gbs": 3.619, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 144, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 69, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 26, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 138.7, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.071, "pcie_tx_gbs": 2.665}], "pulls": {"expired": 0, "need": 46881, "idle": 0, "idle_used": 0, "ram_first": 4158, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 11.46, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 257.39, "ssd_below_cap": 27.19, "nothing_needed": 0.46}, "sd": {"all_readers": 284.63, "some_readers": 10.73, "idle_nothing": 0.95, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.16}, "readers_busy": 12, "h2d_ssd_gbs_cum": 925684051968, "h2d_ram_gbs_cum": 89630728192, "h2d_used_cum": 1015293239296, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 497.5, "phase": "measure", "dt": 1.0, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.083, "ram_to_gpu_gbs": 3.6, "gpu_to_ram_gbs": 3.298, "decode_gbs": 4.253, "ssd_reads": 143, "ram_promotions": 24, "gpu_decodes": 169, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 24, "l2": 260}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 7644, "l2": 16356}, "missing": 185, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 27, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 138.7, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.071, "pcie_tx_gbs": 2.665}], "pulls": {"expired": 0, "need": 47048, "idle": 0, "idle_used": 0, "ram_first": 4182, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 11.53, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 258.16, "ssd_below_cap": 27.34, "nothing_needed": 0.46}, "sd": {"all_readers": 285.56, "some_readers": 10.8, "idle_nothing": 0.95, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.16}, "readers_busy": 12, "h2d_ssd_gbs_cum": 928767148032, "h2d_ram_gbs_cum": 90148179968, "h2d_used_cum": 1018936897536, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 498.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.145, "ram_to_gpu_gbs": 3.145, "gpu_to_ram_gbs": 2.693, "decode_gbs": 3.647, "ssd_reads": 146, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 39, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 27, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 147.9, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.347, "pcie_tx_gbs": 3.846}], "pulls": {"expired": 0, "need": 47194, "idle": 0, "idle_used": 0, "ram_first": 4182, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 11.53, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 259.16, "ssd_below_cap": 27.34, "nothing_needed": 0.46}, "sd": {"all_readers": 286.56, "some_readers": 10.8, "idle_nothing": 0.95, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.16}, "readers_busy": 12, "h2d_ssd_gbs_cum": 931914936320, "h2d_ram_gbs_cum": 90148179968, "h2d_used_cum": 1022063136768, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 499.5, "phase": "measure", "dt": 1.002, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.098, "ram_to_gpu_gbs": 3.528, "gpu_to_ram_gbs": 3.012, "decode_gbs": 4.169, "ssd_reads": 144, "ram_promotions": 20, "gpu_decodes": 166, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 20, "l2": 263}, "lkw": {"l0": 986, "arriving": 0, "ssd_inflight": 0, "l1": 6252, "l2": 16762}, "missing": 158, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 28, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 139.3, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 5.333, "pcie_tx_gbs": 3.745}], "pulls": {"expired": 0, "need": 47358, "idle": 0, "idle_used": 0, "ram_first": 4202, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 11.59, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 259.94, "ssd_below_cap": 27.51, "nothing_needed": 0.47}, "sd": {"all_readers": 287.48, "some_readers": 10.87, "idle_nothing": 0.96, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.16}, "readers_busy": 12, "h2d_ssd_gbs_cum": 935019593728, "h2d_ram_gbs_cum": 90579365888, "h2d_used_cum": 1025620529152, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 500.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.124, "ram_to_gpu_gbs": 3.124, "gpu_to_ram_gbs": 2.865, "decode_gbs": 3.647, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 13, "reads_in_flight": 13, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1064, "ram_cap": 1080, "layer": 28, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 143.4, "power_limit_w": 600.0, "temp_c": 30, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.346, "pcie_tx_gbs": 4.051}], "pulls": {"expired": 0, "need": 47480, "idle": 0, "idle_used": 0, "ram_first": 4202, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 11.64, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 260.78, "ssd_below_cap": 27.63, "nothing_needed": 0.47}, "sd": {"all_readers": 288.48, "some_readers": 10.87, "idle_nothing": 0.96, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.16}, "readers_busy": 12, "h2d_ssd_gbs_cum": 938145574912, "h2d_ram_gbs_cum": 90579365888, "h2d_used_cum": 1028746510336, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 501.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.102, "ram_to_gpu_gbs": 3.554, "gpu_to_ram_gbs": 3.296, "decode_gbs": 4.175, "ssd_reads": 144, "ram_promotions": 21, "gpu_decodes": 166, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 21, "l2": 260}, "lkw": {"l0": 725, "arriving": 0, "ssd_inflight": 0, "l1": 5231, "l2": 18044}, "missing": 129, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 29, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 142.5, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.497, "pcie_tx_gbs": 4.486}], "pulls": {"expired": 0, "need": 47668, "idle": 0, "idle_used": 0, "ram_first": 4223, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 11.66, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 261.7, "ssd_below_cap": 27.68, "nothing_needed": 0.47}, "sd": {"all_readers": 289.42, "some_readers": 10.93, "idle_nothing": 0.96, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.16}, "readers_busy": 12, "h2d_ssd_gbs_cum": 941249769472, "h2d_ram_gbs_cum": 91032113152, "h2d_used_cum": 1032303452160, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 502.5, "phase": "measure", "dt": 1.0, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.147, "ram_to_gpu_gbs": 3.621, "gpu_to_ram_gbs": 3.319, "decode_gbs": 4.279, "ssd_reads": 146, "ram_promotions": 22, "gpu_decodes": 170, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 22, "l2": 259}, "lkw": {"l0": 1519, "arriving": 0, "ssd_inflight": 0, "l1": 6463, "l2": 16018}, "missing": 242, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 30, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 31, "mem_util_pct": 14, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 142.0, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.336, "pcie_tx_gbs": 3.122}], "pulls": {"expired": 0, "need": 47836, "idle": 0, "idle_used": 0, "ram_first": 4245, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 11.73, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 262.46, "ssd_below_cap": 27.85, "nothing_needed": 0.47}, "sd": {"all_readers": 290.35, "some_readers": 10.99, "idle_nothing": 0.97, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.16}, "readers_busy": 12, "h2d_ssd_gbs_cum": 944396914688, "h2d_ram_gbs_cum": 91506409472, "h2d_used_cum": 1035924893696, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 503.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.146, "ram_to_gpu_gbs": 3.146, "gpu_to_ram_gbs": 2.586, "decode_gbs": 3.648, "ssd_reads": 146, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 96, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 30, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 56, "mem_util_pct": 20, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 146.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 2.345, "pcie_tx_gbs": 5.456}], "pulls": {"expired": 0, "need": 47982, "idle": 0, "idle_used": 0, "ram_first": 4245, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 11.73, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 263.46, "ssd_below_cap": 27.85, "nothing_needed": 0.47}, "sd": {"all_readers": 291.35, "some_readers": 10.99, "idle_nothing": 0.97, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.16}, "readers_busy": 12, "h2d_ssd_gbs_cum": 947544457216, "h2d_ram_gbs_cum": 91506409472, "h2d_used_cum": 1039050878976, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 504.5, "phase": "measure", "dt": 1.0, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.104, "ram_to_gpu_gbs": 3.556, "gpu_to_ram_gbs": 3.147, "decode_gbs": 4.228, "ssd_reads": 144, "ram_promotions": 21, "gpu_decodes": 168, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 21, "l2": 260}, "lkw": {"l0": 2509, "arriving": 0, "ssd_inflight": 0, "l1": 5316, "l2": 16175}, "missing": 212, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 31, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 137.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.345, "pcie_tx_gbs": 4.048}], "pulls": {"expired": 0, "need": 48147, "idle": 0, "idle_used": 0, "ram_first": 4266, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 11.8, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 264.22, "ssd_below_cap": 28.01, "nothing_needed": 0.48}, "sd": {"all_readers": 292.29, "some_readers": 11.05, "idle_nothing": 0.97, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.16}, "readers_busy": 12, "h2d_ssd_gbs_cum": 950648705024, "h2d_ram_gbs_cum": 91959140352, "h2d_used_cum": 1042629414912, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 505.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.144, "ram_to_gpu_gbs": 3.144, "gpu_to_ram_gbs": 2.563, "decode_gbs": 3.621, "ssd_reads": 146, "ram_promotions": 0, "gpu_decodes": 144, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 67, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 31, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 146.3, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.356, "pcie_tx_gbs": 1.617}], "pulls": {"expired": 0, "need": 48292, "idle": 0, "idle_used": 0, "ram_first": 4266, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 11.8, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 265.23, "ssd_below_cap": 28.01, "nothing_needed": 0.48}, "sd": {"all_readers": 293.29, "some_readers": 11.05, "idle_nothing": 0.97, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.16}, "readers_busy": 12, "h2d_ssd_gbs_cum": 953796263936, "h2d_ram_gbs_cum": 91959140352, "h2d_used_cum": 1045733847040, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 506.5, "phase": "measure", "dt": 1.002, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.121, "ram_to_gpu_gbs": 3.573, "gpu_to_ram_gbs": 3.121, "decode_gbs": 4.222, "ssd_reads": 145, "ram_promotions": 21, "gpu_decodes": 168, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 21, "l2": 266}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 6102, "l2": 17898}, "missing": 187, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 32, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 137.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 3.903, "pcie_tx_gbs": 3.846}], "pulls": {"expired": 0, "need": 48459, "idle": 0, "idle_used": 0, "ram_first": 4287, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 11.88, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 266.0, "ssd_below_cap": 28.16, "nothing_needed": 0.48}, "sd": {"all_readers": 294.22, "some_readers": 11.11, "idle_nothing": 0.98, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.16}, "readers_busy": 12, "h2d_ssd_gbs_cum": 956921966592, "h2d_ram_gbs_cum": 92411904000, "h2d_used_cum": 1049355440128, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 507.5, "phase": "measure", "dt": 1.0, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.06, "ram_to_gpu_gbs": 3.06, "gpu_to_ram_gbs": 2.672, "decode_gbs": 3.573, "ssd_reads": 142, "ram_promotions": 0, "gpu_decodes": 142, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 45, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 32, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 14, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 143.8, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.679, "pcie_tx_gbs": 6.509}], "pulls": {"expired": 0, "need": 48601, "idle": 0, "idle_used": 0, "ram_first": 4287, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 11.88, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 267.0, "ssd_below_cap": 28.16, "nothing_needed": 0.48}, "sd": {"all_readers": 295.22, "some_readers": 11.11, "idle_nothing": 0.98, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.16}, "readers_busy": 12, "h2d_ssd_gbs_cum": 959983091712, "h2d_ram_gbs_cum": 92411904000, "h2d_used_cum": 1052416565248, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 508.5, "phase": "measure", "dt": 1.0, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.147, "ram_to_gpu_gbs": 3.643, "gpu_to_ram_gbs": 3.319, "decode_gbs": 4.279, "ssd_reads": 146, "ram_promotions": 23, "gpu_decodes": 170, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 23, "l2": 259}, "lkw": {"l0": 1248, "arriving": 0, "ssd_inflight": 0, "l1": 7353, "l2": 15399}, "missing": 158, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 33, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 53, "mem_util_pct": 19, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 146.1, "power_limit_w": 600.0, "temp_c": 33, "sm_clock_mhz": 2325, "pcie_rx_gbs": 4.689, "pcie_tx_gbs": 2.666}], "pulls": {"expired": 0, "need": 48770, "idle": 0, "idle_used": 0, "ram_first": 4310, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 11.95, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 267.77, "ssd_below_cap": 28.32, "nothing_needed": 0.48}, "sd": {"all_readers": 296.15, "some_readers": 11.17, "idle_nothing": 0.99, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.16}, "readers_busy": 12, "h2d_ssd_gbs_cum": 963130425344, "h2d_ram_gbs_cum": 92907782144, "h2d_used_cum": 1056059777024, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 509.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.145, "ram_to_gpu_gbs": 3.145, "gpu_to_ram_gbs": 2.628, "decode_gbs": 3.647, "ssd_reads": 146, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 13, "reads_in_flight": 12, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1065, "ram_cap": 1080, "layer": 33, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 137.3, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 5.852, "pcie_tx_gbs": 3.946}], "pulls": {"expired": 0, "need": 48892, "idle": 0, "idle_used": 0, "ram_first": 4310, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 11.99, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 268.62, "ssd_below_cap": 28.43, "nothing_needed": 0.48}, "sd": {"all_readers": 297.15, "some_readers": 11.17, "idle_nothing": 0.99, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.16}, "readers_busy": 12, "h2d_ssd_gbs_cum": 966277832704, "h2d_ram_gbs_cum": 92907782144, "h2d_used_cum": 1059185627136, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 510.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.123, "ram_to_gpu_gbs": 3.553, "gpu_to_ram_gbs": 3.08, "decode_gbs": 4.199, "ssd_reads": 145, "ram_promotions": 20, "gpu_decodes": 167, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 20, "l2": 261}, "lkw": {"l0": 1979, "arriving": 0, "ssd_inflight": 0, "l1": 6402, "l2": 15619}, "missing": 128, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 34, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 31, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 147.0, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 5.835, "pcie_tx_gbs": 1.285}], "pulls": {"expired": 0, "need": 49081, "idle": 0, "idle_used": 0, "ram_first": 4330, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 12.02, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 269.54, "ssd_below_cap": 28.47, "nothing_needed": 0.48}, "sd": {"all_readers": 298.09, "some_readers": 11.24, "idle_nothing": 0.99, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.16}, "readers_busy": 12, "h2d_ssd_gbs_cum": 969403756544, "h2d_ram_gbs_cum": 93338992640, "h2d_used_cum": 1062742765568, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 511.5, "phase": "measure", "dt": 1.002, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.141, "ram_to_gpu_gbs": 3.506, "gpu_to_ram_gbs": 3.205, "decode_gbs": 4.119, "ssd_reads": 146, "ram_promotions": 17, "gpu_decodes": 164, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 17, "l2": 266}, "lkw": {"l0": 2223, "arriving": 0, "ssd_inflight": 0, "l1": 5371, "l2": 16406}, "missing": 250, "reads_in_flight": 34, "uploads_pending": 2, "gpu_experts": 106, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 35, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 136.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.346, "pcie_tx_gbs": 3.845}], "pulls": {"expired": 0, "need": 49242, "idle": 0, "idle_used": 0, "ram_first": 4347, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 12.08, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 270.31, "ssd_below_cap": 28.64, "nothing_needed": 0.49}, "sd": {"all_readers": 299.02, "some_readers": 11.3, "idle_nothing": 1.0, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.16}, "readers_busy": 12, "h2d_ssd_gbs_cum": 972551168000, "h2d_ram_gbs_cum": 93705482240, "h2d_used_cum": 1066235113472, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 512.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.08, "ram_to_gpu_gbs": 3.08, "gpu_to_ram_gbs": 2.714, "decode_gbs": 3.646, "ssd_reads": 143, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 105, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 35, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 145.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.346, "pcie_tx_gbs": 3.846}], "pulls": {"expired": 0, "need": 49387, "idle": 0, "idle_used": 0, "ram_first": 4347, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 12.08, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 271.31, "ssd_below_cap": 28.64, "nothing_needed": 0.49}, "sd": {"all_readers": 300.02, "some_readers": 11.3, "idle_nothing": 1.0, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.16}, "readers_busy": 12, "h2d_ssd_gbs_cum": 975634006016, "h2d_ram_gbs_cum": 93705482240, "h2d_used_cum": 1069361057792, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 513.5, "phase": "measure", "dt": 1.002, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.163, "ram_to_gpu_gbs": 3.507, "gpu_to_ram_gbs": 3.077, "decode_gbs": 4.095, "ssd_reads": 147, "ram_promotions": 16, "gpu_decodes": 163, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 16, "l2": 268}, "lkw": {"l0": 1871, "arriving": 0, "ssd_inflight": 0, "l1": 5565, "l2": 16564}, "missing": 226, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 36, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 26, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 137.6, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.712, "pcie_tx_gbs": 5.15}], "pulls": {"expired": 0, "need": 49550, "idle": 0, "idle_used": 0, "ram_first": 4363, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 12.15, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 272.07, "ssd_below_cap": 28.81, "nothing_needed": 0.49}, "sd": {"all_readers": 300.94, "some_readers": 11.37, "idle_nothing": 1.0, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.16}, "readers_busy": 12, "h2d_ssd_gbs_cum": 978802909184, "h2d_ram_gbs_cum": 94050426880, "h2d_used_cum": 1072831787008, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 514.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.123, "ram_to_gpu_gbs": 3.123, "gpu_to_ram_gbs": 2.584, "decode_gbs": 3.671, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 146, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 81, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 36, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 143.6, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.355, "pcie_tx_gbs": 3.945}], "pulls": {"expired": 0, "need": 49695, "idle": 0, "idle_used": 0, "ram_first": 4363, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 12.15, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 273.07, "ssd_below_cap": 28.81, "nothing_needed": 0.49}, "sd": {"all_readers": 301.94, "some_readers": 11.37, "idle_nothing": 1.0, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.16}, "readers_busy": 12, "h2d_ssd_gbs_cum": 981928775680, "h2d_ram_gbs_cum": 94050426880, "h2d_used_cum": 1075979214848, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 515.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.101, "ram_to_gpu_gbs": 3.467, "gpu_to_ram_gbs": 3.402, "decode_gbs": 4.048, "ssd_reads": 144, "ram_promotions": 17, "gpu_decodes": 161, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 17, "l2": 269}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 6201, "l2": 17799}, "missing": 207, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 37, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 142.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.33, "pcie_tx_gbs": 2.525}], "pulls": {"expired": 0, "need": 49855, "idle": 0, "idle_used": 0, "ram_first": 4380, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 12.21, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 273.84, "ssd_below_cap": 28.98, "nothing_needed": 0.49}, "sd": {"all_readers": 302.89, "some_readers": 11.42, "idle_nothing": 1.01, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.16}, "readers_busy": 12, "h2d_ssd_gbs_cum": 985032990720, "h2d_ram_gbs_cum": 94416941056, "h2d_used_cum": 1079449944064, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 516.5, "phase": "measure", "dt": 1.0, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.103, "ram_to_gpu_gbs": 3.103, "gpu_to_ram_gbs": 2.629, "decode_gbs": 3.649, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 62, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 37, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 139.3, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 4.682, "pcie_tx_gbs": 0.918}], "pulls": {"expired": 0, "need": 50000, "idle": 0, "idle_used": 0, "ram_first": 4380, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 12.21, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 274.84, "ssd_below_cap": 28.98, "nothing_needed": 0.49}, "sd": {"all_readers": 303.89, "some_readers": 11.42, "idle_nothing": 1.01, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.16}, "readers_busy": 12, "h2d_ssd_gbs_cum": 988137152512, "h2d_ram_gbs_cum": 94416941056, "h2d_used_cum": 1082575663104, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 517.5, "phase": "measure", "dt": 1.002, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.14, "ram_to_gpu_gbs": 3.377, "gpu_to_ram_gbs": 3.076, "decode_gbs": 4.019, "ssd_reads": 146, "ram_promotions": 11, "gpu_decodes": 160, "scratch_hits": 0, "lk": {"l0": 3, "arriving": 0, "ssd_inflight": 0, "l1": 11, "l2": 271}, "lkw": {"l0": 3500, "arriving": 0, "ssd_inflight": 0, "l1": 3979, "l2": 16521}, "missing": 187, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 38, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 46, "mem_util_pct": 16, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 142.7, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 2.344, "pcie_tx_gbs": 3.947}], "pulls": {"expired": 0, "need": 50157, "idle": 0, "idle_used": 0, "ram_first": 4391, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 12.29, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 275.61, "ssd_below_cap": 29.13, "nothing_needed": 0.5}, "sd": {"all_readers": 304.83, "some_readers": 11.48, "idle_nothing": 1.01, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.16}, "readers_busy": 12, "h2d_ssd_gbs_cum": 991284068352, "h2d_ram_gbs_cum": 94654050304, "h2d_used_cum": 1085959688192, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 518.5, "phase": "measure", "dt": 1.0, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.146, "ram_to_gpu_gbs": 3.125, "gpu_to_ram_gbs": 2.607, "decode_gbs": 3.624, "ssd_reads": 146, "ram_promotions": 0, "gpu_decodes": 144, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 43, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 38, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 139.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.527, "pcie_tx_gbs": 0.001}], "pulls": {"expired": 0, "need": 50301, "idle": 0, "idle_used": 0, "ram_first": 4391, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 12.29, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 276.62, "ssd_below_cap": 29.13, "nothing_needed": 0.5}, "sd": {"all_readers": 305.83, "some_readers": 11.48, "idle_nothing": 1.01, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.16}, "readers_busy": 12, "h2d_ssd_gbs_cum": 994409594880, "h2d_ram_gbs_cum": 94654050304, "h2d_used_cum": 1089063665664, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 519.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.1, "ram_to_gpu_gbs": 3.466, "gpu_to_ram_gbs": 3.122, "decode_gbs": 4.098, "ssd_reads": 144, "ram_promotions": 16, "gpu_decodes": 163, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 16, "l2": 266}, "lkw": {"l0": 1066, "arriving": 0, "ssd_inflight": 0, "l1": 5650, "l2": 17284}, "missing": 163, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 39, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 26, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 146.0, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.345, "pcie_tx_gbs": 2.665}], "pulls": {"expired": 0, "need": 50463, "idle": 0, "idle_used": 0, "ram_first": 4407, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 12.36, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 277.38, "ssd_below_cap": 29.29, "nothing_needed": 0.5}, "sd": {"all_readers": 306.78, "some_readers": 11.52, "idle_nothing": 1.02, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.16}, "readers_busy": 12, "h2d_ssd_gbs_cum": 997535150080, "h2d_ram_gbs_cum": 94998966272, "h2d_used_cum": 1092555685888, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 520.5, "phase": "measure", "dt": 1.002, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.119, "ram_to_gpu_gbs": 3.119, "gpu_to_ram_gbs": 2.732, "decode_gbs": 3.618, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 144, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 19, "reads_in_flight": 18, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1057, "ram_cap": 1080, "layer": 39, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 138.6, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.345, "pcie_tx_gbs": 2.562}], "pulls": {"expired": 0, "need": 50590, "idle": 0, "idle_used": 0, "ram_first": 4407, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 12.39, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 278.26, "ssd_below_cap": 29.38, "nothing_needed": 0.5}, "sd": {"all_readers": 307.78, "some_readers": 11.52, "idle_nothing": 1.02, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.16}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1000660439040, "h2d_ram_gbs_cum": 94998966272, "h2d_used_cum": 1095659421696, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 521.5, "phase": "measure", "dt": 1.0, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.102, "ram_to_gpu_gbs": 3.447, "gpu_to_ram_gbs": 3.059, "decode_gbs": 4.076, "ssd_reads": 144, "ram_promotions": 16, "gpu_decodes": 162, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 16, "l2": 269}, "lkw": {"l0": 1109, "arriving": 0, "ssd_inflight": 0, "l1": 6029, "l2": 16862}, "missing": 143, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 40, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 140.8, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 3.515, "pcie_tx_gbs": 2.737}], "pulls": {"expired": 0, "need": 50768, "idle": 0, "idle_used": 0, "ram_first": 4423, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 12.42, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 279.15, "ssd_below_cap": 29.46, "nothing_needed": 0.5}, "sd": {"all_readers": 308.7, "some_readers": 11.6, "idle_nothing": 1.02, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.16}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1003763888128, "h2d_ram_gbs_cum": 95343751168, "h2d_used_cum": 1099129208832, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 522.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.079, "ram_to_gpu_gbs": 3.445, "gpu_to_ram_gbs": 3.208, "decode_gbs": 4.024, "ssd_reads": 143, "ram_promotions": 17, "gpu_decodes": 160, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 17, "l2": 270}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 6249, "l2": 17751}, "missing": 270, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1031, "ram_cap": 1080, "layer": 41, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 140.8, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 3.515, "pcie_tx_gbs": 2.737}], "pulls": {"expired": 0, "need": 50928, "idle": 0, "idle_used": 0, "ram_first": 4440, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 12.49, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 279.92, "ssd_below_cap": 29.62, "nothing_needed": 0.5}, "sd": {"all_readers": 309.64, "some_readers": 11.65, "idle_nothing": 1.03, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.17}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1006845591552, "h2d_ram_gbs_cum": 95710027776, "h2d_used_cum": 1102577188864, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 523.5, "phase": "measure", "dt": 1.0, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.166, "ram_to_gpu_gbs": 3.166, "gpu_to_ram_gbs": 2.8, "decode_gbs": 3.674, "ssd_reads": 147, "ram_promotions": 0, "gpu_decodes": 146, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 124, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 41, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 27, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 139.8, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 17.459, "pcie_tx_gbs": 1.336}], "pulls": {"expired": 0, "need": 51074, "idle": 0, "idle_used": 0, "ram_first": 4440, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 12.49, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 280.92, "ssd_below_cap": 29.62, "nothing_needed": 0.5}, "sd": {"all_readers": 310.64, "some_readers": 11.65, "idle_nothing": 1.03, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.17}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1010012491776, "h2d_ram_gbs_cum": 95710027776, "h2d_used_cum": 1105722540032, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 524.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.142, "ram_to_gpu_gbs": 3.593, "gpu_to_ram_gbs": 3.228, "decode_gbs": 4.249, "ssd_reads": 146, "ram_promotions": 21, "gpu_decodes": 169, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 21, "l2": 261}, "lkw": {"l0": 940, "arriving": 0, "ssd_inflight": 0, "l1": 7443, "l2": 15617}, "missing": 238, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 42, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 141.5, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 5.829, "pcie_tx_gbs": 3.945}], "pulls": {"expired": 0, "need": 51242, "idle": 0, "idle_used": 0, "ram_first": 4461, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 12.56, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 281.69, "ssd_below_cap": 29.78, "nothing_needed": 0.51}, "sd": {"all_readers": 311.57, "some_readers": 11.72, "idle_nothing": 1.04, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.17}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1013157785600, "h2d_ram_gbs_cum": 96162336768, "h2d_used_cum": 1109341691904, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 525.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.076, "ram_to_gpu_gbs": 3.076, "gpu_to_ram_gbs": 2.667, "decode_gbs": 3.57, "ssd_reads": 143, "ram_promotions": 0, "gpu_decodes": 142, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 96, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 42, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 51, "mem_util_pct": 17, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 145.1, "power_limit_w": 600.0, "temp_c": 32, "sm_clock_mhz": 2325, "pcie_rx_gbs": 3.523, "pcie_tx_gbs": 5.121}], "pulls": {"expired": 0, "need": 51384, "idle": 0, "idle_used": 0, "ram_first": 4461, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 12.56, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 282.69, "ssd_below_cap": 29.78, "nothing_needed": 0.51}, "sd": {"all_readers": 312.57, "some_readers": 11.72, "idle_nothing": 1.04, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.17}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1016237809664, "h2d_ram_gbs_cum": 96162336768, "h2d_used_cum": 1112400183296, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 526.5, "phase": "measure", "dt": 1.0, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.144, "ram_to_gpu_gbs": 3.445, "gpu_to_ram_gbs": 3.144, "decode_gbs": 4.052, "ssd_reads": 146, "ram_promotions": 14, "gpu_decodes": 161, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 14, "l2": 273}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 5793, "l2": 18207}, "missing": 222, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 43, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 25, "mem_util_pct": 10, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 138.7, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 2.342, "pcie_tx_gbs": 5.221}], "pulls": {"expired": 0, "need": 51545, "idle": 0, "idle_used": 0, "ram_first": 4475, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 12.62, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 283.46, "ssd_below_cap": 29.94, "nothing_needed": 0.51}, "sd": {"all_readers": 313.5, "some_readers": 11.78, "idle_nothing": 1.04, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.17}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1019382321152, "h2d_ram_gbs_cum": 96463896576, "h2d_used_cum": 1115867787264, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 527.5, "phase": "measure", "dt": 1.002, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.139, "ram_to_gpu_gbs": 3.139, "gpu_to_ram_gbs": 2.645, "decode_gbs": 3.668, "ssd_reads": 146, "ram_promotions": 0, "gpu_decodes": 146, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 76, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 43, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 31, "mem_util_pct": 14, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 141.8, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 5.845, "pcie_tx_gbs": 2.46}], "pulls": {"expired": 0, "need": 51691, "idle": 0, "idle_used": 0, "ram_first": 4475, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 12.62, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 284.46, "ssd_below_cap": 29.94, "nothing_needed": 0.51}, "sd": {"all_readers": 314.5, "some_readers": 11.78, "idle_nothing": 1.04, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.17}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1022527619072, "h2d_ram_gbs_cum": 96463896576, "h2d_used_cum": 1119013085184, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 528.5, "phase": "measure", "dt": 1.003, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.138, "ram_to_gpu_gbs": 3.546, "gpu_to_ram_gbs": 3.03, "decode_gbs": 4.092, "ssd_reads": 146, "ram_promotions": 19, "gpu_decodes": 163, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 19, "l2": 261}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 7560, "l2": 16440}, "missing": 192, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 44, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 136.3, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 5.856, "pcie_tx_gbs": 2.56}], "pulls": {"expired": 0, "need": 51855, "idle": 0, "idle_used": 0, "ram_first": 4494, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 12.69, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 285.23, "ssd_below_cap": 30.11, "nothing_needed": 0.51}, "sd": {"all_readers": 315.44, "some_readers": 11.84, "idle_nothing": 1.05, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.17}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1025673670656, "h2d_ram_gbs_cum": 96873254912, "h2d_used_cum": 1122525413376, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 529.5, "phase": "measure", "dt": 1.001, "tokens": 9869, "tok_s": 0.0, "ssd_gbs": 3.105, "ram_to_gpu_gbs": 3.105, "gpu_to_ram_gbs": 2.673, "decode_gbs": 3.673, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 146, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 47, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 44, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 147.9, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.526, "pcie_tx_gbs": 3.246}], "pulls": {"expired": 0, "need": 52000, "idle": 0, "idle_used": 0, "ram_first": 4494, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 12.69, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 286.23, "ssd_below_cap": 30.11, "nothing_needed": 0.51}, "sd": {"all_readers": 316.44, "some_readers": 11.84, "idle_nothing": 1.05, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.17}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1028780474368, "h2d_ram_gbs_cum": 96873254912, "h2d_used_cum": 1125675298816, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 530.5, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 1627.916, "ssd_gbs": 3.143, "ram_to_gpu_gbs": 3.272, "gpu_to_ram_gbs": 2.863, "decode_gbs": 3.796, "ssd_reads": 146, "ram_promotions": 6, "gpu_decodes": 151, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 6, "l2": 282}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 1130, "l2": 22870}, "missing": 184, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 3, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 136.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.686, "pcie_tx_gbs": 5.123}], "pulls": {"expired": 0, "need": 52151, "idle": 0, "idle_used": 0, "ram_first": 4500, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 12.76, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 286.99, "ssd_below_cap": 30.28, "nothing_needed": 0.51}, "sd": {"all_readers": 317.36, "some_readers": 11.92, "idle_nothing": 1.05, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.17}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1031927517184, "h2d_ram_gbs_cum": 97002532864, "h2d_used_cum": 1128930066432, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 531.5, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.123, "ram_to_gpu_gbs": 3.123, "gpu_to_ram_gbs": 2.628, "decode_gbs": 3.648, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 38, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 3, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 149.1, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.344, "pcie_tx_gbs": 5.325}], "pulls": {"expired": 0, "need": 52297, "idle": 0, "idle_used": 0, "ram_first": 4500, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 12.76, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 287.99, "ssd_below_cap": 30.28, "nothing_needed": 0.51}, "sd": {"all_readers": 318.36, "some_readers": 11.92, "idle_nothing": 1.05, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.17}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1035052425216, "h2d_ram_gbs_cum": 97002532864, "h2d_used_cum": 1132054974464, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 532.5, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.1, "ram_to_gpu_gbs": 3.638, "gpu_to_ram_gbs": 3.401, "decode_gbs": 4.274, "ssd_reads": 144, "ram_promotions": 25, "gpu_decodes": 170, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 25, "l2": 263}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 3489, "l2": 20511}, "missing": 157, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 4, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 141.3, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.355, "pcie_tx_gbs": 2.562}], "pulls": {"expired": 0, "need": 52462, "idle": 0, "idle_used": 0, "ram_first": 4525, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 12.83, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 288.75, "ssd_below_cap": 30.45, "nothing_needed": 0.52}, "sd": {"all_readers": 319.3, "some_readers": 11.97, "idle_nothing": 1.06, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.17}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1038156128256, "h2d_ram_gbs_cum": 97541427200, "h2d_used_cum": 1135719124992, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 533.5, "phase": "measure", "dt": 1.0, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.125, "ram_to_gpu_gbs": 3.125, "gpu_to_ram_gbs": 2.759, "decode_gbs": 3.599, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 143, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 13, "reads_in_flight": 12, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1065, "ram_cap": 1080, "layer": 4, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 144.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.355, "pcie_tx_gbs": 3.945}], "pulls": {"expired": 0, "need": 52583, "idle": 0, "idle_used": 0, "ram_first": 4525, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 12.87, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 289.58, "ssd_below_cap": 30.58, "nothing_needed": 0.52}, "sd": {"all_readers": 320.31, "some_readers": 11.97, "idle_nothing": 1.06, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.17}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1041281605632, "h2d_ram_gbs_cum": 97541427200, "h2d_used_cum": 1138801487872, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 534.5, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.124, "ram_to_gpu_gbs": 3.619, "gpu_to_ram_gbs": 3.296, "decode_gbs": 4.252, "ssd_reads": 145, "ram_promotions": 23, "gpu_decodes": 169, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 23, "l2": 265}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 3410, "l2": 20590}, "missing": 133, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 5, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 25, "mem_util_pct": 10, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 147.8, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 3.263, "pcie_tx_gbs": 5.226}], "pulls": {"expired": 0, "need": 52773, "idle": 0, "idle_used": 0, "ram_first": 4548, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 12.9, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 290.51, "ssd_below_cap": 30.62, "nothing_needed": 0.52}, "sd": {"all_readers": 321.24, "some_readers": 12.03, "idle_nothing": 1.06, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.17}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1044407148544, "h2d_ram_gbs_cum": 98037190656, "h2d_used_cum": 1142444351488, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 535.5, "phase": "measure", "dt": 1.0, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.125, "ram_to_gpu_gbs": 3.771, "gpu_to_ram_gbs": 3.577, "decode_gbs": 4.429, "ssd_reads": 145, "ram_promotions": 30, "gpu_decodes": 176, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 30, "l2": 258}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 4334, "l2": 19666}, "missing": 245, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1033, "ram_cap": 1080, "layer": 6, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 136.6, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.343, "pcie_tx_gbs": 5.227}], "pulls": {"expired": 0, "need": 52942, "idle": 0, "idle_used": 0, "ram_first": 4578, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 12.97, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 291.27, "ssd_below_cap": 30.79, "nothing_needed": 0.52}, "sd": {"all_readers": 322.17, "some_readers": 12.09, "idle_nothing": 1.07, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.17}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1047532638208, "h2d_ram_gbs_cum": 98683858944, "h2d_used_cum": 1146238066688, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 536.5, "phase": "measure", "dt": 1.0, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.104, "ram_to_gpu_gbs": 3.104, "gpu_to_ram_gbs": 2.63, "decode_gbs": 3.625, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 144, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 101, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 6, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 149.0, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 0.021, "pcie_tx_gbs": 2.663}], "pulls": {"expired": 0, "need": 53086, "idle": 0, "idle_used": 0, "ram_first": 4578, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 12.97, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 292.27, "ssd_below_cap": 30.79, "nothing_needed": 0.52}, "sd": {"all_readers": 323.17, "some_readers": 12.09, "idle_nothing": 1.07, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.17}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1050636627968, "h2d_ram_gbs_cum": 98683858944, "h2d_used_cum": 1149342056448, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 537.5, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.102, "ram_to_gpu_gbs": 3.921, "gpu_to_ram_gbs": 3.662, "decode_gbs": 4.629, "ssd_reads": 144, "ram_promotions": 38, "gpu_decodes": 184, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 38, "l2": 248}, "lkw": {"l0": 1415, "arriving": 0, "ssd_inflight": 0, "l1": 6772, "l2": 15813}, "missing": 205, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 7, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 27, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 138.0, "power_limit_w": 600.0, "temp_c": 30, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.344, "pcie_tx_gbs": 3.844}], "pulls": {"expired": 0, "need": 53264, "idle": 0, "idle_used": 0, "ram_first": 4616, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 13.03, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 293.03, "ssd_below_cap": 30.96, "nothing_needed": 0.52}, "sd": {"all_readers": 324.11, "some_readers": 12.15, "idle_nothing": 1.07, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.17}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1053740544000, "h2d_ram_gbs_cum": 99502940160, "h2d_used_cum": 1153265053696, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 538.5, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.124, "ram_to_gpu_gbs": 3.124, "gpu_to_ram_gbs": 2.758, "decode_gbs": 3.648, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 60, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 7, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 31, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 151.7, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.335, "pcie_tx_gbs": 3.846}], "pulls": {"expired": 0, "need": 53409, "idle": 0, "idle_used": 0, "ram_first": 4616, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 13.03, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 294.03, "ssd_below_cap": 30.96, "nothing_needed": 0.52}, "sd": {"all_readers": 325.11, "some_readers": 12.15, "idle_nothing": 1.07, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.17}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1056866013184, "h2d_ram_gbs_cum": 99502940160, "h2d_used_cum": 1156390522880, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 539.5, "phase": "measure", "dt": 1.002, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.163, "ram_to_gpu_gbs": 3.981, "gpu_to_ram_gbs": 3.679, "decode_gbs": 4.598, "ssd_reads": 147, "ram_promotions": 38, "gpu_decodes": 183, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 38, "l2": 250}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 7642, "l2": 16358}, "missing": 165, "reads_in_flight": 34, "uploads_pending": 2, "gpu_experts": 106, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 8, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 149.0, "power_limit_w": 600.0, "temp_c": 33, "sm_clock_mhz": 2325, "pcie_rx_gbs": 3.508, "pcie_tx_gbs": 4.249}], "pulls": {"expired": 0, "need": 53588, "idle": 0, "idle_used": 0, "ram_first": 4654, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 13.1, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 294.8, "ssd_below_cap": 31.12, "nothing_needed": 0.53}, "sd": {"all_readers": 326.03, "some_readers": 12.22, "idle_nothing": 1.08, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.17}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1060034449408, "h2d_ram_gbs_cum": 100321996800, "h2d_used_cum": 1160334905344, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 540.5, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.123, "ram_to_gpu_gbs": 3.123, "gpu_to_ram_gbs": 2.541, "decode_gbs": 3.697, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 147, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 18, "reads_in_flight": 18, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1057, "ram_cap": 1080, "layer": 8, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 31, "mem_util_pct": 15, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 135.6, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.344, "pcie_tx_gbs": 3.845}], "pulls": {"expired": 0, "need": 53717, "idle": 0, "idle_used": 0, "ram_first": 4654, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 13.13, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 295.67, "ssd_below_cap": 31.21, "nothing_needed": 0.53}, "sd": {"all_readers": 327.03, "some_readers": 12.22, "idle_nothing": 1.08, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.17}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1063159853056, "h2d_ram_gbs_cum": 100321996800, "h2d_used_cum": 1163503419392, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 541.5, "phase": "measure", "dt": 1.0, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.147, "ram_to_gpu_gbs": 3.966, "gpu_to_ram_gbs": 3.599, "decode_gbs": 4.606, "ssd_reads": 146, "ram_promotions": 38, "gpu_decodes": 183, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 38, "l2": 248}, "lkw": {"l0": 695, "arriving": 0, "ssd_inflight": 0, "l1": 6815, "l2": 16490}, "missing": 121, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 9, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 31, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 150.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.691, "pcie_tx_gbs": 5.124}], "pulls": {"expired": 0, "need": 53915, "idle": 0, "idle_used": 0, "ram_first": 4692, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 13.16, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 296.56, "ssd_below_cap": 31.3, "nothing_needed": 0.53}, "sd": {"all_readers": 327.96, "some_readers": 12.29, "idle_nothing": 1.08, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.17}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1066306846720, "h2d_ram_gbs_cum": 101141098496, "h2d_used_cum": 1167426412544, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 542.5, "phase": "measure", "dt": 1.0, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.081, "ram_to_gpu_gbs": 3.728, "gpu_to_ram_gbs": 3.448, "decode_gbs": 4.379, "ssd_reads": 143, "ram_promotions": 30, "gpu_decodes": 174, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 30, "l2": 258}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 5312, "l2": 18688}, "missing": 236, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 10, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 141.5, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.68, "pcie_tx_gbs": 2.563}], "pulls": {"expired": 0, "need": 54087, "idle": 0, "idle_used": 0, "ram_first": 4722, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 13.23, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 297.33, "ssd_below_cap": 31.45, "nothing_needed": 0.53}, "sd": {"all_readers": 328.89, "some_readers": 12.35, "idle_nothing": 1.09, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.17}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1069389070336, "h2d_ram_gbs_cum": 101787754496, "h2d_used_cum": 1171176845312, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 543.5, "phase": "measure", "dt": 1.002, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.12, "ram_to_gpu_gbs": 3.12, "gpu_to_ram_gbs": 2.647, "decode_gbs": 3.644, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 90, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 10, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 148.7, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.507, "pcie_tx_gbs": 5.125}], "pulls": {"expired": 0, "need": 54233, "idle": 0, "idle_used": 0, "ram_first": 4722, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 13.23, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 298.33, "ssd_below_cap": 31.45, "nothing_needed": 0.53}, "sd": {"all_readers": 329.89, "some_readers": 12.35, "idle_nothing": 1.09, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.17}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1072514531328, "h2d_ram_gbs_cum": 101787754496, "h2d_used_cum": 1174302298112, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 544.6, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.164, "ram_to_gpu_gbs": 3.724, "gpu_to_ram_gbs": 3.336, "decode_gbs": 4.374, "ssd_reads": 147, "ram_promotions": 26, "gpu_decodes": 174, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 26, "l2": 260}, "lkw": {"l0": 2436, "arriving": 0, "ssd_inflight": 0, "l1": 4731, "l2": 16833}, "missing": 203, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 11, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 24, "mem_util_pct": 10, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 148.5, "power_limit_w": 600.0, "temp_c": 33, "sm_clock_mhz": 2325, "pcie_rx_gbs": 3.525, "pcie_tx_gbs": 2.563}], "pulls": {"expired": 0, "need": 54406, "idle": 0, "idle_used": 0, "ram_first": 4748, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 13.3, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 299.09, "ssd_below_cap": 31.62, "nothing_needed": 0.53}, "sd": {"all_readers": 330.82, "some_readers": 12.42, "idle_nothing": 1.1, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.17}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1075683041280, "h2d_ram_gbs_cum": 102348193792, "h2d_used_cum": 1178009702400, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 545.6, "phase": "measure", "dt": 1.002, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.097, "ram_to_gpu_gbs": 3.097, "gpu_to_ram_gbs": 2.667, "decode_gbs": 3.617, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 144, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 61, "reads_in_flight": 34, "uploads_pending": 2, "gpu_experts": 106, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 11, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 31, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 136.3, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 0.011, "pcie_tx_gbs": 3.844}], "pulls": {"expired": 0, "need": 54548, "idle": 0, "idle_used": 0, "ram_first": 4748, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 13.3, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 300.09, "ssd_below_cap": 31.62, "nothing_needed": 0.53}, "sd": {"all_readers": 331.82, "some_readers": 12.42, "idle_nothing": 1.1, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.17}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1078786949120, "h2d_ram_gbs_cum": 102348193792, "h2d_used_cum": 1181113610240, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 546.6, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.102, "ram_to_gpu_gbs": 3.877, "gpu_to_ram_gbs": 3.619, "decode_gbs": 4.628, "ssd_reads": 144, "ram_promotions": 36, "gpu_decodes": 184, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 36, "l2": 249}, "lkw": {"l0": 1184, "arriving": 0, "ssd_inflight": 0, "l1": 7879, "l2": 14937}, "missing": 164, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 12, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 149.7, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.355, "pcie_tx_gbs": 2.666}], "pulls": {"expired": 0, "need": 54727, "idle": 0, "idle_used": 0, "ram_first": 4784, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 13.37, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 300.87, "ssd_below_cap": 31.78, "nothing_needed": 0.54}, "sd": {"all_readers": 332.74, "some_readers": 12.49, "idle_nothing": 1.1, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.18}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1081891045376, "h2d_ram_gbs_cum": 103124258816, "h2d_used_cum": 1185036873728, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 547.6, "phase": "measure", "dt": 1.0, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.147, "ram_to_gpu_gbs": 3.147, "gpu_to_ram_gbs": 2.586, "decode_gbs": 3.649, "ssd_reads": 146, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 19, "reads_in_flight": 18, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1059, "ram_cap": 1080, "layer": 12, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 32, "mem_util_pct": 14, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 140.3, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 2.352, "pcie_tx_gbs": 5.128}], "pulls": {"expired": 0, "need": 54855, "idle": 0, "idle_used": 0, "ram_first": 4784, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 13.4, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 301.76, "ssd_below_cap": 31.86, "nothing_needed": 0.54}, "sd": {"all_readers": 333.74, "some_readers": 12.49, "idle_nothing": 1.1, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.18}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1085038231552, "h2d_ram_gbs_cum": 103124258816, "h2d_used_cum": 1188162502656, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 548.6, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.123, "ram_to_gpu_gbs": 3.79, "gpu_to_ram_gbs": 3.532, "decode_gbs": 4.552, "ssd_reads": 145, "ram_promotions": 31, "gpu_decodes": 181, "scratch_hits": 0, "lk": {"l0": 4, "arriving": 0, "ssd_inflight": 0, "l1": 31, "l2": 252}, "lkw": {"l0": 3015, "arriving": 0, "ssd_inflight": 0, "l1": 6292, "l2": 14693}, "missing": 125, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 13, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 32, "mem_util_pct": 14, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 140.3, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 2.352, "pcie_tx_gbs": 5.128}], "pulls": {"expired": 0, "need": 55048, "idle": 0, "idle_used": 0, "ram_first": 4815, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 13.44, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 302.64, "ssd_below_cap": 31.93, "nothing_needed": 0.54}, "sd": {"all_readers": 334.67, "some_readers": 12.55, "idle_nothing": 1.11, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.18}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1088163766272, "h2d_ram_gbs_cum": 103792447488, "h2d_used_cum": 1191977783296, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 549.6, "phase": "measure", "dt": 1.0, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.102, "ram_to_gpu_gbs": 3.684, "gpu_to_ram_gbs": 3.447, "decode_gbs": 4.403, "ssd_reads": 144, "ram_promotions": 27, "gpu_decodes": 175, "scratch_hits": 0, "lk": {"l0": 4, "arriving": 0, "ssd_inflight": 0, "l1": 27, "l2": 256}, "lkw": {"l0": 3453, "arriving": 0, "ssd_inflight": 0, "l1": 5605, "l2": 14942}, "missing": 237, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 14, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 25, "mem_util_pct": 10, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 146.8, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 5.86, "pcie_tx_gbs": 1.28}], "pulls": {"expired": 0, "need": 55218, "idle": 0, "idle_used": 0, "ram_first": 4842, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 13.51, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 303.39, "ssd_below_cap": 32.11, "nothing_needed": 0.54}, "sd": {"all_readers": 335.6, "some_readers": 12.62, "idle_nothing": 1.12, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.18}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1091267608576, "h2d_ram_gbs_cum": 104374521856, "h2d_used_cum": 1195663699968, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 550.6, "phase": "measure", "dt": 1.002, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.121, "ram_to_gpu_gbs": 3.121, "gpu_to_ram_gbs": 2.497, "decode_gbs": 3.644, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 92, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 14, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 44, "mem_util_pct": 16, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 146.1, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 2.365, "pcie_tx_gbs": 2.61}], "pulls": {"expired": 0, "need": 55363, "idle": 0, "idle_used": 0, "ram_first": 4842, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 13.51, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 304.39, "ssd_below_cap": 32.11, "nothing_needed": 0.54}, "sd": {"all_readers": 336.6, "some_readers": 12.62, "idle_nothing": 1.12, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.18}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1094393675776, "h2d_ram_gbs_cum": 104374521856, "h2d_used_cum": 1198789767168, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 551.6, "phase": "measure", "dt": 1.002, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.141, "ram_to_gpu_gbs": 3.765, "gpu_to_ram_gbs": 3.356, "decode_gbs": 4.447, "ssd_reads": 146, "ram_promotions": 29, "gpu_decodes": 177, "scratch_hits": 0, "lk": {"l0": 3, "arriving": 0, "ssd_inflight": 0, "l1": 29, "l2": 255}, "lkw": {"l0": 2311, "arriving": 0, "ssd_inflight": 0, "l1": 6884, "l2": 14805}, "missing": 201, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 15, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 136.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.335, "pcie_tx_gbs": 3.947}], "pulls": {"expired": 0, "need": 55538, "idle": 0, "idle_used": 0, "ram_first": 4871, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 13.58, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 305.16, "ssd_below_cap": 32.27, "nothing_needed": 0.54}, "sd": {"all_readers": 337.53, "some_readers": 12.68, "idle_nothing": 1.12, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.18}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1097541185536, "h2d_ram_gbs_cum": 104999870464, "h2d_used_cum": 1202541068288, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 552.6, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.101, "ram_to_gpu_gbs": 3.101, "gpu_to_ram_gbs": 2.714, "decode_gbs": 3.646, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 57, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 15, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 146.6, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.38, "pcie_tx_gbs": 2.666}], "pulls": {"expired": 0, "need": 55682, "idle": 0, "idle_used": 0, "ram_first": 4871, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 13.58, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 306.16, "ssd_below_cap": 32.27, "nothing_needed": 0.54}, "sd": {"all_readers": 338.53, "some_readers": 12.68, "idle_nothing": 1.12, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.18}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1100646023168, "h2d_ram_gbs_cum": 104999870464, "h2d_used_cum": 1205667463168, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 553.6, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.144, "ram_to_gpu_gbs": 4.048, "gpu_to_ram_gbs": 3.876, "decode_gbs": 4.751, "ssd_reads": 146, "ram_promotions": 42, "gpu_decodes": 189, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 42, "l2": 244}, "lkw": {"l0": 841, "arriving": 0, "ssd_inflight": 0, "l1": 9042, "l2": 14117}, "missing": 155, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 16, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 136.7, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.526, "pcie_tx_gbs": 1.284}], "pulls": {"expired": 0, "need": 55863, "idle": 0, "idle_used": 0, "ram_first": 4913, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 13.65, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 306.92, "ssd_below_cap": 32.44, "nothing_needed": 0.55}, "sd": {"all_readers": 339.45, "some_readers": 12.76, "idle_nothing": 1.13, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.18}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1103793950720, "h2d_ram_gbs_cum": 105905512448, "h2d_used_cum": 1209721032704, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 554.6, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.081, "ram_to_gpu_gbs": 3.081, "gpu_to_ram_gbs": 2.715, "decode_gbs": 3.597, "ssd_reads": 143, "ram_promotions": 0, "gpu_decodes": 143, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 12, "reads_in_flight": 12, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1066, "ram_cap": 1080, "layer": 16, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 146.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 7.015, "pcie_tx_gbs": 3.843}], "pulls": {"expired": 0, "need": 55982, "idle": 0, "idle_used": 0, "ram_first": 4913, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 13.7, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 307.75, "ssd_below_cap": 32.56, "nothing_needed": 0.55}, "sd": {"all_readers": 340.45, "some_readers": 12.76, "idle_nothing": 1.13, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.18}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1106877095936, "h2d_ram_gbs_cum": 105905512448, "h2d_used_cum": 1212804177920, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 555.6, "phase": "measure", "dt": 1.0, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.126, "ram_to_gpu_gbs": 3.966, "gpu_to_ram_gbs": 3.643, "decode_gbs": 4.706, "ssd_reads": 145, "ram_promotions": 39, "gpu_decodes": 187, "scratch_hits": 0, "lk": {"l0": 3, "arriving": 0, "ssd_inflight": 0, "l1": 39, "l2": 243}, "lkw": {"l0": 1903, "arriving": 0, "ssd_inflight": 0, "l1": 8449, "l2": 13648}, "missing": 110, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 17, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 65, "mem_util_pct": 20, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 146.6, "power_limit_w": 600.0, "temp_c": 32, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.302, "pcie_tx_gbs": 3.847}], "pulls": {"expired": 0, "need": 56186, "idle": 0, "idle_used": 0, "ram_first": 4952, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 13.73, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 308.69, "ssd_below_cap": 32.6, "nothing_needed": 0.55}, "sd": {"all_readers": 341.39, "some_readers": 12.82, "idle_nothing": 1.13, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.18}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1110003351552, "h2d_ram_gbs_cum": 106746408960, "h2d_used_cum": 1216771330048, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 556.6, "phase": "measure", "dt": 1.0, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.169, "ram_to_gpu_gbs": 3.815, "gpu_to_ram_gbs": 3.492, "decode_gbs": 4.606, "ssd_reads": 147, "ram_promotions": 30, "gpu_decodes": 183, "scratch_hits": 0, "lk": {"l0": 7, "arriving": 0, "ssd_inflight": 0, "l1": 30, "l2": 244}, "lkw": {"l0": 5462, "arriving": 0, "ssd_inflight": 0, "l1": 6880, "l2": 11658}, "missing": 207, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1041, "ram_cap": 1080, "layer": 18, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 27, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 139.1, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 5.844, "pcie_tx_gbs": 2.565}], "pulls": {"expired": 0, "need": 56363, "idle": 0, "idle_used": 0, "ram_first": 4982, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 13.79, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 309.45, "ssd_below_cap": 32.77, "nothing_needed": 0.55}, "sd": {"all_readers": 342.31, "some_readers": 12.89, "idle_nothing": 1.14, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.18}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1113172586496, "h2d_ram_gbs_cum": 107393159168, "h2d_used_cum": 1220565749760, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 557.6, "phase": "measure", "dt": 1.0, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.083, "ram_to_gpu_gbs": 3.083, "gpu_to_ram_gbs": 2.716, "decode_gbs": 3.624, "ssd_reads": 143, "ram_promotions": 0, "gpu_decodes": 144, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 64, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 18, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 150.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 1.183, "pcie_tx_gbs": 3.843}], "pulls": {"expired": 0, "need": 56506, "idle": 0, "idle_used": 0, "ram_first": 4982, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 13.79, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 310.45, "ssd_below_cap": 32.77, "nothing_needed": 0.55}, "sd": {"all_readers": 343.31, "some_readers": 12.89, "idle_nothing": 1.14, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.18}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1116255465472, "h2d_ram_gbs_cum": 107393159168, "h2d_used_cum": 1223670194176, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 558.6, "phase": "measure", "dt": 1.0, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.148, "ram_to_gpu_gbs": 3.795, "gpu_to_ram_gbs": 3.407, "decode_gbs": 4.506, "ssd_reads": 146, "ram_promotions": 30, "gpu_decodes": 179, "scratch_hits": 0, "lk": {"l0": 6, "arriving": 0, "ssd_inflight": 0, "l1": 30, "l2": 242}, "lkw": {"l0": 3979, "arriving": 0, "ssd_inflight": 0, "l1": 7378, "l2": 12643}, "missing": 162, "reads_in_flight": 34, "uploads_pending": 2, "gpu_experts": 106, "gpu_cap": 108, "ram_victims": 1041, "ram_cap": 1080, "layer": 19, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 14, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 136.5, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 4.69, "pcie_tx_gbs": 2.566}], "pulls": {"expired": 0, "need": 56679, "idle": 0, "idle_used": 0, "ram_first": 5012, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 13.86, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 311.21, "ssd_below_cap": 32.93, "nothing_needed": 0.56}, "sd": {"all_readers": 344.23, "some_readers": 12.96, "idle_nothing": 1.14, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.18}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1119403446272, "h2d_ram_gbs_cum": 108040024064, "h2d_used_cum": 1227400355840, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 559.6, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.103, "ram_to_gpu_gbs": 3.103, "gpu_to_ram_gbs": 2.586, "decode_gbs": 3.673, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 146, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 17, "reads_in_flight": 16, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1060, "ram_cap": 1080, "layer": 19, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 149.8, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.69, "pcie_tx_gbs": 2.565}], "pulls": {"expired": 0, "need": 56805, "idle": 0, "idle_used": 0, "ram_first": 5012, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 13.9, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 312.08, "ssd_below_cap": 33.03, "nothing_needed": 0.56}, "sd": {"all_readers": 345.24, "some_readers": 12.96, "idle_nothing": 1.14, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.18}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1122508386304, "h2d_ram_gbs_cum": 108040024064, "h2d_used_cum": 1230548418560, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 560.6, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.144, "ram_to_gpu_gbs": 3.855, "gpu_to_ram_gbs": 3.51, "decode_gbs": 4.577, "ssd_reads": 146, "ram_promotions": 33, "gpu_decodes": 182, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 33, "l2": 247}, "lkw": {"l0": 1428, "arriving": 0, "ssd_inflight": 0, "l1": 7974, "l2": 14598}, "missing": 117, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 20, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 27, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 143.9, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 5.842, "pcie_tx_gbs": 2.563}], "pulls": {"expired": 0, "need": 57003, "idle": 0, "idle_used": 0, "ram_first": 5045, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 13.93, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 312.98, "ssd_below_cap": 33.1, "nothing_needed": 0.56}, "sd": {"all_readers": 346.17, "some_readers": 13.02, "idle_nothing": 1.15, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.18}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1125655609344, "h2d_ram_gbs_cum": 108751355904, "h2d_used_cum": 1234428534784, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 561.6, "phase": "measure", "dt": 1.003, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.117, "ram_to_gpu_gbs": 3.87, "gpu_to_ram_gbs": 3.547, "decode_gbs": 4.544, "ssd_reads": 145, "ram_promotions": 35, "gpu_decodes": 181, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 35, "l2": 246}, "lkw": {"l0": 1180, "arriving": 0, "ssd_inflight": 0, "l1": 8406, "l2": 14414}, "missing": 219, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 21, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 34, "mem_util_pct": 15, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 136.8, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.507, "pcie_tx_gbs": 1.35}], "pulls": {"expired": 0, "need": 57182, "idle": 0, "idle_used": 0, "ram_first": 5080, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 13.99, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 313.75, "ssd_below_cap": 33.25, "nothing_needed": 0.56}, "sd": {"all_readers": 347.1, "some_readers": 13.09, "idle_nothing": 1.15, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.18}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1128780988416, "h2d_ram_gbs_cum": 109505802240, "h2d_used_cum": 1238286815232, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 562.6, "phase": "measure", "dt": 1.0, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.125, "ram_to_gpu_gbs": 3.125, "gpu_to_ram_gbs": 2.651, "decode_gbs": 3.674, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 146, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 73, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 21, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 148.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 5.849, "pcie_tx_gbs": 1.285}], "pulls": {"expired": 0, "need": 57328, "idle": 0, "idle_used": 0, "ram_first": 5080, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 13.99, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 314.76, "ssd_below_cap": 33.25, "nothing_needed": 0.56}, "sd": {"all_readers": 348.1, "some_readers": 13.09, "idle_nothing": 1.15, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.18}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1131906408448, "h2d_ram_gbs_cum": 109505802240, "h2d_used_cum": 1241433780224, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 563.6, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.099, "ram_to_gpu_gbs": 3.831, "gpu_to_ram_gbs": 3.68, "decode_gbs": 4.474, "ssd_reads": 144, "ram_promotions": 34, "gpu_decodes": 178, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 34, "l2": 244}, "lkw": {"l0": 825, "arriving": 0, "ssd_inflight": 0, "l1": 7711, "l2": 15464}, "missing": 173, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 22, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 138.5, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 5.85, "pcie_tx_gbs": 3.846}], "pulls": {"expired": 0, "need": 57503, "idle": 0, "idle_used": 0, "ram_first": 5114, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 14.06, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 315.53, "ssd_below_cap": 33.41, "nothing_needed": 0.56}, "sd": {"all_readers": 349.03, "some_readers": 13.15, "idle_nothing": 1.16, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.18}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1135010041856, "h2d_ram_gbs_cum": 110238523392, "h2d_used_cum": 1245248569344, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 564.6, "phase": "measure", "dt": 1.002, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.12, "ram_to_gpu_gbs": 3.12, "gpu_to_ram_gbs": 2.647, "decode_gbs": 3.669, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 146, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 28, "reads_in_flight": 28, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1050, "ram_cap": 1080, "layer": 22, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 138.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.508, "pcie_tx_gbs": 2.666}], "pulls": {"expired": 0, "need": 57640, "idle": 0, "idle_used": 0, "ram_first": 5114, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 14.08, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 316.46, "ssd_below_cap": 33.46, "nothing_needed": 0.56}, "sd": {"all_readers": 350.03, "some_readers": 13.15, "idle_nothing": 1.16, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.18}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1138135306240, "h2d_ram_gbs_cum": 110238523392, "h2d_used_cum": 1248395399168, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 565.6, "phase": "measure", "dt": 1.0, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.125, "ram_to_gpu_gbs": 3.663, "gpu_to_ram_gbs": 3.426, "decode_gbs": 4.304, "ssd_reads": 145, "ram_promotions": 25, "gpu_decodes": 171, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 25, "l2": 260}, "lkw": {"l0": 671, "arriving": 0, "ssd_inflight": 0, "l1": 5859, "l2": 17470}, "missing": 143, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 23, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 143.6, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.344, "pcie_tx_gbs": 3.845}], "pulls": {"expired": 0, "need": 57818, "idle": 0, "idle_used": 0, "ram_first": 5139, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 14.13, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 317.28, "ssd_below_cap": 33.59, "nothing_needed": 0.57}, "sd": {"all_readers": 350.95, "some_readers": 13.22, "idle_nothing": 1.17, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.18}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1141260333056, "h2d_ram_gbs_cum": 110777237504, "h2d_used_cum": 1252059140096, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 566.6, "phase": "measure", "dt": 1.0, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.082, "ram_to_gpu_gbs": 3.556, "gpu_to_ram_gbs": 3.081, "decode_gbs": 4.229, "ssd_reads": 143, "ram_promotions": 22, "gpu_decodes": 168, "scratch_hits": 0, "lk": {"l0": 3, "arriving": 0, "ssd_inflight": 0, "l1": 22, "l2": 259}, "lkw": {"l0": 2155, "arriving": 0, "ssd_inflight": 0, "l1": 5995, "l2": 15850}, "missing": 259, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1030, "ram_cap": 1080, "layer": 24, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 136.8, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 8.183, "pcie_tx_gbs": 0.92}], "pulls": {"expired": 0, "need": 57983, "idle": 0, "idle_used": 0, "ram_first": 5161, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 14.21, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 318.05, "ssd_below_cap": 33.75, "nothing_needed": 0.57}, "sd": {"all_readers": 351.87, "some_readers": 13.3, "idle_nothing": 1.17, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.18}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1144342130688, "h2d_ram_gbs_cum": 111251468288, "h2d_used_cum": 1255615168512, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 567.6, "phase": "measure", "dt": 1.002, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.183, "ram_to_gpu_gbs": 3.183, "gpu_to_ram_gbs": 2.667, "decode_gbs": 3.692, "ssd_reads": 148, "ram_promotions": 0, "gpu_decodes": 147, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 112, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 24, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 32, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 144.9, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.6, "pcie_tx_gbs": 1.282}], "pulls": {"expired": 0, "need": 58130, "idle": 0, "idle_used": 0, "ram_first": 5161, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 14.21, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 319.05, "ssd_below_cap": 33.75, "nothing_needed": 0.57}, "sd": {"all_readers": 352.87, "some_readers": 13.3, "idle_nothing": 1.17, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.18}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1147532509184, "h2d_ram_gbs_cum": 111251468288, "h2d_used_cum": 1258783981568, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 568.6, "phase": "measure", "dt": 1.003, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.096, "ram_to_gpu_gbs": 3.676, "gpu_to_ram_gbs": 3.375, "decode_gbs": 4.393, "ssd_reads": 144, "ram_promotions": 27, "gpu_decodes": 175, "scratch_hits": 0, "lk": {"l0": 3, "arriving": 0, "ssd_inflight": 0, "l1": 27, "l2": 252}, "lkw": {"l0": 2593, "arriving": 0, "ssd_inflight": 0, "l1": 6937, "l2": 14470}, "missing": 219, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 25, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 25, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 138.9, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 26.766, "pcie_tx_gbs": 6.464}], "pulls": {"expired": 0, "need": 58302, "idle": 0, "idle_used": 0, "ram_first": 5188, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 14.28, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 319.82, "ssd_below_cap": 33.9, "nothing_needed": 0.57}, "sd": {"all_readers": 353.8, "some_readers": 13.37, "idle_nothing": 1.18, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.19}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1150636748800, "h2d_ram_gbs_cum": 111833673728, "h2d_used_cum": 1262491992064, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 569.6, "phase": "measure", "dt": 1.002, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.163, "ram_to_gpu_gbs": 3.163, "gpu_to_ram_gbs": 2.69, "decode_gbs": 3.668, "ssd_reads": 147, "ram_promotions": 0, "gpu_decodes": 146, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 72, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 25, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 145.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 1.173, "pcie_tx_gbs": 3.844}], "pulls": {"expired": 0, "need": 58449, "idle": 0, "idle_used": 0, "ram_first": 5188, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 14.28, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 320.83, "ssd_below_cap": 33.9, "nothing_needed": 0.57}, "sd": {"all_readers": 354.8, "some_readers": 13.37, "idle_nothing": 1.18, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.19}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1153806295040, "h2d_ram_gbs_cum": 111833673728, "h2d_used_cum": 1265639981056, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 570.6, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.079, "ram_to_gpu_gbs": 3.595, "gpu_to_ram_gbs": 3.208, "decode_gbs": 4.248, "ssd_reads": 143, "ram_promotions": 24, "gpu_decodes": 169, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 24, "l2": 259}, "lkw": {"l0": 845, "arriving": 0, "ssd_inflight": 0, "l1": 6578, "l2": 16577}, "missing": 188, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 26, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 145.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 2.335, "pcie_tx_gbs": 1.49}], "pulls": {"expired": 0, "need": 58616, "idle": 0, "idle_used": 0, "ram_first": 5212, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 14.35, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 321.6, "ssd_below_cap": 34.06, "nothing_needed": 0.57}, "sd": {"all_readers": 355.73, "some_readers": 13.43, "idle_nothing": 1.18, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.19}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1156889325568, "h2d_ram_gbs_cum": 112351145984, "h2d_used_cum": 1269262041088, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 571.6, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.124, "ram_to_gpu_gbs": 3.124, "gpu_to_ram_gbs": 2.671, "decode_gbs": 3.647, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 43, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 26, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 138.3, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.69, "pcie_tx_gbs": 3.844}], "pulls": {"expired": 0, "need": 58761, "idle": 0, "idle_used": 0, "ram_first": 5212, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 14.35, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 322.6, "ssd_below_cap": 34.06, "nothing_needed": 0.57}, "sd": {"all_readers": 356.73, "some_readers": 13.43, "idle_nothing": 1.18, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.19}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1160015564800, "h2d_ram_gbs_cum": 112351145984, "h2d_used_cum": 1272388280320, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 572.6, "phase": "measure", "dt": 1.0, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.126, "ram_to_gpu_gbs": 3.665, "gpu_to_ram_gbs": 3.256, "decode_gbs": 4.279, "ssd_reads": 145, "ram_promotions": 25, "gpu_decodes": 170, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 25, "l2": 257}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 7794, "l2": 16206}, "missing": 155, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 27, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 145.0, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.499, "pcie_tx_gbs": 3.847}], "pulls": {"expired": 0, "need": 58929, "idle": 0, "idle_used": 0, "ram_first": 5237, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 14.42, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 323.36, "ssd_below_cap": 34.22, "nothing_needed": 0.58}, "sd": {"all_readers": 357.65, "some_readers": 13.51, "idle_nothing": 1.19, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.19}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1163141791744, "h2d_ram_gbs_cum": 112890159104, "h2d_used_cum": 1276053520384, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 573.6, "phase": "measure", "dt": 1.002, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.162, "ram_to_gpu_gbs": 3.141, "gpu_to_ram_gbs": 2.818, "decode_gbs": 3.617, "ssd_reads": 147, "ram_promotions": 0, "gpu_decodes": 144, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 11, "reads_in_flight": 9, "uploads_pending": 2, "gpu_experts": 106, "gpu_cap": 108, "ram_victims": 1067, "ram_cap": 1080, "layer": 27, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 31, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 138.6, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.354, "pcie_tx_gbs": 0.07}], "pulls": {"expired": 0, "need": 59048, "idle": 0, "idle_used": 0, "ram_first": 5237, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 14.47, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 324.18, "ssd_below_cap": 34.35, "nothing_needed": 0.58}, "sd": {"all_readers": 358.65, "some_readers": 13.51, "idle_nothing": 1.19, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.19}, "readers_busy": 8, "h2d_ssd_gbs_cum": 1166289440768, "h2d_ram_gbs_cum": 112890159104, "h2d_used_cum": 1279158063104, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 574.6, "phase": "measure", "dt": 1.0, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.082, "ram_to_gpu_gbs": 3.535, "gpu_to_ram_gbs": 3.168, "decode_gbs": 4.152, "ssd_reads": 143, "ram_promotions": 20, "gpu_decodes": 165, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 20, "l2": 263}, "lkw": {"l0": 972, "arriving": 0, "ssd_inflight": 0, "l1": 6219, "l2": 16809}, "missing": 130, "reads_in_flight": 34, "uploads_pending": 2, "gpu_experts": 106, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 28, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 31, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 138.6, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.354, "pcie_tx_gbs": 0.07}], "pulls": {"expired": 0, "need": 59237, "idle": 0, "idle_used": 0, "ram_first": 5257, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 14.49, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 325.12, "ssd_below_cap": 34.39, "nothing_needed": 0.58}, "sd": {"all_readers": 359.6, "some_readers": 13.56, "idle_nothing": 1.19, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.19}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1169394257920, "h2d_ram_gbs_cum": 113321345024, "h2d_used_cum": 1282694062080, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 575.6, "phase": "measure", "dt": 1.003, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.138, "ram_to_gpu_gbs": 3.611, "gpu_to_ram_gbs": 3.288, "decode_gbs": 4.241, "ssd_reads": 146, "ram_promotions": 22, "gpu_decodes": 169, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 22, "l2": 260}, "lkw": {"l0": 706, "arriving": 0, "ssd_inflight": 0, "l1": 5491, "l2": 17803}, "missing": 243, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 29, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 33, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 144.8, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 5.843, "pcie_tx_gbs": 2.563}], "pulls": {"expired": 0, "need": 59406, "idle": 0, "idle_used": 0, "ram_first": 5279, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 14.56, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 325.89, "ssd_below_cap": 34.55, "nothing_needed": 0.58}, "sd": {"all_readers": 360.52, "some_readers": 13.63, "idle_nothing": 1.2, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.19}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1172541595648, "h2d_ram_gbs_cum": 113795653632, "h2d_used_cum": 1286315692032, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 576.6, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.103, "ram_to_gpu_gbs": 3.103, "gpu_to_ram_gbs": 2.586, "decode_gbs": 3.648, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 99, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 29, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 56, "mem_util_pct": 18, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 150.2, "power_limit_w": 600.0, "temp_c": 33, "sm_clock_mhz": 2325, "pcie_rx_gbs": 4.502, "pcie_tx_gbs": 1.284}], "pulls": {"expired": 0, "need": 59550, "idle": 0, "idle_used": 0, "ram_first": 5279, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 14.56, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 326.89, "ssd_below_cap": 34.55, "nothing_needed": 0.58}, "sd": {"all_readers": 361.52, "some_readers": 13.63, "idle_nothing": 1.2, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.19}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1175645831168, "h2d_ram_gbs_cum": 113795653632, "h2d_used_cum": 1289441497088, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 577.6, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.143, "ram_to_gpu_gbs": 3.616, "gpu_to_ram_gbs": 3.186, "decode_gbs": 4.298, "ssd_reads": 146, "ram_promotions": 22, "gpu_decodes": 171, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 22, "l2": 258}, "lkw": {"l0": 1528, "arriving": 0, "ssd_inflight": 0, "l1": 6311, "l2": 16161}, "missing": 210, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 30, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 26, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 139.5, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 2.346, "pcie_tx_gbs": 5.126}], "pulls": {"expired": 0, "need": 59719, "idle": 0, "idle_used": 0, "ram_first": 5301, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 14.63, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 327.66, "ssd_below_cap": 34.71, "nothing_needed": 0.59}, "sd": {"all_readers": 362.46, "some_readers": 13.69, "idle_nothing": 1.2, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.19}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1178792984576, "h2d_ram_gbs_cum": 114269949952, "h2d_used_cum": 1293084504064, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 578.6, "phase": "measure", "dt": 1.0, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.104, "ram_to_gpu_gbs": 3.104, "gpu_to_ram_gbs": 2.673, "decode_gbs": 3.624, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 144, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 66, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 30, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 27, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 145.1, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 1.173, "pcie_tx_gbs": 5.189}], "pulls": {"expired": 0, "need": 59863, "idle": 0, "idle_used": 0, "ram_first": 5301, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 14.63, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 328.66, "ssd_below_cap": 34.71, "nothing_needed": 0.59}, "sd": {"all_readers": 363.46, "some_readers": 13.69, "idle_nothing": 1.2, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.19}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1181897416704, "h2d_ram_gbs_cum": 114269949952, "h2d_used_cum": 1296188936192, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 579.6, "phase": "measure", "dt": 1.0, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.168, "ram_to_gpu_gbs": 3.686, "gpu_to_ram_gbs": 3.211, "decode_gbs": 4.329, "ssd_reads": 147, "ram_promotions": 24, "gpu_decodes": 172, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 24, "l2": 253}, "lkw": {"l0": 2422, "arriving": 0, "ssd_inflight": 0, "l1": 5930, "l2": 15648}, "missing": 172, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 31, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 26, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 139.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 5.851, "pcie_tx_gbs": 1.283}], "pulls": {"expired": 0, "need": 60034, "idle": 0, "idle_used": 0, "ram_first": 5325, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 14.7, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 329.42, "ssd_below_cap": 34.87, "nothing_needed": 0.59}, "sd": {"all_readers": 364.4, "some_readers": 13.74, "idle_nothing": 1.21, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.19}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1185066397696, "h2d_ram_gbs_cum": 114787356672, "h2d_used_cum": 1299853766656, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 580.6, "phase": "measure", "dt": 1.0, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.082, "ram_to_gpu_gbs": 3.082, "gpu_to_ram_gbs": 2.63, "decode_gbs": 3.624, "ssd_reads": 143, "ram_promotions": 0, "gpu_decodes": 144, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 29, "reads_in_flight": 29, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1049, "ram_cap": 1080, "layer": 31, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 32, "mem_util_pct": 15, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 148.7, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 5.843, "pcie_tx_gbs": 2.562}], "pulls": {"expired": 0, "need": 60170, "idle": 0, "idle_used": 0, "ram_first": 5325, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 14.72, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 330.37, "ssd_below_cap": 34.91, "nothing_needed": 0.59}, "sd": {"all_readers": 365.4, "some_readers": 13.74, "idle_nothing": 1.21, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.19}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1188149235712, "h2d_ram_gbs_cum": 114787356672, "h2d_used_cum": 1302958161920, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 581.6, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.123, "ram_to_gpu_gbs": 3.575, "gpu_to_ram_gbs": 3.209, "decode_gbs": 4.174, "ssd_reads": 145, "ram_promotions": 21, "gpu_decodes": 166, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 21, "l2": 266}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 6098, "l2": 17902}, "missing": 150, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 32, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 135.6, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.529, "pcie_tx_gbs": 2.565}], "pulls": {"expired": 0, "need": 60343, "idle": 0, "idle_used": 0, "ram_first": 5346, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 14.77, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 331.18, "ssd_below_cap": 35.05, "nothing_needed": 0.59}, "sd": {"all_readers": 366.34, "some_readers": 13.8, "idle_nothing": 1.21, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.19}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1191274942464, "h2d_ram_gbs_cum": 115240120320, "h2d_used_cum": 1306536632320, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 582.6, "phase": "measure", "dt": 1.0, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.125, "ram_to_gpu_gbs": 3.125, "gpu_to_ram_gbs": 2.716, "decode_gbs": 3.624, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 144, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 6, "reads_in_flight": 5, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1071, "ram_cap": 1080, "layer": 32, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 140.9, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.241, "pcie_tx_gbs": 2.562}], "pulls": {"expired": 0, "need": 60457, "idle": 0, "idle_used": 0, "ram_first": 5346, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 14.83, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 331.98, "ssd_below_cap": 35.19, "nothing_needed": 0.59}, "sd": {"all_readers": 367.3, "some_readers": 13.84, "idle_nothing": 1.21, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.19}, "readers_busy": 5, "h2d_ssd_gbs_cum": 1194400628736, "h2d_ram_gbs_cum": 115240120320, "h2d_used_cum": 1309640765440, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 583.6, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.08, "ram_to_gpu_gbs": 3.576, "gpu_to_ram_gbs": 3.253, "decode_gbs": 4.226, "ssd_reads": 143, "ram_promotions": 23, "gpu_decodes": 168, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 23, "l2": 258}, "lkw": {"l0": 1220, "arriving": 0, "ssd_inflight": 0, "l1": 7360, "l2": 15420}, "missing": 120, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 33, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 41, "mem_util_pct": 14, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 148.3, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 2.934, "pcie_tx_gbs": 4.464}], "pulls": {"expired": 0, "need": 60654, "idle": 0, "idle_used": 0, "ram_first": 5369, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 14.84, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 332.95, "ssd_below_cap": 35.2, "nothing_needed": 0.59}, "sd": {"all_readers": 368.28, "some_readers": 13.85, "idle_nothing": 1.22, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.19}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1197483393024, "h2d_ram_gbs_cum": 115735998464, "h2d_used_cum": 1313240961024, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 584.6, "phase": "measure", "dt": 1.0, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.168, "ram_to_gpu_gbs": 3.599, "gpu_to_ram_gbs": 3.34, "decode_gbs": 4.252, "ssd_reads": 147, "ram_promotions": 20, "gpu_decodes": 169, "scratch_hits": 0, "lk": {"l0": 3, "arriving": 0, "ssd_inflight": 0, "l1": 20, "l2": 260}, "lkw": {"l0": 2672, "arriving": 0, "ssd_inflight": 0, "l1": 5809, "l2": 15519}, "missing": 233, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 34, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 138.7, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.345, "pcie_tx_gbs": 4.049}], "pulls": {"expired": 0, "need": 60821, "idle": 0, "idle_used": 0, "ram_first": 5389, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 14.91, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 333.71, "ssd_below_cap": 35.37, "nothing_needed": 0.6}, "sd": {"all_readers": 369.21, "some_readers": 13.91, "idle_nothing": 1.23, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.19}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1200652361728, "h2d_ram_gbs_cum": 116167204864, "h2d_used_cum": 1316819578880, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 585.6, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.124, "ram_to_gpu_gbs": 3.124, "gpu_to_ram_gbs": 2.607, "decode_gbs": 3.673, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 146, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 88, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 34, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 145.6, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 5.845, "pcie_tx_gbs": 1.384}], "pulls": {"expired": 0, "need": 60966, "idle": 0, "idle_used": 0, "ram_first": 5389, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 14.91, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 334.71, "ssd_below_cap": 35.37, "nothing_needed": 0.6}, "sd": {"all_readers": 370.21, "some_readers": 13.91, "idle_nothing": 1.23, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.19}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1203778306048, "h2d_ram_gbs_cum": 116167204864, "h2d_used_cum": 1319967080448, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 586.6, "phase": "measure", "dt": 1.0, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.126, "ram_to_gpu_gbs": 3.514, "gpu_to_ram_gbs": 3.212, "decode_gbs": 4.128, "ssd_reads": 145, "ram_promotions": 18, "gpu_decodes": 164, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 18, "l2": 266}, "lkw": {"l0": 2155, "arriving": 0, "ssd_inflight": 0, "l1": 5460, "l2": 16385}, "missing": 210, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 35, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 32, "mem_util_pct": 14, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 139.9, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.165, "pcie_tx_gbs": 5.123}], "pulls": {"expired": 0, "need": 61128, "idle": 0, "idle_used": 0, "ram_first": 5407, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 14.98, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 335.48, "ssd_below_cap": 35.53, "nothing_needed": 0.6}, "sd": {"all_readers": 371.15, "some_readers": 13.97, "idle_nothing": 1.23, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.19}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1206904156160, "h2d_ram_gbs_cum": 116555251712, "h2d_used_cum": 1323459420160, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 587.6, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.123, "ram_to_gpu_gbs": 3.123, "gpu_to_ram_gbs": 2.757, "decode_gbs": 3.647, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 64, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 35, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 25, "mem_util_pct": 9, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 148.5, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.347, "pcie_tx_gbs": 1.283}], "pulls": {"expired": 0, "need": 61274, "idle": 0, "idle_used": 0, "ram_first": 5407, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 14.98, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 336.48, "ssd_below_cap": 35.53, "nothing_needed": 0.6}, "sd": {"all_readers": 372.15, "some_readers": 13.97, "idle_nothing": 1.23, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.19}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1210030071808, "h2d_ram_gbs_cum": 116555251712, "h2d_used_cum": 1326585331712, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 588.6, "phase": "measure", "dt": 1.002, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.12, "ram_to_gpu_gbs": 3.508, "gpu_to_ram_gbs": 3.077, "decode_gbs": 4.146, "ssd_reads": 145, "ram_promotions": 18, "gpu_decodes": 165, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 18, "l2": 266}, "lkw": {"l0": 1799, "arriving": 0, "ssd_inflight": 0, "l1": 6047, "l2": 16154}, "missing": 185, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 36, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 33, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 144.5, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 3.516, "pcie_tx_gbs": 2.491}], "pulls": {"expired": 0, "need": 61437, "idle": 0, "idle_used": 0, "ram_first": 5425, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 15.05, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 337.24, "ssd_below_cap": 35.7, "nothing_needed": 0.6}, "sd": {"all_readers": 373.09, "some_readers": 14.03, "idle_nothing": 1.24, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.19}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1213155917824, "h2d_ram_gbs_cum": 116943314944, "h2d_used_cum": 1330099240960, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 589.6, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.124, "ram_to_gpu_gbs": 3.124, "gpu_to_ram_gbs": 2.65, "decode_gbs": 3.648, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 40, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 36, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 136.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 5.843, "pcie_tx_gbs": 1.283}], "pulls": {"expired": 0, "need": 61582, "idle": 0, "idle_used": 0, "ram_first": 5425, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 15.05, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 338.24, "ssd_below_cap": 35.7, "nothing_needed": 0.6}, "sd": {"all_readers": 374.09, "some_readers": 14.03, "idle_nothing": 1.24, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.19}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1216281698304, "h2d_ram_gbs_cum": 116943314944, "h2d_used_cum": 1333225025536, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 590.6, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.102, "ram_to_gpu_gbs": 3.446, "gpu_to_ram_gbs": 3.123, "decode_gbs": 4.024, "ssd_reads": 144, "ram_promotions": 16, "gpu_decodes": 160, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 16, "l2": 270}, "lkw": {"l0": 826, "arriving": 0, "ssd_inflight": 0, "l1": 5381, "l2": 17793}, "missing": 168, "reads_in_flight": 34, "uploads_pending": 2, "gpu_experts": 106, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 37, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 26, "mem_util_pct": 10, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 142.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.691, "pcie_tx_gbs": 2.563}], "pulls": {"expired": 0, "need": 61740, "idle": 0, "idle_used": 0, "ram_first": 5441, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 15.11, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 339.01, "ssd_below_cap": 35.86, "nothing_needed": 0.61}, "sd": {"all_readers": 375.02, "some_readers": 14.09, "idle_nothing": 1.24, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.19}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1219385974784, "h2d_ram_gbs_cum": 117288271872, "h2d_used_cum": 1336652709888, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 591.6, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.145, "ram_to_gpu_gbs": 3.145, "gpu_to_ram_gbs": 2.735, "decode_gbs": 3.723, "ssd_reads": 146, "ram_promotions": 0, "gpu_decodes": 148, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 20, "reads_in_flight": 20, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1057, "ram_cap": 1080, "layer": 37, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 137.7, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.346, "pcie_tx_gbs": 5.125}], "pulls": {"expired": 0, "need": 61872, "idle": 0, "idle_used": 0, "ram_first": 5441, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 15.15, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 339.9, "ssd_below_cap": 35.94, "nothing_needed": 0.61}, "sd": {"all_readers": 376.02, "some_readers": 14.09, "idle_nothing": 1.24, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.19}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1222533103616, "h2d_ram_gbs_cum": 117288271872, "h2d_used_cum": 1339842945024, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 592.6, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.122, "ram_to_gpu_gbs": 3.38, "gpu_to_ram_gbs": 3.014, "decode_gbs": 3.998, "ssd_reads": 145, "ram_promotions": 12, "gpu_decodes": 159, "scratch_hits": 0, "lk": {"l0": 3, "arriving": 0, "ssd_inflight": 0, "l1": 12, "l2": 269}, "lkw": {"l0": 3431, "arriving": 0, "ssd_inflight": 0, "l1": 4140, "l2": 16429}, "missing": 144, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 38, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 31, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 143.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.353, "pcie_tx_gbs": 3.944}], "pulls": {"expired": 0, "need": 62045, "idle": 0, "idle_used": 0, "ram_first": 5453, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 15.18, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 340.77, "ssd_below_cap": 36.03, "nothing_needed": 0.61}, "sd": {"all_readers": 376.96, "some_readers": 14.14, "idle_nothing": 1.25, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.19}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1225658441728, "h2d_ram_gbs_cum": 117546942464, "h2d_used_cum": 1343205404672, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 593.6, "phase": "measure", "dt": 1.004, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.09, "ram_to_gpu_gbs": 3.434, "gpu_to_ram_gbs": 3.048, "decode_gbs": 4.035, "ssd_reads": 144, "ram_promotions": 16, "gpu_decodes": 161, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 16, "l2": 268}, "lkw": {"l0": 1057, "arriving": 0, "ssd_inflight": 0, "l1": 5580, "l2": 17363}, "missing": 268, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1034, "ram_cap": 1080, "layer": 39, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 139.0, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.687, "pcie_tx_gbs": 1.285}], "pulls": {"expired": 0, "need": 62205, "idle": 0, "idle_used": 0, "ram_first": 5469, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 15.25, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 341.53, "ssd_below_cap": 36.2, "nothing_needed": 0.61}, "sd": {"all_readers": 377.9, "some_readers": 14.2, "idle_nothing": 1.25, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.19}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1228762406912, "h2d_ram_gbs_cum": 117891858432, "h2d_used_cum": 1346654269440, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 594.6, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.123, "ram_to_gpu_gbs": 3.123, "gpu_to_ram_gbs": 2.692, "decode_gbs": 3.672, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 146, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 123, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 39, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 140.5, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2340, "pcie_rx_gbs": 2.345, "pcie_tx_gbs": 5.229}], "pulls": {"expired": 0, "need": 62350, "idle": 0, "idle_used": 0, "ram_first": 5469, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 15.25, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 342.54, "ssd_below_cap": 36.2, "nothing_needed": 0.61}, "sd": {"all_readers": 378.91, "some_readers": 14.2, "idle_nothing": 1.25, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.19}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1231887921152, "h2d_ram_gbs_cum": 117891858432, "h2d_used_cum": 1349801349120, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 595.6, "phase": "measure", "dt": 1.002, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.12, "ram_to_gpu_gbs": 3.464, "gpu_to_ram_gbs": 3.034, "decode_gbs": 4.045, "ssd_reads": 145, "ram_promotions": 16, "gpu_decodes": 161, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 16, "l2": 266}, "lkw": {"l0": 1944, "arriving": 0, "ssd_inflight": 0, "l1": 5322, "l2": 16734}, "missing": 246, "reads_in_flight": 34, "uploads_pending": 2, "gpu_experts": 106, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 40, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 34, "mem_util_pct": 14, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 139.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 7.647, "pcie_tx_gbs": 17.587}], "pulls": {"expired": 0, "need": 62509, "idle": 0, "idle_used": 0, "ram_first": 5485, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 15.31, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 343.32, "ssd_below_cap": 36.36, "nothing_needed": 0.61}, "sd": {"all_readers": 379.84, "some_readers": 14.26, "idle_nothing": 1.26, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.19}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1235013152768, "h2d_ram_gbs_cum": 118236659712, "h2d_used_cum": 1353228251136, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 596.6, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.144, "ram_to_gpu_gbs": 3.144, "gpu_to_ram_gbs": 2.563, "decode_gbs": 3.723, "ssd_reads": 146, "ram_promotions": 0, "gpu_decodes": 148, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 98, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 40, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 27, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 145.7, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.344, "pcie_tx_gbs": 4.147}], "pulls": {"expired": 0, "need": 62657, "idle": 0, "idle_used": 0, "ram_first": 5485, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 15.31, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 344.32, "ssd_below_cap": 36.36, "nothing_needed": 0.61}, "sd": {"all_readers": 380.84, "some_readers": 14.26, "idle_nothing": 1.26, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.19}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1238159585280, "h2d_ram_gbs_cum": 118236659712, "h2d_used_cum": 1356417814528, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 597.6, "phase": "measure", "dt": 1.0, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.081, "ram_to_gpu_gbs": 3.447, "gpu_to_ram_gbs": 2.973, "decode_gbs": 4.027, "ssd_reads": 143, "ram_promotions": 17, "gpu_decodes": 160, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 17, "l2": 270}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 6189, "l2": 17811}, "missing": 225, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 41, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 44, "mem_util_pct": 16, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 145.3, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.343, "pcie_tx_gbs": 5.122}], "pulls": {"expired": 0, "need": 62817, "idle": 0, "idle_used": 0, "ram_first": 5502, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 15.38, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 345.09, "ssd_below_cap": 36.52, "nothing_needed": 0.62}, "sd": {"all_readers": 381.78, "some_readers": 14.32, "idle_nothing": 1.27, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.19}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1241240965120, "h2d_ram_gbs_cum": 118602936320, "h2d_used_cum": 1359865470976, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 598.6, "phase": "measure", "dt": 1.0, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.145, "ram_to_gpu_gbs": 3.145, "gpu_to_ram_gbs": 2.628, "decode_gbs": 3.674, "ssd_reads": 146, "ram_promotions": 0, "gpu_decodes": 146, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 79, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 41, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 24, "mem_util_pct": 10, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 139.3, "power_limit_w": 600.0, "temp_c": 30, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.353, "pcie_tx_gbs": 1.385}], "pulls": {"expired": 0, "need": 62963, "idle": 0, "idle_used": 0, "ram_first": 5502, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 15.38, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 346.09, "ssd_below_cap": 36.52, "nothing_needed": 0.62}, "sd": {"all_readers": 382.78, "some_readers": 14.32, "idle_nothing": 1.27, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.19}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1244386430976, "h2d_ram_gbs_cum": 118602936320, "h2d_used_cum": 1363010936832, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 599.6, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.142, "ram_to_gpu_gbs": 3.615, "gpu_to_ram_gbs": 3.228, "decode_gbs": 4.25, "ssd_reads": 146, "ram_promotions": 22, "gpu_decodes": 169, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 22, "l2": 261}, "lkw": {"l0": 899, "arriving": 0, "ssd_inflight": 0, "l1": 7740, "l2": 15361}, "missing": 194, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 42, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 24, "mem_util_pct": 10, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 139.3, "power_limit_w": 600.0, "temp_c": 30, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.353, "pcie_tx_gbs": 1.385}], "pulls": {"expired": 0, "need": 63131, "idle": 0, "idle_used": 0, "ram_first": 5524, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 15.44, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 346.87, "ssd_below_cap": 36.67, "nothing_needed": 0.62}, "sd": {"all_readers": 383.7, "some_readers": 14.39, "idle_nothing": 1.27, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.2}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1247531409408, "h2d_ram_gbs_cum": 119076777984, "h2d_used_cum": 1366629756928, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 600.6, "phase": "measure", "dt": 1.002, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.097, "ram_to_gpu_gbs": 3.097, "gpu_to_ram_gbs": 2.581, "decode_gbs": 3.619, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 144, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 50, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 42, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 144.8, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.676, "pcie_tx_gbs": 1.383}], "pulls": {"expired": 0, "need": 63275, "idle": 0, "idle_used": 0, "ram_first": 5524, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 15.44, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 347.87, "ssd_below_cap": 36.67, "nothing_needed": 0.62}, "sd": {"all_readers": 384.7, "some_readers": 14.39, "idle_nothing": 1.27, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.2}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1250633035776, "h2d_ram_gbs_cum": 119076777984, "h2d_used_cum": 1369731383296, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 601.6, "phase": "measure", "dt": 1.002, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.16, "ram_to_gpu_gbs": 3.482, "gpu_to_ram_gbs": 3.246, "decode_gbs": 4.045, "ssd_reads": 147, "ram_promotions": 15, "gpu_decodes": 161, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 15, "l2": 273}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 5995, "l2": 18005}, "missing": 177, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 43, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 140.3, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 5.837, "pcie_tx_gbs": 3.823}], "pulls": {"expired": 0, "need": 63436, "idle": 0, "idle_used": 0, "ram_first": 5539, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 15.51, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 348.64, "ssd_below_cap": 36.84, "nothing_needed": 0.62}, "sd": {"all_readers": 385.63, "some_readers": 14.45, "idle_nothing": 1.28, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.2}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1253799182336, "h2d_ram_gbs_cum": 119399874560, "h2d_used_cum": 1373199089664, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 602.6, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.077, "ram_to_gpu_gbs": 3.077, "gpu_to_ram_gbs": 2.689, "decode_gbs": 3.62, "ssd_reads": 143, "ram_promotions": 0, "gpu_decodes": 144, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 33, "reads_in_flight": 33, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1046, "ram_cap": 1080, "layer": 43, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 26, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 142.4, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 4.685, "pcie_tx_gbs": 3.84}], "pulls": {"expired": 0, "need": 63577, "idle": 0, "idle_used": 0, "ram_first": 5539, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 15.52, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 349.62, "ssd_below_cap": 36.85, "nothing_needed": 0.62}, "sd": {"all_readers": 386.64, "some_readers": 14.45, "idle_nothing": 1.28, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.2}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1256879702016, "h2d_ram_gbs_cum": 119399874560, "h2d_used_cum": 1376301146112, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 603.6, "phase": "measure", "dt": 1.001, "tokens": 11499, "tok_s": 0.0, "ssd_gbs": 3.124, "ram_to_gpu_gbs": 3.577, "gpu_to_ram_gbs": 3.124, "decode_gbs": 4.176, "ssd_reads": 145, "ram_promotions": 21, "gpu_decodes": 166, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 21, "l2": 260}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 8027, "l2": 15973}, "missing": 148, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 44, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 27, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 138.7, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.684, "pcie_tx_gbs": 3.841}], "pulls": {"expired": 0, "need": 63746, "idle": 0, "idle_used": 0, "ram_first": 5560, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 15.58, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 350.41, "ssd_below_cap": 36.99, "nothing_needed": 0.62}, "sd": {"all_readers": 387.57, "some_readers": 14.52, "idle_nothing": 1.28, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.2}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1260005761024, "h2d_ram_gbs_cum": 119852314624, "h2d_used_cum": 1379879645184, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 604.6, "phase": "measure", "dt": 1.014, "tokens": 13107, "tok_s": 1586.133, "ssd_gbs": 3.149, "ram_to_gpu_gbs": 3.149, "gpu_to_ram_gbs": 2.809, "decode_gbs": 3.675, "ssd_reads": 148, "ram_promotions": 0, "gpu_decodes": 148, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 2, "l2": 286}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 429, "l2": 23571}, "missing": 0, "reads_in_flight": 0, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1077, "ram_cap": 1080, "layer": 3, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 144.3, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.534, "pcie_tx_gbs": 3.955}], "pulls": {"expired": 0, "need": 63858, "idle": 0, "idle_used": 0, "ram_first": 5560, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 15.65, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 351.18, "ssd_below_cap": 37.14, "nothing_needed": 0.63}, "sd": {"all_readers": 388.5, "some_readers": 14.57, "idle_nothing": 1.29, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.2}, "readers_busy": 0, "h2d_ssd_gbs_cum": 1263198445568, "h2d_ram_gbs_cum": 119852314624, "h2d_used_cum": 1383072329728, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 605.6, "phase": "measure", "dt": 1.0, "tokens": 13107, "tok_s": 0.0, "ssd_gbs": 3.081, "ram_to_gpu_gbs": 3.124, "gpu_to_ram_gbs": 2.542, "decode_gbs": 3.649, "ssd_reads": 143, "ram_promotions": 2, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 143, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 3, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 34, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 147.6, "power_limit_w": 600.0, "temp_c": 32, "sm_clock_mhz": 2325, "pcie_rx_gbs": 2.344, "pcie_tx_gbs": 2.647}], "pulls": {"expired": 0, "need": 64039, "idle": 0, "idle_used": 0, "ram_first": 5562, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 15.65, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 352.18, "ssd_below_cap": 37.16, "nothing_needed": 0.63}, "sd": {"all_readers": 389.49, "some_readers": 14.6, "idle_nothing": 1.29, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.2}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1266280243200, "h2d_ram_gbs_cum": 119895392256, "h2d_used_cum": 1386197204992, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 606.6, "phase": "measure", "dt": 1.001, "tokens": 13107, "tok_s": 0.0, "ssd_gbs": 3.1, "ram_to_gpu_gbs": 3.552, "gpu_to_ram_gbs": 3.014, "decode_gbs": 4.149, "ssd_reads": 144, "ram_promotions": 21, "gpu_decodes": 165, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 21, "l2": 267}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 2856, "l2": 21144}, "missing": 266, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1028, "ram_cap": 1080, "layer": 4, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 138.4, "power_limit_w": 600.0, "temp_c": 30, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.344, "pcie_tx_gbs": 3.777}], "pulls": {"expired": 0, "need": 64200, "idle": 0, "idle_used": 0, "ram_first": 5583, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 15.73, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 352.94, "ssd_below_cap": 37.33, "nothing_needed": 0.63}, "sd": {"all_readers": 390.42, "some_readers": 14.66, "idle_nothing": 1.29, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.2}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1269383479296, "h2d_ram_gbs_cum": 120348061696, "h2d_used_cum": 1389753110528, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 607.6, "phase": "measure", "dt": 1.001, "tokens": 13107, "tok_s": 0.0, "ssd_gbs": 3.188, "ram_to_gpu_gbs": 3.188, "gpu_to_ram_gbs": 2.671, "decode_gbs": 3.698, "ssd_reads": 148, "ram_promotions": 0, "gpu_decodes": 147, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 119, "reads_in_flight": 35, "uploads_pending": 1, "gpu_experts": 107, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 4, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 142.7, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.516, "pcie_tx_gbs": 4.546}], "pulls": {"expired": 0, "need": 64347, "idle": 0, "idle_used": 0, "ram_first": 5583, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 15.73, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 353.94, "ssd_below_cap": 37.33, "nothing_needed": 0.63}, "sd": {"all_readers": 391.42, "some_readers": 14.66, "idle_nothing": 1.29, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.2}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1272573669376, "h2d_ram_gbs_cum": 120348061696, "h2d_used_cum": 1392921747456, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 608.6, "phase": "measure", "dt": 1.003, "tokens": 13107, "tok_s": 0.0, "ssd_gbs": 3.138, "ram_to_gpu_gbs": 3.568, "gpu_to_ram_gbs": 3.375, "decode_gbs": 4.167, "ssd_reads": 146, "ram_promotions": 20, "gpu_decodes": 166, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 20, "l2": 268}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 2918, "l2": 21082}, "missing": 240, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 5, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 138.2, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.687, "pcie_tx_gbs": 3.843}], "pulls": {"expired": 0, "need": 64514, "idle": 0, "idle_used": 0, "ram_first": 5603, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 15.79, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 354.7, "ssd_below_cap": 37.5, "nothing_needed": 0.63}, "sd": {"all_readers": 392.35, "some_readers": 14.73, "idle_nothing": 1.3, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.2}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1275720663040, "h2d_ram_gbs_cum": 120779157504, "h2d_used_cum": 1396499836928, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 609.6, "phase": "measure", "dt": 1.0, "tokens": 13107, "tok_s": 0.0, "ssd_gbs": 3.103, "ram_to_gpu_gbs": 3.103, "gpu_to_ram_gbs": 2.586, "decode_gbs": 3.648, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 145, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 96, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 5, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 24, "mem_util_pct": 10, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 147.0, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 2.631, "pcie_tx_gbs": 1.282}], "pulls": {"expired": 0, "need": 64658, "idle": 0, "idle_used": 0, "ram_first": 5603, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 15.79, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 355.7, "ssd_below_cap": 37.5, "nothing_needed": 0.63}, "sd": {"all_readers": 393.35, "some_readers": 14.73, "idle_nothing": 1.3, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.2}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1278824677376, "h2d_ram_gbs_cum": 120779157504, "h2d_used_cum": 1399625404416, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 610.6, "phase": "measure", "dt": 1.002, "tokens": 13107, "tok_s": 0.0, "ssd_gbs": 3.142, "ram_to_gpu_gbs": 3.701, "gpu_to_ram_gbs": 3.486, "decode_gbs": 4.272, "ssd_reads": 146, "ram_promotions": 26, "gpu_decodes": 170, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 26, "l2": 262}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 3835, "l2": 20165}, "missing": 214, "reads_in_flight": 34, "uploads_pending": 2, "gpu_experts": 106, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 6, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 137.9, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 5.483, "pcie_tx_gbs": 2.619}], "pulls": {"expired": 0, "need": 64825, "idle": 0, "idle_used": 0, "ram_first": 5629, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 15.86, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 356.47, "ssd_below_cap": 37.67, "nothing_needed": 0.64}, "sd": {"all_readers": 394.28, "some_readers": 14.8, "idle_nothing": 1.3, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.21}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1281971744768, "h2d_ram_gbs_cum": 121339604992, "h2d_used_cum": 1403289812992, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 611.6, "phase": "measure", "dt": 1.0, "tokens": 13107, "tok_s": 0.0, "ssd_gbs": 3.082, "ram_to_gpu_gbs": 3.082, "gpu_to_ram_gbs": 2.586, "decode_gbs": 3.624, "ssd_reads": 143, "ram_promotions": 0, "gpu_decodes": 144, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 69, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 6, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 27, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 144.3, "power_limit_w": 600.0, "temp_c": 30, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.689, "pcie_tx_gbs": 5.126}], "pulls": {"expired": 0, "need": 64970, "idle": 0, "idle_used": 0, "ram_first": 5629, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 15.86, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 357.47, "ssd_below_cap": 37.67, "nothing_needed": 0.64}, "sd": {"all_readers": 395.28, "some_readers": 14.8, "idle_nothing": 1.3, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.21}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1285054152704, "h2d_ram_gbs_cum": 121339604992, "h2d_used_cum": 1406393774080, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 612.6, "phase": "measure", "dt": 1.001, "tokens": 13107, "tok_s": 0.0, "ssd_gbs": 3.122, "ram_to_gpu_gbs": 3.94, "gpu_to_ram_gbs": 3.639, "decode_gbs": 4.627, "ssd_reads": 145, "ram_promotions": 38, "gpu_decodes": 184, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 38, "l2": 250}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 7729, "l2": 16271}, "missing": 174, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 7, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 26, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 137.2, "power_limit_w": 600.0, "temp_c": 30, "sm_clock_mhz": 2332, "pcie_rx_gbs": 19.783, "pcie_tx_gbs": 28.2}], "pulls": {"expired": 0, "need": 65147, "idle": 0, "idle_used": 0, "ram_first": 5667, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 15.93, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 358.23, "ssd_below_cap": 37.83, "nothing_needed": 0.64}, "sd": {"all_readers": 396.21, "some_readers": 14.86, "idle_nothing": 1.31, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.21}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1288179609600, "h2d_ram_gbs_cum": 122158686208, "h2d_used_cum": 1410359865344, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 613.6, "phase": "measure", "dt": 1.002, "tokens": 13107, "tok_s": 0.0, "ssd_gbs": 3.097, "ram_to_gpu_gbs": 3.097, "gpu_to_ram_gbs": 2.753, "decode_gbs": 3.617, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 144, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 30, "reads_in_flight": 30, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1048, "ram_cap": 1080, "layer": 7, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 25, "mem_util_pct": 11, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 149.8, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.679, "pcie_tx_gbs": 2.501}], "pulls": {"expired": 0, "need": 65285, "idle": 0, "idle_used": 0, "ram_first": 5667, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 15.94, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 359.19, "ssd_below_cap": 37.86, "nothing_needed": 0.64}, "sd": {"all_readers": 397.21, "some_readers": 14.86, "idle_nothing": 1.31, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.21}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1291283488768, "h2d_ram_gbs_cum": 122158686208, "h2d_used_cum": 1413463744512, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 614.6, "phase": "measure", "dt": 1.001, "tokens": 13107, "tok_s": 0.0, "ssd_gbs": 3.142, "ram_to_gpu_gbs": 3.96, "gpu_to_ram_gbs": 3.681, "decode_gbs": 4.625, "ssd_reads": 146, "ram_promotions": 38, "gpu_decodes": 184, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 38, "l2": 249}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 7487, "l2": 16513}, "missing": 133, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 8, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 58, "mem_util_pct": 18, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 147.4, "power_limit_w": 600.0, "temp_c": 32, "sm_clock_mhz": 2325, "pcie_rx_gbs": 2.355, "pcie_tx_gbs": 2.665}], "pulls": {"expired": 0, "need": 65471, "idle": 0, "idle_used": 0, "ram_first": 5705, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 16.0, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 360.0, "ssd_below_cap": 37.99, "nothing_needed": 0.64}, "sd": {"all_readers": 398.13, "some_readers": 14.93, "idle_nothing": 1.32, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.21}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1294430420992, "h2d_ram_gbs_cum": 122977742848, "h2d_used_cum": 1417429733376, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 615.6, "phase": "measure", "dt": 1.001, "tokens": 13107, "tok_s": 0.0, "ssd_gbs": 3.123, "ram_to_gpu_gbs": 3.92, "gpu_to_ram_gbs": 3.64, "decode_gbs": 4.603, "ssd_reads": 145, "ram_promotions": 37, "gpu_decodes": 183, "scratch_hits": 0, "lk": {"l0": 1, "arriving": 0, "ssd_inflight": 0, "l1": 37, "l2": 249}, "lkw": {"l0": 691, "arriving": 0, "ssd_inflight": 0, "l1": 6577, "l2": 16732}, "missing": 237, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1026, "ram_cap": 1080, "layer": 9, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 29, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 139.0, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.678, "pcie_tx_gbs": 3.846}], "pulls": {"expired": 0, "need": 65649, "idle": 0, "idle_used": 0, "ram_first": 5742, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 16.08, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 360.76, "ssd_below_cap": 38.15, "nothing_needed": 0.64}, "sd": {"all_readers": 399.05, "some_readers": 15.0, "idle_nothing": 1.32, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.21}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1297555804160, "h2d_ram_gbs_cum": 123775291392, "h2d_used_cum": 1421352665088, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 616.6, "phase": "measure", "dt": 1.0, "tokens": 13107, "tok_s": 0.0, "ssd_gbs": 3.124, "ram_to_gpu_gbs": 3.124, "gpu_to_ram_gbs": 2.672, "decode_gbs": 3.623, "ssd_reads": 145, "ram_promotions": 0, "gpu_decodes": 144, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 92, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 9, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 31, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 149.6, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 4.679, "pcie_tx_gbs": 2.666}], "pulls": {"expired": 0, "need": 65794, "idle": 0, "idle_used": 0, "ram_first": 5742, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 16.08, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 361.77, "ssd_below_cap": 38.15, "nothing_needed": 0.64}, "sd": {"all_readers": 400.05, "some_readers": 15.0, "idle_nothing": 1.32, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.21}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1300681244672, "h2d_ram_gbs_cum": 123775291392, "h2d_used_cum": 1424456552448, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 617.6, "phase": "measure", "dt": 1.002, "tokens": 13107, "tok_s": 0.0, "ssd_gbs": 3.14, "ram_to_gpu_gbs": 3.828, "gpu_to_ram_gbs": 3.548, "decode_gbs": 4.496, "ssd_reads": 146, "ram_promotions": 32, "gpu_decodes": 179, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 32, "l2": 256}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 5514, "l2": 18486}, "missing": 202, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 10, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 30, "mem_util_pct": 13, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 140.6, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2325, "pcie_rx_gbs": 6.706, "pcie_tx_gbs": 2.512}], "pulls": {"expired": 0, "need": 65969, "idle": 0, "idle_used": 0, "ram_first": 5774, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 16.15, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 362.53, "ssd_below_cap": 38.32, "nothing_needed": 0.65}, "sd": {"all_readers": 400.98, "some_readers": 15.07, "idle_nothing": 1.33, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.21}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1303828160512, "h2d_ram_gbs_cum": 124465061888, "h2d_used_cum": 1428314791936, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 618.6, "phase": "measure", "dt": 1.0, "tokens": 13107, "tok_s": 0.0, "ssd_gbs": 3.104, "ram_to_gpu_gbs": 3.104, "gpu_to_ram_gbs": 2.737, "decode_gbs": 3.625, "ssd_reads": 144, "ram_promotions": 0, "gpu_decodes": 144, "scratch_hits": 0, "lk": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "lkw": {"l0": 0, "arriving": 0, "ssd_inflight": 0, "l1": 0, "l2": 0}, "missing": 58, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1042, "ram_cap": 1080, "layer": 10, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 146.3, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.515, "pcie_tx_gbs": 3.847}], "pulls": {"expired": 0, "need": 66113, "idle": 0, "idle_used": 0, "ram_first": 5774, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 16.15, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 363.53, "ssd_below_cap": 38.32, "nothing_needed": 0.65}, "sd": {"all_readers": 401.98, "some_readers": 15.07, "idle_nothing": 1.33, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.21}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1306932011008, "h2d_ram_gbs_cum": 124465061888, "h2d_used_cum": 1431418642432, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}, {"type": "s", "t": 619.6, "phase": "measure", "dt": 1.003, "tokens": 13107, "tok_s": 0.0, "ssd_gbs": 3.139, "ram_to_gpu_gbs": 3.698, "gpu_to_ram_gbs": 3.333, "decode_gbs": 4.369, "ssd_reads": 146, "ram_promotions": 26, "gpu_decodes": 174, "scratch_hits": 0, "lk": {"l0": 2, "arriving": 0, "ssd_inflight": 0, "l1": 26, "l2": 260}, "lkw": {"l0": 2382, "arriving": 0, "ssd_inflight": 0, "l1": 4546, "l2": 17072}, "missing": 172, "reads_in_flight": 36, "uploads_pending": 0, "gpu_experts": 108, "gpu_cap": 108, "ram_victims": 1043, "ram_cap": 1080, "layer": 11, "fail": {"no_gpu_slot": 0, "promote_no_slot": 0, "no_ram_slot": 0}, "gpus": [{"index": 0, "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "util_pct": 28, "mem_util_pct": 12, "mem_used_gb": 8.2, "mem_total_gb": 102.64, "power_w": 146.9, "power_limit_w": 600.0, "temp_c": 31, "sm_clock_mhz": 2332, "pcie_rx_gbs": 3.704, "pcie_tx_gbs": 2.577}], "pulls": {"expired": 0, "need": 66283, "idle": 0, "idle_used": 0, "ram_first": 5800, "ram_prefetch": 0, "ram_prefetch_used": 0}, "rg": {"copying": 16.22, "copy_cap": 0, "no_gpu_slot": 0, "waiting_on_ssd": 364.3, "ssd_below_cap": 38.48, "nothing_needed": 0.65}, "sd": {"all_readers": 402.92, "some_readers": 15.13, "idle_nothing": 1.34, "idle_no_ram_slot": 0.04, "idle_upload_backlog": 0.21}, "readers_busy": 12, "h2d_ssd_gbs_cum": 1310079012864, "h2d_ram_gbs_cum": 125025501184, "h2d_used_cum": 1435126083584, "h2d_wasted_cum": 0, "node": "", "pod": "expert-lab"}]}