# HELP expert_cache_run_info Current engine run (labels describe it). # TYPE expert_cache_run_info gauge expert_cache_run_info{gpu="0",node="ip-10-43-3-139.ec2.internal",run_id="1794760960969",mode="wait",pull="heap",evict="lookahead",gpu_store="packed",gpu_pool="global",streams="600",guesses="4",phase="measure"} 1 # HELP expert_cache_tokens_total Tokens kept in the current run. # TYPE expert_cache_tokens_total counter expert_cache_tokens_total{gpu="0",node="ip-10-43-3-139.ec2.internal"} 13107 # HELP expert_cache_tokens_per_second Tokens per second, all streams, last 10 s. # TYPE expert_cache_tokens_per_second gauge expert_cache_tokens_per_second{gpu="0",node="ip-10-43-3-139.ec2.internal"} 0.0 # HELP expert_cache_lookups_total Expert lookups in the measured part of the run by outcome: l0 GPU hit, arriving, ssd_inflight, l1 RAM hit, l2 SSD miss. count=expert counts each distinct expert once per layer visit, count=token once per token. # TYPE expert_cache_lookups_total counter expert_cache_lookups_total{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="l0",count="expert"} 359 expert_cache_lookups_total{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="arriving",count="expert"} 0 expert_cache_lookups_total{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="ssd_inflight",count="expert"} 0 expert_cache_lookups_total{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="l1",count="expert"} 5800 expert_cache_lookups_total{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="l2",count="expert"} 60760 expert_cache_lookups_total{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="l0",count="token"} 300381 expert_cache_lookups_total{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="arriving",count="token"} 0 expert_cache_lookups_total{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="ssd_inflight",count="token"} 0 expert_cache_lookups_total{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="l1",count="token"} 1435108 expert_cache_lookups_total{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="l2",count="token"} 3904511 # HELP expert_cache_hit_ratio Share of lookups by outcome. # TYPE expert_cache_hit_ratio gauge expert_cache_hit_ratio{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="l0",count="expert",window="10s"} 0.0017 expert_cache_hit_ratio{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="arriving",count="expert",window="10s"} 0.0 expert_cache_hit_ratio{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="ssd_inflight",count="expert",window="10s"} 0.0 expert_cache_hit_ratio{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="l1",count="expert",window="10s"} 0.1141 expert_cache_hit_ratio{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="l2",count="expert",window="10s"} 0.8841 expert_cache_hit_ratio{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="l0",count="expert",window="run"} 0.0054 expert_cache_hit_ratio{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="arriving",count="expert",window="run"} 0.0 expert_cache_hit_ratio{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="ssd_inflight",count="expert",window="run"} 0.0 expert_cache_hit_ratio{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="l1",count="expert",window="run"} 0.0867 expert_cache_hit_ratio{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="l2",count="expert",window="run"} 0.908 expert_cache_hit_ratio{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="l0",count="token",window="10s"} 0.0213 expert_cache_hit_ratio{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="arriving",count="token",window="10s"} 0.0 expert_cache_hit_ratio{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="ssd_inflight",count="token",window="10s"} 0.0 expert_cache_hit_ratio{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="l1",count="token",window="10s"} 0.2478 expert_cache_hit_ratio{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="l2",count="token",window="10s"} 0.7308 expert_cache_hit_ratio{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="l0",count="token",window="run"} 0.0533 expert_cache_hit_ratio{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="arriving",count="token",window="run"} 0.0 expert_cache_hit_ratio{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="ssd_inflight",count="token",window="run"} 0.0 expert_cache_hit_ratio{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="l1",count="token",window="run"} 0.2545 expert_cache_hit_ratio{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="l2",count="token",window="run"} 0.6923 # HELP expert_cache_link_bytes_total Bytes moved per link in the measured part of the run (gpu_decode = FP8 bytes written). # TYPE expert_cache_link_bytes_total counter expert_cache_link_bytes_total{gpu="0",node="ip-10-43-3-139.ec2.internal",link="ssd_to_ram"} 1310066423000 expert_cache_link_bytes_total{gpu="0",node="ip-10-43-3-139.ec2.internal",link="ram_to_gpu"} 1435082766000 expert_cache_link_bytes_total{gpu="0",node="ip-10-43-3-139.ec2.internal",link="gpu_to_ram"} 1273033135000 expert_cache_link_bytes_total{gpu="0",node="ip-10-43-3-139.ec2.internal",link="gpu_decode"} 1684920947000 # HELP expert_cache_link_gigabytes_per_second Link throughput, last 10 s. # TYPE expert_cache_link_gigabytes_per_second gauge expert_cache_link_gigabytes_per_second{gpu="0",node="ip-10-43-3-139.ec2.internal",link="ssd_to_ram"} 3.122 expert_cache_link_gigabytes_per_second{gpu="0",node="ip-10-43-3-139.ec2.internal",link="ram_to_gpu"} 3.546 expert_cache_link_gigabytes_per_second{gpu="0",node="ip-10-43-3-139.ec2.internal",link="gpu_to_ram"} 3.208 expert_cache_link_gigabytes_per_second{gpu="0",node="ip-10-43-3-139.ec2.internal",link="gpu_decode"} 4.148 # HELP expert_cache_link_max_gigabytes_per_second Measured maximum of each link on this box. # TYPE expert_cache_link_max_gigabytes_per_second gauge expert_cache_link_max_gigabytes_per_second{gpu="0",node="ip-10-43-3-139.ec2.internal",link="ssd_to_ram"} 3.12 expert_cache_link_max_gigabytes_per_second{gpu="0",node="ip-10-43-3-139.ec2.internal",link="ram_to_gpu"} 55.4 expert_cache_link_max_gigabytes_per_second{gpu="0",node="ip-10-43-3-139.ec2.internal",link="gpu_to_ram"} 49.8 expert_cache_link_max_gigabytes_per_second{gpu="0",node="ip-10-43-3-139.ec2.internal",link="gpu_decode"} 55.0 # HELP expert_cache_link_utilization_ratio Link throughput over its measured maximum, last 10 s. # TYPE expert_cache_link_utilization_ratio gauge expert_cache_link_utilization_ratio{gpu="0",node="ip-10-43-3-139.ec2.internal",link="ssd_to_ram"} 1.0006 expert_cache_link_utilization_ratio{gpu="0",node="ip-10-43-3-139.ec2.internal",link="ram_to_gpu"} 0.064 expert_cache_link_utilization_ratio{gpu="0",node="ip-10-43-3-139.ec2.internal",link="gpu_to_ram"} 0.0644 expert_cache_link_utilization_ratio{gpu="0",node="ip-10-43-3-139.ec2.internal",link="gpu_decode"} 0.0754 # HELP expert_cache_ram_to_gpu_useful_gigabytes_per_second RAM->GPU bytes of experts that were used after arriving (last 10 s); wasted = evicted before any use. # TYPE expert_cache_ram_to_gpu_useful_gigabytes_per_second gauge expert_cache_ram_to_gpu_useful_gigabytes_per_second{gpu="0",node="ip-10-43-3-139.ec2.internal",kind="useful"} 3.55 expert_cache_ram_to_gpu_useful_gigabytes_per_second{gpu="0",node="ip-10-43-3-139.ec2.internal",kind="wasted"} 0.0 # HELP expert_cache_ssd_state_seconds_total Seconds the SSD->RAM path spent in each state: all_readers, some_readers, idle_upload_backlog, idle_no_ram_slot, idle_nothing. # TYPE expert_cache_ssd_state_seconds_total counter expert_cache_ssd_state_seconds_total{gpu="0",node="ip-10-43-3-139.ec2.internal",state="all_readers"} 402.92 expert_cache_ssd_state_seconds_total{gpu="0",node="ip-10-43-3-139.ec2.internal",state="some_readers"} 15.13 expert_cache_ssd_state_seconds_total{gpu="0",node="ip-10-43-3-139.ec2.internal",state="idle_nothing"} 1.34 expert_cache_ssd_state_seconds_total{gpu="0",node="ip-10-43-3-139.ec2.internal",state="idle_no_ram_slot"} 0.04 expert_cache_ssd_state_seconds_total{gpu="0",node="ip-10-43-3-139.ec2.internal",state="idle_upload_backlog"} 0.21 # HELP expert_cache_ram_to_gpu_state_seconds_total Seconds the RAM->GPU link spent in each state: copying, copy_cap, no_gpu_slot, waiting_on_ssd (nothing needed is in RAM), ssd_below_cap, nothing_needed. # TYPE expert_cache_ram_to_gpu_state_seconds_total counter expert_cache_ram_to_gpu_state_seconds_total{gpu="0",node="ip-10-43-3-139.ec2.internal",state="copying"} 16.22 expert_cache_ram_to_gpu_state_seconds_total{gpu="0",node="ip-10-43-3-139.ec2.internal",state="copy_cap"} 0 expert_cache_ram_to_gpu_state_seconds_total{gpu="0",node="ip-10-43-3-139.ec2.internal",state="no_gpu_slot"} 0 expert_cache_ram_to_gpu_state_seconds_total{gpu="0",node="ip-10-43-3-139.ec2.internal",state="waiting_on_ssd"} 364.3 expert_cache_ram_to_gpu_state_seconds_total{gpu="0",node="ip-10-43-3-139.ec2.internal",state="ssd_below_cap"} 38.48 expert_cache_ram_to_gpu_state_seconds_total{gpu="0",node="ip-10-43-3-139.ec2.internal",state="nothing_needed"} 0.65 # HELP expert_cache_events_total Engine events in the measured part of the run. # TYPE expert_cache_events_total counter expert_cache_events_total{gpu="0",node="ip-10-43-3-139.ec2.internal",event="ssd_read"} 60777 expert_cache_events_total{gpu="0",node="ip-10-43-3-139.ec2.internal",event="ram_promotion"} 5800 expert_cache_events_total{gpu="0",node="ip-10-43-3-139.ec2.internal",event="gpu_decode"} 66937 expert_cache_events_total{gpu="0",node="ip-10-43-3-139.ec2.internal",event="scratch_hit"} 0 # HELP expert_cache_missing Experts some waiting token needs that are not on the GPU. # TYPE expert_cache_missing gauge expert_cache_missing{gpu="0",node="ip-10-43-3-139.ec2.internal"} 172 # HELP expert_cache_reads_in_flight SSD reads in flight (including reads waiting for a GPU slot). # TYPE expert_cache_reads_in_flight gauge expert_cache_reads_in_flight{gpu="0",node="ip-10-43-3-139.ec2.internal"} 36 # HELP expert_cache_uploads_pending Copies to the GPU in flight. # TYPE expert_cache_uploads_pending gauge expert_cache_uploads_pending{gpu="0",node="ip-10-43-3-139.ec2.internal"} 0 # HELP expert_cache_tier_experts Experts held per tier. # TYPE expert_cache_tier_experts gauge expert_cache_tier_experts{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="l0"} 108 expert_cache_tier_experts{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="l1"} 1043 # HELP expert_cache_tier_slots Expert slots per tier. # TYPE expert_cache_tier_slots gauge expert_cache_tier_slots{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="l0"} 108 expert_cache_tier_slots{gpu="0",node="ip-10-43-3-139.ec2.internal",tier="l1"} 1080 # HELP expert_cache_failures_total Retries: no free GPU slot for an upload, RAM hit waiting for a GPU slot, no free RAM slot for a read. # TYPE expert_cache_failures_total counter expert_cache_failures_total{gpu="0",node="ip-10-43-3-139.ec2.internal",reason="no_gpu_slot"} 0 expert_cache_failures_total{gpu="0",node="ip-10-43-3-139.ec2.internal",reason="promote_no_slot"} 0 expert_cache_failures_total{gpu="0",node="ip-10-43-3-139.ec2.internal",reason="no_ram_slot"} 0 # HELP expert_cache_gpu_lifetime_seconds Seconds an expert stayed on the GPU before eviction. (_sum is estimated from bucket midpoints) # TYPE expert_cache_gpu_lifetime_seconds histogram expert_cache_gpu_lifetime_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="0.1"} 87630 expert_cache_gpu_lifetime_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="0.25"} 89524 expert_cache_gpu_lifetime_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="0.5"} 90156 expert_cache_gpu_lifetime_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="1"} 90575 expert_cache_gpu_lifetime_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="2"} 96147 expert_cache_gpu_lifetime_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="5"} 96613 expert_cache_gpu_lifetime_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="10"} 97026 expert_cache_gpu_lifetime_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="20"} 97219 expert_cache_gpu_lifetime_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="50"} 97465 expert_cache_gpu_lifetime_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="100"} 97499 expert_cache_gpu_lifetime_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="200"} 97502 expert_cache_gpu_lifetime_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="+Inf"} 97514 expert_cache_gpu_lifetime_seconds_sum{gpu="0",node="ip-10-43-3-139.ec2.internal"} 35255.7 expert_cache_gpu_lifetime_seconds_count{gpu="0",node="ip-10-43-3-139.ec2.internal"} 97514 # HELP expert_cache_gpu_idle_before_evict_seconds Seconds since last use when an expert was evicted from the GPU. (_sum is estimated from bucket midpoints) # TYPE expert_cache_gpu_idle_before_evict_seconds histogram expert_cache_gpu_idle_before_evict_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="0.1"} 87625 expert_cache_gpu_idle_before_evict_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="0.25"} 89528 expert_cache_gpu_idle_before_evict_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="0.5"} 90161 expert_cache_gpu_idle_before_evict_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="1"} 90580 expert_cache_gpu_idle_before_evict_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="2"} 96154 expert_cache_gpu_idle_before_evict_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="5"} 96621 expert_cache_gpu_idle_before_evict_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="10"} 97040 expert_cache_gpu_idle_before_evict_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="20"} 97235 expert_cache_gpu_idle_before_evict_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="50"} 97483 expert_cache_gpu_idle_before_evict_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="100"} 97514 expert_cache_gpu_idle_before_evict_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="200"} 97514 expert_cache_gpu_idle_before_evict_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="+Inf"} 97514 expert_cache_gpu_idle_before_evict_seconds_sum{gpu="0",node="ip-10-43-3-139.ec2.internal"} 32333.9 expert_cache_gpu_idle_before_evict_seconds_count{gpu="0",node="ip-10-43-3-139.ec2.internal"} 97514 # HELP expert_cache_reuse_gap_seconds Seconds between consecutive uses of an expert. (_sum is estimated from bucket midpoints) # TYPE expert_cache_reuse_gap_seconds histogram expert_cache_reuse_gap_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="0.1"} 0 expert_cache_reuse_gap_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="0.25"} 0 expert_cache_reuse_gap_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="0.5"} 0 expert_cache_reuse_gap_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="1"} 0 expert_cache_reuse_gap_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="2"} 0 expert_cache_reuse_gap_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="5"} 0 expert_cache_reuse_gap_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="10"} 0 expert_cache_reuse_gap_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="20"} 0 expert_cache_reuse_gap_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="50"} 0 expert_cache_reuse_gap_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="100"} 85955 expert_cache_reuse_gap_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="200"} 86027 expert_cache_reuse_gap_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="+Inf"} 86074 expert_cache_reuse_gap_seconds_sum{gpu="0",node="ip-10-43-3-139.ec2.internal"} 6466825.0 expert_cache_reuse_gap_seconds_count{gpu="0",node="ip-10-43-3-139.ec2.internal"} 86074 # HELP expert_cache_evict_to_reuse_gap_seconds Seconds from eviction to the next load of the same expert. (_sum is estimated from bucket midpoints) # TYPE expert_cache_evict_to_reuse_gap_seconds histogram expert_cache_evict_to_reuse_gap_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="0.1"} 0 expert_cache_evict_to_reuse_gap_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="0.25"} 0 expert_cache_evict_to_reuse_gap_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="0.5"} 0 expert_cache_evict_to_reuse_gap_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="1"} 0 expert_cache_evict_to_reuse_gap_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="2"} 0 expert_cache_evict_to_reuse_gap_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="5"} 0 expert_cache_evict_to_reuse_gap_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="10"} 2 expert_cache_evict_to_reuse_gap_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="20"} 10 expert_cache_evict_to_reuse_gap_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="50"} 184 expert_cache_evict_to_reuse_gap_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="100"} 85502 expert_cache_evict_to_reuse_gap_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="200"} 85574 expert_cache_evict_to_reuse_gap_seconds_bucket{gpu="0",node="ip-10-43-3-139.ec2.internal",le="+Inf"} 85621 expert_cache_evict_to_reuse_gap_seconds_sum{gpu="0",node="ip-10-43-3-139.ec2.internal"} 6425275.0 expert_cache_evict_to_reuse_gap_seconds_count{gpu="0",node="ip-10-43-3-139.ec2.internal"} 85621 # HELP gpu_utilization_ratio GPU busy share (NVML). # TYPE gpu_utilization_ratio gauge gpu_utilization_ratio{gpu="0",name="NVIDIA RTX PRO 6000 Blackwell Server Edition",node="ip-10-43-3-139.ec2.internal"} 0.0 # HELP gpu_memory_used_bytes GPU memory used. # TYPE gpu_memory_used_bytes gauge gpu_memory_used_bytes{gpu="0",name="NVIDIA RTX PRO 6000 Blackwell Server Edition",node="ip-10-43-3-139.ec2.internal"} 670000000.0 # HELP gpu_memory_total_bytes GPU memory total. # TYPE gpu_memory_total_bytes gauge gpu_memory_total_bytes{gpu="0",name="NVIDIA RTX PRO 6000 Blackwell Server Edition",node="ip-10-43-3-139.ec2.internal"} 102640000000.0 # HELP gpu_power_watts GPU power draw. # TYPE gpu_power_watts gauge gpu_power_watts{gpu="0",name="NVIDIA RTX PRO 6000 Blackwell Server Edition",node="ip-10-43-3-139.ec2.internal"} 29.7 # HELP gpu_power_limit_watts GPU power limit. # TYPE gpu_power_limit_watts gauge gpu_power_limit_watts{gpu="0",name="NVIDIA RTX PRO 6000 Blackwell Server Edition",node="ip-10-43-3-139.ec2.internal"} 600.0 # HELP gpu_temperature_celsius GPU temperature. # TYPE gpu_temperature_celsius gauge gpu_temperature_celsius{gpu="0",name="NVIDIA RTX PRO 6000 Blackwell Server Edition",node="ip-10-43-3-139.ec2.internal"} 21 # HELP gpu_sm_clock_hertz SM clock. # TYPE gpu_sm_clock_hertz gauge gpu_sm_clock_hertz{gpu="0",name="NVIDIA RTX PRO 6000 Blackwell Server Edition",node="ip-10-43-3-139.ec2.internal"} 180000000.0 # HELP gpu_pcie_rx_bytes_per_second PCIe bytes into the GPU per second. # TYPE gpu_pcie_rx_bytes_per_second gauge gpu_pcie_rx_bytes_per_second{gpu="0",name="NVIDIA RTX PRO 6000 Blackwell Server Edition",node="ip-10-43-3-139.ec2.internal"} 0.0 # HELP gpu_pcie_tx_bytes_per_second PCIe bytes out of the GPU per second. # TYPE gpu_pcie_tx_bytes_per_second gauge gpu_pcie_tx_bytes_per_second{gpu="0",name="NVIDIA RTX PRO 6000 Blackwell Server Edition",node="ip-10-43-3-139.ec2.internal"} 1000000.0 # HELP expert_cache_finished_run_tokens_per_second Tokens per second of finished runs. # TYPE expert_cache_finished_run_tokens_per_second gauge expert_cache_finished_run_tokens_per_second{run="engine-s100-park-prefetch",mode="park",gpu_store="fp8",gpu_pool="layer",guesses="0"} 0.65 expert_cache_finished_run_tokens_per_second{run="engine-s1",mode="park",gpu_store="fp8",gpu_pool="layer",guesses="0"} 1.208 expert_cache_finished_run_tokens_per_second{run="engine-s100-wait",mode="wait",gpu_store="fp8",gpu_pool="layer",guesses="0"} 2.5 expert_cache_finished_run_tokens_per_second{run="engine-s100-park",mode="park",gpu_store="fp8",gpu_pool="layer",guesses="0"} 0.658 expert_cache_finished_run_tokens_per_second{run="engine-s1-victim",mode="park",gpu_store="fp8",gpu_pool="layer",guesses="0"} 1.85 expert_cache_finished_run_tokens_per_second{run="engine-s100-wait-victim",mode="wait",gpu_store="fp8",gpu_pool="layer",guesses="0"} 2.5 expert_cache_finished_run_tokens_per_second{run="engine-s10-wait",mode="wait",gpu_store="fp8",gpu_pool="layer",guesses="0"} 1.5 expert_cache_finished_run_tokens_per_second{run="engine-s10-park",mode="park",gpu_store="fp8",gpu_pool="layer",guesses="0"} 1.383 expert_cache_finished_run_tokens_per_second{run="engine-s100-wait-lru",mode="wait",gpu_store="fp8",gpu_pool="layer",guesses="0"} 1.844 expert_cache_finished_run_tokens_per_second{run="engine-s100-wait-keep",mode="wait",gpu_store="fp8",gpu_pool="layer",guesses="0"} 2.21 expert_cache_finished_run_tokens_per_second{run="engine-s200-wait-keep",mode="wait",gpu_store="fp8",gpu_pool="layer",guesses="0"} 3.435 expert_cache_finished_run_tokens_per_second{run="engine-s400-wait-keep",mode="wait",gpu_store="fp8",gpu_pool="layer",guesses="0"} 5.805 expert_cache_finished_run_tokens_per_second{run="engine-pull-park-sort-keep",mode="park",gpu_store="fp8",gpu_pool="layer",guesses="0"} 1.072 expert_cache_finished_run_tokens_per_second{run="engine-pull-park-heap-keep",mode="park",gpu_store="fp8",gpu_pool="layer",guesses="0"} 1.233 expert_cache_finished_run_tokens_per_second{run="engine-pull-park-heap-heap75",mode="park",gpu_store="fp8",gpu_pool="layer",guesses="0"} 1.2 expert_cache_finished_run_tokens_per_second{run="engine-pull-wait-heap-keep",mode="wait",gpu_store="fp8",gpu_pool="layer",guesses="0"} 2.014 expert_cache_finished_run_tokens_per_second{run="engine-pull-wait-heap-heap75",mode="wait",gpu_store="fp8",gpu_pool="layer",guesses="0"} 1.982 expert_cache_finished_run_tokens_per_second{run="engine-pull-wait-heap-keep-noidle",mode="wait",gpu_store="fp8",gpu_pool="layer",guesses="0"} 2.047 expert_cache_finished_run_tokens_per_second{run="engine-fix-wait-sort-keep",mode="wait",gpu_store="fp8",gpu_pool="layer",guesses="0"} 2.049 expert_cache_finished_run_tokens_per_second{run="engine-fix-wait-heap-keep",mode="wait",gpu_store="fp8",gpu_pool="layer",guesses="0"} 2.049 expert_cache_finished_run_tokens_per_second{run="engine-fix-park-heap-keep",mode="park",gpu_store="fp8",gpu_pool="layer",guesses="0"} 1.228 expert_cache_finished_run_tokens_per_second{run="engine-pe-wait-sort-keep",mode="wait",gpu_store="fp8",gpu_pool="layer",guesses="0"} 2.05 expert_cache_finished_run_tokens_per_second{run="engine-store-wait-packed-layer",mode="wait",gpu_store="packed",gpu_pool="layer",guesses="0"} 2.172 expert_cache_finished_run_tokens_per_second{run="engine-store-wait-packed-global",mode="wait",gpu_store="packed",gpu_pool="global",guesses="0"} 2.18 expert_cache_finished_run_tokens_per_second{run="engine-store-park-packed-global",mode="park",gpu_store="packed",gpu_pool="global",guesses="0"} 1.667 expert_cache_finished_run_tokens_per_second{run="engine-spec-0",mode="wait",gpu_store="packed",gpu_pool="global",guesses="0"} 2.332 expert_cache_finished_run_tokens_per_second{run="engine-spec-3",mode="wait",gpu_store="packed",gpu_pool="global",guesses="3"} 4.417 expert_cache_finished_run_tokens_per_second{run="engine-spec-7",mode="wait",gpu_store="packed",gpu_pool="global",guesses="7"} 5.47 expert_cache_finished_run_tokens_per_second{run="engine-spec-20",mode="wait",gpu_store="packed",gpu_pool="global",guesses="20"} 5.546 expert_cache_finished_run_tokens_per_second{run="engine-tel-wait-fp8-layer",mode="wait",gpu_store="fp8",gpu_pool="layer",guesses="0"} 2.049 expert_cache_finished_run_tokens_per_second{run="engine-tel-wait-packed-global",mode="wait",gpu_store="packed",gpu_pool="global",guesses="0"} 2.181 expert_cache_finished_run_tokens_per_second{run="engine-tel-park-fp8-layer",mode="park",gpu_store="fp8",gpu_pool="layer",guesses="0"} 1.239 expert_cache_finished_run_tokens_per_second{run="engine-tel-park-packed-global",mode="park",gpu_store="packed",gpu_pool="global",guesses="0"} 1.678 expert_cache_finished_run_tokens_per_second{run="engine-rf-wait-ram-first",mode="wait",gpu_store="packed",gpu_pool="global",guesses="0"} 2.167 expert_cache_finished_run_tokens_per_second{run="engine-rf-park-ram-first",mode="park",gpu_store="packed",gpu_pool="global",guesses="0"} 1.672 expert_cache_finished_run_tokens_per_second{run="engine-rf-wait-ram-first+prefetch",mode="wait",gpu_store="packed",gpu_pool="global",guesses="0"} 2.162 expert_cache_finished_run_tokens_per_second{run="engine-rf-park-ram-first+prefetch",mode="park",gpu_store="packed",gpu_pool="global",guesses="0"} 1.672 expert_cache_finished_run_tokens_per_second{run="engine-la-wait",mode="wait",gpu_store="packed",gpu_pool="global",guesses="0"} 2.578 expert_cache_finished_run_tokens_per_second{run="engine-la-park",mode="park",gpu_store="packed",gpu_pool="global",guesses="0"} 2.2 expert_cache_finished_run_tokens_per_second{run="engine-l1-wait",mode="wait",gpu_store="packed",gpu_pool="global",guesses="0"} 2.616 expert_cache_finished_run_tokens_per_second{run="engine-l1-park",mode="park",gpu_store="packed",gpu_pool="global",guesses="0"} 2.2 expert_cache_finished_run_tokens_per_second{run="engine-sub-wait-l1",mode="wait",gpu_store="packed",gpu_pool="global",guesses="0"} 80.925 expert_cache_finished_run_tokens_per_second{run="engine-sub-park-l1",mode="park",gpu_store="packed",gpu_pool="global",guesses="0"} 29.871 expert_cache_finished_run_tokens_per_second{run="engine-sub-wait-move",mode="wait",gpu_store="packed",gpu_pool="global",guesses="0"} 76.911 expert_cache_finished_run_tokens_per_second{run="engine-sub-park-move",mode="park",gpu_store="packed",gpu_pool="global",guesses="0"} 28.905 expert_cache_finished_run_tokens_per_second{run="engine-sub2-wait-128",mode="wait",gpu_store="packed",gpu_pool="global",guesses="0"} 85.531 expert_cache_finished_run_tokens_per_second{run="engine-sub2-park-128",mode="park",gpu_store="packed",gpu_pool="global",guesses="0"} 58.363 expert_cache_finished_run_tokens_per_second{run="engine-sub2-wait-256",mode="wait",gpu_store="packed",gpu_pool="global",guesses="0"} 84.399 expert_cache_finished_run_tokens_per_second{run="engine-spec2-full-4",mode="wait",gpu_store="packed",gpu_pool="global",guesses="4"} 5.509 expert_cache_finished_run_tokens_per_second{run="engine-bd-wait",mode="wait",gpu_store="packed",gpu_pool="global",guesses="0"} 2.55 expert_cache_finished_run_tokens_per_second{run="engine-bd-park",mode="park",gpu_store="packed",gpu_pool="global",guesses="0"} 2.2 expert_cache_finished_run_tokens_per_second{run="engine-nx-100-wait-r48",mode="wait",gpu_store="packed",gpu_pool="global",guesses="0"} 2.709 expert_cache_finished_run_tokens_per_second{run="engine-nx-100-park-r48",mode="park",gpu_store="packed",gpu_pool="global",guesses="0"} 2.183 expert_cache_finished_run_tokens_per_second{run="engine-nx-200-wait-r48",mode="wait",gpu_store="packed",gpu_pool="global",guesses="0"} 4.182 expert_cache_finished_run_tokens_per_second{run="engine-nx-400-wait-r48",mode="wait",gpu_store="packed",gpu_pool="global",guesses="0"} 6.553 expert_cache_finished_run_tokens_per_second{run="engine-fifo-wait-base",mode="wait",gpu_store="packed",gpu_pool="global",guesses="0"} 2.71 expert_cache_finished_run_tokens_per_second{run="engine-fifo-park-base",mode="park",gpu_store="packed",gpu_pool="global",guesses="0"} 2.194 expert_cache_finished_run_tokens_per_second{run="engine-fifo-wait-fifo",mode="wait",gpu_store="packed",gpu_pool="global",guesses="0"} 2.496 expert_cache_finished_run_tokens_per_second{run="engine-fifo-park-fifo",mode="park",gpu_store="packed",gpu_pool="global",guesses="0"} 2.189 expert_cache_finished_run_tokens_per_second{run="engine-fifo-wait-fifo+pred",mode="wait",gpu_store="packed",gpu_pool="global",guesses="0"} 2.496 expert_cache_finished_run_tokens_per_second{run="engine-fifo-park-fifo+pred",mode="park",gpu_store="packed",gpu_pool="global",guesses="0"} 2.2 expert_cache_finished_run_tokens_per_second{run="engine-pg-wait-base",mode="wait",gpu_store="packed",gpu_pool="global",guesses="0"} 3.743 expert_cache_finished_run_tokens_per_second{run="engine-pg-park-base",mode="park",gpu_store="packed",gpu_pool="global",guesses="0"} 2.389 expert_cache_finished_run_tokens_per_second{run="engine-pg-wait-keep",mode="wait",gpu_store="packed",gpu_pool="global",guesses="0"} 3.743 expert_cache_finished_run_tokens_per_second{run="engine-pg-park-keep",mode="park",gpu_store="packed",gpu_pool="global",guesses="0"} 2.428 expert_cache_finished_run_tokens_per_second{run="engine-pg-wait-keep+prefetch",mode="wait",gpu_store="packed",gpu_pool="global",guesses="0"} 3.361 expert_cache_finished_run_tokens_per_second{run="engine-pg-park-keep+prefetch",mode="park",gpu_store="packed",gpu_pool="global",guesses="0"} 2.567 expert_cache_finished_run_tokens_per_second{run="engine-wp-wait-predfifo",mode="wait",gpu_store="packed",gpu_pool="global",guesses="0"} 2.536 expert_cache_finished_run_tokens_per_second{run="engine-wp-park-predfifo",mode="park",gpu_store="packed",gpu_pool="global",guesses="0"} 1.606 expert_cache_finished_run_tokens_per_second{run="engine-wp-wait-waves2",mode="wait",gpu_store="packed",gpu_pool="global",guesses="0"} 2.006 expert_cache_finished_run_tokens_per_second{run="engine-wp-wait-waves4",mode="wait",gpu_store="packed",gpu_pool="global",guesses="0"} 1.639 expert_cache_finished_run_tokens_per_second{run="engine-scale-700-spec0",mode="wait",gpu_store="packed",gpu_pool="global",guesses="0"} 8.879 expert_cache_finished_run_tokens_per_second{run="engine-scale-700-spec4",mode="wait",gpu_store="packed",gpu_pool="global",guesses="4"} 23.71 expert_cache_finished_run_tokens_per_second{run="engine-scale-500-spec4",mode="wait",gpu_store="packed",gpu_pool="global",guesses="4"} 19.535 expert_cache_finished_run_tokens_per_second{run="engine-scale-600-spec4",mode="wait",gpu_store="packed",gpu_pool="global",guesses="4"} 21.804