Skip to content

Commit 8aade45

Browse files
committed
fix(host-agent): net prewarm pending charge, gate ledger gauges on measured, real statfs tmpfs-used, pin watch-channel invariant (findings 2, 3, 4, 5)
- finding 2: RamLedger::register_pending_base_shm now stores the target file path alongside the expected byte count. sample() nets each pending charge against that file's own st_blocks (file_allocated_bytes) instead of holding the full charge outstanding for the whole write window — the fix for the transient double-charge where a ~19 GiB dev-brain prewarm at 90% written was under-reporting allocatable_mib by ~17 GiB (MemAvailable already dropping AND the full charge still subtracted). New regression test pending_charge_nets_against_bytes_already_written proves the charge shrinks as bytes actually land. - finding 3: the whole HOST_RAM_LEDGER_MIB / HOST_RAM_ALLOCATABLE_MIB / HOST_BASE_SHM_TMPFS_* gauge-emission block is now gated on ram_snapshot.measured, so VZ/Process/non-Linux hosts (and a genuine /proc/meminfo parse failure) stop reporting a permanently-zero gauge family that reads as "this host has 0 MiB of everything" instead of "unmeasured" — matches the PR's own acceptance criterion. - finding 5: HOST_BASE_SHM_TMPFS_USED_MIB was byte-identical to the base_shm category gauge (a read_dir/st_blocks walk over known files). tmpfs_stat_mib now also does a real statfs read (f_blocks - f_bfree) and RamLedgerSnapshot carries a new base_shm_tmpfs_used_mib field for it, so an unlinked-but-open file or stray subdir the dir walk can't see is no longer invisible to the gauge that exists specifically to debug ENOSPC-class tmpfs incidents. - finding 4: added ram_ledger_snapshot_round_trips_through_watch_channel, publishing a RamLedgerSnapshot on a tokio::sync::watch channel and asserting two independent readers observe byte-identical values to what was published — pins the "one source of truth" mechanism the heartbeat and pressure gate rely on. Review threads: #561 (comment) (finding 2) #561 (comment) (finding 3) #561 (comment) (finding 4) #561 (comment) (finding 5)
1 parent 6db9843 commit 8aade45

4 files changed

Lines changed: 264 additions & 60 deletions

File tree

‎crates/engram-host-agent/src/image_prefetch.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -660,7 +660,11 @@ async fn prefetch_one(
660660
the handler's lazy path backstops",
661661
);
662662
} else {
663-
ram_ledger.register_pending_base_shm(memory_ref, pending_bytes);
663+
ram_ledger.register_pending_base_shm(
664+
memory_ref,
665+
base_path.clone(),
666+
pending_bytes,
667+
);
664668
let prewarm_result =
665669
prewarm_base_shm(&base_path, &memory_manifest, chunk_store, chunk_cache)
666670
.await;

‎crates/engram-host-agent/src/lib.rs‎

Lines changed: 37 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1173,26 +1173,46 @@ impl HostAgent {
11731173
&guest_mem.unwrap_or_default(),
11741174
);
11751175
ram_ledger_tx_for_heartbeat.send_replace(ram_snapshot);
1176-
::metrics::gauge!(crate::metrics::HOST_RAM_LEDGER_MIB, "category" => "running_vms")
1177-
.set(ram_snapshot.running_vm_pss_mib as f64);
1178-
::metrics::gauge!(crate::metrics::HOST_RAM_LEDGER_MIB, "category" => "parked_paused")
1179-
.set(ram_snapshot.parked_paused_pss_mib as f64);
1180-
::metrics::gauge!(crate::metrics::HOST_RAM_LEDGER_MIB, "category" => "base_shm")
1181-
.set(ram_snapshot.base_shm_mib as f64);
1182-
::metrics::gauge!(crate::metrics::HOST_RAM_LEDGER_MIB, "category" => "base_shm_pending")
1183-
.set(ram_snapshot.base_shm_pending_mib as f64);
1184-
::metrics::gauge!(crate::metrics::HOST_RAM_LEDGER_MIB, "category" => "parked_local_memfiles")
1185-
.set(ram_snapshot.parked_local_memfile_mib as f64);
1186-
::metrics::gauge!(crate::metrics::HOST_RAM_ALLOCATABLE_MIB)
1187-
.set(ram_snapshot.allocatable_mib() as f64);
1188-
::metrics::gauge!(crate::metrics::HOST_BASE_SHM_TMPFS_TOTAL_MIB)
1189-
.set(ram_snapshot.base_shm_tmpfs_total_mib as f64);
1190-
::metrics::gauge!(crate::metrics::HOST_BASE_SHM_TMPFS_USED_MIB)
1191-
.set(ram_snapshot.base_shm_mib as f64);
1176+
// Issue #540 review finding 3: gate every ledger gauge
1177+
// on `measured` — VZ/Process/non-Linux backends (and a
1178+
// genuine `/proc/meminfo` parse failure) never took a
1179+
// real sample, so `ram_snapshot` is the all-zero
1180+
// default. Emitting that as a value would look like
1181+
// "this host has 0 MiB of everything" on a dashboard
1182+
// instead of "unmeasured" — matches the acceptance
1183+
// criterion's "gauges not emitted" posture.
1184+
if ram_snapshot.measured {
1185+
::metrics::gauge!(crate::metrics::HOST_RAM_LEDGER_MIB, "category" => "running_vms")
1186+
.set(ram_snapshot.running_vm_pss_mib as f64);
1187+
::metrics::gauge!(crate::metrics::HOST_RAM_LEDGER_MIB, "category" => "parked_paused")
1188+
.set(ram_snapshot.parked_paused_pss_mib as f64);
1189+
::metrics::gauge!(crate::metrics::HOST_RAM_LEDGER_MIB, "category" => "base_shm")
1190+
.set(ram_snapshot.base_shm_mib as f64);
1191+
::metrics::gauge!(crate::metrics::HOST_RAM_LEDGER_MIB, "category" => "base_shm_pending")
1192+
.set(ram_snapshot.base_shm_pending_mib as f64);
1193+
::metrics::gauge!(crate::metrics::HOST_RAM_LEDGER_MIB, "category" => "parked_local_memfiles")
1194+
.set(ram_snapshot.parked_local_memfile_mib as f64);
1195+
::metrics::gauge!(crate::metrics::HOST_RAM_ALLOCATABLE_MIB)
1196+
.set(ram_snapshot.allocatable_mib() as f64);
1197+
::metrics::gauge!(crate::metrics::HOST_BASE_SHM_TMPFS_TOTAL_MIB)
1198+
.set(ram_snapshot.base_shm_tmpfs_total_mib as f64);
1199+
// Issue #540 review finding 5: this is the tmpfs
1200+
// mount's own `statfs` used figure (`f_blocks -
1201+
// f_bfree`), NOT `base_shm_mib` (this ledger's
1202+
// st_blocks walk over known files) — the two can
1203+
// legitimately diverge (an unlinked-but-open file,
1204+
// a stray subdir) and this gauge exists specifically
1205+
// to catch that divergence during an ENOSPC-class
1206+
// incident.
1207+
::metrics::gauge!(crate::metrics::HOST_BASE_SHM_TMPFS_USED_MIB)
1208+
.set(ram_snapshot.base_shm_tmpfs_used_mib as f64);
1209+
}
11921210
// Issue #540: single emission site for this gauge (was
11931211
// previously only set inside the idle-evictor's
11941212
// pressure-aware branch, so it read stale/unset when
1195-
// that mode was off). Every tick now, unconditionally.
1213+
// that mode was off). Every tick now, unconditionally
1214+
// (still gated on `measured` via `free_pct()`'s own
1215+
// `None` return).
11961216
if let Some(pct) = ram_snapshot.free_pct() {
11971217
::metrics::gauge!(crate::metrics::HOST_MEM_FREE_PCT).set(f64::from(pct));
11981218
}

0 commit comments

Comments
 (0)