jcp/SPEC.md

@@ -675,7 +675,15 @@ Recommended ladder, best first:
    pid is another process. The count is folded only when a thread leaves a
    core, so at each end of the span a member's count MUST be read with none
    of its threads on a core or queued for one, or the member is a gap
-   (below). A walk of the tree reads a parent before it lists and reads
+   (below). Nor is a member at rest while one of its threads is in a wait
+   that no signal ends (the kernel's own waits: for a block being written,
+   a page being brought in, a lock). That thread is off its core and
+   folded, and it runs again when the kernel is done, with nothing
+   arriving; a stop that took the member for rest there would read the
+   span short by the rest of its work, with nothing seen missing. So at
+   each end a member with a thread in such a wait MUST be waited out as a
+   running one is, or is the same gap. A walk of the tree reads a parent
+   before it lists and reads
    the parent's children, so a child that exits and is reaped in between
    is in that walk neither as a member nor in its parent's reaped time;
    the walk at each end MUST therefore be confirmed by another, one after
@@ -708,10 +716,13 @@ Recommended ladder, best first:
      separated, so none of it is counted;
    - a member that could not be read (a process of another user), or
      whose children could not be listed;
-   - a member still running at an end: at the stop its reading lacks up to
-     a scheduler quantum of energy per running thread; at the START its
-     reading is short by as much, so the figure may hold up to that much
-     energy spent just BEFORE the span. This one can make the figure
+   - a member not at rest at an end. Still running: at the stop its
+     reading lacks up to a scheduler quantum of energy per running thread;
+     at the START its reading is short by as much, so the figure may hold
+     up to that much energy spent just BEFORE the span. Or with a thread
+     in a wait no signal ends, in the middle of its work: what that thread
+     goes on to spend falls after the stop, or, at the START, inside the
+     span though the work began before it. This one can make the figure
      exceed the tree's energy over the span, so a `Partial` tree figure
      with it is not a lower bound, and an implementation that names the
      gaps beside the figure SHOULD name the end (the reference gateway:

jcp/jcp-rs/crates/jcp-gateway/tests/upstream_tree.rs

@@ -11,7 +11,8 @@
 //! and 272), read before and after each relay once the server is quiet
 //! on the oracle's own terms: no thread on a core or queued
 //! (`proc_pidinfo(PROC_PIDTASKINFO)` into a raw buffer, `pti_numrunning` at
-//! byte 88) at two reads 20 ms apart whose energy agrees. A server blocked
+//! byte 88) and none in a wait no signal ends (each thread's `pth_run_state`,
+//! read raw as well) at two reads 20 ms apart whose energy agrees. A server blocked
 //! waiting for its next request does not move its count, so between the
 //! meter's reads and the oracle's nothing is spent: the call's figure must
 //! be the oracle's advance across the relay, in nanojoules rounded UP to
@@ -105,17 +106,44 @@ fn oracle_running(pid: i32) -> Option<i32> {
     (n == 96).then(|| i32::from_ne_bytes(buf[88..92].try_into().unwrap()))
 }
 
-/// `pid`'s rusage once it is quiet by the oracle's rule: not running or
-/// queued at two reads 20 ms apart, with the same energy at both. Panics
+/// Whether every thread of `pid` is at rest by its own run state: none on a
+/// core or queued (`TH_STATE_RUNNING`, 1) and none in a wait that no signal
+/// ends (`TH_STATE_UNINTERRUPTIBLE`, 4: waiting on the disk, the pager or a
+/// kernel lock, to run again without anything arriving), the rule the meter
+/// settles by. `proc_pidinfo(pid, PROC_PIDLISTTHREADS = 6)` into a raw buffer
+/// of 64-bit thread handles, then `proc_pidinfo(pid, PROC_PIDTHREADINFO = 5,
+/// handle)` into a raw 112-byte buffer, `pth_run_state` the int32 at byte 24.
+/// `None` if the listing fails (an exited process has no task).
+fn oracle_at_rest(pid: i32) -> Option<bool> {
+    let mut handles = [0u64; 512];
+    // SAFETY: `handles` is writable for the bytes passed.
+    let n = unsafe { libc::proc_pidinfo(pid, 6, 0, handles.as_mut_ptr().cast(), (handles.len() * 8) as libc::c_int) };
+    if n <= 0 {
+        return None;
+    }
+    let n = n as usize / 8;
+    assert!(n < handles.len(), "{pid} has more threads than the oracle's buffer holds");
+    Some(handles[..n].iter().all(|&handle| {
+        let mut buf = [0u8; 112];
+        // SAFETY: `buf` is writable for the 112 bytes passed.
+        let got = unsafe { libc::proc_pidinfo(pid, 5, handle, buf.as_mut_ptr().cast(), 112) };
+        // A thread gone since the listing waits on nothing.
+        got != 112 || !matches!(i32::from_ne_bytes(buf[24..28].try_into().unwrap()), 1 | 4)
+    }))
+}
+
+/// `pid`'s rusage once it is quiet by the oracle's rule: no thread running,
+/// queued or in a wait no signal ends at two reads 20 ms apart, with the
+/// same energy at both. Panics
 /// after 20 s.
 fn quiet(pid: i32) -> [u64; 64] {
     let deadline = Instant::now() + Duration::from_secs(20);
     loop {
         let first = oracle_rusage(pid).expect("oracle read");
-        let before = oracle_running(pid) == Some(0);
+        let before = oracle_running(pid) == Some(0) && oracle_at_rest(pid) == Some(true);
         std::thread::sleep(Duration::from_millis(20));
         let second = oracle_rusage(pid).expect("oracle read");
-        let after = oracle_running(pid) == Some(0);
+        let after = oracle_running(pid) == Some(0) && oracle_at_rest(pid) == Some(true);
         if before && after && energy_nj(&first) == energy_nj(&second) {
             return second;
         }

proof/proof-rs/crates/proof-solvers/tests/z3_tree.rs

@@ -7,7 +7,9 @@
 //! taken at the SDK's byte offset (energy at 336), read before and after each
 //! attempt once `z3` is quiet on the oracle's own terms: no thread on a core
 //! or queued (`proc_pidinfo(PROC_PIDTASKINFO)` into a raw buffer,
-//! `pti_numrunning` at byte 88) at two reads 20 ms apart whose energy agrees.
+//! `pti_numrunning` at byte 88) and none in a wait no signal ends (each
+//! thread's `pth_run_state`, read raw as well) at two reads 20 ms apart
+//! whose energy agrees.
 //! A resident `z3` blocked on its next read does not move its count, so
 //! between the meter's reads and the oracle's nothing is spent: a decision's
 //! reading must EQUAL the oracle's advance across the attempt, to the
@@ -67,17 +69,44 @@ fn oracle_running(pid: i32) -> Option<i32> {
     (n == 96).then(|| i32::from_ne_bytes(buf[88..92].try_into().unwrap()))
 }
 
-/// `pid`'s energy count once it is quiet by the oracle's rule: not running
-/// or queued at two reads 20 ms apart, with the same energy at both. Panics
+/// Whether every thread of `pid` is at rest by its own run state: none on a
+/// core or queued (`TH_STATE_RUNNING`, 1) and none in a wait that no signal
+/// ends (`TH_STATE_UNINTERRUPTIBLE`, 4: waiting on the disk, the pager or a
+/// kernel lock, to run again without anything arriving), the rule the meter
+/// settles by. `proc_pidinfo(pid, PROC_PIDLISTTHREADS = 6)` into a raw buffer
+/// of 64-bit thread handles, then `proc_pidinfo(pid, PROC_PIDTHREADINFO = 5,
+/// handle)` into a raw 112-byte buffer, `pth_run_state` the int32 at byte 24.
+/// `None` if the listing fails (an exited process has no task).
+fn oracle_at_rest(pid: i32) -> Option<bool> {
+    let mut handles = [0u64; 512];
+    // SAFETY: `handles` is writable for the bytes passed.
+    let n = unsafe { libc::proc_pidinfo(pid, 6, 0, handles.as_mut_ptr().cast(), (handles.len() * 8) as libc::c_int) };
+    if n <= 0 {
+        return None;
+    }
+    let n = n as usize / 8;
+    assert!(n < handles.len(), "{pid} has more threads than the oracle's buffer holds");
+    Some(handles[..n].iter().all(|&handle| {
+        let mut buf = [0u8; 112];
+        // SAFETY: `buf` is writable for the 112 bytes passed.
+        let got = unsafe { libc::proc_pidinfo(pid, 5, handle, buf.as_mut_ptr().cast(), 112) };
+        // A thread gone since the listing waits on nothing.
+        got != 112 || !matches!(i32::from_ne_bytes(buf[24..28].try_into().unwrap()), 1 | 4)
+    }))
+}
+
+/// `pid`'s energy count once it is quiet by the oracle's rule: no thread
+/// running, queued or in a wait no signal ends at two reads 20 ms apart,
+/// with the same energy at both. Panics
 /// after 20 s.
 fn quiet_energy(pid: i32) -> u64 {
     let deadline = Instant::now() + Duration::from_secs(20);
     loop {
         let first = oracle_rusage(pid).expect("oracle read");
-        let before = oracle_running(pid) == Some(0);
+        let before = oracle_running(pid) == Some(0) && oracle_at_rest(pid) == Some(true);
         std::thread::sleep(Duration::from_millis(20));
         let second = oracle_rusage(pid).expect("oracle read");
-        let after = oracle_running(pid) == Some(0);
+        let after = oracle_running(pid) == Some(0) && oracle_at_rest(pid) == Some(true);
         if before && after && energy_nj(&first) == energy_nj(&second) {
             return energy_nj(&second);
         }

sandbox/sandbox-rs/crates/sandbox-meter/src/probe/tree.rs

@@ -94,7 +94,32 @@
 //! the core in between, and leaving folds its time and energy, so the
 //! second read would differ. The same runs read with that rule were settled
 //! and exact in 58 of 58 (the other 2 still running at 50 ms on a host at
-//! load ~200). A zombie has no task to ask and no thread to run: it is
+//! load ~200).
+//!
+//! A thread off its core is not always at rest, though. One in a wait that
+//! no signal ends (`TH_STATE_UNINTERRUPTIBLE` in its own run state: the
+//! kernel's waits, for a block being written, a page being brought in, a
+//! lock) is neither on a core nor queued, so `pti_numrunning` does not count
+//! it, and it has folded, so the two reads agree; but it runs again when the
+//! kernel is done, with nothing arriving from outside. It is in the middle
+//! of the member's work, not at its end. Measured (M4 Max, macOS 27.0.1): a
+//! perl server whose reply is followed by twenty rounds of a 4 KiB write, an
+//! `fsync` and a short burn, read with the run-state count alone, came out
+//! short in 24 of 60 round trips, at 16 to 92% of the oracle's advance,
+//! every one of those readings whole; a thread in an `fsync` loop was in
+//! that state at 18 of 60 looks, one asleep or blocked reading a pipe at
+//! none of 20 each, and of the 4,200 threads of an idle host none was at
+//! any of 5 looks. So a process is settled only when, as well, none of its
+//! threads is in such a wait: asked of each thread (`PROC_PIDLISTTHREADS`,
+//! then `PROC_PIDTHREADINFO` for each), and only once `pti_numrunning` is
+//! zero. The same server then read exact in 60 of 60. The check costs a
+//! listing and one read per thread, paid only by a process with no thread
+//! running: the start of a span on a tree of 61 single-threaded processes
+//! took 1.73 to 1.75 ms with it against 1.68 to 1.72 ms without (the least
+//! of 300 starts, three runs each; the means lay within each other's
+//! spread).
+//!
+//! A zombie has no task to ask and no thread to run: it is
 //! settled when its two reads agree — its record is final then, though not
 //! complete (below). The meter walks again, every
 //! [`SETTLE_POLL`], until every member is settled, for at most
@@ -140,11 +165,14 @@
 //! blocking first; one 2 ms exit in 12 read low); the process's own exit is
 //! not. So an exited member is a
 //! gap ([`TreeGap::Exited`]) for its energy as well as its GPU time. Nor
-//! does settling make the tree's work finished. A member waiting on I/O in the
-//! middle of its work, or a process still being started, is on no core and
-//! reads as settled. Measured: a tree still starting a helper after its
-//! reply was read, with every member off-core, 1.05 mJ short of a read
-//! taken once it had gone quiet. So a span is stopped after the reply, and
+//! does settling make the tree's work finished. A member waiting, in the
+//! middle of its work, on something a signal could end (a reply on a pipe
+//! or a socket, a child, a timer) is on no core and in no uninterruptible
+//! wait, and reads as settled: the wait-state rule above narrows this and
+//! does not close it. Measured with the run-state count alone: a tree still
+//! starting a helper after its reply was read, with every member off-core,
+//! 1.05 mJ short of a read taken once it had gone quiet. So a span is
+//! stopped after the reply, and
 //! what the tree does after the stop is not in this span's figure — but it
 //! is not in no span's either: a span reads everything the tree spends
 //! inside it, whatever for. Work a server defers past one reply, or does on
@@ -358,24 +386,41 @@ pub(super) struct Usage {
     pub(super) serviced_cpu: u64,
 }
 
-/// A process's threads at one instant, from `proc_pidinfo(PROC_PIDTASKINFO)`.
+/// A process's threads at one instant: `proc_pidinfo(PROC_PIDTASKINFO)`'s
+/// count of the running ones and, when that is zero, each thread's own run
+/// state (`PROC_PIDLISTTHREADS`, then `PROC_PIDTHREADINFO` per thread).
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub(super) enum Threads {
-    /// `pti_numrunning`: this many threads were on a core or queued for one.
+    /// This many threads were on a core or queued for one (`pti_numrunning`,
+    /// or, when that was zero, the threads then found in
+    /// `TH_STATE_RUNNING`). `Running(0)` is a process at rest: none was, and
+    /// none was in an uninterruptible wait either.
     Running(u32),
+    /// No thread was on a core or queued, and this many (one at least) were
+    /// in a wait that no signal ends (`TH_STATE_UNINTERRUPTIBLE`): the
+    /// kernel's own waits, for the disk, the pager or one of its locks,
+    /// which end without anything arriving from outside.
+    Uninterruptible(u32),
     /// The process has no task to ask (`ESRCH`): it has exited.
     NoTask,
-    /// The call failed for another reason.
+    /// A call failed for another reason.
     Unknown,
 }
 
 /// Whether one process's reading is complete: two reads that agree in every
 /// field the process's OWN threads move with, between them, no thread on a
-/// core or queued for one — or, for a process that has exited (it has no
-/// task), two agreeing reads of a process whose exit time is set. A thread on
-/// a core at the first read and off it at the run-state read left the core in
-/// between, and leaving folds its CPU time and energy, so the second read
-/// would differ.
+/// core or queued for one, and none in an uninterruptible wait — or, for a
+/// process that has exited (it has no task), two agreeing reads of a process
+/// whose exit time is set. A thread on a core at the first read and off it
+/// at the run-state read left the core in between, and leaving folds its CPU
+/// time and energy, so the second read would differ.
+///
+/// The uninterruptible wait is another matter: such a thread has left its
+/// core and is folded, so the two reads agree and the count is exact for
+/// that instant. But it is not at rest. It waits on the kernel (a block
+/// being written, a page being brought in, a lock), and runs again when the
+/// kernel is done, with no request arriving: the middle of the member's
+/// work, not its end. See the [module docs](self), "Settling".
 ///
 /// The two billed fields are left out of the comparison: they are credited
 /// by ANOTHER process's thread when it changes voucher, whatever this one's
@@ -395,6 +440,7 @@ pub(super) fn settled(first: &Usage, threads: Threads, second: &Usage) -> bool {
     }
     match threads {
         Threads::Running(n) => n == 0,
+        Threads::Uninterruptible(_) => false,
         Threads::NoTask => second.exit_abstime != 0,
         Threads::Unknown => false,
     }
@@ -437,7 +483,7 @@ pub(super) struct Snapshot {
     /// before the settle bound ran out, and this, the last, is returned all
     /// the same: the tree kept changing, so it is not known to hold every
     /// process the tree had ([`TreeGap::Changing`], or
-    /// [`TreeGap::Unsettled`] where a member was still running).
+    /// [`TreeGap::Unsettled`] where a member was not at rest).
     pub(super) unconfirmed: bool,
 }
 
@@ -549,12 +595,16 @@ pub enum TreeGap {
     /// A member's children could not be listed: any descendant below it is
     /// unseen.
     Unlisted { pid: u32, errno: i32, end: SpanEnd },
-    /// A member still had a thread on a core, or queued for one, when the
-    /// meter's settle bound ran out: its reading at that end may lack up to
-    /// a quantum of energy per thread, so the figure may count too much (at
-    /// the start: energy spent just before the span, folded inside it) or
-    /// too little (at the stop). Its figure is counted, so with this gap at
-    /// the start the figure is no lower bound. The
+    /// A member was not at rest when the meter's settle bound ran out. It
+    /// still had a thread on a core, or queued for one: its reading at that
+    /// end may lack up to a quantum of energy per thread, so the figure may
+    /// count too much (at the start: energy spent just before the span,
+    /// folded inside it) or too little (at the stop). Or it had a thread in
+    /// a wait no signal ends, in the middle of its work: its reading is
+    /// exact for that instant, and what the thread goes on to spend falls
+    /// after the stop, outside the figure, or, at the start, inside the span
+    /// though the work began before it. Its figure is counted, so with this
+    /// gap at the start the figure is no lower bound. The
     /// walk at that end was not confirmed by a second one either, so a
     /// process born, exited or reaped while it was read may be missing too.
     Unsettled { pid: u32, end: SpanEnd },
@@ -671,7 +721,10 @@ impl std::fmt::Display for TreeGap {
                 write!(f, "pid {pid}'s children could not be listed at the {end} (errno {errno})")
             }
             TreeGap::Unsettled { pid, end } => {
-                write!(f, "pid {pid} was still running at the {end}, so its reading may be stale")
+                write!(
+                    f,
+                    "pid {pid} was not at rest at the {end} (a thread running, or waiting on the kernel in the middle of its work), so its reading may be stale or short of that work"
+                )
             }
             TreeGap::Changing { end } => {
                 write!(f, "the tree kept changing at the {end}, so a process that came and went may be missing")
@@ -1314,7 +1367,11 @@ mod sys {
         // SAFETY: `ti` is writable for `size` bytes, the size of the flavor's struct.
         let n = unsafe { libc::proc_pidinfo(pid, libc::PROC_PIDTASKINFO, 0, (&mut ti as *mut libc::proc_taskinfo).cast(), size) };
         if n == size {
-            return u32::try_from(ti.pti_numrunning).map_or(Threads::Unknown, Threads::Running);
+            return match u32::try_from(ti.pti_numrunning) {
+                Ok(0) => waits(pid, usize::try_from(ti.pti_threadnum).unwrap_or(0)),
+                Ok(running) => Threads::Running(running),
+                Err(_) => Threads::Unknown,
+            };
         }
         if errno() == libc::ESRCH {
             Threads::NoTask
@@ -1323,6 +1380,74 @@ mod sys {
         }
     }
 
+    /// `proc_pidinfo`'s flavor that lists a process's threads, as 64-bit
+    /// handles (`sys/proc_info.h`; `libc` binds its neighbours and not it).
+    const PROC_PIDLISTTHREADS: libc::c_int = 6;
+    /// `TH_STATE_RUNNING`: on a core or queued for one.
+    const TH_STATE_RUNNING: i32 = 1;
+    /// `TH_STATE_UNINTERRUPTIBLE`: in a wait that no signal ends.
+    const TH_STATE_UNINTERRUPTIBLE: i32 = 4;
+
+    /// The threads of a process none of whose threads was running a moment
+    /// ago (`threads`), by each one's own run state: [`Threads::Running`]
+    /// with the count now running (a thread woke since), else
+    /// [`Threads::Uninterruptible`] with the count in an uninterruptible
+    /// wait, else `Running(0)`, a process at rest. `hint` is the task's
+    /// thread count, to size the listing.
+    ///
+    /// `PROC_PIDLISTTHREADS` fills as many 64-bit thread handles as fit and
+    /// returns the bytes it filled, so a full buffer is grown and listed
+    /// again, as the child listing is. A thread gone between the listing
+    /// and its own read (`ESRCH`) waits on nothing and is passed over: its
+    /// exit folded it, and the member's second read then differs.
+    fn waits(pid: libc::pid_t, hint: usize) -> Threads {
+        let mut cap = hint + 8;
+        let handles = loop {
+            let mut buf: Vec<u64> = vec![0; cap];
+            let Ok(bytes) = libc::c_int::try_from(cap * std::mem::size_of::<u64>()) else { return Threads::Unknown };
+            // SAFETY: `buf` is writable for `bytes` bytes.
+            let n = unsafe { libc::proc_pidinfo(pid, PROC_PIDLISTTHREADS, 0, buf.as_mut_ptr().cast(), bytes) };
+            if n <= 0 {
+                return if errno() == libc::ESRCH { Threads::NoTask } else { Threads::Unknown };
+            }
+            let n = n as usize / std::mem::size_of::<u64>();
+            if n < cap {
+                buf.truncate(n);
+                break buf;
+            }
+            if cap >= 1 << 16 {
+                return Threads::Unknown;
+            }
+            cap *= 4;
+        };
+        let (mut running, mut uninterruptible) = (0u32, 0u32);
+        for handle in handles {
+            // SAFETY: an all-zero `proc_threadinfo` is a valid struct of integers.
+            let mut ti: libc::proc_threadinfo = unsafe { std::mem::zeroed() };
+            let size = std::mem::size_of::<libc::proc_threadinfo>() as libc::c_int;
+            // SAFETY: `ti` is writable for `size` bytes, the size of the flavor's struct.
+            let n = unsafe {
+                libc::proc_pidinfo(pid, libc::PROC_PIDTHREADINFO, handle, (&mut ti as *mut libc::proc_threadinfo).cast(), size)
+            };
+            if n != size {
+                match errno() {
+                    libc::ESRCH => continue,
+                    _ => return Threads::Unknown,
+                }
+            }
+            match ti.pth_run_state {
+                TH_STATE_RUNNING => running += 1,
+                TH_STATE_UNINTERRUPTIBLE => uninterruptible += 1,
+                _ => {}
+            }
+        }
+        match (running, uninterruptible) {
+            (0, 0) => Threads::Running(0),
+            (0, n) => Threads::Uninterruptible(n),
+            (n, _) => Threads::Running(n),
+        }
+    }
+
     /// One member: two reads around its run state. `Ok(None)`: it is gone
     /// (no such process, or reaped between the reads).
     fn member(pid: u32, parent: u32) -> Result<Option<Member>, i32> {
@@ -2218,6 +2343,7 @@ mod tests {
         zombie.exit_abstime = 77;
         assert!(settled(&a, Threads::Running(0), &a));
         assert!(!settled(&a, Threads::Running(1), &a), "a thread on a core or queued");
+        assert!(!settled(&a, Threads::Uninterruptible(1), &a), "a thread waiting on the kernel, to run again unasked");
         assert!(!settled(&a, Threads::Running(0), &moved), "the reads disagree");
         assert!(settled(&zombie, Threads::NoTask, &zombie));
         assert!(!settled(&a, Threads::NoTask, &a), "no task, yet no exit time");

sandbox/sandbox-rs/crates/sandbox-meter/tests/tree_energy.rs

@@ -11,7 +11,9 @@
 //! 272), read only
 //! once the oracle has seen every member quiet on its own terms: no thread on
 //! a core or queued (`proc_pidinfo(PROC_PIDTASKINFO)` into a raw buffer,
-//! `pti_numrunning` at byte 88) at two reads 20 ms apart whose energy agrees.
+//! `pti_numrunning` at byte 88) and none in a wait no signal ends (each
+//! thread's `pth_run_state`, read raw as well) at two reads 20 ms apart
+//! whose energy agrees.
 //! A server blocked waiting for its next request does not move its count, so
 //! the meter's reading of a round trip must EQUAL the oracle's advance across
 //! it, to the nanojoule: a settle that read a member still finishing its
@@ -93,6 +95,32 @@ fn oracle_running(pid: i32) -> Option<i32> {
     (n == 96).then(|| i32::from_ne_bytes(buf[88..92].try_into().unwrap()))
 }
 
+/// Whether every thread of `pid` is at rest by its own run state: none on a
+/// core or queued (`TH_STATE_RUNNING`, 1) and none in a wait that no signal
+/// ends (`TH_STATE_UNINTERRUPTIBLE`, 4: waiting on the disk, the pager or a
+/// kernel lock, to run again without anything arriving).
+/// `proc_pidinfo(pid, PROC_PIDLISTTHREADS = 6)` into a raw buffer of 64-bit
+/// thread handles, then `proc_pidinfo(pid, PROC_PIDTHREADINFO = 5, handle)`
+/// into a raw 112-byte buffer, `pth_run_state` the int32 at byte 24. `None`
+/// if the listing fails (an exited process has no task).
+fn oracle_at_rest(pid: i32) -> Option<bool> {
+    let mut handles = [0u64; 512];
+    // SAFETY: `handles` is writable for the bytes passed.
+    let n = unsafe { libc::proc_pidinfo(pid, 6, 0, handles.as_mut_ptr().cast(), (handles.len() * 8) as libc::c_int) };
+    if n <= 0 {
+        return None;
+    }
+    let n = n as usize / 8;
+    assert!(n < handles.len(), "{pid} has more threads than the oracle's buffer holds");
+    Some(handles[..n].iter().all(|&handle| {
+        let mut buf = [0u8; 112];
+        // SAFETY: `buf` is writable for the 112 bytes passed.
+        let got = unsafe { libc::proc_pidinfo(pid, 5, handle, buf.as_mut_ptr().cast(), 112) };
+        // A thread gone since the listing waits on nothing.
+        got != 112 || !matches!(i32::from_ne_bytes(buf[24..28].try_into().unwrap()), 1 | 4)
+    }))
+}
+
 /// The mach timebase, `(numer, denom)`: ticks × numer / denom = ns.
 fn timebase() -> (u64, u64) {
     #[repr(C)]
@@ -156,10 +184,13 @@ fn settled_tree(root: i32, want: usize) -> Vec<i32> {
 }
 
 /// Each of `pids`' rusage once every one of them is quiet by the oracle's
-/// rule: not running or queued (or exited) at two reads 20 ms apart, with
-/// the same energy at both. Panics after 20 s.
+/// rule: no thread running, queued or in a wait no signal ends (or the
+/// process exited) at two reads 20 ms apart, with the same energy at both.
+/// Panics after 20 s.
 fn quiet(pids: &[i32]) -> Vec<[u64; 64]> {
-    let still = |pid: i32, ru: &[u64; 64]| oracle_running(pid) == Some(0) || exit_abstime(ru) != 0;
+    let still = |pid: i32, ru: &[u64; 64]| {
+        (oracle_running(pid) == Some(0) && oracle_at_rest(pid) == Some(true)) || exit_abstime(ru) != 0
+    };
     let deadline = Instant::now() + Duration::from_secs(20);
     loop {
         let first: Vec<[u64; 64]> = pids.iter().map(|&p| oracle_rusage(p).expect("oracle read")).collect();
@@ -311,11 +342,22 @@ fn a_server_that_reaps_its_worker_is_not_whole() {
 /// server reaps it after the reply has gone (perl, with 60 idle children
 /// beside it, so a walk of its tree takes long enough for the reap to land
 /// inside one). Born in the span, the worker spent energy in it, and by the
-/// time the stop is read it is still running (a gap), exited and unreaped
-/// (`Exited`), or reaped (`ReapedDescendant` on the server). So no reading
-/// is whole — in particular not one taken by a walk that read the server
-/// before the reap and looked for the worker after it, which finds neither
-/// the worker nor its reaped time.
+/// time the stop is read it is in one of four states. Three are gaps: still
+/// running (`Unsettled`), exited and unreaped (`Exited`), or reaped
+/// (`ReapedDescendant` on the server). The fourth is not: alive and on no
+/// core, blocked (a process can be, at any point of its work or of its
+/// exit), which reads as settled. A walk that reads it there counts its
+/// record as it stands, and the reading is whole and names it as a member.
+/// Seen on a loaded host: one round trip of 21, in a run where a round trip
+/// took up to 1.4 s, read the worker so, at 7.08 mJ, where the round trips
+/// beside it that read the same work as `Exited` read 3.76 to 3.88 mJ (an
+/// exited record lacks the process's last time on a core).
+///
+/// So what holds of every reading is the test's name: the worker is never
+/// missed. A reading that does not hold the worker as a member is not whole
+/// — in particular not one taken by a walk that read the server before the
+/// reap and looked for the worker after it, which finds neither the worker
+/// nor its reaped time.
 #[test]
 fn a_worker_reaped_while_its_tree_is_walked_is_never_missed() {
     let _one = alone();
@@ -343,7 +385,7 @@ fn a_worker_reaped_while_its_tree_is_walked_is_never_missed() {
     let tree = settled_tree(s.pid(), 61);
     quiet(&tree);
     let meter = s.meter();
-    let (mut exited, mut reaped, mut unsettled) = (0, 0, 0);
+    let (mut exited, mut reaped, mut unsettled, mut live) = (0, 0, 0, 0);
     for rep in 0..40 {
         let (reply, r) = meter.measure(|| s.ask("go"));
         let worker: u32 = reply.parse().expect("the worker's pid, as its reply");
@@ -355,7 +397,14 @@ fn a_worker_reaped_while_its_tree_is_walked_is_never_missed() {
             if r.pids.contains(&worker) { "read" } else { "not read" },
             r.gaps
         );
-        assert!(!r.is_whole(), "a reading without the worker born in the span read whole: {r:?}");
+        if r.is_whole() {
+            // Whole only with the worker read as a member: alive, settled,
+            // its record counted from its birth.
+            assert!(r.pids.contains(&worker), "a reading without the worker born in the span read whole: {r:?}");
+            assert_eq!(r.processes, 62, "the server, its 60 idle children and the worker: {r:?}");
+            live += 1;
+            continue;
+        }
         if r.gaps.iter().any(|g| matches!(g, TreeGap::Unsettled { .. } | TreeGap::Changing { .. })) {
             unsettled += 1;
             continue;
@@ -369,7 +418,9 @@ fn a_worker_reaped_while_its_tree_is_walked_is_never_missed() {
             panic!("the worker is in no gap of a settled reading: {r:?}");
         }
     }
-    eprintln!("tree meter: worker reaped by its server: {exited} read exited, {reaped} reaped, {unsettled} unsettled at the bound");
+    eprintln!(
+        "tree meter: worker reaped by its server: {exited} read exited, {reaped} reaped, {unsettled} unsettled at the bound, {live} read alive and settled"
+    );
 }
 
 /// A two-process tree that lives across the span (a shell waiting on its
@@ -477,6 +528,57 @@ fn a_reply_with_a_tail_of_work_is_read_once_the_tail_is_done() {
     }
 }
 
+/// A server whose reply has a tail that waits on the disk (perl: burn,
+/// reply, then twenty times write a block, `fsync` it and burn a little,
+/// before it reads the next line). While the kernel writes a block the
+/// server's one thread is on no core and queued for none, and two reads of
+/// its counts agree — but it is not waiting for a request: it is in a wait
+/// no signal ends, and runs again when the disk is done. A stop that took
+/// that for rest would read the round trip short by the rest of its tail,
+/// with no gap. So a member with a thread in such a wait is not settled, and
+/// the stop waits the tail out: whole, and equal to the oracle's advance.
+///
+/// The settle bound here is far past the others': a write waits on the disk
+/// for as long as the host's other writers make it, and the meter returns
+/// as soon as the tail is done whatever the bound.
+#[test]
+fn a_reply_whose_tail_waits_on_the_disk_is_read_once_the_tail_is_done() {
+    let _one = alone();
+    let file = std::env::temp_dir().join(format!("tree-energy-disk-tail-{}", std::process::id()));
+    let mut s = Server::spawn(
+        "/usr/bin/perl",
+        &[
+            "-e",
+            r#"use IO::Handle;
+               $| = 1;
+               open(my $f, '>', $ARGV[0]) or die;
+               while (my $l = <STDIN>) {
+                   my $x = 0; $x += $_ for 1 .. 20000;
+                   print "$x\n";
+                   for my $k (1 .. 20) { print $f 'x' x 4096; $f->flush; $f->sync; $x += $_ for 1 .. 3000 }
+               }"#,
+            file.to_str().expect("a UTF-8 temporary path"),
+        ],
+    );
+    let meter = s.meter().with_settle_bound(Duration::from_secs(20));
+    let pid = s.pid();
+    // perl has compiled the script and opened the file once it has answered.
+    let _ = s.ask("warm up");
+    let mut before = energy_nj(&quiet(&[pid])[0]);
+    for rep in 0..6 {
+        let (reply, r) = meter.measure(|| s.ask("go"));
+        let r = r.expect("a reading");
+        let after = energy_nj(&quiet(&[pid])[0]);
+        log("perl with a tail that waits on the disk", rep, &r, after - before);
+        assert!(!reply.is_empty());
+        assert!(r.is_whole(), "{:?}", r.gaps);
+        assert_eq!(r.cpu_nj, after - before, "the meter's reading against the oracle's advance");
+        before = after;
+    }
+    drop(s);
+    let _ = std::fs::remove_file(&file);
+}
+
 /// Set in the environment of this test binary when it is run again inside a
 /// sandbox that denies listing processes (the test below).
 const IN_LISTING_SANDBOX: &str = "TREE_ENERGY_IN_LISTING_SANDBOX";