From ce76e029b16f5dd2559c179de8c521c660965596 Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Sun, 25 Oct 2026 04:06:17 -0400 Subject: [PATCH] wai-quantum-service: sandboxed workers, IPv6 clients counted by /64, a per-client memory cap The audit of the public job service found three ways past its limits: - Each IPv6 address counted as its own client. A subscriber is usually given a /64 whole, so rotating through it handed out fresh per-client limits with every request. A client is now an IPv4 address or an IPv6 /64; an IPv4 address carried in IPv6 counts as itself. - Three active jobs from one client could hold the whole 96 GiB budget. One client's queued and running jobs may now hold at most one job's worth together (WAI_QSVC_CLIENT_GB, default the job size), so any client can still run its largest job, and no client holds the rest. - A worker ran as the service's user with everything that user can reach, so a flaw in parsing a submitted circuit would have been code running as the account that owns the build host. On macOS each worker now runs under sandbox-exec: no network, no file writes, no program but itself. It needs none of them, since it reads its job on stdin and writes its result on stdout. WAI_QSVC_SANDBOX=0 turns it off for debugging. GET /v1 and the MCP server publish the new limit, and the README documents all three. Co-Authored-By: Claude Opus 5.5 --- wai/CHANGELOG.md | 18 +++++ wai/wai-quantum-service/README.md | 6 ++ wai/wai-quantum-service/src/http.rs | 49 +++++++++++++- wai/wai-quantum-service/src/jobs.rs | 100 +++++++++++++++++++++++++--- wai/wai-quantum-service/src/mcp.rs | 2 +- 5 files changed, 163 insertions(+), 12 deletions(-) diff --git a/wai/CHANGELOG.md b/wai/CHANGELOG.md index 71ee2439f..32166e8d8 100644 --- a/wai/CHANGELOG.md +++ b/wai/CHANGELOG.md @@ -1092,6 +1092,24 @@ wire format may change and such changes are recorded here. - **`tools/ops-gpu-bench`** times each capability's decode with its graphs on the CPU and on each adapter, and reads the energy counters the host has. - **CI:** `wai-ops-gpu.yml` runs the Metal and browser legs on the `macos` runner, and requires on a runner with no GPU that the device tests skip without `WAI_OPS_GPU_REQUIRE` and fail with it. +### wai-quantum-service: sandboxed workers, and limits no client can multiply + +- **Workers run sandboxed** (macOS). Each job's worker runs under `sandbox-exec`: no network, + no file writes, and no program but itself. It reads its job on stdin and writes its result on + stdout, so it needs none of them. A flaw in parsing a submitted circuit then reaches neither + the host's files nor the network nor a shell. The pipes it was given stay writable, and its + energy is read as before. `WAI_QSVC_SANDBOX=0` turns it off for debugging. +- **An IPv6 client is its /64.** Limits were counted per address, and a subscriber is usually + given a whole /64, so rotating through it handed out fresh limits with every request. A + client is now an IPv4 address or an IPv6 /64. An IPv4 address carried in IPv6 counts as + itself. +- **One client holds at most one job's worth of memory.** Three active jobs from one client + could hold the whole 96 GiB budget. A client's queued and running jobs may now hold together + at most `WAI_QSVC_CLIENT_GB`, by default the size one job may take. Any client can still run + its largest job, and no client holds the rest. +- `GET /v1` and the MCP server publish `memory_per_client_bytes`. `GET /v1` also publishes how + a client is counted and whether workers are sandboxed. + ### wai-quantum 0.6.14 and wai-quantum-cli 0.1.2: contraction trees that sweep along the line, and the design checks on the job service 0.6.13 was not published on its own: this release carries it to the package registry, and `wai-quantum-cli` 0.1.2 requires it. diff --git a/wai/wai-quantum-service/README.md b/wai/wai-quantum-service/README.md index 5467601c6..63cc04a62 100644 --- a/wai/wai-quantum-service/README.md +++ b/wai/wai-quantum-service/README.md @@ -163,6 +163,10 @@ bash smoke.sh http://127.0.0.1:8241 Each job runs in a child process of the same binary (`worker`), at a lower scheduling priority than the host's other work, and is watched for its resident memory and its time. +On macOS the worker runs in a sandbox (`sandbox-exec`): no network, no file writes, and no program but itself. It reads its job on stdin and writes its result on stdout, so it needs none of them, and a flaw in parsing a submitted circuit then reaches neither the host's files, nor the network, nor a shell. `WAI_QSVC_SANDBOX=0` turns the sandbox off, for debugging. + +Limits are counted per client. A client is an IPv4 address, or the /64 an IPv6 address is in: one subscriber is usually given a /64 whole, so counting single addresses would hand out fresh limits to anyone who rotates through theirs. Behind the front proxy, the client is the first `X-Forwarded-For` address, which the proxy sets from the edge network's own client-address header rather than passing on what the request carried. + These variables set the limits: | variable | default | meaning | @@ -173,11 +177,13 @@ These variables set the limits: | `WAI_QSVC_MIN_FREE_PCT` | 15 | share of the host's memory a starting job must leave free | | `WAI_QSVC_JOB_SECS` | 900 | the longest a job may run | | `WAI_QSVC_CLIENT_ACTIVE` | 3 | queued or running jobs one client may hold | +| `WAI_QSVC_CLIENT_GB` | the job size | memory one client's queued and running jobs may hold together | | `WAI_QSVC_CLIENT_BURST` | 30 | submissions one client may make per window | | `WAI_QSVC_CLIENT_WINDOW_SECS` | 600 | the window | | `WAI_QSVC_MAX_QUEUE` | 256 | jobs waiting at once | | `WAI_QSVC_RETAIN_SECS` | 86400 | how long results are kept | | `WAI_QSVC_TRUSTED_PROXIES` | `127.0.0.1,::1` | peers whose `X-Forwarded-For` names the client | +| `WAI_QSVC_SANDBOX` | 1 | run each worker sandboxed (macOS); 0 runs it unsandboxed | | `WAI_QSVC_KEY` | `~/.config/wai-quantum-service/signing.key` | the signing key, created on first start | `wai-quantum-service run job.json` runs one job in the current process and prints its plan and its result. diff --git a/wai/wai-quantum-service/src/http.rs b/wai/wai-quantum-service/src/http.rs index 80bbad619..a24893d65 100644 --- a/wai/wai-quantum-service/src/http.rs +++ b/wai/wai-quantum-service/src/http.rs @@ -71,15 +71,33 @@ fn error(status: u16, why: &str) -> Response { } /// The client a request is from: the first `X-Forwarded-For` address when the -/// peer is a trusted proxy, otherwise the peer. +/// peer is a trusted proxy, otherwise the peer, as [`identity`] counts it. fn client(app: &App, peer: SocketAddr, headers: &HeaderMap) -> String { if app.trusted.contains(&peer.ip()) && let Some(first) = headers.get("x-forwarded-for").and_then(|v| v.to_str().ok()).and_then(|v| v.split(',').next()) && let Ok(ip) = first.trim().parse::() { - return ip.to_string(); + return identity(ip); + } + identity(peer.ip()) +} + +/// The name a client's limits are counted under: an IPv4 address, or the /64 +/// an IPv6 address is in. A subscriber is usually given a /64 whole and may +/// use any address in it, so counting addresses would let one client take a +/// fresh set of limits with every request. An IPv4 address carried in IPv6 +/// counts as itself. +fn identity(ip: IpAddr) -> String { + match ip { + IpAddr::V4(v4) => v4.to_string(), + IpAddr::V6(v6) => match v6.to_ipv4_mapped() { + Some(v4) => v4.to_string(), + None => { + let s = v6.segments(); + format!("{}/64", std::net::Ipv6Addr::new(s[0], s[1], s[2], s[3], 0, 0, 0, 0)) + } + }, } - peer.ip().to_string() } async fn describe(State(app): State) -> Response { @@ -97,6 +115,9 @@ async fn describe(State(app): State) -> Response { "job_memory_bytes": c.job_mib << 20, "job_seconds": c.job_timeout.as_secs(), "active_jobs_per_client": c.per_client_active, + "memory_per_client_bytes": c.per_client_mib << 20, + "client": "an IPv4 address, or an IPv6 /64", + "worker_sandbox": c.sandbox && cfg!(target_os = "macos"), "submissions_per_window": c.per_client_burst, "window_seconds": c.per_client_window.as_secs(), "queue": c.max_queue, @@ -280,3 +301,25 @@ pub fn router(app: Shared) -> Router { .layer(DefaultBodyLimit::max(MAX_BODY)) .with_state(app) } + +#[cfg(test)] +mod tests { + use super::identity; + use std::net::IpAddr; + + fn id(s: &str) -> String { + identity(s.parse::().unwrap()) + } + + #[test] + fn an_ipv6_client_is_its_64_and_ipv4_is_itself() { + // Every address in one /64 is one client, however it rotates. + assert_eq!(id("2001:db8:aa:bb:1:2:3:4"), "2001:db8:aa:bb::/64"); + assert_eq!(id("2001:db8:aa:bb:ffff:0:dead:beef"), "2001:db8:aa:bb::/64"); + assert_ne!(id("2001:db8:aa:bc::1"), id("2001:db8:aa:bb::1")); + assert_eq!(id("203.0.113.7"), "203.0.113.7"); + // An IPv4 address carried in IPv6 is the IPv4 client, not a /64. + assert_eq!(id("::ffff:203.0.113.7"), "203.0.113.7"); + assert_eq!(id("::1"), "::/64"); + } +} diff --git a/wai/wai-quantum-service/src/jobs.rs b/wai/wai-quantum-service/src/jobs.rs index 1a814e936..a20bc6115 100644 --- a/wai/wai-quantum-service/src/jobs.rs +++ b/wai/wai-quantum-service/src/jobs.rs @@ -14,7 +14,12 @@ //! - **Isolation.** Each job runs in a child process (this binary's `worker` //! mode), at a lower scheduling priority, polled for its resident memory. //! One that passes its admitted memory or its time limit is killed, and the -//! job fails saying which. +//! job fails saying which. On macOS the worker runs sandboxed: no network, +//! no file writes, no further processes ([`worker_profile`]). +//! - **Fairness.** One client may hold a few jobs at once, and together no +//! more memory than one job may take ([`Config::per_client_mib`]), so no +//! client holds the whole budget. A client is an IPv4 address or an IPv6 +//! /64 (`http::client`), since one subscriber is usually given a /64 whole. //! - **Energy.** A finished worker is read before it is reaped, and its //! result carries the CPU energy its process spent, with the counter's //! class and declared uncertainty, inside the signed body ([`energy`]). @@ -55,6 +60,12 @@ pub struct Config { pub max_queue: usize, /// Queued or running jobs one client may hold. pub per_client_active: usize, + /// The memory one client's queued and running jobs may hold together, + /// MiB: one job's worth by default, so no client holds the whole budget. + pub per_client_mib: u64, + /// Whether each worker runs in a sandbox with no network, no file writes + /// and no further processes (macOS; see [`worker_profile`]). + pub sandbox: bool, /// Submissions one client may make in a window. pub per_client_burst: usize, pub per_client_window: Duration, @@ -68,18 +79,22 @@ impl Config { /// `WAI_QSVC_BUDGET_GB` (96), `WAI_QSVC_JOB_GB` (72), `WAI_QSVC_WORKERS` /// (4), `WAI_QSVC_MIN_FREE_PCT` (15), `WAI_QSVC_JOB_SECS` (900), /// `WAI_QSVC_RETAIN_SECS` (86400), `WAI_QSVC_MAX_QUEUE` (256), - /// `WAI_QSVC_CLIENT_ACTIVE` (3), `WAI_QSVC_CLIENT_BURST` (30 in - /// `WAI_QSVC_CLIENT_WINDOW_SECS`, 600). + /// `WAI_QSVC_CLIENT_ACTIVE` (3), `WAI_QSVC_CLIENT_GB` (the job size), + /// `WAI_QSVC_CLIENT_BURST` (30 in `WAI_QSVC_CLIENT_WINDOW_SECS`, 600), + /// `WAI_QSVC_SANDBOX` (1; 0 runs workers unsandboxed). pub fn from_env() -> Config { + let job_gb = env_num("WAI_QSVC_JOB_GB", 72u64); Config { budget_mib: env_num("WAI_QSVC_BUDGET_GB", 96u64) * 1024, - job_mib: env_num("WAI_QSVC_JOB_GB", 72u64) * 1024, + job_mib: job_gb * 1024, workers: env_num("WAI_QSVC_WORKERS", 4usize).max(1), min_free_pct: env_num("WAI_QSVC_MIN_FREE_PCT", 15u32), job_timeout: Duration::from_secs(env_num("WAI_QSVC_JOB_SECS", 900u64)), retain: Duration::from_secs(env_num("WAI_QSVC_RETAIN_SECS", 86_400u64)), max_queue: env_num("WAI_QSVC_MAX_QUEUE", 256usize), per_client_active: env_num("WAI_QSVC_CLIENT_ACTIVE", 3usize), + per_client_mib: env_num("WAI_QSVC_CLIENT_GB", job_gb) * 1024, + sandbox: env_num("WAI_QSVC_SANDBOX", 1u8) != 0, per_client_burst: env_num("WAI_QSVC_CLIENT_BURST", 30usize), per_client_window: Duration::from_secs(env_num("WAI_QSVC_CLIENT_WINDOW_SECS", 600u64)), } @@ -244,10 +259,18 @@ impl Service { } { let jobs = self.jobs.lock().unwrap(); - let active = jobs.values().filter(|j| j.client == client && j.status.active()).count(); + let mine = || jobs.values().filter(|j| j.client == client && j.status.active()); + let active = mine().count(); if active >= self.config.per_client_active { return Err(Refused(429, format!("{active} of your jobs are queued or running; wait for one to finish (at most {} at once)", self.config.per_client_active))); } + let held: u64 = mine().map(|j| j.plan.memory_bytes.div_ceil(1 << 20)).sum(); + if held + job_mib > self.config.per_client_mib { + return Err(Refused( + 429, + format!("your queued and running jobs already hold about {held} MiB, and this one needs {job_mib} MiB; one client may hold at most {} MiB at once", self.config.per_client_mib), + )); + } } self.count(client)?; let id = job_id(); @@ -428,7 +451,7 @@ impl Service { Ok(e) => e, Err(e) => return (Err(format!("the service could not find itself: {e}")), 0), }; - let mut cmd = low_priority(&exe); + let mut cmd = low_priority(&exe, self.config.sandbox); cmd.stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::piped()).kill_on_drop(true); let mut child = match cmd.spawn() { Ok(c) => c, @@ -512,10 +535,18 @@ impl Service { } /// The worker command at a lower priority than the host's other work. -fn low_priority(exe: &std::path::Path) -> tokio::process::Command { +fn low_priority(exe: &std::path::Path, sandbox: bool) -> tokio::process::Command { let mut cmd = if cfg!(target_os = "macos") { let mut c = tokio::process::Command::new("/usr/sbin/taskpolicy"); - c.args(["-c", "utility", "/usr/bin/nice", "-n", "10"]).arg(exe); + c.args(["-c", "utility", "/usr/bin/nice", "-n", "10"]); + if sandbox { + // The rule names the path the exec resolves to, so the worker is + // started by that same path. + let exe = std::fs::canonicalize(exe).unwrap_or_else(|_| exe.to_path_buf()); + c.args(["/usr/bin/sandbox-exec", "-p"]).arg(worker_profile(&exe)).arg(exe); + } else { + c.arg(exe); + } c } else { let mut c = tokio::process::Command::new("nice"); @@ -526,6 +557,20 @@ fn low_priority(exe: &std::path::Path) -> tokio::process::Command { cmd } +/// The sandbox a worker runs in on macOS. A worker reads its job on stdin and +/// writes its result on stdout, so it needs no network, writes no file and +/// starts no process: a flaw in parsing a submitted circuit then reaches +/// neither the host's files nor the network nor a shell. Network, file writes +/// (bar `/dev/null`) and fork are denied; exec is allowed for the worker +/// binary alone, which `sandbox-exec` starts once the profile is in force. +/// The pipes it was given stay writable, since they are already open. +pub fn worker_profile(exe: &std::path::Path) -> String { + let path = exe.to_string_lossy().replace('\\', "\\\\").replace('"', "\\\""); + format!( + "(version 1)(allow default)(deny network*)(deny file-write*)(allow file-write-data (literal \"/dev/null\"))(deny process-fork)(deny process-exec)(allow process-exec (literal \"{path}\"))" + ) +} + fn kill(pid: u32) { if pid > 0 { let _ = std::process::Command::new("kill").args(["-9", &pid.to_string()]).status(); @@ -603,11 +648,50 @@ mod tests { retain: Duration::from_secs(60), max_queue: 4, per_client_active: 2, + per_client_mib: 512, + sandbox: true, per_client_burst: 3, per_client_window: Duration::from_secs(60), } } + #[test] + fn a_client_holds_at_most_one_jobs_worth_of_memory_at_once() { + let s = Service::new(Config { per_client_active: 5, per_client_burst: 10, ..config() }); + assert!(s.submit("a", json!({}), plan(300 << 20)).is_ok()); + // 300 + 300 MiB would pass the 512 MiB a client may hold. + let Err(Refused(code, why)) = s.submit("a", json!({}), plan(300 << 20)) else { panic!("admitted past the client's memory") }; + assert_eq!(code, 429); + assert!(why.contains("300 MiB") && why.contains("512 MiB"), "{why}"); + // What still fits is admitted, and another client is not affected. + assert!(s.submit("a", json!({}), plan(200 << 20)).is_ok()); + assert!(s.submit("b", json!({}), plan(500 << 20)).is_ok()); + // A cancelled job gives its memory back. + let first: Vec = s.jobs.lock().unwrap().values().filter(|j| j.client == "a" && j.plan.memory_bytes == 300 << 20).map(|j| j.id.clone()).collect(); + assert_eq!(s.cancel(&first[0]), Some(true)); + assert!(s.submit("a", json!({}), plan(300 << 20)).is_ok()); + } + + #[cfg(target_os = "macos")] + #[test] + fn the_worker_sandbox_denies_files_network_and_processes() { + let run = |exe: &str, args: &[&str]| { + std::process::Command::new("/usr/bin/sandbox-exec").arg("-p").arg(worker_profile(std::path::Path::new(exe))).arg(exe).args(args).output().expect("sandbox-exec runs") + }; + let probe = std::env::temp_dir().join(format!("wai-qsvc-sandbox-{}", std::process::id())); + let _ = std::fs::remove_file(&probe); + let out = run("/usr/bin/touch", &[probe.to_str().unwrap()]); + assert!(!out.status.success() && !probe.exists(), "a sandboxed worker wrote a file"); + let out = run("/usr/bin/curl", &["-s", "-m", "5", "-o", "/dev/null", "-w", "%{http_code}", "https://example.com"]); + assert_eq!(String::from_utf8_lossy(&out.stdout), "000", "a sandboxed worker reached the network"); + // /bin/sh hands off to bash, a second exec, which the profile refuses. + let out = run("/bin/sh", &["-c", "echo ran"]); + assert!(!String::from_utf8_lossy(&out.stdout).contains("ran"), "a sandboxed worker started another program"); + // What a worker does need: writing its result to the pipe it was given. + let out = run("/bin/echo", &["result"]); + assert_eq!(String::from_utf8_lossy(&out.stdout).trim(), "result"); + } + #[test] fn a_job_over_the_per_job_cap_is_refused_with_its_estimate() { let s = Service::new(config()); diff --git a/wai/wai-quantum-service/src/mcp.rs b/wai/wai-quantum-service/src/mcp.rs index 9a205c9dd..28f468092 100644 --- a/wai/wai-quantum-service/src/mcp.rs +++ b/wai/wai-quantum-service/src/mcp.rs @@ -278,7 +278,7 @@ async fn call(app: &App, who: &str, name: &str, args: Value, base: &str) -> Valu "amplitudes": "bit strings with one character per qubit, qubit n-1 first", "observables": "Pauli strings such as \"Z0 Z1\" or {\"pauli\": \"X3\", \"coeff\": 0.5}; qubits count from 0" }, - "limits": { "job_seconds": c.job_timeout.as_secs(), "active_jobs_per_client": c.per_client_active, "submissions_per_window": c.per_client_burst, "window_seconds": c.per_client_window.as_secs() }, + "limits": { "job_seconds": c.job_timeout.as_secs(), "active_jobs_per_client": c.per_client_active, "memory_per_client_bytes": c.per_client_mib << 20, "submissions_per_window": c.per_client_burst, "window_seconds": c.per_client_window.as_secs() }, "load": { "queued": queued, "running": running }, "api": discovery::index(base, &app.console), }); -- 2.54.0 (Apple Git-157)