c6cbb73843
`wai-pack live` used to gather every object and window in memory and write them all after the end of the stream. It now writes: - each object when it is published; - each window's index, then its receipt, when the window is sealed; - the plan file and LISTING at the end. Each file is written to a hidden temporary file beside it, then renamed into place. If the source fails mid-stream (a short frame or a frame of another size): - every sealed window and the objects it lists are kept; - the objects only unsealed pieces published are removed; - no plan file and no LISTING are written; - it exits 1, naming what it kept. Once an object is written, its bytes are no longer held. A coded piece's stages now carry their entry lengths (`pack::Lens`), so window_index builds an entry without the envelope. The live packer moves each envelope into its publication. The packer hands out sealed windows with `take_sealed()` and keeps none. With --window 0, the packer still holds every piece's entry fields until the end of the stream. The module doc, the CLI help and packaging §9.2 now say exactly what is held. wai::pin::listing_from_sha256 builds the listing from per-file digests (the same lines as listing_from), so LISTING needs no file's bytes. Tests: - A CLI test feeds the stream through a pipe and checks that windows 0 and 1 and their objects are on disk while the stream is still open. - A second CLI test checks that a stream cut inside a frame keeps window 0 and its objects and nothing else. - A unit test checks that no open piece holds an envelope. - A wai-rs test checks that the listing from digests equals the listing from bytes. Co-Authored-By: Claude Opus 5.5 <[email protected]>
690 lines
32 KiB
Rust
690 lines
32 KiB
Rust
//! Verifying a parameter-set pin (SPEC §3.1) against the parameter sets a sink
|
||
//! holds, and reading and writing the listing a set of several files is pinned
|
||
//! through (`wai.prior.set`).
|
||
//!
|
||
//! The core, [`verify_bytes`], works on bytes in memory, so a sink with no
|
||
//! filesystem (a browser) runs the same checks as one that reads files; [`verify`]
|
||
//! reads the held artifacts from disk and calls it. Both run SPEC §3.1's checks in
|
||
//! order and report the first that fails as a [`PinRefusal`].
|
||
//!
|
||
//! A **listing** has one line per file: the file's SHA-256 as 64 lowercase
|
||
//! hexadecimal digits, two spaces, the file's path below the set's root, and a line
|
||
//! feed. Paths are `/`-joined segments of `A`–`Z`, `a`–`z`, `0`–`9`, `_`, `-` and
|
||
//! `.` (a segment starts with a letter, digit, `_` or `-` and does not end with
|
||
//! `.`); lines are sorted by path as byte strings; no two paths are equal once
|
||
//! `A`–`Z` are folded to `a`–`z`; no path followed by `/` begins another path. The
|
||
//! listing has at least one line and nothing else. It is the text common checksum
|
||
//! utilities write and check.
|
||
|
||
use std::collections::BTreeMap;
|
||
use std::path::{Path, PathBuf};
|
||
|
||
use sha2::{Digest, Sha256};
|
||
|
||
use crate::codecs::{ParameterSetShape, CAP_PRIOR_SET};
|
||
use crate::container::{is_sha256_hex, pin_shape, PinRefusal, Prior};
|
||
|
||
/// The prior forms this crate reads: `wai.prior.set`. It reads no coded form.
|
||
pub const READABLE_FORMS: &[&str] = &[CAP_PRIOR_SET];
|
||
|
||
/// SHA-256 of `bytes`.
|
||
pub fn sha256(bytes: &[u8]) -> [u8; 32] {
|
||
Sha256::digest(bytes).into()
|
||
}
|
||
|
||
/// SHA-256 of `bytes` as 64 lowercase hexadecimal digits.
|
||
pub fn sha256_hex(bytes: &[u8]) -> String {
|
||
hex(&sha256(bytes))
|
||
}
|
||
|
||
fn hex(d: &[u8]) -> String {
|
||
const H: &[u8; 16] = b"0123456789abcdef";
|
||
let mut s = String::with_capacity(d.len() * 2);
|
||
for &b in d {
|
||
s.push(H[(b >> 4) as usize] as char);
|
||
s.push(H[(b & 15) as usize] as char);
|
||
}
|
||
s
|
||
}
|
||
|
||
/// A digest shortened for display: the first 16 digits and an ellipsis when it is
|
||
/// a SHA-256 in WAI's grammar ([`is_sha256_hex`]); otherwise the whole value,
|
||
/// quoted and escaped, so a malformed digest is shown as it is rather than cut
|
||
/// at a byte that may fall inside a character.
|
||
pub fn digest_preview(d: &str) -> String {
|
||
if is_sha256_hex(d) {
|
||
format!("{}\u{2026}", &d[..16])
|
||
} else {
|
||
format!("{d:?} (not a SHA-256)")
|
||
}
|
||
}
|
||
|
||
/// One line describing a pin for people: its `id`, its digest
|
||
/// ([`digest_preview`]), and the optional members it states.
|
||
pub fn describe_pin(p: &Prior) -> String {
|
||
let mut s = format!("id {} sha256 {}", p.id, digest_preview(&p.sha256));
|
||
if let Some(f) = p.form() {
|
||
s += &format!(" form {f}");
|
||
}
|
||
if let Some(t) = p.tier() {
|
||
s += &format!(" tier {t}");
|
||
}
|
||
if let Some(sup) = p.supersedes() {
|
||
s += &format!(" supersedes {}", digest_preview(&sup.sha256));
|
||
}
|
||
s
|
||
}
|
||
|
||
fn malformed(why: impl Into<String>) -> PinRefusal {
|
||
PinRefusal::SetMalformed { why: why.into() }
|
||
}
|
||
|
||
/// Whether `path` is a listing path: one or more `/`-joined segments, each made of
|
||
/// `A`–`Z`, `a`–`z`, `0`–`9`, `_`, `-` and `.`, starting with a character other
|
||
/// than `.` and not ending with `.`.
|
||
pub fn is_listing_path(path: &str) -> bool {
|
||
!path.is_empty()
|
||
&& path.split('/').all(|seg| {
|
||
let b = seg.as_bytes();
|
||
!b.is_empty()
|
||
&& b[0] != b'.'
|
||
&& b[b.len() - 1] != b'.'
|
||
&& b.iter().all(|&c| c.is_ascii_alphanumeric() || c == b'_' || c == b'-' || c == b'.')
|
||
})
|
||
}
|
||
|
||
/// Check that a sorted list of paths obeys the listing rules beyond the per-path
|
||
/// grammar: strictly ascending as byte strings, no case-folded duplicates, and no
|
||
/// path that is a directory prefix of another.
|
||
fn check_paths(paths: &[&str]) -> Result<(), PinRefusal> {
|
||
if paths.is_empty() {
|
||
return Err(malformed("a listing has at least one line"));
|
||
}
|
||
for p in paths {
|
||
if !is_listing_path(p) {
|
||
return Err(malformed(format!("'{p}' is not a listing path")));
|
||
}
|
||
}
|
||
for w in paths.windows(2) {
|
||
if w[0].as_bytes() >= w[1].as_bytes() {
|
||
return Err(malformed(format!("'{}' does not sort strictly before '{}'", w[0], w[1])));
|
||
}
|
||
}
|
||
let mut folded: Vec<String> = paths.iter().map(|p| p.to_ascii_lowercase()).collect();
|
||
folded.sort_unstable();
|
||
if let Some(w) = folded.windows(2).find(|w| w[0] == w[1]) {
|
||
return Err(malformed(format!("two paths fold to '{}'", w[0])));
|
||
}
|
||
for a in paths {
|
||
let dir = format!("{a}/");
|
||
if let Some(b) = paths.iter().find(|b| b.starts_with(&dir)) {
|
||
return Err(malformed(format!("'{a}' is a file and the directory of '{b}'")));
|
||
}
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
/// Parse a listing into `(path, digest)` lines, enforcing every listing rule.
|
||
/// Any rule broken gives `set_malformed`.
|
||
pub fn parse_listing(bytes: &[u8]) -> Result<Vec<(String, [u8; 32])>, PinRefusal> {
|
||
if bytes.is_empty() {
|
||
return Err(malformed("a listing has at least one line"));
|
||
}
|
||
if *bytes.last().unwrap() != b'\n' {
|
||
return Err(malformed("a listing ends with a line feed"));
|
||
}
|
||
let mut out = Vec::new();
|
||
for (n, line) in bytes[..bytes.len() - 1].split(|&b| b == b'\n').enumerate() {
|
||
// 64 hex digits, two spaces, a path of at least one byte.
|
||
if line.len() < 67 || &line[64..66] != b" " {
|
||
return Err(malformed(format!("line {} is not '<64 hex> <path>'", n + 1)));
|
||
}
|
||
let digest = std::str::from_utf8(&line[..64]).ok().filter(|d| is_sha256_hex(d))
|
||
.ok_or_else(|| malformed(format!("line {} does not start with 64 lowercase hexadecimal digits", n + 1)))?;
|
||
let path = std::str::from_utf8(&line[66..])
|
||
.map_err(|_| malformed(format!("line {} has a path that is not ASCII", n + 1)))?;
|
||
let mut d = [0u8; 32];
|
||
for (i, out) in d.iter_mut().enumerate() {
|
||
*out = u8::from_str_radix(&digest[2 * i..2 * i + 2], 16).unwrap();
|
||
}
|
||
out.push((path.to_string(), d));
|
||
}
|
||
let paths: Vec<&str> = out.iter().map(|(p, _)| p.as_str()).collect();
|
||
check_paths(&paths)?;
|
||
Ok(out)
|
||
}
|
||
|
||
/// Write the listing of the files `files` (path → bytes). The paths must obey the
|
||
/// listing rules; the lines come out sorted.
|
||
pub fn listing_from(files: &BTreeMap<String, &[u8]>) -> Result<Vec<u8>, PinRefusal> {
|
||
listing_from_sha256(&files.iter().map(|(p, b)| (p.clone(), sha256_hex(b))).collect())
|
||
}
|
||
|
||
/// The listing [`listing_from`] writes, of files given by their SHA-256 (path → 64
|
||
/// lowercase hexadecimal digits): for a writer that hashed each file as it wrote it and
|
||
/// no longer holds the bytes.
|
||
pub fn listing_from_sha256(files: &BTreeMap<String, String>) -> Result<Vec<u8>, PinRefusal> {
|
||
let mut paths: Vec<&str> = files.keys().map(String::as_str).collect();
|
||
paths.sort_unstable_by(|a, b| a.as_bytes().cmp(b.as_bytes()));
|
||
check_paths(&paths)?;
|
||
let mut out = Vec::new();
|
||
for p in paths {
|
||
if !is_sha256_hex(&files[p]) {
|
||
return Err(malformed(format!("{p}: its digest is not 64 lowercase hexadecimal digits")));
|
||
}
|
||
out.extend_from_slice(files[p].as_bytes());
|
||
out.extend_from_slice(b" ");
|
||
out.extend_from_slice(p.as_bytes());
|
||
out.push(b'\n');
|
||
}
|
||
Ok(out)
|
||
}
|
||
|
||
/// Write the listing of the files at `paths` below `root`, hashing each one.
|
||
pub fn write_listing(root: &Path, paths: &[String]) -> Result<Vec<u8>, PinRefusal> {
|
||
let mut bytes = BTreeMap::new();
|
||
for p in paths {
|
||
if !is_listing_path(p) {
|
||
return Err(malformed(format!("'{p}' is not a listing path")));
|
||
}
|
||
let b = std::fs::read(root.join(p)).map_err(|e| malformed(format!("{p}: {e}")))?;
|
||
bytes.insert(p.clone(), b);
|
||
}
|
||
let view: BTreeMap<String, &[u8]> = bytes.iter().map(|(k, v)| (k.clone(), v.as_slice())).collect();
|
||
listing_from(&view)
|
||
}
|
||
|
||
/// The listing of every regular file below `root`, skipping names that begin with
|
||
/// `.` and the `*.sha256` files directly in `root` (the listings themselves). A
|
||
/// symbolic link, or a name a listing path cannot carry, is refused.
|
||
pub fn listing_of_dir(root: &Path) -> Result<Vec<u8>, PinRefusal> {
|
||
let files = read_dir_set(root)?;
|
||
let view: BTreeMap<String, &[u8]> = files.iter().map(|(k, v)| (k.clone(), v.as_slice())).collect();
|
||
listing_from(&view)
|
||
}
|
||
|
||
/// Every file [`listing_of_dir`] lists below `root`, read once, by its listing path:
|
||
/// the set a sink holds as that directory, with the bytes its listing is derived
|
||
/// from. A symbolic link, or a name a listing path cannot carry, is refused
|
||
/// (`set_malformed`).
|
||
pub fn read_dir_set(root: &Path) -> Result<BTreeMap<String, Vec<u8>>, PinRefusal> {
|
||
fn walk(root: &Path, rel: &str, out: &mut BTreeMap<String, Vec<u8>>) -> Result<(), PinRefusal> {
|
||
let dir = if rel.is_empty() { root.to_path_buf() } else { root.join(rel) };
|
||
let rd = std::fs::read_dir(&dir).map_err(|e| malformed(format!("{}: {e}", dir.display())))?;
|
||
for entry in rd {
|
||
let entry = entry.map_err(|e| malformed(e.to_string()))?;
|
||
let name = entry.file_name().into_string()
|
||
.map_err(|n| malformed(format!("{n:?} is not a listing path")))?;
|
||
if name.starts_with('.') {
|
||
continue;
|
||
}
|
||
let ty = entry.file_type().map_err(|e| malformed(e.to_string()))?;
|
||
let path = if rel.is_empty() { name.clone() } else { format!("{rel}/{name}") };
|
||
if ty.is_symlink() {
|
||
return Err(malformed(format!("'{path}' is a symbolic link")));
|
||
} else if ty.is_dir() {
|
||
walk(root, &path, out)?;
|
||
} else if rel.is_empty() && name.ends_with(".sha256") {
|
||
continue;
|
||
} else {
|
||
if !is_listing_path(&path) {
|
||
return Err(malformed(format!("'{path}' is not a listing path")));
|
||
}
|
||
let b = std::fs::read(root.join(&path)).map_err(|e| malformed(format!("{path}: {e}")))?;
|
||
out.insert(path, b);
|
||
}
|
||
}
|
||
Ok(())
|
||
}
|
||
let mut files = BTreeMap::new();
|
||
walk(root, "", &mut files)?;
|
||
Ok(files)
|
||
}
|
||
|
||
/// A parameter set a sink holds for a capability, in memory: one file, or a
|
||
/// listing together with the files below its root (path → bytes; a file the sink
|
||
/// could not read is simply absent).
|
||
#[derive(Debug, Clone, Copy)]
|
||
pub enum HeldBytes<'a> {
|
||
File(&'a [u8]),
|
||
Set { listing: &'a [u8], files: &'a BTreeMap<String, &'a [u8]> },
|
||
}
|
||
|
||
/// SPEC §3.1's check 5, run by a sink that holds a key log before it looks for the set
|
||
/// a pin names among what it holds: a set whose digest the key log revokes is refused,
|
||
/// and where the sink's relying party holds the capability's pinned parameter sets to
|
||
/// jwp-receipts §7.9, a set §7.9 does not hold is refused too.
|
||
/// `crate::key_log::SinkKeyLog` (feature `key_discovery`) implements it from the key
|
||
/// log a sink holds. A sink given no authority skips the check.
|
||
pub trait PinAuthority: std::fmt::Debug {
|
||
/// `Ok` when the key log does not revoke the set `prior` names — its `sha256` in
|
||
/// its `form` — and either the sink does not hold `capability`'s pinned sets to
|
||
/// §7.9 or §7.9 holds that set. Otherwise the refusal: `prior_revoked`
|
||
/// ([`PinRefusal::Revoked`]) or `prior_not_authorised`
|
||
/// ([`PinRefusal::NotAuthorised`]).
|
||
fn check(&self, capability: &str, prior: &Prior) -> Result<(), PinRefusal>;
|
||
|
||
/// The same check for an unpinned decode through `capability` (SPEC §4, *Module
|
||
/// authorisation*), over the set it reads, identified by `digests`: each digest a
|
||
/// pin over that set could name, as 64 lowercase hexadecimal digits. Refused with
|
||
/// `prior_revoked` when any of them is revoked, and, for a capability held to
|
||
/// §7.9, with `prior_not_authorised` unless §7.9 holds the set under one of them.
|
||
fn check_unpinned(&self, capability: &str, digests: &[String]) -> Result<(), PinRefusal>;
|
||
|
||
/// Whether the sink holds `capability`'s sets to §7.9, so that an unpinned decode
|
||
/// through it reads a set only under a digest §7.9 holds. The default asks
|
||
/// [`Self::check_unpinned`] over no digest, which such a sink refuses and any other
|
||
/// accepts (no digest is revoked).
|
||
fn requires_grant(&self, capability: &str) -> bool {
|
||
self.check_unpinned(capability, &[]).is_err()
|
||
}
|
||
}
|
||
|
||
/// Which held artifact satisfied a pin: its index in the `held` slice given to
|
||
/// [`verify_bytes`], and, for a set, the verified files' paths.
|
||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||
pub enum VerifiedRef {
|
||
/// Form-less: the one file at `held[index]`.
|
||
File { index: usize },
|
||
/// `wai.prior.set`: the set at `held[index]`, every file its listing names
|
||
/// verified.
|
||
Set { index: usize, paths: Vec<String> },
|
||
}
|
||
|
||
/// Run SPEC §3.1's checks for `prior` over `capability` against the parameter sets
|
||
/// in `held`, reading the forms in `readable`, at a sink with no key log: check 5 is
|
||
/// skipped. See [`verify_bytes_with`].
|
||
pub fn verify_bytes(
|
||
capability: &str,
|
||
prior: &Prior,
|
||
held: &[HeldBytes<'_>],
|
||
readable: &[&str],
|
||
) -> Result<VerifiedRef, PinRefusal> {
|
||
verify_bytes_with(capability, prior, held, readable, None)
|
||
}
|
||
|
||
/// Run SPEC §3.1's checks for `prior` over `capability` against the parameter sets
|
||
/// in `held`, reading the forms in `readable`, with check 5 run by `authority` where
|
||
/// the sink has one. The first check that fails is the reason: `pin_malformed`,
|
||
/// `form_not_registered`, `form_unreadable`, `form_required`, `prior_revoked` /
|
||
/// `prior_not_authorised`, `prior_not_held` / `digest_superseded`, `set_malformed`,
|
||
/// `part_mismatch`. Checks other than 5 read only the pin, the registry, `readable`
|
||
/// and the bytes, so two sinks that hold the same bytes and read the same forms
|
||
/// report the same reason; check 5 also reads what `authority` holds and its clock.
|
||
pub fn verify_bytes_with(
|
||
capability: &str,
|
||
prior: &Prior,
|
||
held: &[HeldBytes<'_>],
|
||
readable: &[&str],
|
||
authority: Option<&dyn PinAuthority>,
|
||
) -> Result<VerifiedRef, PinRefusal> {
|
||
// 1. pin_malformed (only `sha256`: a bad `supersedes` is ignored).
|
||
prior.check_syntax()?;
|
||
// 2. form_not_registered, 3. form_unreadable.
|
||
prior.check_form(readable)?;
|
||
// 4. form_required.
|
||
if prior.form().is_none() && pin_shape(capability) != ParameterSetShape::OneFile {
|
||
return Err(PinRefusal::FormRequired { capability: capability.to_string() });
|
||
}
|
||
// 5. prior_revoked / prior_not_authorised, keyed on (form, sha256): jwp-receipts
|
||
// §7.9 over the set the pin names, at a sink that holds the capability to it.
|
||
if let Some(a) = authority {
|
||
a.check(capability, prior)?;
|
||
}
|
||
let set_form = prior.form() == Some(CAP_PRIOR_SET);
|
||
let digest_of = |h: &HeldBytes<'_>| -> Option<String> {
|
||
match (h, set_form) {
|
||
(HeldBytes::File(b), false) => Some(sha256_hex(b)),
|
||
(HeldBytes::Set { listing, .. }, true) => Some(sha256_hex(listing)),
|
||
_ => None,
|
||
}
|
||
};
|
||
let digests: Vec<Option<String>> = held.iter().map(digest_of).collect();
|
||
let matching: Vec<usize> = (0..held.len())
|
||
.filter(|&i| digests[i].as_deref() == Some(prior.sha256.as_str()))
|
||
.collect();
|
||
// 6. prior_not_held / digest_superseded.
|
||
if matching.is_empty() {
|
||
if let Some(pre) = prior.usable_supersedes() {
|
||
let pre_set = pre.form.as_deref() == Some(CAP_PRIOR_SET);
|
||
let holds_pre = held.iter().any(|h| match (h, pre_set) {
|
||
(HeldBytes::File(b), false) => sha256_hex(b) == pre.sha256,
|
||
(HeldBytes::Set { listing, .. }, true) => sha256_hex(listing) == pre.sha256,
|
||
_ => false,
|
||
});
|
||
if holds_pre {
|
||
return Err(PinRefusal::Superseded {
|
||
capability: capability.to_string(),
|
||
predecessor: pre.sha256.clone(),
|
||
});
|
||
}
|
||
}
|
||
return Err(PinRefusal::NotHeld { capability: capability.to_string() });
|
||
}
|
||
if !set_form {
|
||
return Ok(VerifiedRef::File { index: matching[0] });
|
||
}
|
||
// 7. set_malformed: every matching listing has the same bytes, so parse one.
|
||
let HeldBytes::Set { listing, .. } = held[matching[0]] else { unreachable!() };
|
||
let lines = parse_listing(listing)?;
|
||
// 8. part_mismatch: the first copy whose files are all intact wins.
|
||
let mut first_failure: Option<String> = None;
|
||
for &i in &matching {
|
||
let HeldBytes::Set { files, .. } = held[i] else { unreachable!() };
|
||
let bad = lines.iter().find(|(path, d)| files.get(path).is_none_or(|b| sha256(b) != *d));
|
||
match bad {
|
||
None => {
|
||
return Ok(VerifiedRef::Set { index: i, paths: lines.into_iter().map(|(p, _)| p).collect() });
|
||
}
|
||
Some((path, _)) => {
|
||
first_failure.get_or_insert_with(|| path.clone());
|
||
}
|
||
}
|
||
}
|
||
Err(PinRefusal::PartMismatch { path: first_failure.unwrap_or_default() })
|
||
}
|
||
|
||
/// An artifact a sink holds, on disk: a file (for a form-less pin), or a listing
|
||
/// whose root is the directory it sits in (for a `wai.prior.set` pin).
|
||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||
pub enum Held {
|
||
File(PathBuf),
|
||
Listing(PathBuf),
|
||
}
|
||
|
||
/// The parameter set a pin verified, as the very bytes that were hashed. A pinned
|
||
/// decode reads its parameters only from here.
|
||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||
pub enum Verified {
|
||
/// Form-less: the one file.
|
||
File(Vec<u8>),
|
||
/// `wai.prior.set`: every file the listing names, by path.
|
||
Set { root: PathBuf, files: BTreeMap<String, Vec<u8>> },
|
||
}
|
||
|
||
impl Verified {
|
||
/// The one file of a form-less pin.
|
||
pub fn one(&self) -> Option<&[u8]> {
|
||
match self {
|
||
Self::File(b) => Some(b),
|
||
Self::Set { .. } => None,
|
||
}
|
||
}
|
||
|
||
/// The file at `path` of a verified set; `set_incomplete` when the set does
|
||
/// not contain it (or the pin was form-less).
|
||
pub fn file(&self, path: &str) -> Result<&[u8], PinRefusal> {
|
||
match self {
|
||
Self::Set { files, .. } => files.get(path).map(Vec::as_slice),
|
||
Self::File(_) => None,
|
||
}
|
||
.ok_or_else(|| PinRefusal::SetIncomplete { path: path.to_string() })
|
||
}
|
||
}
|
||
|
||
/// [`verify_bytes`] over artifacts on disk. Each held file is read once and the
|
||
/// bytes that were hashed are the bytes returned; a file that cannot be read is not
|
||
/// held. For a listing, only the files of a listing whose digest matches the pin
|
||
/// are read. Check 5 is skipped; see [`verify_with`].
|
||
pub fn verify(capability: &str, prior: &Prior, held: &[Held], readable: &[&str]) -> Result<Verified, PinRefusal> {
|
||
verify_with(capability, prior, held, readable, None)
|
||
}
|
||
|
||
/// [`verify_bytes_with`] over artifacts on disk, as [`verify`] reads them. Check 5
|
||
/// runs before any file is read, so a set the sink's key log refuses is never read.
|
||
pub fn verify_with(
|
||
capability: &str,
|
||
prior: &Prior,
|
||
held: &[Held],
|
||
readable: &[&str],
|
||
authority: Option<&dyn PinAuthority>,
|
||
) -> Result<Verified, PinRefusal> {
|
||
// The cheap checks first, so a malformed or form-less pin, or one check 5
|
||
// refuses, reads no file.
|
||
prior.check_syntax()?;
|
||
prior.check_form(readable)?;
|
||
if prior.form().is_none() && pin_shape(capability) != ParameterSetShape::OneFile {
|
||
return Err(PinRefusal::FormRequired { capability: capability.to_string() });
|
||
}
|
||
if let Some(a) = authority {
|
||
a.check(capability, prior)?;
|
||
}
|
||
let set_form = prior.form() == Some(CAP_PRIOR_SET);
|
||
struct Loaded {
|
||
bytes: Vec<u8>,
|
||
root: Option<PathBuf>,
|
||
files: BTreeMap<String, Vec<u8>>,
|
||
}
|
||
let mut loaded: Vec<Loaded> = Vec::new();
|
||
for h in held {
|
||
match (h, set_form) {
|
||
(Held::File(p), false) => {
|
||
if let Ok(bytes) = std::fs::read(p) {
|
||
loaded.push(Loaded { bytes, root: None, files: BTreeMap::new() });
|
||
}
|
||
}
|
||
(Held::Listing(p), true) => {
|
||
let Ok(bytes) = std::fs::read(p) else { continue };
|
||
let root = p.parent().map(Path::to_path_buf).unwrap_or_default();
|
||
let mut files = BTreeMap::new();
|
||
if sha256_hex(&bytes) == prior.sha256 {
|
||
if let Ok(lines) = parse_listing(&bytes) {
|
||
for (path, _) in lines {
|
||
if let Ok(b) = std::fs::read(root.join(&path)) {
|
||
files.insert(path, b);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
loaded.push(Loaded { bytes, root: Some(root), files });
|
||
}
|
||
_ => {}
|
||
}
|
||
}
|
||
let views: Vec<BTreeMap<String, &[u8]>> = loaded.iter()
|
||
.map(|l| l.files.iter().map(|(k, v)| (k.clone(), v.as_slice())).collect())
|
||
.collect();
|
||
let held_bytes: Vec<HeldBytes<'_>> = loaded.iter().zip(views.iter())
|
||
.map(|(l, v)| if l.root.is_some() {
|
||
HeldBytes::Set { listing: &l.bytes, files: v }
|
||
} else {
|
||
HeldBytes::File(&l.bytes)
|
||
})
|
||
.collect();
|
||
// Check 5 ran above, before any file was read.
|
||
match verify_bytes_with(capability, prior, &held_bytes, readable, None)? {
|
||
VerifiedRef::File { index } => Ok(Verified::File(loaded.swap_remove(index).bytes)),
|
||
VerifiedRef::Set { index, paths } => {
|
||
let l = loaded.swap_remove(index);
|
||
let files = paths.into_iter().filter_map(|p| l.files.get(&p).cloned().map(|b| (p, b))).collect();
|
||
Ok(Verified::Set { root: l.root.unwrap_or_default(), files })
|
||
}
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
use crate::codecs::{CAP_NEURAL_BMSHJ2018, CAP_NEURAL_INT_MLICPP, CAP_PRIOR_NNC};
|
||
use crate::container::Supersedes;
|
||
|
||
fn line(d: &[u8], path: &str) -> Vec<u8> {
|
||
format!("{} {path}\n", sha256_hex(d)).into_bytes()
|
||
}
|
||
|
||
/// A digest that is not 64 lowercase hex digits is shown whole and escaped, so
|
||
/// a multi-byte character across byte 16 does not panic the preview.
|
||
#[test]
|
||
fn digest_preview_never_slices_inside_a_character() {
|
||
let good = "ab".repeat(32);
|
||
assert_eq!(digest_preview(&good), format!("{}\u{2026}", &good[..16]));
|
||
let bad = format!("a{}", "\u{e9}".repeat(8)); // byte 16 falls inside a character
|
||
assert!(!bad.is_char_boundary(16));
|
||
assert_eq!(digest_preview(&bad), format!("{bad:?} (not a SHA-256)"));
|
||
let p = Prior::new("m", bad.clone(), "int")
|
||
.with_supersedes(crate::container::Supersedes::new(bad.clone()));
|
||
let line = describe_pin(&p);
|
||
assert!(line.starts_with("id m sha256 \"a"), "{line}");
|
||
assert!(line.contains("supersedes \"a"), "{line}");
|
||
}
|
||
|
||
/// A listing from digests is the listing from bytes, and a digest outside WAI's
|
||
/// grammar is refused, not written.
|
||
#[test]
|
||
fn a_listing_from_digests_is_the_listing_from_bytes() {
|
||
let files: BTreeMap<String, &[u8]> = [("z".to_string(), &b"z"[..]), ("a/c.onnx".into(), b"c"), ("a.b".into(), b"a")].into_iter().collect();
|
||
let digests: BTreeMap<String, String> = files.iter().map(|(p, b)| (p.clone(), sha256_hex(b))).collect();
|
||
assert_eq!(listing_from_sha256(&digests).unwrap(), listing_from(&files).unwrap());
|
||
let mut upper = digests.clone();
|
||
upper.insert("z".into(), sha256_hex(b"z").to_uppercase());
|
||
assert!(listing_from_sha256(&upper).is_err());
|
||
}
|
||
|
||
#[test]
|
||
fn listing_rules() {
|
||
let ok: Vec<u8> = [line(b"a", "a.b"), line(b"c", "a/c.onnx"), line(b"z", "z")].concat();
|
||
assert_eq!(parse_listing(&ok).unwrap().len(), 3, "'a.b' sorts before 'a/c.onnx' as bytes");
|
||
let h = sha256_hex(b"x");
|
||
let bad: &[(&str, Vec<u8>)] = &[
|
||
("CRLF", format!("{h} a\r\n").into_bytes()),
|
||
("BOM", [b"\xef\xbb\xbf".to_vec(), line(b"x", "a")].concat()),
|
||
("no final LF", format!("{h} a").into_bytes()),
|
||
("blank line", [line(b"x", "a"), b"\n".to_vec(), line(b"x", "b")].concat()),
|
||
("one-space separator", format!("{h} a\n").into_bytes()),
|
||
("uppercase hex", format!("{} a\n", h.to_ascii_uppercase()).into_bytes()),
|
||
("unsorted", [line(b"x", "b"), line(b"x", "a")].concat()),
|
||
("duplicate", [line(b"x", "a"), line(b"x", "a")].concat()),
|
||
("case fold", [line(b"x", "A.json"), line(b"x", "a.json")].concat()),
|
||
("file and directory", [line(b"x", "w"), line(b"x", "w/x")].concat()),
|
||
("leading dot", line(b"x", ".hidden")),
|
||
("trailing dot", line(b"x", "a.")),
|
||
("dot-dot", line(b"x", "a/../b")),
|
||
("empty segment", line(b"x", "a//b")),
|
||
("absolute", line(b"x", "/a")),
|
||
("space in path", line(b"x", "a b")),
|
||
("empty", Vec::new()),
|
||
];
|
||
for (what, b) in bad {
|
||
assert_eq!(parse_listing(b).map(|_| ()).unwrap_err().code(), "set_malformed", "{what}");
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn write_listing_matches_the_checksum_format() {
|
||
let d = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/vectors/int_mlicpp");
|
||
let listing = write_listing(&d, &["model.wmb".to_string()]).unwrap();
|
||
assert_eq!(listing.len(), 76);
|
||
assert_eq!(listing, std::fs::read(d.join("model.sha256")).unwrap(), "the committed listing");
|
||
assert_eq!(sha256_hex(&listing), "85a05b0dac15566c5a657ddeee253eaa9f54ce852ae04924f5b04151901d9d1b");
|
||
assert_eq!(parse_listing(&listing).unwrap()[0].0, "model.wmb");
|
||
}
|
||
|
||
fn files<'a>(pairs: &[(&str, &'a [u8])]) -> BTreeMap<String, &'a [u8]> {
|
||
pairs.iter().map(|&(k, v)| (k.to_string(), v)).collect()
|
||
}
|
||
|
||
/// Every precedence edge of SPEC §3.1's order, table-driven.
|
||
#[test]
|
||
fn refusals_come_in_order() {
|
||
let model: &[u8] = b"one-file model";
|
||
let m_hex = sha256_hex(model);
|
||
let f = files(&[("model.wmb", model)]);
|
||
let listing = listing_from(&f).unwrap();
|
||
let l_hex = sha256_hex(&listing);
|
||
let set = |p: Prior| p.with_form(CAP_PRIOR_SET);
|
||
let one = Prior::new("m", m_hex.clone(), "int");
|
||
let held_file = [HeldBytes::File(model)];
|
||
let held_set = [HeldBytes::Set { listing: &listing, files: &f }];
|
||
let r = |cap: &str, p: &Prior, h: &[HeldBytes<'_>], readable: &[&str]| {
|
||
verify_bytes(cap, p, h, readable).map(|_| "ok").unwrap_or_else(|e| e.code())
|
||
};
|
||
let mlic = CAP_NEURAL_INT_MLICPP;
|
||
// ok, both shapes.
|
||
assert_eq!(r(mlic, &one, &held_file, READABLE_FORMS), "ok");
|
||
assert_eq!(r(mlic, &set(Prior::new("m", l_hex.clone(), "int")), &held_set, READABLE_FORMS), "ok");
|
||
// 1 before everything: an uppercase digest over an unreadable form.
|
||
let upper = Prior::new("m", m_hex.to_ascii_uppercase(), "int").with_form(CAP_PRIOR_NNC);
|
||
assert_eq!(r(mlic, &upper, &held_file, &[]), "pin_malformed");
|
||
assert_eq!(r(mlic, &Prior::new("m", &m_hex[..63], "int"), &held_file, READABLE_FORMS), "pin_malformed");
|
||
// 2 before 3.
|
||
assert_eq!(r(mlic, &one.clone().with_form("wai.image.png"), &held_file, &[]), "form_not_registered");
|
||
assert_eq!(r(mlic, &one.clone().with_form("wai.prior.nope"), &held_file, &[]), "form_not_registered");
|
||
// 3 before 4 and 6.
|
||
assert_eq!(r(CAP_NEURAL_BMSHJ2018, &one.clone().with_form(CAP_PRIOR_NNC), &[], READABLE_FORMS), "form_unreadable");
|
||
// 4 before 6: a several-file capability with a form-less pin, even when a
|
||
// held file matches.
|
||
assert_eq!(r(CAP_NEURAL_BMSHJ2018, &one, &held_file, READABLE_FORMS), "form_required");
|
||
// An unregistered capability is several files: fail closed.
|
||
assert_eq!(r("wai.neural.future", &one, &held_file, READABLE_FORMS), "form_required");
|
||
// 6: not held; superseded when the predecessor is held.
|
||
let other = Prior::new("m", sha256_hex(b"other"), "int");
|
||
assert_eq!(r(mlic, &other, &held_file, READABLE_FORMS), "prior_not_held");
|
||
let succ = other.clone().with_supersedes(Supersedes::new(m_hex.clone()));
|
||
assert_eq!(r(mlic, &succ, &held_file, READABLE_FORMS), "digest_superseded");
|
||
// A malformed supersedes is ignored: not held, never superseded.
|
||
let bad_sup = other.clone().with_supersedes(Supersedes::new(m_hex.clone()).with_form("wai.image.png"));
|
||
assert_eq!(r(mlic, &bad_sup, &held_file, READABLE_FORMS), "prior_not_held");
|
||
// A successor in set form whose predecessor is a set.
|
||
let succ_set = set(other.clone()).with_supersedes(Supersedes::new(l_hex.clone()).with_form(CAP_PRIOR_SET));
|
||
assert_eq!(r(mlic, &succ_set, &held_set, READABLE_FORMS), "digest_superseded");
|
||
// A file never satisfies a set pin, a listing never a file pin.
|
||
assert_eq!(r(mlic, &set(one.clone()), &held_file, READABLE_FORMS), "prior_not_held");
|
||
assert_eq!(r(mlic, &Prior::new("m", l_hex.clone(), "int"), &held_set, READABLE_FORMS), "prior_not_held");
|
||
// 7: the pinned listing is malformed (CRLF).
|
||
let crlf = String::from_utf8(listing.clone()).unwrap().replace('\n', "\r\n").into_bytes();
|
||
let held_crlf = [HeldBytes::Set { listing: &crlf, files: &f }];
|
||
assert_eq!(r(mlic, &set(Prior::new("m", sha256_hex(&crlf), "int")), &held_crlf, READABLE_FORMS), "set_malformed");
|
||
// 8: a file the listing names is flipped, or missing.
|
||
let flipped = files(&[("model.wmb", b"one-file modeL")]);
|
||
let held_flip = [HeldBytes::Set { listing: &listing, files: &flipped }];
|
||
let pin = set(Prior::new("m", l_hex.clone(), "int"));
|
||
assert_eq!(r(mlic, &pin, &held_flip, READABLE_FORMS), "part_mismatch");
|
||
let empty = BTreeMap::new();
|
||
assert_eq!(r(mlic, &pin, &[HeldBytes::Set { listing: &listing, files: &empty }], READABLE_FORMS), "part_mismatch");
|
||
// A flipped listing is a different digest: not held.
|
||
let mut lflip = listing.clone();
|
||
lflip[0] ^= 1;
|
||
assert_eq!(r(mlic, &pin, &[HeldBytes::Set { listing: &lflip, files: &f }], READABLE_FORMS), "prior_not_held");
|
||
}
|
||
|
||
#[test]
|
||
fn two_copies_one_intact_resolves() {
|
||
let model: &[u8] = b"weights";
|
||
let good = files(&[("model.wmb", model)]);
|
||
let bad = files(&[("model.wmb", b"weightz")]);
|
||
let listing = listing_from(&good).unwrap();
|
||
let pin = Prior::new("m", sha256_hex(&listing), "int").with_form(CAP_PRIOR_SET);
|
||
let held = [
|
||
HeldBytes::Set { listing: &listing, files: &bad },
|
||
HeldBytes::Set { listing: &listing, files: &good },
|
||
];
|
||
assert_eq!(
|
||
verify_bytes(CAP_NEURAL_INT_MLICPP, &pin, &held, READABLE_FORMS),
|
||
Ok(VerifiedRef::Set { index: 1, paths: vec!["model.wmb".into()] })
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn verified_bytes_are_the_bytes_hashed() {
|
||
let dir = std::env::temp_dir().join(format!("wai-pin-verified-{}", std::process::id()));
|
||
std::fs::create_dir_all(dir.join("weights")).unwrap();
|
||
std::fs::write(dir.join("a.json"), b"{\"a\":1}").unwrap();
|
||
std::fs::write(dir.join("weights/b.json"), b"{\"b\":2}").unwrap();
|
||
std::fs::write(dir.join(".hidden"), b"skipped").unwrap();
|
||
let listing = listing_of_dir(&dir).unwrap();
|
||
assert_eq!(parse_listing(&listing).unwrap().iter().map(|(p, _)| p.as_str()).collect::<Vec<_>>(),
|
||
["a.json", "weights/b.json"], "hidden files are not listed");
|
||
std::fs::write(dir.join("set.sha256"), &listing).unwrap();
|
||
let pin = Prior::new("s", sha256_hex(&listing), "int").with_form(CAP_PRIOR_SET);
|
||
let v = verify("wai.neural.elic", &pin, &[Held::Listing(dir.join("set.sha256"))], READABLE_FORMS).unwrap();
|
||
// Overwrite a file after verification: the verified bytes do not change.
|
||
std::fs::write(dir.join("a.json"), b"{\"a\":2}").unwrap();
|
||
assert_eq!(v.file("a.json").unwrap(), b"{\"a\":1}");
|
||
assert_eq!(v.file("weights/b.json").unwrap(), b"{\"b\":2}");
|
||
assert_eq!(v.file("c.json").unwrap_err().code(), "set_incomplete");
|
||
// Now the held copy is damaged.
|
||
let again = verify("wai.neural.elic", &pin, &[Held::Listing(dir.join("set.sha256"))], READABLE_FORMS);
|
||
assert_eq!(again.unwrap_err(), PinRefusal::PartMismatch { path: "a.json".into() });
|
||
std::fs::remove_dir_all(&dir).ok();
|
||
}
|
||
}
|