open-standards/joule-code/joule-code-rs/crates/joule-moq-relay/tests/too_far_behind.rs

//! A subscriber that does not read is ended TOO_FAR_BEHIND (`draft-ietf-moq-transport-23`
//! §5.2, §12.4, §12.5; `draft-englishm-moq-relay-dos-02` §5.1).
//!
//! One publisher sends a track with no delivery timeout, a group a stream, each object larger
//! than the stream window a slow subscriber gives the relay. A fast subscriber reads as it
//! goes, and the publisher keeps in step with it. The slow one reads nothing, so each of the
//! relay's writes to it stops on its stream's flow control and what arrives for its
//! subscription waits in its session's scheduler. The relay is given a bound of 128 items a
//! subscription (`limits.max_queued_per_subscription`). Past it the relay drops what the
//! subscription had queued, resets its streams TOO_FAR_BEHIND, those its writes were stopped
//! on, and sends PUBLISH_DONE TOO_FAR_BEHIND; it counts the subscription and the objects
//! dropped. The fast subscriber loses nothing, the slow session and its other subscription
//! stand, and the slow subscriber subscribes again and is served.
//!
//! The slow subscriber's connection window is left wide: one that held the whole connection's
//! credit in streams it had not read could be sent nothing on any stream, its PUBLISH_DONE
//! included, until it read (the README, "A subscriber that does not read"); what bounds that
//! one is `limits.stalled_session_ms` (`tests/stalled_session.rs`).

mod common;

use common::*;
use joule_moq::draft22::data::{SubgroupHeader, SubgroupId, SubgroupObject};
use joule_moq::draft22::{ControlMessage, Parameters, PublishDoneCode, StreamResetCode};
use joule_moq_relay::client::ClientOptions;
use joule_moq_relay::transport::TransportError;
use joule_moq_relay::wire::StreamError;
use std::time::Duration;
use tokio::time::timeout;

const BOUND: u64 = 128;
const GROUPS: u64 = 200;
/// Larger than the slow subscriber's stream window.
const BYTES: usize = 48 * 1024;
const STREAM_WINDOW: u32 = 32 * 1024;
/// Smaller than it: what the slow subscriber is sent once it reads again.
const SMALL: usize = 4 * 1024;

#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_subscriber_that_does_not_read_is_ended_too_far_behind() {
    timeout(Duration::from_secs(120), run()).await.expect("the test finishes in time");
}

fn group(alias: u64, g: u64, bytes: usize) -> (SubgroupHeader, [SubgroupObject; 1]) {
    let h = SubgroupHeader { track_alias: alias, group_id: g, subgroup_id: SubgroupId::Zero, publisher_priority: Some(128), properties: false, end_of_group: true, first_object: true };
    (h, [SubgroupObject::normal(0, vec![g as u8; bytes])])
}

async fn run() {
    let server = relay(|c| c.limits.max_queued_per_subscription = BOUND).await;
    let (publisher, _ep) = connect(&server, Via::Quic, ClientOptions::default()).await;
    let (space, video, audio) = (ns(&["behind"]), name("video"), name("audio"));
    let _v = publisher.publish(space.clone(), video.clone(), 1, Parameters::new(), Default::default()).await.expect("PUBLISH video");
    let _a = publisher.publish(space.clone(), audio.clone(), 2, Parameters::new(), Default::default()).await.expect("PUBLISH audio");

    let (fast, _ef) = connect(&server, Via::Quic, ClientOptions::default()).await;
    let mut small = quinn::TransportConfig::default();
    small.stream_receive_window(quinn::VarInt::from_u32(STREAM_WINDOW));
    let (slow, _es) = connect_with(&server, Via::Quic, ClientOptions { lazy_data: true, ..ClientOptions::default() }, Some(small)).await;

    let (fast_ok, _ftx, mut frx) = fast.subscribe(space.clone(), video.clone(), Parameters::new()).await.expect("SUBSCRIBE");
    let (slow_ok, _stx, mut srx) = slow.subscribe(space.clone(), video.clone(), Parameters::new()).await.expect("SUBSCRIBE");
    let (slow_audio, _satx, mut sarx) = slow.subscribe(space.clone(), audio.clone(), Parameters::new()).await.expect("SUBSCRIBE audio");

    // The publisher keeps within eight groups of the subscriber that reads, so that one's
    // queue stays far inside the bound however loaded the host is.
    let (seen_tx, mut seen) = tokio::sync::watch::channel(0u64);
    let publish = async {
        for g in 0..GROUPS {
            timeout(STEP, seen.wait_for(|n| n + 8 > g)).await.expect("the fast subscriber keeps up").expect("it is reading");
            let (h, o) = group(1, g, BYTES);
            timeout(STEP, publisher.send_subgroup(h, &o)).await.expect("in time").expect("a send");
        }
    };
    let fast_reads = async {
        let mut got = Vec::new();
        while (got.len() as u64) < GROUPS {
            let r = fast.next_data(STEP).await.expect("the fast subscriber's next stream");
            got.push(read_stream(r).await.expect("a subgroup stream"));
            let _ = seen_tx.send(got.len() as u64);
        }
        got
    };
    let ((), fast_got) = tokio::join!(publish, fast_reads);

    // The slow subscription is told, by name, with the streams it was opened.
    let done = match timeout(STEP, srx.next()).await {
        Ok(Ok(Some(ControlMessage::PublishDone(d)))) => d,
        other => panic!("no PUBLISH_DONE for the subscription that fell behind: {other:?}"),
    };
    assert_eq!(done.status, PublishDoneCode::TOO_FAR_BEHIND);
    assert_eq!(done.reason, b"queue_bound");
    assert!(done.stream_count >= 1 && done.stream_count < GROUPS, "streams opened for it: {}", done.stream_count);
    assert!(matches!(timeout(STEP, srx.next()).await, Ok(Ok(None))), "and its request stream ends");

    // The fast subscriber has every group whole, and its subscription stands.
    assert_eq!(fast_got.iter().filter(|x| x.header.track_alias == fast_ok.track_alias && x.reset.is_none() && x.objects.len() == 1).count() as u64, GROUPS);
    assert!(timeout(Duration::from_millis(300), frx.next()).await.is_err(), "the fast subscription was not ended");

    // Counted, once, with what was dropped; nothing of it is left queued.
    let relay = server.relay().clone();
    let s = relay.stats();
    assert_eq!(s.too_far_behind, 1, "{s:?}");
    // The bound's items are objects and their streams' ends, two a group, but for the ends of
    // the streams being written (`write_concurrency`, 8), whose objects were with the transport.
    assert!(s.dropped_too_far_behind >= (BOUND - 8) / 2 && s.dropped_too_far_behind <= BOUND / 2 + 1, "{s:?}");
    assert_eq!(server.relay().session_diagnostics().len(), 3, "no session ended");
    assert!(until(STEP, || relay.stats().queued == 0).await);

    // What the slow subscriber finds when it reads at last: the streams the relay's writes were
    // stopped on, each reset TOO_FAR_BEHIND, and no more streams than the PUBLISH_DONE counted.
    // No object of the track was ever whole on a stream: each is larger than the window.
    let (mut whole, mut reset) = (0u64, 0u64);
    while let Some(mut r) = slow.next_stream(Duration::from_millis(500)).await {
        match timeout(STEP, r.start()).await.expect("in time") {
            Ok(_) => match read_stream(r).await {
                Some(g) if g.reset.is_none() => {
                    assert_eq!(g.header.track_alias, slow_ok.track_alias);
                    whole += 1;
                }
                Some(g) => {
                    assert_eq!(g.reset, Some(StreamResetCode::TOO_FAR_BEHIND.0), "group {}", g.header.group_id);
                    reset += 1;
                }
                None => panic!("not a subgroup stream"),
            },
            Err(StreamError::Transport(TransportError::Reset(code))) => {
                assert_eq!(code, StreamResetCode::TOO_FAR_BEHIND.0);
                reset += 1;
            }
            Err(e) => panic!("a data stream: {e}"),
        }
    }
    assert!(reset >= 1 && whole == 0, "{reset} streams reset TOO_FAR_BEHIND, {whole} whole");
    assert!(reset <= done.stream_count, "{reset} reset of {} opened", done.stream_count);

    // The session stands, and its other subscription: a group of the audio track arrives.
    assert!(timeout(Duration::from_millis(100), sarx.next()).await.is_err(), "the session's other subscription was not ended");
    let (h, o) = group(2, 0, SMALL);
    timeout(STEP, publisher.send_subgroup(h, &o)).await.expect("in time").expect("a send");
    let got = collect(&slow, STEP, |g| !g.is_empty()).await;
    assert_eq!((got[0].header.track_alias, got[0].objects[0].payload.len(), got[0].reset), (slow_audio.track_alias, SMALL, None));

    // And the subscriber may subscribe again: it is served from the next group.
    let (again, _atx, _arx) = slow.subscribe(space.clone(), video.clone(), Parameters::new()).await.expect("SUBSCRIBE again");
    assert_ne!(again.track_alias, slow_ok.track_alias);
    let (h, o) = group(1, GROUPS, SMALL);
    timeout(STEP, publisher.send_subgroup(h, &o)).await.expect("in time").expect("a send");
    let got = collect(&slow, STEP, |g| !g.is_empty()).await;
    assert_eq!((got[0].header.track_alias, got[0].header.group_id, got[0].reset), (again.track_alias, GROUPS, None));
    assert_eq!(relay.stats().too_far_behind, 1);

    for c in [&publisher, &fast, &slow] {
        c.close(0, "done");
    }
    server.shutdown().await;
}