dcharlot
bbc5e531ce
Some checks failed
repository checks / lockfiles, advisories, tests in CI, workflow hygiene, spec and section references, public text (push) Has been cancelled
255 lines
16 KiB
YAML
255 lines
16 KiB
YAML
name: wai-quantum signal processor
|
|
|
|
# The signal processor (DSP) as a processor and as a fabric member, under its
|
|
# user-mode emulator:
|
|
#
|
|
# - the processor: the byte-pinned bit-identity digests, the frame fixtures and
|
|
# the byte-pinning library tests, built for the processor's Linux target (a
|
|
# tier-3 target: the standard library is built from source);
|
|
# - the member: the vector unit (128-byte vectors) runs the gate's
|
|
# arithmetic, and its device tests must find it, agree with the CPU, reproduce
|
|
# all four pinned certificates (each naming the emulator) and catch three
|
|
# kernel mutants;
|
|
# - the same build on x86_64, where the member is absent and says so.
|
|
#
|
|
# The emulator is QEMU 11.1.1, built here from its release tarball (SHA-256
|
|
# pinned; its signature was checked against the release key when it was pinned).
|
|
# Releases before 11.0 test a whole predicate register, not its low bit, when a
|
|
# branch reads a predicate written in the same packet, and the processor
|
|
# branches on the low bit. Compiled code relies on that (a predicate built from
|
|
# a general register's 1 and xored with a comparison's 0xff, so 0xfe means
|
|
# "equal"), and a library test fails under such an emulator though the code is
|
|
# right. A probe shows which emulator does what before any suite runs on it.
|
|
#
|
|
# The cross toolchain (clang, lld and the musl sysroot for the target) is the
|
|
# open-source release for this processor, SHA-256 pinned.
|
|
#
|
|
# RUSTC_BOOTSTRAP: building the standard library for a tier-3 target, and the
|
|
# vector unit's intrinsics, are unstable; this workflow's builds for this
|
|
# processor are the only ones that use it.
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
on:
|
|
pull_request:
|
|
paths:
|
|
- 'wai/wai-quantum/**'
|
|
- 'rust-toolchain.toml'
|
|
- '.github/workflows/wai-quantum-dsp.yml'
|
|
schedule:
|
|
- cron: '17 4 * * *'
|
|
|
|
jobs:
|
|
# Does this pull request's own diff touch the paths below? This forge matches a
|
|
# workflow's `paths:` against the push, so a merge of main into a branch fires the
|
|
# workflow whatever the pull request changes; every job here waits on this answer.
|
|
# A push to main, a schedule, or anything the script cannot determine answers
|
|
# `true` (scripts/pr-touches-paths.py).
|
|
relevant:
|
|
name: does this change touch it
|
|
runs-on: gate
|
|
outputs:
|
|
run: ${{ steps.gate.outputs.run }}
|
|
steps:
|
|
- uses: actions/checkout@v5
|
|
with:
|
|
fetch-depth: 0
|
|
- id: gate
|
|
run: |
|
|
set -euo pipefail
|
|
r=$(python3 scripts/pr-touches-paths.py .github/workflows/wai-quantum-dsp.yml \
|
|
"${{ github.event.pull_request.base.sha }}" \
|
|
"${{ github.event.pull_request.head.sha }}")
|
|
echo "this change touches wai-quantum-dsp: $r"
|
|
echo "run=$r" >> "$GITHUB_OUTPUT"
|
|
|
|
dsp-emulated:
|
|
name: processor and member · signal processor, emulated
|
|
needs: relevant
|
|
if: needs.relevant.outputs.run == 'true'
|
|
runs-on: linux
|
|
timeout-minutes: 180
|
|
container:
|
|
image: debian@sha256:913f6706df59a68922d1dd08f78c2476560a8d367897200a6005b00e5f67c2d5
|
|
options: --memory=6g --memory-swap=6g
|
|
env:
|
|
TOOLCHAIN_VERSION: 22.1.8
|
|
TOOLCHAIN_SHA256: bc122fe6473ead9693613a6ab3a177c5583653ff76b937e7578cf7de0273315b
|
|
# Not QEMU_*: qemu-user reads every QEMU_<option> variable as that
|
|
# option, and QEMU_VERSION is -version (print it and exit).
|
|
HEXAGON_EMULATOR_VERSION: 11.1.1
|
|
HEXAGON_EMULATOR_SHA256: 079ffbff8a7111bbc89022107cbabf3bbfd614d5fc9d7cc675991196aca12482
|
|
WAI_QUANTUM_DSP_EXPECT: dsp-vector
|
|
WAI_QUANTUM_DSP_EMULATOR: qemu-user 11.1.1 (built from the signed release)
|
|
steps:
|
|
- name: Tools
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
apt-get update -qq
|
|
apt-get install -y -qq --no-install-recommends ca-certificates curl git xz-utils zstd build-essential \
|
|
ninja-build meson python3 python3-venv pkg-config libglib2.0-dev flex bison >/dev/null
|
|
# No Node for a checkout action: the forge's archive of the exact commit,
|
|
# its embedded commit id checked.
|
|
- name: Fetch the source
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
want="${{ github.event.pull_request.head.sha || github.sha }}"
|
|
rm -rf src src.tar.gz src.tar && mkdir src
|
|
curl -sSf -o src.tar.gz "http://100.64.224.0:3400/Transaction-Science/open-standards/archive/$want.tar.gz"
|
|
gzip -dc src.tar.gz > src.tar
|
|
got="$(git get-tar-commit-id < src.tar)"
|
|
[ "$got" = "$want" ] || { echo "::error::the archive is of $got, not $want"; exit 1; }
|
|
# -m: every file gets the time it was extracted, not its commit's. The builds
|
|
# reuse target directories kept between runs, and cargo takes a crate as
|
|
# built when no source file is newer than its last build: with the commit's
|
|
# times, an older commit ran the binary another commit had built there.
|
|
tar -xmf src.tar -C src --strip-components=1
|
|
rm src.tar.gz src.tar
|
|
- name: The cross toolchain, the emulator and Rust (cached, pinned)
|
|
shell: bash
|
|
working-directory: src/wai/wai-quantum
|
|
run: |
|
|
set -euo pipefail
|
|
cache=/ci-cache/wai-quantum-hexagon
|
|
mkdir -p "$cache"
|
|
exec 9>"$cache/.lock"
|
|
flock 9
|
|
tc="$cache/toolchain-$TOOLCHAIN_VERSION"
|
|
if [ ! -x "$tc/clang+llvm-$TOOLCHAIN_VERSION-cross-hexagon-unknown-linux-musl/x86_64-linux-gnu/bin/hexagon-linux-musl-clang" ]; then
|
|
f="clang+llvm-$TOOLCHAIN_VERSION-cross-hexagon-unknown-linux-musl_x86_64-linux-gnu.tar.zst"
|
|
# The release directory's name carries trailing underscores.
|
|
curl -sSfL -o "$cache/$f" "https://artifacts.codelinaro.org/artifactory/codelinaro-toolchain-for-hexagon/${TOOLCHAIN_VERSION}________/$f"
|
|
echo "$TOOLCHAIN_SHA256 $cache/$f" | sha256sum -c -
|
|
rm -rf "$tc.tmp" && mkdir "$tc.tmp"
|
|
tar --zstd -xf "$cache/$f" -C "$tc.tmp"
|
|
rm -rf "$tc" && mv "$tc.tmp" "$tc" && rm "$cache/$f"
|
|
fi
|
|
q="$cache/qemu-hexagon-$HEXAGON_EMULATOR_VERSION"
|
|
if [ ! -x "$q" ]; then
|
|
curl -sSfL -o "$cache/qemu.tar.xz" "https://download.qemu.org/qemu-$HEXAGON_EMULATOR_VERSION.tar.xz"
|
|
echo "$HEXAGON_EMULATOR_SHA256 $cache/qemu.tar.xz" | sha256sum -c -
|
|
b="$cache/qemu-build"
|
|
rm -rf "$b" && mkdir "$b" && tar -xJf "$cache/qemu.tar.xz" -C "$b" --strip-components=1
|
|
(cd "$b" && ./configure --target-list=hexagon-linux-user --static --disable-docs --disable-tools --disable-werror > configure.log 2>&1 && ninja -C build -j 4 qemu-hexagon > ninja.log 2>&1) || { tail -30 "$b/configure.log" "$b"/ninja.log 2>/dev/null; exit 1; }
|
|
cp "$b/build/qemu-hexagon" "$q.tmp" && mv "$q.tmp" "$q"
|
|
rm -rf "$b" "$cache/qemu.tar.xz"
|
|
fi
|
|
"$q" --version | head -1
|
|
export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo"
|
|
export PATH="$CARGO_HOME/bin:$PATH"
|
|
if ! command -v rustup >/dev/null; then
|
|
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --profile minimal --default-toolchain none
|
|
fi
|
|
rustup toolchain install
|
|
rustup component add rust-src
|
|
- name: Which emulator branches on a predicate's low bit
|
|
shell: bash
|
|
working-directory: src/wai/wai-quantum
|
|
run: |
|
|
set -euo pipefail
|
|
cache=/ci-cache/wai-quantum-hexagon
|
|
bin="$cache/toolchain-$TOOLCHAIN_VERSION/clang+llvm-$TOOLCHAIN_VERSION-cross-hexagon-unknown-linux-musl/x86_64-linux-gnu/bin"
|
|
"$bin/hexagon-linux-musl-clang" -O1 -mv68 -static -o "$RUNNER_TEMP/predicate_branch" ci/dsp/predicate_branch.c
|
|
echo "== the pinned emulator"
|
|
# The probe must say what it found: an emulator that exits without
|
|
# running the program (as qemu-user does for an environment variable
|
|
# it reads as -version) would otherwise pass this check.
|
|
"$cache/qemu-hexagon-$HEXAGON_EMULATOR_VERSION" "$RUNNER_TEMP/predicate_branch" > "$RUNNER_TEMP/predicate.log" 2>&1 || true
|
|
cat "$RUNNER_TEMP/predicate.log"
|
|
grep -qx "this emulator branches on the low bit" "$RUNNER_TEMP/predicate.log" || { echo "::error::the pinned emulator does not branch on the low bit, or did not run the probe"; exit 1; }
|
|
# The host's own emulator, where it has one registered: shown, not required.
|
|
if [ -e /proc/sys/fs/binfmt_misc/qemu-hexagon ]; then
|
|
echo "== the host's emulator"
|
|
"$RUNNER_TEMP/predicate_branch" || echo "(the host's emulator differs; the suites below run on the pinned one)"
|
|
fi
|
|
- name: The processor's suites
|
|
shell: bash
|
|
working-directory: src/wai/wai-quantum
|
|
run: |
|
|
set -euo pipefail
|
|
cache=/ci-cache/wai-quantum-hexagon
|
|
exec 9>"$cache/.lock"
|
|
flock 9
|
|
# Every source file newer than every build in the cache, now that the lock is
|
|
# held: another run may have built here while this one waited for it, after
|
|
# this one's extraction. (../.. is the extracted tree.) Each step below that
|
|
# builds takes the lock afresh, and refreshes again.
|
|
find ../.. -type f -exec touch {} +
|
|
tc="$cache/toolchain-$TOOLCHAIN_VERSION/clang+llvm-$TOOLCHAIN_VERSION-cross-hexagon-unknown-linux-musl/x86_64-linux-gnu"
|
|
export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target"
|
|
export PATH="$CARGO_HOME/bin:$tc/bin:$PATH" RUSTC_BOOTSTRAP=1
|
|
export CARGO_TARGET_HEXAGON_UNKNOWN_LINUX_MUSL_LINKER=hexagon-linux-musl-clang
|
|
export CARGO_TARGET_HEXAGON_UNKNOWN_LINUX_MUSL_RUNNER="$cache/qemu-hexagon-$HEXAGON_EMULATOR_VERSION -L $tc/target/hexagon-unknown-linux-musl"
|
|
export CC_hexagon_unknown_linux_musl=hexagon-linux-musl-clang AR_hexagon_unknown_linux_musl=hexagon-linux-musl-ar
|
|
logs="$cache/logs"; mkdir -p "$logs"
|
|
# FEATURES names the build (full, unless a step says otherwise).
|
|
t() {
|
|
name="$1"; shift
|
|
timeout 2700 cargo test --locked --release -j 6 -Zbuild-std -Zbuild-std-features=system-llvm-libunwind \
|
|
--target hexagon-unknown-linux-musl --features "${FEATURES:-full}" "$@" 2>&1 | tee "$logs/$name.log"
|
|
}
|
|
t bit_identity --test bit_identity -- --test-threads=1
|
|
# The first build after the refresh compiles wai-quantum; one that compiled
|
|
# none ran a binary another commit left in the cache.
|
|
grep -qE '^ *Compiling wai-quantum v[0-9]' "$logs/bit_identity.log" || { echo "::error::the build compiled no wai-quantum: its tests ran a binary already in the cache"; exit 1; }
|
|
grep -qE '^test result: ok\. 23 passed; 0 failed' "$logs/bit_identity.log" || { echo "::error::the 23 bit-identity digests did not all pass"; exit 1; }
|
|
t frame --test frame_reference -- --test-threads=1
|
|
grep -qE '^test result: ok\. 2 passed; 0 failed' "$logs/frame.log" || { echo "::error::the fixtures did not pass"; exit 1; }
|
|
# The library's tests with the features their modules need, not full: the
|
|
# whole engine's test binary is past the processor's direct-call reach
|
|
# (R_HEX_B22_PCREL, ±8 MiB), and these modules need none of the rest.
|
|
FEATURES="quantum_fabric quantum_receipt quantum_qbom" t lib --lib -- quantum:: quantum_fabric:: ijson:: quantum_receipt quantum_energy quantum_qbom merkle
|
|
grep -qE '^test result: ok\. [1-9][0-9]+ passed; 0 failed' "$logs/lib.log" || { echo "::error::the byte-pinning library tests did not pass"; exit 1; }
|
|
- name: The vector-unit member's tests
|
|
shell: bash
|
|
working-directory: src/wai/wai-quantum
|
|
run: |
|
|
set -euo pipefail
|
|
cache=/ci-cache/wai-quantum-hexagon
|
|
exec 9>"$cache/.lock"
|
|
flock 9
|
|
find ../.. -type f -exec touch {} +
|
|
tc="$cache/toolchain-$TOOLCHAIN_VERSION/clang+llvm-$TOOLCHAIN_VERSION-cross-hexagon-unknown-linux-musl/x86_64-linux-gnu"
|
|
# The vector unit in 128-byte mode, for the whole build (the standard
|
|
# library included): a separate target directory.
|
|
export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target-vector"
|
|
export PATH="$CARGO_HOME/bin:$tc/bin:$PATH" RUSTC_BOOTSTRAP=1 RUSTFLAGS="-C target-feature=+hvxv68,+hvx-length128b"
|
|
export CARGO_TARGET_HEXAGON_UNKNOWN_LINUX_MUSL_LINKER=hexagon-linux-musl-clang
|
|
export CARGO_TARGET_HEXAGON_UNKNOWN_LINUX_MUSL_RUNNER="$cache/qemu-hexagon-$HEXAGON_EMULATOR_VERSION -L $tc/target/hexagon-unknown-linux-musl"
|
|
export CC_hexagon_unknown_linux_musl=hexagon-linux-musl-clang AR_hexagon_unknown_linux_musl=hexagon-linux-musl-ar
|
|
b="-Zbuild-std -Zbuild-std-features=system-llvm-libunwind --target hexagon-unknown-linux-musl --features quantum_fabric_dsp"
|
|
timeout 1800 cargo test --locked --release -j 6 $b --lib -- quantum_fabric_dsp:: 2>&1 | tee "$RUNNER_TEMP/unit.log"
|
|
grep -qE '^ *Compiling wai-quantum v[0-9]' "$RUNNER_TEMP/unit.log" || { echo "::error::the build compiled no wai-quantum: its tests ran a binary already in the cache"; exit 1; }
|
|
grep -qE '^test result: ok\. [1-9][0-9]* passed; 0 failed' "$RUNNER_TEMP/unit.log" || { echo "::error::no unit test ran"; exit 1; }
|
|
report="$RUNNER_TEMP/dsp-report.txt"; : > "$report"
|
|
WAI_QUANTUM_DSP_REPORT="$report" timeout 5400 cargo test --locked --release -j 6 $b --test dsp_member -- --test-threads=1 2>&1 | tee "$RUNNER_TEMP/device.log"
|
|
grep -qE '^test result: ok\. 9 passed; 0 failed' "$RUNNER_TEMP/device.log" || { echo "::error::not every device test ran and passed"; exit 1; }
|
|
cat "$report"
|
|
key="$WAI_QUANTUM_DSP_EXPECT"
|
|
grep -qx "dsp devices: as declared ($key)" "$report" || { echo "::error::the member found is not the declared one"; exit 1; }
|
|
grep -q "^dsp member: key=$key .*emulator:qemu-user 11.1.1" "$report" || { echo "::error::the member does not say it ran under the pinned emulator"; exit 1; }
|
|
n="$(grep -c "^dsp certified: key=$key .* parity=bit-exact " "$report")"
|
|
[ "$n" = 4 ] || { echo "::error::$n of 4 certificates were earned"; exit 1; }
|
|
grep -qx "dsp mutants: key=$key killed 3 of 3" "$report" || { echo "::error::a kernel mutant survived"; exit 1; }
|
|
- name: Elsewhere the member is absent and says so (x86_64)
|
|
shell: bash
|
|
working-directory: src/wai/wai-quantum
|
|
run: |
|
|
set -euo pipefail
|
|
cache=/ci-cache/wai-quantum-hexagon
|
|
exec 9>"$cache/.lock"
|
|
flock 9
|
|
find ../.. -type f -exec touch {} +
|
|
export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target-host"
|
|
export PATH="$CARGO_HOME/bin:$PATH"
|
|
cargo test --locked -j 6 --features quantum_fabric_dsp --lib -- quantum_fabric_dsp:: 2>&1 | tee "$RUNNER_TEMP/host-unit.log"
|
|
grep -qE '^ *Compiling wai-quantum v[0-9]' "$RUNNER_TEMP/host-unit.log" || { echo "::error::the build compiled no wai-quantum: its tests ran a binary already in the cache"; exit 1; }
|
|
grep -qE '^test result: ok\. [1-9][0-9]* passed; 0 failed' "$RUNNER_TEMP/host-unit.log" || { echo "::error::no unit test ran"; exit 1; }
|
|
report="$RUNNER_TEMP/dsp-absent.txt"; : > "$report"
|
|
WAI_QUANTUM_DSP_EXPECT=none WAI_QUANTUM_DSP_REPORT="$report" cargo test --locked -j 6 --features quantum_fabric_dsp --test dsp_member -- --test-threads=1 2>&1 | tee "$RUNNER_TEMP/host-device.log"
|
|
grep -qE '^test result: ok\. 9 passed; 0 failed' "$RUNNER_TEMP/host-device.log" || { echo "::error::not every device test ran and passed"; exit 1; }
|
|
grep -qx 'dsp devices: as declared (none)' "$report" || { echo "::error::the absent case was not established"; exit 1; }
|