pax_global_header00006660000000000000000000000064152523254550014522gustar00rootroot0000000000000052 comment=58bda8798c2735cadb5aa36695bab9a5e5e2fbd5 myguard-labs-nginx-zstd-module-3486b5b/000077500000000000000000000000001525232545500200365ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/.clang-format000066400000000000000000000016561525232545500224210ustar00rootroot00000000000000# nginx coding style for C modules # Based on nginx source conventions: 4-space indent, 80-col limit, # aligned struct members and macros (nginx hallmark style). BasedOnStyle: LLVM IndentWidth: 4 TabWidth: 4 UseTab: Never ColumnLimit: 80 BreakBeforeBraces: Linux AllowShortFunctionsOnASingleLine: None AllowShortIfStatementsOnASingleLine: Never AllowShortLoopsOnASingleLine: false IndentCaseLabels: false SpaceBeforeParens: ControlStatements PointerAlignment: Right AlignConsecutiveDeclarations: Enabled: true AcrossEmptyLines: false AcrossComments: false AlignConsecutiveMacros: Enabled: true AcrossEmptyLines: false AcrossComments: false AlignConsecutiveAssignments: false AlignTrailingComments: Kind: Always OverEmptyLines: 0 SortIncludes: false ReflowComments: false BreakBeforeTernaryOperators: true AlwaysBreakAfterReturnType: None SpacesInParentheses: false SpacesInSquareBrackets: false SpaceInEmptyParentheses: false myguard-labs-nginx-zstd-module-3486b5b/.editorconfig000066400000000000000000000003711525232545500225140ustar00rootroot00000000000000root = true [*] end_of_line = lf insert_final_newline = true charset = utf-8 trim_trailing_whitespace = true indent_style = space indent_size = 4 [*.{sh,bash}] indent_style = space indent_size = 4 switch_case_indent = true binary_next_line = true myguard-labs-nginx-zstd-module-3486b5b/.gitattributes000066400000000000000000000034471525232545500227410ustar00rootroot00000000000000*.t linguist-language=Text # Files that are executed or sourced by POSIX sh/perl on Linux (including # WSL builds from a Windows checkout) must stay LF: nginx's ./configure # sources `config`/`auto/zstd` with sh, which treats a CR as part of the # line and fails with "$'\r': command not found" when core.autocrlf=true # has smudged the checkout to CRLF. config text eol=lf auto/* text eol=lf */config text eol=lf *.sh text eol=lf *.t text eol=lf # Byte-exact fixtures: fuzz corpus entries are raw Accept-Encoding header # bytes and .zst files are compressed frames — any EOL conversion corrupts # them. ci/fuzz/corpus/* -text ci/fuzz/fuzz.dict -text *.dict -text *.zst binary # t/suite/test's byte length is asserted verbatim by the static suite # (Content-Length and the size half of the ETag) — CRLF smudging changes # the on-disk size and fails every identity-response assertion. ci/t/suite/* -text # More byte-exact fuzz fixtures: corpus_dcz/* are raw dcz-dict header # bytes, and regressions{,_dcz}/* are saved crash-reproducer inputs — # core.autocrlf=true would smudge either and corrupt the fuzzer's inputs # or make a saved crash stop reproducing. Each regressions dir also has a # real README.md, which must stay text; order it after the wildcard so # the more specific rule wins. ci/fuzz/corpus_dcz/* -text ci/fuzz/regressions/* -text ci/fuzz/regressions/README.md text ci/fuzz/regressions_dcz/* -text ci/fuzz/regressions_dcz/README.md text # CI-only stub module (linkage-isolation job); keep it out of release # tarballs (git archive). Its former sibling, the vary-suppression # stub, no longer exists in the tree. /ci/tools/ci-linkcheck-module export-ignore # Patches are applied with git apply against LF checkouts; CRLF # smudging would make every context line mismatch. ci/tools/patches/** text eol=lf myguard-labs-nginx-zstd-module-3486b5b/.githooks/000077500000000000000000000000001525232545500217435ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/.githooks/pre-commit000077500000000000000000000061551525232545500237540ustar00rootroot00000000000000#!/usr/bin/env bash # .githooks/pre-commit -- tracked, reviewable commit gate. # # Lives in the repo (unlike .git/hooks/, which is per-clone and invisible to # review), so a change to what the gate checks arrives as a diff. # # Enable once per clone: # git config core.hooksPath .githooks # Emergency bypass: # git commit --no-verify # # It lints only the STAGED files, so a commit is never blocked by a finding in # a file it does not touch. Run the whole tree yourself with: # ci/linter/run-all.sh # pre-commit run --all-files # # TWO GATES, ONE ENTRY POINT, AND THE REASON THIS FILE CALLS BOTH. # # `core.hooksPath` REPLACES .git/hooks/ -- git does not consult both. So the # hook that `pre-commit install` writes into .git/hooks/pre-commit becomes # unreachable the moment core.hooksPath is set, and .pre-commit-config.yaml # stops running on commit while still looking installed from every angle a # human checks: the config is present, `pre-commit install` reported success, # and the file is right there in .git/hooks/. Measured 2026-08-02 in this repo: # a file with trailing whitespace and no closing newline committed clean, past # hooks whose entire job is those two things -- and past detect-private-key, # flawfinder and semgrep with it. # # Neither list is a superset of the other (ci/linter/ owns the workflow-policy # and docs-drift checks; .pre-commit-config.yaml owns secret detection and the # C SAST gates), so this hook runs BOTH and fails on either. # # Exit 2 (a tool is not installed) blocks the commit on purpose: a gate that # skips itself when its tool is missing reports green while checking nothing. # That applies to pre-commit itself, which is why its absence is fatal here # rather than a warning. # # VERIFY BEFORE TRUSTING (this gate was green and inert for days): # printf 'probe \n\nno newline at eof' > _p.txt # git add _p.txt && git commit -m probe -- _p.txt # expect BLOCKED # git reset -q HEAD _p.txt; rm -f _p.txt # Both `trim trailing whitespace` and `fix end of files` must report Failed. If # the commit lands, .pre-commit-config.yaml is not being consulted. set -uo pipefail ROOT="$(git rev-parse --show-toplevel)" "$ROOT/ci/linter/run-all.sh" --staged rc=$? if [ "$rc" -eq 2 ]; then echo "pre-commit: missing linters -- run ci/linter/install-linters.sh" >&2 exit 2 elif [ "$rc" -ne 0 ]; then echo "pre-commit: lint failed -- fix, or commit with --no-verify" >&2 exit "$rc" fi if [ -f "$ROOT/.pre-commit-config.yaml" ]; then if ! command -v pre-commit >/dev/null 2>&1; then echo "pre-commit: .pre-commit-config.yaml present but pre-commit is not" \ "installed -- pipx install pre-commit" >&2 exit 2 fi # No `pre-commit install` needed and none is wanted: that command writes to # .git/hooks/, which core.hooksPath has already taken out of the picture. pre-commit run --hook-stage pre-commit rc=$? if [ "$rc" -ne 0 ]; then echo "pre-commit: .pre-commit-config.yaml hooks failed -- fix, or commit" \ "with --no-verify. Fixer hooks may have rewritten files; re-stage." >&2 exit "$rc" fi fi exit 0 myguard-labs-nginx-zstd-module-3486b5b/.github/000077500000000000000000000000001525232545500213765ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/.github/actionlint.yaml000066400000000000000000000000671525232545500244310ustar00rootroot00000000000000self-hosted-runner: labels: [builder02, lxc, docker] myguard-labs-nginx-zstd-module-3486b5b/.github/scripts/000077500000000000000000000000001525232545500230655ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/.github/scripts/configure-msvc.sh000066400000000000000000000012621525232545500263510ustar00rootroot00000000000000#!/bin/bash # Configure the Windows/MSVC CI build after vcvars64.bat has initialized the # parent cmd process. Kept in a real bash file because a GitHub custom `cmd` # shell wraps inline run blocks in a .cmd file, which bash cannot parse. set -euo pipefail module_dir=$(cygpath -m "$GITHUB_WORKSPACE") zstd_root=$(cygpath -m \ "$VCPKG_INSTALLATION_ROOT/installed/x64-windows-static") ZSTD_INC="$zstd_root/include" ZSTD_LIB="$zstd_root/lib" \ ./configure \ --crossbuild=win32 \ --with-cc=cl \ --with-cc-opt='-MT -DFD_SETSIZE=1024' \ --with-pcre="$module_dir/pcre2-$PCRE2_VERSION" \ --with-zlib="$module_dir/zlib-$ZLIB_VERSION" \ --add-module="$module_dir" myguard-labs-nginx-zstd-module-3486b5b/.github/workflows/000077500000000000000000000000001525232545500234335ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/.github/workflows/arch-package.yml000066400000000000000000000017221525232545500264660ustar00rootroot00000000000000--- # yamllint disable rule:line-length rule:truthy name: Arch package on: workflow_dispatch: {} workflow_call: {} concurrency: group: arch-package-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true permissions: contents: read env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true jobs: package: name: Build package and checkout runs-on: ubuntu-latest timeout-minutes: 20 steps: - name: Checkout module uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Build and test Arch package and checkout run: packaging/arch/build.sh - name: Upload Arch package uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: nginx-mod-zstd-release-arch-package path: packaging/arch/dist/*.pkg.tar.zst if-no-files-found: error retention-days: 7 myguard-labs-nginx-zstd-module-3486b5b/.github/workflows/asan.yml000066400000000000000000000126431525232545500251060ustar00rootroot00000000000000name: A/UBSan # Dedicated ASan+UBSan soak — mirrors the fast build-asan job in build-test.yml # (same configure flags, same PGP-verified tarball) but drives it through # soak.sh's sustained mixed-load traffic instead of the Test::Nginx suite, the # same way valgrind.yml's memcheck-lite job soaks the debug build. Two # independent dynamic-analysis lenses on the same binary shape catch different # things: ASan/UBSan is fast enough to run a real request storm in CI Deep # and catches heap overflow / UAF outside nginx's pool blocks; Valgrind (20x # slower) still catches the uninitialised reads and intra-pool UAF that ASan # cannot see. Complementary, not redundant -- keep both. # # The asan build is a STATIC --add-module binary (the module is linked into # the nginx binary, there is no separate .so), so the sanitizer runtime is # present on every code path soak.sh exercises. # # Action pins (keep in sync with build-test.yml / valgrind.yml headers): # actions/checkout@v5.1.0 -> fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true NGINX_VERSION: "" on: workflow_dispatch: {} # Called by ci.yml, the single PR entry point. workflow_call: {} concurrency: group: asan-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true permissions: contents: read jobs: asan-soak: name: ASan/UBSan soak (60s) runs-on: ${{ fromJSON(vars.POOL || '["ubuntu-latest"]') }} timeout-minutes: 45 steps: - name: Checkout module uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false # `vars.POOL` is absent in forks, so do not rely on the hosted image # carrying curl before the full dependency installation below. - name: Install bootstrap dependencies run: | sudo apt-get update sudo apt-get install -y ca-certificates curl python3 - name: Resolve latest mainline nginx env: GH_TOKEN: ${{ github.token }} run: | set -euo pipefail ver=$(bash "$GITHUB_WORKSPACE/ci/tools/nginx-releases.sh" mainline) [ -n "$ver" ] || { echo "::error::cannot resolve nginx version"; exit 1; } echo "NGINX_VERSION=$ver" >> "$GITHUB_ENV" - name: Install dependencies run: | # builder02 apt profile: loose timeouts — the box runs jobs # concurrently and never talks to the Azure mirror (PR #121) printf '%s\n' \ 'DPkg::Lock::Timeout "120";' \ 'Acquire::Retries "3";' \ 'Acquire::http::Timeout "20";' \ 'Acquire::https::Timeout "20";' \ | sudo tee /etc/apt/apt.conf.d/99-ci-net > /dev/null sudo apt-get update sudo apt-get install -y \ build-essential curl gnupg libpcre2-dev libzstd-dev openssl pkg-config \ wget zlib1g-dev zstd - name: Cache nginx source tarball id: nginx-tarball uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: nginx-${{ env.NGINX_VERSION }}.tar.gz key: nginx-src-${{ env.NGINX_VERSION }} - name: Download nginx source tarball if: steps.nginx-tarball.outputs.cache-hit != 'true' run: wget -q "https://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz" - name: Verify nginx tarball PGP signature run: | set -euo pipefail wget -q "https://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz.asc" \ -O "nginx-${NGINX_VERSION}.tar.gz.asc" gnupghome="$(mktemp -d)" export GNUPGHOME="$gnupghome" chmod 700 "$gnupghome" for keyfile in ci/tools/keys/*.key; do gpg --quiet --import "$keyfile" 2>/dev/null done gpg --quiet --verify "nginx-${NGINX_VERSION}.tar.gz.asc" "nginx-${NGINX_VERSION}.tar.gz" rm -rf "$gnupghome" # soak.sh requires ok>0 and bad==0 per worker, where a zstd-magic # response must ALSO decode cleanly (ci/tools/soak.sh's worker()): # a green result proves the module actually compressed real # responses under the sanitizer, not merely that nothing crashed. # It sets its own ASAN_OPTIONS/UBSAN_OPTIONS with halt_on_error=1 # so the soak fails loudly on the first report. - name: Configure & build nginx with ASAN + UBSAN (static --add-module) env: # Keep the portable SHA-256 fallback covered under the sanitizer -- # see build-test.yml's identical build-asan job for why. NGX_ZSTD_NO_LIBCRYPTO: 1 run: | tar -xzf "nginx-${NGINX_VERSION}.tar.gz" cd "nginx-${NGINX_VERSION}" SAN="-fsanitize=address,undefined -fno-sanitize-recover=undefined -fno-omit-frame-pointer -g3 -O1" ./configure \ --with-compat \ --with-debug \ --with-cc-opt="-DZSTD_STATIC_LINKING_ONLY $SAN $(pkg-config --cflags libzstd zlib libpcre2-8)" \ --with-ld-opt="$SAN $(pkg-config --libs libzstd zlib libpcre2-8)" \ --add-module="$GITHUB_WORKSPACE" make -j"$(nproc)" if grep -q 'NGX_HTTP_ZSTD_HAVE_LIBCRYPTO' objs/ngx_auto_config.h; then echo "libcrypto leaked into the portable-fallback build"; exit 1 fi - name: ASan/UBSan soak (60s @ 4 conc) run: | bash ci/tools/soak.sh \ "$GITHUB_WORKSPACE/nginx-${NGINX_VERSION}/objs/nginx" 60 4 myguard-labs-nginx-zstd-module-3486b5b/.github/workflows/build-test.yml000066400000000000000000003521661525232545500262470ustar00rootroot00000000000000name: Build & Test # Third-party actions are pinned to commit SHAs for supply-chain safety. # This pin table is the canonical reference for ALL workflow files # (build-test.yml, codeql.yml, security-scanners.yml, fuzzing.yml) — # keep them consistent. # actions/checkout@v5.1.0 -> fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # actions/cache@v5.1.0 -> caa296126883cff596d87d8935842f9db880ef25 # actions/upload-artifact@v6.0.0 -> b7c566a772e6b6bfb58ed0dc250532a479d7789f # actions/download-artifact@v7.0.0 -> 37930b1c2abaa49bbe596cd826c3c89aef350131 # github/codeql-action/*@v4 -> 8aad20d150bbac5944a9f9d289da16a4b0d87c1e env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true # NGINX_VERSION is NOT pinned here. The `resolve` job reads the current # mainline release from the GitHub releases feed (ci/tools/nginx-releases.sh) # and feeds it via needs.resolve.outputs.*; per-job env: blocks below # override this placeholder with the resolved value. NGINX_VERSION: "" # Single source of truth for every `cpanm ... Test::Nginx::Socket` call # site in this file (keep in sync with ci-deep.yml's own copy). An # unpinned `cpanm Test::Nginx::Socket` resolves to whatever CPAN serves # at run time -- a cold runner can silently change test behaviour with # no repository diff, and a warm cache can retain a different historical # release, so two runners can disagree with nothing in git explaining # it. Pinned to 0.32 because that is what every install site already # resolved to as of this pin (verified against a recent successful CI # run's "Install Test::Nginx::Socket" log: "Test::Nginx::Socket is up to # date. (0.32)"), which is also CPAN's current latest release -- this # pin is a no-op in behaviour, not a dependency bump. TEST_NGINX_SOCKET_VERSION: "0.32" TEST_NGINX_SOCKET_DIST: "AGENT/Test-Nginx-0.32.tar.gz" # The `resolve` job reads the current mainline release from the GitHub # releases feed at run time and feeds it to every job via `needs.resolve.outputs.nginx_version` # (and the build matrix JSON). This makes the weekly cron actually test the # module against new releases as they ship, instead of failing once nginx.org # drops an old tarball. on: schedule: # Weekly Monday 04:17 UTC — catches nginx API drift against new releases # even when nothing is pushed. - cron: "17 4 * * 1" workflow_dispatch: {} # Called by ci.yml, the single PR entry point (step 15). workflow_call: {} # Cancel superseded runs on the same ref instead of piling up runner minutes. concurrency: group: ci-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true # Least-privilege default token. Jobs that need more request it explicitly. permissions: contents: read jobs: # ── Resolve the current mainline nginx version once per run ─────────────── # Reads the GitHub releases feed and exposes a bare version string and a single-entry # build-matrix JSON (nginx mainline only). Every other job consumes # needs.resolve.outputs.* so the whole run agrees on one version. # nginx.org only keeps the newest mainline tarball around, so resolving # at run time is what keeps CI from 404ing after a release. resolve: name: Resolve nginx version runs-on: ${{ fromJSON(vars.POOL || '["ubuntu-latest"]') }} timeout-minutes: 5 outputs: nginx_version: ${{ steps.v.outputs.nginx_version }} matrix: ${{ steps.v.outputs.matrix }} steps: # The resolver is a repo script, so this job checks out too. - name: Checkout uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false # `vars.POOL` is unavailable in forks; curl and python3 are needed by # the very next step, before any job-specific package installation # can run. - name: Install bootstrap dependencies run: | sudo apt-get update sudo apt-get install -y ca-certificates curl python3 - name: Resolve latest mainline nginx env: GH_TOKEN: ${{ github.token }} id: v run: | set -euo pipefail ver=$(bash "$GITHUB_WORKSPACE/ci/tools/nginx-releases.sh" mainline) if [ -z "$ver" ]; then echo "::error::could not resolve the mainline nginx release (see the log)" exit 1 fi echo "Resolved nginx mainline: $ver" matrix=$(printf 'matrix={"include":[{"flavor":"nginx","version":"%s","url":"https://nginx.org/download/nginx-%s.tar.gz","dir":"nginx-%s"}]}' \ "$ver" "$ver" "$ver") { echo "nginx_version=$ver" echo "$matrix" } >> "$GITHUB_OUTPUT" # ── Lint & static analysis — no nginx build, fully independent ──────────── validation: name: Validation runs-on: ${{ fromJSON(vars.POOL || '["ubuntu-latest"]') }} timeout-minutes: 15 steps: - name: Checkout module uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false # actionlint is downloaded in the next step, so install curl before it. - name: Install bootstrap dependencies run: | sudo apt-get update sudo apt-get install -y ca-certificates curl python3 # Lint the workflow files themselves. actionlint validates GitHub # Actions expression contexts (e.g. "env is not allowed in a job # name:") that a plain YAML parser cannot catch — exactly the class # of error that previously took the whole pipeline down. Runs first # so a broken workflow fails fast and obviously. - name: Lint workflows (actionlint) run: | set -euo pipefail ACTIONLINT_VERSION=1.7.7 # Pinned sha256 for the linux_amd64 release asset, computed once # from an HTTPS fetch and cross-checked against the checksums.txt # rhysd publishes alongside each release -- same "reviewed pin, # verify before use" shape as ci/tools/ci-build.sh's NGINX_SHA256 / # ANGIE_SHA256 tables. A version bump is a reviewed PR that updates # this hash, not a runtime fetch of a fresher one. ACTIONLINT_SHA256=023070a287cd8cccd71515fedc843f1985bf96c436b7effaecce67290e7e0757 curl -fsSL -o /tmp/actionlint.tgz \ "https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}/actionlint_${ACTIONLINT_VERSION}_linux_amd64.tar.gz" actual="$(sha256sum /tmp/actionlint.tgz | awk '{print $1}')" if [ "$actual" != "$ACTIONLINT_SHA256" ]; then echo "✗ sha256 mismatch for actionlint_${ACTIONLINT_VERSION}_linux_amd64.tar.gz" >&2 echo " expected: $ACTIONLINT_SHA256" >&2 echo " actual: $actual" >&2 rm -f /tmp/actionlint.tgz exit 1 fi echo "✓ sha256 verified for actionlint_${ACTIONLINT_VERSION}_linux_amd64.tar.gz" tar -xzf /tmp/actionlint.tgz -C /tmp actionlint /tmp/actionlint -color "$GITHUB_WORKSPACE"/.github/workflows/*.yml echo "✓ actionlint: workflows valid" - name: Cache apt packages uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: # User-owned dir, not the system /var/cache/apt/archives: apt runs # under sudo, so the post-step tar (unprivileged runner user) cannot # read the root-owned archives/lock or archives/partial (c2). # # Glob the .deb files rather than the directory: apt recreates # `partial/` (0700 _apt:root) and `lock` (root:root) INSIDE whatever # Dir::Cache::archives points at, so caching the dir itself still # gives tar two unreadable entries and it exits 2 -> "Failed to # save" (a warning, so CI stays green while nothing is ever cached). # Only the .deb files are worth caching anyway. path: ${{ runner.temp }}/apt-archives/*.deb key: apt-validation-${{ runner.os }}-shellcheck-cppcheck-clang-tools-nginx-dev-gcc-multilib restore-keys: apt-validation-${{ runner.os }}- - name: Install dependencies run: | # apt profile the bare apt-get lines below rely on: loose timeouts # for the shared self-hosted box; Azure mirror only on the # ubuntu-latest fallback when vars.POOL is unset (PR #121) mkdir -p "$RUNNER_TEMP/apt-archives" printf '%s\n' \ 'DPkg::Lock::Timeout "120";' \ 'Acquire::Retries "3";' \ 'Acquire::http::Timeout "20";' \ 'Acquire::https::Timeout "20";' \ "Dir::Cache::archives \"$RUNNER_TEMP/apt-archives\";" \ | sudo tee /etc/apt/apt.conf.d/99-ci-net > /dev/null sudo apt-get update sudo apt-get install -y \ build-essential \ curl \ git \ gcc-multilib \ libpcre2-dev \ zlib1g-dev \ shellcheck \ cppcheck \ clang-tools \ python3 \ python3-yaml \ nginx-dev - name: Validate tool scripts (syntax only) run: | cd "$GITHUB_WORKSPACE/ci/tools" bash -n ci-build.sh echo "✓ All tool scripts: syntax OK" - name: Python tool validation run: | cd "$GITHUB_WORKSPACE/ci/tools" python3 test_encoding.py --help | head -3 echo "✓ Python tools validated" - name: Python unit tests (test harness self-checks) run: | cd "$GITHUB_WORKSPACE/ci/tools" python3 -m unittest -v \ test_test_encoding \ test_test_package_artifact \ test_perf_stat_recipe # Cheapest test layer over the real pure decision functions -- the # Accept-Encoding parser and the .zst frame-header probe: no nginx # build tree, no network, sub-second. Links the shipped # src/ngx_http_zstd_common.h via ci/fuzz/extract_parser.sh, and the # frame probe by including src/ngx_http_zstd_frame_probe.h directly # (#270 -- the probe's authoritative copy is a header now, so there # is no extraction left to go stale). - name: Pure-function unit tests (Accept-Encoding parser + frame probe) run: | bash "$GITHUB_WORKSPACE/ci/tests/unit/run.sh" echo "✓ Python harness unit tests passed" # Audit sha e289021 F1/F7 regression coverage: no nginx build needed, # so these run here rather than in the build/tests jobs. - name: fuzz_seconds dispatch-input validation (F1 regression) run: bash "$GITHUB_WORKSPACE"/ci/tools/test_fuzz_seconds_validation.sh - name: README/SECURITY/CONTRIBUTING vs live workflows (F7 regression) run: bash "$GITHUB_WORKSPACE"/ci/tools/test_docs_ci_drift.sh - name: Fork-runner dependency bootstrap regression run: bash "$GITHUB_WORKSPACE"/ci/tools/test_ci_dependency_bootstrap.sh - name: Scanner selectors and fuzz-boundary regressions run: bash "$GITHUB_WORKSPACE"/ci/tools/test_ci_scanner_fuzz_boundaries.sh - name: Soak schedule and max-length diagnostic contracts run: | bash "$GITHUB_WORKSPACE"/ci/tools/test_soak_schedule_unit.sh bash "$GITHUB_WORKSPACE"/ci/tools/test_max_length_diagnostics.sh # Audit sha e289021 F2/F3 regression coverage: exercises ci-build.sh's # mv-guard and provenance checks directly (offline cases + one live # clean-root angie build). - name: ci-build.sh clean-root + provenance regressions (F2/F3) run: bash "$GITHUB_WORKSPACE"/ci/tools/test_ci_build_provenance.sh - name: Persistent build-input invalidation regressions run: bash "$GITHUB_WORKSPACE"/ci/tools/test_build_input_manifest.sh - name: Build configuration policy regressions run: bash "$GITHUB_WORKSPACE"/ci/tools/test_build_config_policy.sh - name: Windows build policy regressions run: | bash "$GITHUB_WORKSPACE"/ci/tools/test_build_windows_zstd_default.sh bash "$GITHUB_WORKSPACE"/ci/tools/test_build_windows_inputs.sh bash "$GITHUB_WORKSPACE"/ci/tools/test_build_windows_openssl_recovery.sh - name: Frame-probe header seam regressions run: bash "$GITHUB_WORKSPACE"/ci/tools/test_probe_header_seam.sh - name: nginx tarball PGP verifier controls run: bash "$GITHUB_WORKSPACE"/ci/tools/test_verify_nginx_tarball.sh - name: nginx-module-testkit scenario runner controls run: bash "$GITHUB_WORKSPACE"/ci/tools/test_run_testkit_scenarios.sh - name: .gitattributes byte-exactness and archive-exclusion regressions (A30-F5) run: bash "$GITHUB_WORKSPACE"/ci/tools/test_gitattributes.sh - name: coverage.sh stale-prober rebuild guard regression (A30-F7) run: bash "$GITHUB_WORKSPACE"/ci/tools/test_coverage_prober_rebuild.sh - name: bump-versions.sh pin-coverage and fail-loud regressions (A30-F4) run: bash "$GITHUB_WORKSPACE"/ci/tools/test_bump_versions.sh - name: nginx-releases.sh resolver controls run: bash "$GITHUB_WORKSPACE"/ci/tools/test_nginx_releases.sh - name: Shell script linting (shellcheck) run: | # Gate on error-severity findings (real bugs); style/info noise does # not fail CI. The previous blanket "|| true" is dropped so a real # error-level finding now fails the job. See audit CI1. shellcheck --severity=error "$GITHUB_WORKSPACE"/ci/tools/*.sh echo "✓ shellcheck complete (no error-severity findings)" - name: SHA-256 unit fixture (portable vectors + EVP failure injection) run: | # Issue #100 item 3: the EVP-to-portable fallback has no runtime # coverage in the nginx-level jobs (mainline runs EVP success, # the sanitizer/coverage builds preprocess the wrapper out). # This fixture compiles ngx_http_zstd_sha256.h against shim # headers with an injectable fake EVP_Digest — needs only a C # compiler. bash "$GITHUB_WORKSPACE"/ci/tools/test_sha256_unit.sh - name: ceil_log2 unit fixture (dcz window-log helper equivalence) run: | # grind-g6-nits: ngx_http_zstd_ceil_log2() (dcz window-log sizing) # was shipped with an inverted power-of-two test and fixed in # c264f4b. static inside the filter module, so this extracts the # function verbatim by line range and checks it against a linear # shift-and-count reference over 0..2^24 (every power of two # through 2^23 included) — needs only a C compiler. bash "$GITHUB_WORKSPACE"/ci/tools/test_ceil_log2_unit.sh - name: directio probe log rate limiter unit fixture (A33-F4(b)) run: | # PR #314 review: the directio hard-error probe arm in # ngx_http_zstd_static_module.c logged NGX_LOG_ERR on every # request for a misaligned-directio sidecar. static inside the # static module, so this extracts # ngx_http_zstd_static_dio_err_should_log() verbatim by line # range and exercises it against a fake, test-controlled clock — # needs only a C compiler. bash "$GITHUB_WORKSPACE"/ci/tools/test_dio_err_ratelimit_unit.sh - name: CCtx profile key unit fixture (collision-free + reversible) run: | # The worker CCtx ring partitions slots on a packed 64-bit key # built from (level, long_mode, window_log). The key must be # collision-free and reversible, NOT a hash: two profiles sharing # a key would share a CCtx and silently change bytes on the wire. # Extracts the packer, unpacker and their layout defines verbatim # from the module and sweeps the full in-domain tuple space, # asserting reversibility (hence injectivity) plus fail-closed # refusal of out-of-domain values — needs only a C compiler. bash "$GITHUB_WORKSPACE"/ci/tools/test_cctx_profile_pack.sh - name: dcz dict lookup unit fixture (linear/bsearch equivalence) run: | # A33-P4: the dcz dictionary negotiation lookup switches from a # linear ngx_memcmp scan to a binary search over a hash-sorted # array once the configured dictionary count exceeds # NGX_HTTP_ZSTD_DCZ_BSEARCH_THRESHOLD. Extracts the lookup, its # sort comparator and the dict struct verbatim from the module # and differentially tests both branches against a brute-force # linear oracle across sizes straddling the threshold, hits on # the first/middle/last element, and misses sorting before, # after and within the configured range — needs only a C # compiler. bash "$GITHUB_WORKSPACE"/ci/tools/test_dcz_dict_lookup_unit.sh - name: dcz window-log unit fixture (clamps + range) env: # gcc-multilib is installed above, so a missing -m32 toolchain # here means the runner image regressed, not that multilib is # optionally absent -- fail loudly instead of silently losing # the 32-bit off_t lens (see the scripts' own comments). REQUIRE_M32: "1" run: | # ngx_http_zstd_dcz_window_log() is the single source of truth for # a value derived in TWO places that must never disagree: the CCtx # ring key and the windowLog pushed into libzstd. Extracts it (and # ngx_http_zstd_ceil_log2(), its dependency) verbatim and sweeps # every dictionary size across the power-of-two boundaries x both # pledged-size branches x every value of both operator ceilings, # asserting the default is unchanged when neither is set, that a # ceiling only ever lowers the window, that the lower of the two # wins, and that the result stays inside the 6-bit window_log # field the profile packer refuses to mask. Needs only a C compiler. bash "$GITHUB_WORKSPACE"/ci/tools/test_dcz_window_log_unit.sh # ngx_http_zstd_dict_file_read(): the shared read-to-completion # loop both dictionary loaders use, included from its header, # plus the module's logging shell around it extracted verbatim. # Drives both against a scripted ngx_read_fd stub, because the # three behaviours that matter -- a short count is resumed, EINTR # is retried without losing progress, and early EOF is fatal -- # are not reachable from a config-level fixture on a local # filesystem (see the note in test_dict_path_hardening.sh). # Needs only a C compiler. bash "$GITHUB_WORKSPACE"/ci/tools/test_read_dict_file_unit.sh # ngx_http_zstd_dict_file_open_strict(): the strict-mode # component-by-component dictionary open, included from its # header and driven against scripted open/openat/fstat/geteuid # fakes. Reaches the exits the config-level matrix cannot stage # (open("/") failing, fstat() failing mid-walk, a foreign-owned # ancestor without root, a component of NGX_MAX_PATH bytes) and # pins the openat() flag words, errno-before-close, and that # every fd the walk opened is closed exactly once. Also checks # that every refusal code has a diagnostic in the module. # Needs only a C compiler. bash "$GITHUB_WORKSPACE"/ci/tools/test_dict_walk_unit.sh # ngx_http_zstd_max_length_exceeded(): the length-independent # zstd_max_length cap checked per iteration of the streaming body # filter. Extracts the predicate verbatim and asserts an unset cap # never fires, a body exactly at the cap does not, one byte over # does, and a bytes_in far past INT32_MAX still compares correctly # against a small cap. That last case only diverges from the # historical signed-cast form on a genuine 32-bit off_t build # (plain -m32, no _FILE_OFFSET_BITS override); a native 64-bit # build cannot reach it, so the -m32 pass is not optional # coverage. Needs only a C compiler. bash "$GITHUB_WORKSPACE"/ci/tools/test_max_length_cap_unit.sh # The normal body-filter path walks caller-owned chain links through # a local cursor and allocates wrappers only for a tail retained # across callbacks. Extracts the shipped cursor/copy helpers and # pins fully drained zero-allocation, partial/backpressured input, # retained-before-new ordering, many links, flush/last carriers and # lazy-copy allocation failure without mutating caller links. bash "$GITHUB_WORKSPACE"/ci/tools/test_input_chain_cursor_unit.sh - name: pledged-length compressBound guard unit fixture (ILP32/LFS) env: # gcc-multilib is installed above, so a missing -m32 toolchain # here means the runner image regressed, not that multilib is # optionally absent. The ILP32 pass is the ONLY one that can # observe this defect at all, so losing it silently would leave a # green tick over untested code -- fail loudly instead. REQUIRE_M32: "1" run: | # A30-N3: get_buf() sizes the FIRST output buffer from the pledged # body length (r->headers_out.content_length_n, an off_t) via # ZSTD_compressBound() (which takes a size_t). On an ILP32 build # with large-file support those are different widths, so a bare # cast truncates modulo 2^32: a 4 GiB + 389 pledge becomes 389 and # the first buffer is allocated ~389 bytes for a 4 GiB body. # ngx_http_zstd_pledge_to_bound_input() decides representability # and the ZSTD_MAX_INPUT_SIZE ceiling at full off_t width before # any narrowing. Extracted verbatim and driven against an # independent uintmax_t oracle over hand-picked boundaries # (SIZE_MAX -1/+0/+1, ZSTD_MAX_INPUT_SIZE -1/+0/+1, 4 GiB, 4 GiB + # 389), every power of two +/- 1, and a deterministic randomized # sweep. Compiled twice -- natively, and under # -m32 -D_FILE_OFFSET_BITS=64, the shape that reproduces the # truncation. Header-only (uses zstd.h's ZSTD_COMPRESSBOUND macro), # so the 32-bit pass needs no 32-bit libzstd. bash "$GITHUB_WORKSPACE"/ci/tools/test_pledge_bound_input_unit.sh - name: $zstd_ratio scaling unit fixture (#294, against the shared header) run: | # ngx_http_zstd_ratio_parts() against an unsigned __int128 oracle: # the boundary table, the near-UINT64_MAX and expanding-stream # witnesses, a magnitude sweep and a seeded random sweep. The # fixture includes src/ngx_http_zstd_ratio.h directly, and the # script pins that $zstd_ratio still routes through it and that # the header stays C89-clean. bash "$GITHUB_WORKSPACE"/ci/tools/test_ratio_scaling_unit.sh - name: soak.sh ASAN/UBSAN verdict unit fixture (A27-F9) run: | # A27-F9: ci/tools/soak.sh's ASan/UBSan check used to fail on # whether "$WORK/logs/asan*" merely EXISTED, never reading its # content -- so LeakSanitizer's own "ptrace appears to be # blocked" startup diagnostic (this runner pool's LXC # seccomp/yama profile blocks its exit-time tracer; see # test_reload_leak.sh and the smoke-tests step above for the # same signature) failed the soak exactly like a real finding. # Observed in CI: PR #214 run 33117464973 job "A/UBSan" attempt # 1 failed on pure ptrace noise with all four soak workers # clean; attempt 2 (identical build) passed. # ci/tools/soak_asan_verdict.sh now extracts the fixed triage # (real finding -> fail, ptrace-blocked -> INDETERMINATE warn, # no log/no finding -> pass) into one function soak.sh sources, # and this fixture drives it directly against ptrace-only, # real-ASan, real-UBSan and absent-log cases, plus a negative # control proving the original ls-only gate fails on the # ptrace-only fixture. No nginx, no libzstd -- pure Bash. bash "$GITHUB_WORKSPACE"/ci/tools/test_soak_asan_verdict_unit.sh - name: LSan positive control unit fixture (smoke-tests clean-vs-could-not-run) run: | # The "Run smoke tests under ASAN+UBSAN" step's leak verdict # already told a real finding from a ptrace-blocked log (the # soak_asan_verdict() shape, A27-F9), but a log with NEITHER # signature only means "no finding was written" -- it does not # prove LSan's exit-time check ran at all. Observed in CI: run # 33084048220 ("Tests ASAN+UBSAN" job) printed "All smoke tests # clean under ASAN+UBSAN" from a /tmp/lsan-smoke.* log with # neither shape, and SECONDS later in the SAME job the # zstd_dict_file reload-leak step hit the ptrace-blocked # signature 4/4 -- same runner, same environment. That "clean" # was indistinguishable from "never ran". # ci/tools/lsan_positive_control.sh (extracted from the canary # test_reload_leak.sh already used) now gates the smoke step's # "clean" branch on independently proving a deliberate leak IS # caught here; if the control fails, the step demotes to a # loud, distinct "INDETERMINATE (could not run)" instead of a # silent green. This fixture drives the real if/elif/else + # control shape against finding / ptrace-signature / # clean-but-control-fails / clean-and-control-passes fixtures, # runs the control function for real, and ends with a negative # control proving the pre-fix (no control) shape would have # reported the run-33084048220 case as clean. Needs a working # `cc`; skips (does not fail) if none is on PATH. bash "$GITHUB_WORKSPACE"/ci/tools/test_lsan_positive_control_unit.sh - name: static pread retry/accumulate unit fixture (EINTR, short read, O_DIRECT alignment) run: | # ngx_http_zstd_static_pread(): the static probe's retry/accumulate # loop. Extracts the helper verbatim and drives it against a # scripted pread(2) stub, because EINTR, a legal short read, true # EOF and a hard error are not reachable from ci/t/01-static.t on a # local filesystem -- and neither is the O_DIRECT constraint the # loop's alignment round-down exists to satisfy. The stub asserts # buffer/offset/length alignment on every call, so a continuation # that resumed mid-block fails here instead of on an operator's # directio mount. Needs only a C compiler. bash "$GITHUB_WORKSPACE"/ci/tools/test_static_pread_unit.sh - name: static probe-verdict cache unit fixture (identity key, bound, eviction) run: | # ngx_http_zstd_static_cached_verdict() / # ngx_http_zstd_static_remember(): the worker-local cache that lets # a repeat request skip the frame probe. Extracts both helpers plus # their table verbatim and states the key directly, because a # live-nginx fixture cannot vary st_ino independently of mtime and # size, and cannot tell "the cache hit" apart from "the probe ran # again and agreed". Pins that mtime, size, uniq and path each # invalidate a cached POSITIVE verdict -- the dangerous direction, # where a stale answer would serve an unvalidated sidecar -- and # that the fixed table stays bounded under more distinct files than # slots. Needs only a C compiler. bash "$GITHUB_WORKSPACE"/ci/tools/test_static_verdict_cache_unit.sh - name: static dictionary-bypass unit fixture run: | # ngx_http_zstd_static_should_bypass() must scan until a second # Available-Dictionary line and stand aside only when exactly one # is present, matching the dynamic filter, which refuses dcz # outright on duplicates. A predicate that returned on the first # match handed duplicate-header requests to a filter that then # declined, forfeiting a usable .zst sidecar for identity -- a # client-triggerable path ci/t/01-static.t cannot reach without a # built nginx. Verbatim extraction, so a regression in the shipped # walk fails here. Needs only a C compiler. bash "$GITHUB_WORKSPACE"/ci/tools/test_static_should_bypass_unit.sh - name: static probe read-log policy unit fixture run: | # Nonnegative short reads must never attach stale errno; genuine # failures retain it, and only ordinary failures rise to CRIT. bash "$GITHUB_WORKSPACE"/ci/tools/test_static_probe_log_unit.sh - name: static frame-header probe unit fixture run: | # ngx_http_zstd_static_probe_frame(): pure frame-header arithmetic. # Includes the authoritative shipped header directly and checks the # reserved descriptor bit, a passing small frame and the oversized- # window verdict without standing up nginx. Needs only a C compiler. bash "$GITHUB_WORKSPACE"/ci/tools/test_static_probe_unit.sh - name: A33 P7/P9/P10 bounded-scan performance witnesses run: | # Verbatim production helpers checked against independent behavior # oracles plus deterministic work-count/source-shape controls. This # keeps the micro-optimizations falsifiable without wall-clock noise. bash "$GITHUB_WORKSPACE"/ci/tools/test_perf_micro_witnesses.sh - name: freeCCtxParams six-exit unit fixture (config-time leak-on-error) run: | # ngx_http_zstd_estimate_cctx_memory() runs during "nginx -t" # config load, not on any per-request path, so the HTTP # fault-injection probe cannot reach it. It has six early-return # error exits plus one success exit, and every one of the seven # must call ZSTD_freeCCtxParams(cp) before returning -- a leak on # any single exit would pass CI silently, since the other six # still free correctly. Extracts the function verbatim and drives # it against scripted ZSTD_CCtx_params fakes that force each exit # independently, asserting the free count is exactly 1 (not 0, a # leak; not 2+, a double free) on every one of the seven paths. # Needs only a C compiler. bash "$GITHUB_WORKSPACE"/ci/tools/test_freecctxparams_unit.sh - name: Configure a real nginx source tree (for scan-build headers) env: GH_TOKEN: ${{ github.token }} run: | # scan-build needs the COMPLETE nginx headers — including the # generated objs/ngx_auto_config.h. The nginx-dev package ships # headers but NOT the build system (no auto/configure), so it can # never produce ngx_auto_config.h; the old approach silently failed # and scan-build "passed" on incomplete headers (the CI2 complaint). # Download + configure a real nginx source instead. Verified via # ci/tools/fetch-verified-nginx.sh -- same PGP-against-vendored-key # check the build matrix uses, so this download is no less trusted # than the one that actually ships in the test binary. set -euo pipefail ver=$(bash "$GITHUB_WORKSPACE/ci/tools/nginx-releases.sh" mainline) echo "Using nginx $ver to generate lint headers" "$GITHUB_WORKSPACE/ci/tools/fetch-verified-nginx.sh" "$ver" cd "nginx-${ver}" CC=gcc ./configure --with-compat \ --add-dynamic-module="$GITHUB_WORKSPACE" > /dev/null test -f objs/ngx_auto_config.h echo "NGINX_SRC_TREE=$PWD" >> "$GITHUB_ENV" echo "✓ ngx_auto_config.h generated" - name: Vary fusion differential fixture run: bash "$GITHUB_WORKSPACE/ci/tools/test_vary_fusion.sh" - name: cppcheck (static analysis) run: | set -o pipefail # Gate on real bug categories (warning/performance/portability), not # the full --enable=all style/unusedFunction noise. error-exitcode=1 # fails the job on any such finding; a checked-in suppressions file is # the baseline for accepted exceptions. The blanket continue-on-error # that previously neutered this is removed. See audit CI1. cppcheck --enable=warning,performance,portability \ --error-exitcode=1 \ --suppress=missingIncludeSystem \ --suppressions-list="$GITHUB_WORKSPACE/ci/tools/cppcheck-suppressions.txt" \ "$GITHUB_WORKSPACE/src/ngx_http_zstd_filter_module.c" \ "$GITHUB_WORKSPACE/src/ngx_http_zstd_static_module.c" \ 2>&1 | tee cppcheck.log echo "✓ cppcheck complete (no gated findings)" - name: clang static analyzer run: | set -o pipefail N="$NGINX_SRC_TREE" INCLUDE_FLAGS="-I$N/src/core -I$N/src/event -I$N/src/event/modules \ -I$N/src/event/quic -I$N/src/os/unix -I$N/objs -I$N/src/http \ -I$N/src/http/modules -I$N/src/http/v2 -I/usr/include" # Scan BOTH module sources (the static module was previously # unscanned), against the COMPLETE configured nginx headers, and let # --status-bugs actually fail the job: the old "|| echo" swallowed its # non-zero exit so a finding (or a compile failure on missing headers) # never gated. See audit CI1/CI2. # INCLUDE_FLAGS is a list of -I flags that MUST word-split into # separate gcc arguments; quoting it would pass one bogus arg. # shellcheck disable=SC2086 scan-build --status-bugs \ gcc -c $INCLUDE_FLAGS \ "$GITHUB_WORKSPACE/src/ngx_http_zstd_filter_module.c" \ "$GITHUB_WORKSPACE/src/ngx_http_zstd_static_module.c" \ 2>&1 | tee clang-scan.log rm -f ngx_http_zstd_filter_module.o ngx_http_zstd_static_module.o - name: Upload lint reports if: always() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: lint-reports path: | ${{ github.workspace }}/cppcheck.log ${{ github.workspace }}/clang-scan.log retention-days: 7 # ── Build the module against nginx mainline (amd64) ───────────────────── release-build-fanout: name: Release build fan-out runs-on: ubuntu-24.04 timeout-minutes: 2 needs: resolve steps: - name: Give mainline a best-effort head start before fan-out # GitHub does not promise ready-queue order. This gate delays the three # fan-out chains by ten seconds after resolve, giving build a bounded, # best-effort eligibility lead without guaranteeing scheduler priority. run: sleep 10 build: # Job name must stay VERSION-FREE. It was # `Build (${{ matrix.flavor }} ${{ matrix.version }})`, which renamed the # check on every nginx mainline release -- so this job could never be a # required context: a version-pinned required check stops reporting the # day nginx ships a new version and blocks every PR with no obvious # cause. # # It interpolates `matrix.flavor` and nothing else, and that is load- # bearing in BOTH directions. A matrix job whose name contains no matrix # expression does not keep the bare name: GitHub appends the entire # matrix cell to disambiguate it, so a plain `name: Build` reports as # `Build (nginx, 1.31.3, https://nginx.org/..., nginx-1.31.3)` -- version # back in the check name, plus the url and dir. `flavor` is the only # matrix key stable across nginx releases, so naming it explicitly is # what suppresses the auto-suffix and keeps the name constant. # # The resolved version is surfaced by the "Report resolved version" step # below and in the job summary. Do NOT add `matrix.version` here, and do # NOT strip the expression back to a bare literal. name: Build (${{ matrix.flavor }}) runs-on: ${{ fromJSON(vars.POOL || '["ubuntu-latest"]') }} timeout-minutes: 30 # Lane 1 starts as soon as the five-second resolver finishes. Validation # remains parallel: serializing the main build behind a multi-minute lint # job left one of the four Linux lanes idle on every PR. needs: resolve strategy: fail-fast: false # nginx mainline, assembled by the resolve job. matrix: ${{ fromJSON(needs.resolve.outputs.matrix) }} steps: # The job title is fixed so the check name stays stable, so the # resolved version has to be visible somewhere -- here and in the job # summary, which is where you look when a build fails and you need to # know which nginx it was. # # Values go through `env:` rather than straight into the `run:` body: # they originate from the resolve job's read of the GitHub releases feed, so # interpolating them into shell is a template-injection sink (zizmor # flags it). As env vars they are data, never code. - name: Report resolved version env: FLAVOR: ${{ matrix.flavor }} VERSION: ${{ matrix.version }} run: | echo "Building $FLAVOR $VERSION" echo "### Build: $FLAVOR $VERSION" >> "$GITHUB_STEP_SUMMARY" - name: Checkout module uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Cache apt packages uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: # User-owned dir, not the system /var/cache/apt/archives: apt runs # under sudo, so the post-step tar (unprivileged runner user) cannot # read the root-owned archives/lock or archives/partial (c2). # # Glob the .deb files rather than the directory: apt recreates # `partial/` (0700 _apt:root) and `lock` (root:root) INSIDE whatever # Dir::Cache::archives points at, so caching the dir itself still # gives tar two unreadable entries and it exits 2 -> "Failed to # save" (a warning, so CI stays green while nothing is ever cached). # Only the .deb files are worth caching anyway. path: ${{ runner.temp }}/apt-archives/*.deb key: apt-build-${{ runner.os }}-build-essential-libzstd-libpcre2-zlib restore-keys: apt-build-${{ runner.os }}- - name: Install dependencies run: | # apt profile the bare apt-get lines below rely on: loose timeouts # for the shared self-hosted box; Azure mirror only on the # ubuntu-latest fallback when vars.POOL is unset (PR #121) mkdir -p "$RUNNER_TEMP/apt-archives" printf '%s\n' \ 'DPkg::Lock::Timeout "120";' \ 'Acquire::Retries "3";' \ 'Acquire::http::Timeout "20";' \ 'Acquire::https::Timeout "20";' \ "Dir::Cache::archives \"$RUNNER_TEMP/apt-archives\";" \ | sudo tee /etc/apt/apt.conf.d/99-ci-net > /dev/null sudo apt-get update sudo apt-get install -y \ build-essential \ pkg-config \ libzstd-dev \ gnupg \ libpcre2-dev \ zlib1g-dev \ ccache \ wget - name: Set up ccache run: | echo "/usr/lib/ccache" >> "$GITHUB_PATH" echo "CCACHE_DIR=$HOME/.ccache" >> "$GITHUB_ENV" - name: Hash module build inputs id: build-inputs run: echo "hash=$(bash ci/tools/build-input-hash.sh)" >> "$GITHUB_OUTPUT" - name: Cache ccache objects uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ~/.ccache key: ccache-${{ runner.os }}-${{ matrix.flavor }}-${{ matrix.version }}-${{ steps.build-inputs.outputs.hash }} restore-keys: | ccache-${{ runner.os }}-${{ matrix.flavor }}-${{ matrix.version }}- ccache-${{ runner.os }}- - name: Cache server source tarball id: src-tarball uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ${{ matrix.dir }}.tar.gz key: src-${{ matrix.flavor }}-${{ matrix.version }} - name: Download server source tarball if: steps.src-tarball.outputs.cache-hit != 'true' env: MATRIX_DIR: ${{ matrix.dir }} MATRIX_URL: ${{ matrix.url }} run: wget -q -O "$MATRIX_DIR.tar.gz" "$MATRIX_URL" - name: Verify nginx tarball PGP signature if: matrix.flavor == 'nginx' env: MATRIX_DIR: ${{ matrix.dir }} MATRIX_URL: ${{ matrix.url }} run: ci/tools/verify-nginx-tarball.sh "$MATRIX_DIR.tar.gz" "$MATRIX_URL.asc" - name: Configure (full debug build) env: MATRIX_DIR: ${{ matrix.dir }} run: | tar -xzf "$MATRIX_DIR.tar.gz" cd "$MATRIX_DIR" # Full debug compile flags: # --with-debug nginx debug logging (ngx_log_debug*) # -g3 max debug info incl. macro definitions # -O0 no optimisation — accurate line/var info # -fno-omit-frame-pointer reliable backtraces under gdb/valgrind # -funwind-tables unwind info for crash backtraces # -DNGX_DEBUG_PALLOC=1 pool-allocator debug bookkeeping # Extra HTTP modules the zstd filter realistically runs alongside, # so the module is exercised in a realistic nginx (not a stripped one): # ssl, v2 (HTTP/2), gzip_static, realip, sub, addition, stub_status ./configure \ --with-compat \ --with-debug \ --with-threads \ --with-file-aio \ --with-http_ssl_module \ --with-http_v2_module \ --with-http_v3_module \ --with-http_gzip_static_module \ --with-http_realip_module \ --with-http_sub_module \ --with-http_addition_module \ --with-http_stub_status_module \ --with-http_auth_request_module \ --with-cc-opt="-DZSTD_STATIC_LINKING_ONLY -DNGX_DEBUG_PALLOC=1 -g3 -O0 -fno-omit-frame-pointer -funwind-tables $(pkg-config --cflags libzstd zlib libpcre2-8)" \ --with-ld-opt="$(pkg-config --libs libzstd zlib libpcre2-8)" \ --add-module="$GITHUB_WORKSPACE" - name: Strict compile of module sources (warnings as errors) env: MATRIX_DIR: ${{ matrix.dir }} run: | cd "$MATRIX_DIR" NGINX_CFLAGS=$(awk '/^CFLAGS[[:space:]]*=/{sub(/^CFLAGS[[:space:]]*=[[:space:]]*/,""); print; exit}' objs/Makefile) ALL_INCS=$(awk '/^ALL_INCS[[:space:]]*=/{found=1} found{line=$0; sub(/^ALL_INCS[[:space:]]*=[[:space:]]*/,"",line); gsub(/\\$/, "", line); printf "%s ", line; if (!/\\$/) exit}' objs/Makefile) for src in \ "$GITHUB_WORKSPACE/src/ngx_http_zstd_filter_module.c" \ "$GITHUB_WORKSPACE/src/ngx_http_zstd_static_module.c"; do echo "--- strict compile: $(basename "$src") ---" # NGINX_CFLAGS and ALL_INCS are flag lists that MUST word-split # into individual gcc arguments. # shellcheck disable=SC2086 gcc -c -fPIC $NGINX_CFLAGS \ -Wall -Wextra -Wshadow -Wstrict-aliasing \ -Wunreachable-code -Wunused -Wwrite-strings -Werror \ $ALL_INCS \ "$src" -o "/tmp/$(basename "$src" .c).o" done echo "✓ All module sources compile cleanly with strict flags" - name: Build binary with module env: MATRIX_DIR: ${{ matrix.dir }} run: | cd "$MATRIX_DIR" NGINX_CFLAGS=$(awk '/^CFLAGS[[:space:]]*=/{sub(/^CFLAGS[[:space:]]*=[[:space:]]*/,""); print; exit}' objs/Makefile) make -j"$(nproc)" modules \ CFLAGS="$NGINX_CFLAGS -Wall -Wextra -Wshadow -Wstrict-aliasing \ -Wunreachable-code -Wunused -Werror" make -j"$(nproc)" ccache -s || true # This binary feeds the full test suites, which are meant to run # the accelerated dictionary-hash path: libcrypto must have been # detected (the runner has OpenSSL headers — the build above uses # --with-http_ssl_module), or hashing silently regressed to the # portable fallback (the ASAN job covers that one deliberately). grep -q 'NGX_HTTP_ZSTD_HAVE_LIBCRYPTO' objs/ngx_auto_config.h echo "✓ nginx binary built (libcrypto EVP SHA-256 detected)" - name: Verify built binary env: MATRIX_DIR: ${{ matrix.dir }} MATRIX_FLAVOR: ${{ matrix.flavor }} run: | ls -la "$MATRIX_DIR/objs/$MATRIX_FLAVOR" "$MATRIX_DIR/objs/$MATRIX_FLAVOR" -V 2>&1 - name: Module must not veto an http-less configuration env: MATRIX_DIR: ${{ matrix.dir }} MATRIX_FLAVOR: ${{ matrix.flavor }} run: | bash "$GITHUB_WORKSPACE"/ci/tools/test_no_http_block.sh \ "$MATRIX_DIR/objs/$MATRIX_FLAVOR" - name: Upload server binary artifact uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: server-binary-${{ matrix.flavor }}-${{ matrix.version }} path: ${{ matrix.dir }}/objs/${{ matrix.flavor }} retention-days: 1 # ── Native arm64 build and decoded runtime smoke with current libzstd ─── build-arm64: name: Build (nginx arm64) runs-on: ubuntu-24.04-arm timeout-minutes: 30 needs: resolve env: NGINX_VERSION: ${{ needs.resolve.outputs.nginx_version }} steps: - name: Checkout module uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Install dependencies run: | set -euo pipefail printf '%s\n' \ 'DPkg::Lock::Timeout "120";' \ 'Acquire::Retries "3";' \ 'Acquire::http::Timeout "20";' \ 'Acquire::https::Timeout "20";' \ | sudo tee /etc/apt/apt.conf.d/99-ci-net > /dev/null sudo apt-get update sudo apt-get install -y \ build-essential \ pkg-config \ libzstd-dev \ libpcre2-dev \ zlib1g-dev \ python3 \ python3-requests \ gnupg \ wget \ zstd - name: Cache nginx source tarball id: nginx-tarball-arm64 uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: nginx-${{ env.NGINX_VERSION }}.tar.gz key: nginx-src-${{ env.NGINX_VERSION }} - name: Download nginx if: steps.nginx-tarball-arm64.outputs.cache-hit != 'true' run: wget -q "https://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz" - name: Verify nginx tarball PGP signature run: ci/tools/verify-nginx-tarball.sh "nginx-${NGINX_VERSION}.tar.gz" - name: Configure arm64 build run: | set -euo pipefail tar -xzf "nginx-${NGINX_VERSION}.tar.gz" cd "nginx-${NGINX_VERSION}" ./configure \ --with-compat \ --with-debug \ --with-http_ssl_module \ --with-http_gzip_static_module \ --with-http_realip_module \ --with-http_v2_module \ --with-http_sub_module \ --with-http_auth_request_module \ --with-cc-opt="$(pkg-config --cflags libzstd zlib libpcre2-8)" \ --with-ld-opt="$(pkg-config --libs libzstd zlib libpcre2-8)" \ --add-dynamic-module="$GITHUB_WORKSPACE" - name: Strict compile module sources run: | set -euo pipefail cd "nginx-${NGINX_VERSION}" NGINX_CFLAGS=$(awk '/^CFLAGS[[:space:]]*=/{sub(/^CFLAGS[[:space:]]*=[[:space:]]*/,""); print; exit}' objs/Makefile) ALL_INCS=$(awk '/^ALL_INCS[[:space:]]*=/{found=1} found{line=$0; sub(/^ALL_INCS[[:space:]]*=[[:space:]]*/,"",line); gsub(/\\$/, "", line); printf "%s ", line; if (!/\\$/) exit}' objs/Makefile) for src in \ "$GITHUB_WORKSPACE/src/ngx_http_zstd_filter_module.c" \ "$GITHUB_WORKSPACE/src/ngx_http_zstd_static_module.c"; do # NGINX_CFLAGS and ALL_INCS are flag lists that MUST word-split. # shellcheck disable=SC2086 gcc -c -fPIC $NGINX_CFLAGS \ -Wall -Wextra -Wshadow -Wstrict-aliasing \ -Wunreachable-code -Wunused -Werror \ $ALL_INCS \ "$src" -o "/tmp/$(basename "$src" .c)-arm64.o" done - name: Build and verify arm64 binary run: | set -euo pipefail cd "nginx-${NGINX_VERSION}" make -j"$(nproc)" "$GITHUB_WORKSPACE/ci/tools/check-zstd-dynamic-linkage.sh" \ objs/ngx_http_zstd_filter_module.so file objs/nginx | grep -Eq 'ARM aarch64|ARM64' for module in \ objs/ngx_http_zstd_filter_module.so \ objs/ngx_http_zstd_static_module.so; do test -f "$module" grep -q "$module" objs/Makefile done grep -q -- '-lzstd' objs/Makefile objs/nginx -V 2>&1 echo "TEST_NGINX_BINARY=$PWD/objs/nginx" >> "$GITHUB_ENV" - name: Decoded runtime smoke on arm64 run: | set -euo pipefail python3 ci/tools/test_encoding.py \ --nginx-binary "$TEST_NGINX_BINARY" --port 18094 smoke_root="$(mktemp -d)" module_dir="$(dirname "$TEST_NGINX_BINARY")" mkdir -p "$smoke_root/conf" "$smoke_root/logs" "$smoke_root/html" printf 'origin body\n' > "$smoke_root/html/bad.txt" printf 'not a zstd frame' > "$smoke_root/html/bad.txt.zst" cat > "$smoke_root/conf/nginx.conf" </dev/null 2>&1 || true' EXIT python3 - <<'PY' > /tmp/bad-static-response.zst import http.client, time for _ in range(40): try: conn = http.client.HTTPConnection("127.0.0.1", 18095, timeout=2) conn.request("GET", "/bad.txt.zst", headers={"Accept-Encoding": "zstd"}) resp = conn.getresponse() body = resp.read() if resp.status != 200: raise SystemExit(f"unexpected malformed-static response: {resp.status} {resp.getheaders()!r}") import sys sys.stdout.buffer.write(body) break except OSError: time.sleep(0.25) finally: try: conn.close() except Exception: pass else: raise SystemExit("nginx did not answer malformed static request") PY if zstd -dc /tmp/bad-static-response.zst >/dev/null 2>&1; then echo "invalid zstd frame served by nginx decoded successfully" >&2 exit 1 fi echo "✓ native arm64 build and decoded runtime smoke passed" # ── Build against libzstd 1.4.x — exercises the documented fallback # paths (#ifdef ZSTD_c_targetCBlockSize and the ZSTD_VERSION_NUMBER # boundary in zstd_comp_level validation) that the main 1.5.x build # never touches. ZSTD_compressStream2 was added in 1.4.0, so that # is the genuine module floor; we build it from source against the # pinned nginx and run the truncation smoke test to prove the # fallback paths still compile and produce correct compression. ───── build-old-libzstd: name: Build (libzstd 1.4.x — fallback paths) runs-on: ${{ fromJSON(vars.POOL || '["ubuntu-latest"]') }} timeout-minutes: 25 # Released after the barrier's bounded best-effort eligibility lead for # build; it may contend with build and validation once the delay expires. needs: [resolve, release-build-fanout] env: # Resolved mainline nginx (used by the cache keys + download/build steps # below, both as ${{ env.NGINX_VERSION }} and the shell $NGINX_VERSION). NGINX_VERSION: ${{ needs.resolve.outputs.nginx_version }} # 1.4.5 is < 1.5.6 (so ZSTD_c_targetCBlockSize is absent and the # #ifdef guard's fallback compiles) and >= 1.4.0 (the module's # hard floor — ZSTD_compressStream2 is present). 1.4.0 itself # bumps the version macro to 10400 so ZSTD_minCLevel() is # available — the negative-level path of zstd_comp_level() # therefore takes the supported branch. OLD_ZSTD_VERSION: "1.4.5" steps: - name: Checkout module uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Install build dependencies run: | # apt profile the bare apt-get lines below rely on: loose timeouts # for the shared self-hosted box; Azure mirror only on the # ubuntu-latest fallback when vars.POOL is unset (PR #121) printf '%s\n' \ 'DPkg::Lock::Timeout "120";' \ 'Acquire::Retries "3";' \ 'Acquire::http::Timeout "20";' \ 'Acquire::https::Timeout "20";' \ | sudo tee /etc/apt/apt.conf.d/99-ci-net > /dev/null sudo apt-get update sudo apt-get install -y build-essential gnupg pkg-config \ libpcre2-dev zlib1g-dev wget cmake python3 \ python3-requests zstd - name: Cache libzstd 1.4.x source tarball id: zstd-tarball uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: zstd-${{ env.OLD_ZSTD_VERSION }}.tar.gz key: zstd-src-${{ env.OLD_ZSTD_VERSION }} - name: Download libzstd 1.4.x if: steps.zstd-tarball.outputs.cache-hit != 'true' run: | wget -q -O "zstd-${OLD_ZSTD_VERSION}.tar.gz" \ "https://github.com/facebook/zstd/releases/download/v${OLD_ZSTD_VERSION}/zstd-${OLD_ZSTD_VERSION}.tar.gz" - name: Verify libzstd 1.4.x tarball sha256 # Runs on EVERY run, cache hit or fresh download -- a cached tarball # from actions/cache is exactly as untrusted as a freshly-downloaded # one until re-checked (same rule ci-build.sh's header documents for # nginx/angie tarballs, audit sha e289021 F3). facebook/zstd does not # publish a detached signature for release tarballs, so this is # pinned sha256 only, computed once from an HTTPS fetch -- same # approach as ci-build.sh's ANGIE_SHA256 table. env: ZSTD_145_SHA256: 98e91c7c6bf162bf90e4e70fdbc41a8188b9fa8de5ad840c401198014406ce9e run: | set -euo pipefail actual="$(sha256sum "zstd-${OLD_ZSTD_VERSION}.tar.gz" | awk '{print $1}')" if [ "$actual" != "$ZSTD_145_SHA256" ]; then echo "✗ sha256 mismatch for zstd-${OLD_ZSTD_VERSION}.tar.gz" >&2 echo " expected: $ZSTD_145_SHA256" >&2 echo " actual: $actual" >&2 rm -f "zstd-${OLD_ZSTD_VERSION}.tar.gz" exit 1 fi echo "✓ sha256 verified for zstd-${OLD_ZSTD_VERSION}.tar.gz" - name: Build libzstd 1.4.x to a private prefix run: | tar -xzf "zstd-${OLD_ZSTD_VERSION}.tar.gz" cd "zstd-${OLD_ZSTD_VERSION}" # Build only the library; tests/binaries not needed. make -j"$(nproc)" -C lib PREFIX="$HOME/zstd14" \ install-pc install-includes install-shared ls "$HOME/zstd14/lib/libzstd.so"* || \ (echo "libzstd not installed" && exit 1) echo "ZSTD_INC=$HOME/zstd14/include" >> "$GITHUB_ENV" echo "ZSTD_LIB=$HOME/zstd14/lib" >> "$GITHUB_ENV" # The header's ZSTD_VERSION_NUMBER is what gates the fallbacks: grep -E 'ZSTD_VERSION_(MAJOR|MINOR|RELEASE)[[:space:]]' \ "$HOME/zstd14/include/zstd.h" | head -3 - name: Cache nginx source tarball id: nginx-tarball-old uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: nginx-${{ env.NGINX_VERSION }}.tar.gz key: nginx-src-${{ env.NGINX_VERSION }} - name: Download nginx if: steps.nginx-tarball-old.outputs.cache-hit != 'true' run: wget -q "https://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz" - name: Verify nginx tarball PGP signature run: ci/tools/verify-nginx-tarball.sh "nginx-${NGINX_VERSION}.tar.gz" - name: Configure & build nginx with module against libzstd 1.4.x run: | tar -xzf "nginx-${NGINX_VERSION}.tar.gz" cd "nginx-${NGINX_VERSION}" # ZSTD_INC/ZSTD_LIB tell the module's filter/config to pick up # the private libzstd 1.4.x build, not the system one. This # prefix has no libzstd.a (install-shared only, above), so # auto/zstd's static probe fails and it falls back to the # dynamic -lzstd retry, which auto/zstd no longer builds with # -DZSTD_STATIC_LINKING_ONLY (see the symbol-import assertion # below): the fallback paths must compile clean under the plain # dynamic-link build too, without ZSTDLIB_STATIC_API imports. ZSTD_INC="$ZSTD_INC" ZSTD_LIB="$ZSTD_LIB" \ ./configure \ --with-compat \ --with-debug \ --with-cc-opt="-Wall -Wextra -Wno-unused-parameter -Werror $(pkg-config --cflags zlib libpcre2-8)" \ --with-ld-opt="$(pkg-config --libs zlib libpcre2-8)" \ --add-module="$GITHUB_WORKSPACE" make -j"$(nproc)" # Verify the binary links against the *old* libzstd we built, # not the system one — otherwise the test is meaningless. ldd objs/nginx | grep libzstd | tee /tmp/libzstd.linkage if ! grep -q "$ZSTD_LIB" /tmp/libzstd.linkage; then echo "FAIL: nginx not linked against private libzstd 1.4.x" exit 1 fi echo "✓ nginx built against libzstd 1.4.x at $ZSTD_LIB" # Make the freshly built libzstd visible at runtime for the # smoke test below (rpath is not set since the build went # through the dynamic auto-discovery path). echo "LD_LIBRARY_PATH=$ZSTD_LIB" >> "$GITHUB_ENV" echo "TEST_NGINX_BINARY=$PWD/objs/nginx" >> "$GITHUB_ENV" - name: Assert no static-only libzstd symbols leaked into the dynamic build run: | set -euo pipefail cd "nginx-${NGINX_VERSION}" # This job's private libzstd was built with install-shared only # (no libzstd.a), so auto/zstd's static probe must have failed # and fallen back to the dynamic -lzstd retry. That retry must # NOT carry -DZSTD_STATIC_LINKING_ONLY: those declarations # (ZSTDLIB_STATIC_API) have no stable ABI, so importing them # into a dynamically linked module is unsupported even when the # symbols happen to resolve today. Keep the denied-symbol list # shared with the dynamic-module artifact gate. ls "$HOME"/zstd14/lib/libzstd.a 2>/dev/null && \ { echo "FAIL: static archive present, this leg no longer tests the dynamic-only fallback"; exit 1; } "$GITHUB_WORKSPACE/ci/tools/check-zstd-dynamic-linkage.sh" objs/nginx - name: Assert the private 1.4.x HEADERS were really compiled against run: | set -euo pipefail cd "nginx-${NGINX_VERSION}" # Regression pin for the flags-in-incs bug: ngx_module_incs # tokens are each prefixed with the include option by # auto/make, so compiler flags stored there came out mangled # ("-I -I") and gcc silently fell back to the runner's # SYSTEM zstd.h — this job compiled 1.5.x headers while # linking the private 1.4.x lib, the exact mismatch it exists # to prevent. Assert the generated compile lines carry the # private include cleanly and nothing mangled remains. grep -q -- "-I $ZSTD_INC" objs/Makefile if grep -q -- "-I -" objs/Makefile; then echo "❌ mangled include tokens in objs/Makefile"; exit 1 fi # Compile-time witness, observable at runtime: the # zstd_target_cblock_size config-load warning is emitted ONLY # when the module was compiled against headers < 1.5.6. If # the system header ever leaks back in, the warning vanishes # and this fails. Verified both directions before adding: a # system-1.5.x build stays silent, a private-1.4.9 build # warns. mkdir -p /tmp/hdr-witness/logs printf '%s\n' \ 'pid logs/nginx.pid; error_log logs/error.log; events {}' \ 'http { zstd on; zstd_target_cblock_size 16k; }' \ > /tmp/hdr-witness/w.conf # No "|| true": the witness config must VALIDATE, not merely # warn — nginx -t's exit status survives the tee via pipefail. ./objs/nginx -t -p /tmp/hdr-witness -c w.conf 2>&1 \ | tee /tmp/hdr-witness/out grep -q "without ZSTD_c_targetCBlockSize support" /tmp/hdr-witness/out echo "✓ compiled against the private libzstd 1.4.x headers" - name: Smoke test (decode-and-compare) against libzstd 1.4.x run: | cd "$GITHUB_WORKSPACE" # Reuse the existing decode-and-byte-compare smoke test so # the fallback paths are not just compiled but actually # exercised at run time — compression must produce a # byte-exact body when decompressed. python3 ci/tools/test_encoding.py \ --nginx-binary "$TEST_NGINX_BINARY" --port 18200 echo "✓ libzstd 1.4.x fallback paths produce correct output" # ── Dynamic-module linkage isolation — the NGX_LD_OPT regression test. # Builds the zstd modules alongside an unrelated stub dynamic module # (ci/tools/ci-linkcheck-module) in ONE configure and asserts both # directions: the zstd .so's DO carry a libzstd DT_NEEDED (positive # control — proves the per-module ngx_module_libs channel works and # that the assertions are not vacuously passing) and the stub does # NOT (the actual fix: an addon appending to build-global NGX_LD_OPT # signs its neighbours up for its libraries). ───────────────────────── linkage: name: Dynamic-module linkage isolation runs-on: ${{ fromJSON(vars.POOL || '["ubuntu-latest"]') }} timeout-minutes: 15 # Lane 4 is cvary-interop -> linkage. The explicit result expression keeps # linkage diagnostic coverage even when cvary itself fails; `needs` here is # an ordering edge, not a reason to hide a second independent failure. needs: [resolve, cvary-interop] if: >- ${{ !cancelled() && needs.resolve.result == 'success' }} env: NGINX_VERSION: ${{ needs.resolve.outputs.nginx_version }} steps: - name: Checkout module uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Install dependencies run: | # apt profile the bare apt-get lines below rely on: loose timeouts # for the shared self-hosted box; Azure mirror only on the # ubuntu-latest fallback when vars.POOL is unset (PR #121) printf '%s\n' \ 'DPkg::Lock::Timeout "120";' \ 'Acquire::Retries "3";' \ 'Acquire::http::Timeout "20";' \ 'Acquire::https::Timeout "20";' \ | sudo tee /etc/apt/apt.conf.d/99-ci-net > /dev/null sudo apt-get update sudo apt-get install -y build-essential gnupg pkg-config libzstd-dev \ libpcre2-dev zlib1g-dev wget binutils - name: Cache nginx source tarball id: nginx-tarball uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: nginx-${{ env.NGINX_VERSION }}.tar.gz key: nginx-src-${{ env.NGINX_VERSION }} - name: Download nginx source tarball if: steps.nginx-tarball.outputs.cache-hit != 'true' run: wget -q "https://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz" - name: Verify nginx tarball PGP signature run: ci/tools/verify-nginx-tarball.sh "nginx-${NGINX_VERSION}.tar.gz" - name: Configure with zstd + stubs as dynamic modules, build # Full build (not just `make modules`): the vary-suppression # witness below runs `nginx -t`, which needs the binary. The # linkage assertions still read objs/Makefile and the .so files, # both of which this produces. run: | set -euo pipefail tar -xzf "nginx-${NGINX_VERSION}.tar.gz" cd "nginx-${NGINX_VERSION}" ./configure \ --with-compat \ --add-dynamic-module="$GITHUB_WORKSPACE" \ --add-dynamic-module="$GITHUB_WORKSPACE/ci/tools/ci-linkcheck-module" \ > /dev/null make -j"$(nproc)" - name: Assert linkage isolation (both directions) run: | set -euo pipefail cd "nginx-${NGINX_VERSION}" # Existence first: a renamed/missing .so must fail here, not # let the negative assertions below pass vacuously. for so in ngx_http_zstd_filter_module.so \ ngx_http_zstd_static_module.so \ ngx_http_linkcheck_module.so; do test -f "objs/$so" || { echo "✗ missing objs/$so"; exit 1; } done # PRIMARY assertions target the GENERATED LINK COMMAND, not # the linked artifact: a toolchain defaulting to --as-needed # (the gcc on these runners included) drops an unused libzstd # NEEDED from the stub's .so even when the NGX_LD_OPT leak is # present — a readelf-only negative therefore passes either # way and tests nothing. The Makefile records what nginx's # build system emitted, before the linker gets a vote. Grab # the "$(LINK) -o objs/" recipe line plus its backslash # continuations (a plain target-to-blank-line sed range stops # at the dependency list and never reaches the recipe — # nginx emits a blank line between the two). index() rather # than a regex match, so the dots in ".so" are literal. rule() { awk -v so="objs/$1.so" \ 'index($0, "-o " so) { grab = 1 } grab { print; if ($0 !~ /\\$/) exit }' objs/Makefile } # Every assertion below is only as good as rule() returning # the recipe it was asked for: an empty block makes a # negative grep pass for free. Check extraction itself first, # for both targets, before reading anything into the result. for mod in ngx_http_zstd_filter_module ngx_http_linkcheck_module; do rule "$mod" | grep -q . || { echo "✗ no link rule extracted for $mod — objs/Makefile shape changed?" exit 1 } done # Positive control: the filter module's rule must carry # -lzstd via ngx_module_libs. Proves the per-module channel # the fix relies on is live, so the negative below is a # statement about NGX_LD_OPT and not about a build that # linked no zstd at all. if ! rule ngx_http_zstd_filter_module | grep -q -- '-lzstd'; then echo "✗ filter module link rule lacks -lzstd (ngx_module_libs broken?)" rule ngx_http_zstd_filter_module exit 1 fi echo "✓ filter module link rule carries -lzstd" # The fix under test: an unrelated module's link rule must # not mention zstd in any form. Matching bare "zstd" rather # than the "-lzstd" spelling on purpose — pkg-config can hand # back an absolute /usr/lib/.../libzstd.so or -l:libzstd.a, # and a leak wearing either spelling is still a leak. Safe # only for this target: the filter module's own rule names # objs/ngx_http_zstd_filter_module.so, so the same pattern # would match its filename and assert nothing. if rule ngx_http_linkcheck_module | grep -qi zstd; then echo "✗ stub module link rule references zstd (NGX_LD_OPT leak)" rule ngx_http_linkcheck_module exit 1 fi echo "✓ stub module link rule free of zstd — no NGX_LD_OPT leak" # readelf belt under the Makefile suspenders: the filter's # DT_NEEDED may not be dropped by any toolchain (it references # libzstd symbols), and on toolchains without --as-needed the # stub check additionally guards link inputs that reach the # command line outside the module's own rule text. The static # module is deliberately not asserted either way: it calls no # libzstd function, so its NEEDED reflects the linker default, # not module behaviour. if ! readelf -d objs/ngx_http_zstd_filter_module.so \ | grep -E 'NEEDED.*libzstd' > /dev/null; then echo "✗ filter module .so does not link libzstd" readelf -d objs/ngx_http_zstd_filter_module.so | grep NEEDED || true exit 1 fi if readelf -d objs/ngx_http_linkcheck_module.so \ | grep -E 'NEEDED.*libzstd' > /dev/null; then echo "✗ stub module .so inherited a libzstd DT_NEEDED" readelf -d objs/ngx_http_linkcheck_module.so | grep NEEDED exit 1 fi echo "✓ readelf agrees on both" - name: "Assert Vary: Accept-Encoding is emitted without gzip_vary" # G5: the modules emit "Vary: Accept-Encoding" themselves, so the # header no longer depends on the operator's "gzip_vary" # directive and the old gzip_vary-off warnings are gone. This # step is the config-load half of that contract: the fixture # below has NO gzip_vary anywhere, and must load with no # gzip_vary advice of any kind. The runtime half (exactly one # Vary line in every gzip_vary / compression_vary state) is the # cvary-interop job and the vary-matrix job. run: | set -euo pipefail cd "nginx-${NGINX_VERSION}" mkdir -p wprefix/logs cat > wprefix/warn.conf <&1) echo "$out" echo "$out" | grep -q 'test is successful' # No gzip_vary advice may survive: neither the old # per-location lines nor the old per-module summary. A stale # warning telling operators to set a directive that no longer # affects the header is worse than none. if echo "$out" | grep -qi 'gzip_vary'; then echo "✗ config load still mentions gzip_vary:" echo "$out" | grep -i 'gzip_vary' exit 1 fi echo "✓ loads clean with no gzip_vary anywhere in the config" - name: Dictionary path hardening + read-completeness fixtures run: | set -euo pipefail cd "$GITHUB_WORKSPACE" # Fixture matrix over both dictionary loaders: non-regular # inputs (FIFO/dir/socket/device) are refused without blocking # the config-parsing master, zstd_dict_strict_path refuses a # symlink and a group/world-writable target, a rejected reload # leaves the old cycle serving, and the config-level half of # the read contract (a large dictionary loads; empty and # unreadable are refused). Every fixture is bounded by # `timeout`, so a blocking-open regression fails the job # instead of hanging it. # # This script existed but was wired into no workflow, so none # of the above was actually being checked on any PR. bash ci/tools/test_dict_path_hardening.sh \ "nginx-${NGINX_VERSION}/objs/nginx" \ "nginx-${NGINX_VERSION}/objs" # ── Interop against the REAL compression_vary module — the stub above # proves name detection; this proves the semantic contract the # summary warning describes: with compression_vary on, exactly one # Vary: Accept-Encoding is emitted from r->gzip_vary alone; with it # off (its default), no Vary appears — the residual risk the # summary warning exists for. ───────────────────────────────────── cvary-interop: name: compression_vary interop (real module) runs-on: ${{ fromJSON(vars.POOL || '["ubuntu-latest"]') }} timeout-minutes: 15 needs: [resolve, release-build-fanout] env: NGINX_VERSION: ${{ needs.resolve.outputs.nginx_version }} # HanadaLee/ngx_http_compression_vary_filter_module, pinned by # commit like every other external input. Bumping the pin is the # moment to re-read that module's r->gzip_vary handling — the # warning suppression banks on its documented contract. CVARY_COMMIT: 018810484d6554c5b9278749a8939697e9501f59 steps: - name: Checkout module uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Install dependencies # A slow-rather-than-dead mirror can still crawl past every # Acquire timeout; cap the step so a stall fails HERE, named, # with the log intact — instead of the 15-minute job cap # swallowing it (the #117 failure shape). timeout-minutes: 5 run: | # ubuntu-latest resolves apt through azure.archive.ubuntu.com, # whose outage shape is ~2 minutes of default Acquire timeout # per index file before the runner's mirror list falls back # (the #117 cancellations); the bare apt-get lines below # rely on this file. printf '%s\n' \ 'DPkg::Lock::Timeout "120";' \ 'Acquire::Retries "1";' \ 'Acquire::http::Timeout "10";' \ 'Acquire::https::Timeout "10";' \ | sudo tee /etc/apt/apt.conf.d/99-ci-net > /dev/null sudo apt-get update sudo apt-get install -y build-essential git gnupg libzstd-dev \ libpcre2-dev zlib1g-dev wget curl - name: Cache nginx source tarball id: nginx-tarball uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: nginx-${{ env.NGINX_VERSION }}.tar.gz key: nginx-src-${{ env.NGINX_VERSION }} - name: Download nginx source tarball if: steps.nginx-tarball.outputs.cache-hit != 'true' run: wget -q "https://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz" - name: Verify nginx tarball PGP signature run: ci/tools/verify-nginx-tarball.sh "nginx-${NGINX_VERSION}.tar.gz" - name: Clone compression_vary module at the pin run: | set -euo pipefail git clone \ https://github.com/HanadaLee/ngx_http_compression_vary_filter_module.git \ cvary git -C cvary switch --detach "$CVARY_COMMIT" - name: Configure & build nginx with both modules run: | set -euo pipefail tar -xzf "nginx-${NGINX_VERSION}.tar.gz" cd "nginx-${NGINX_VERSION}" ./configure \ --with-compat \ --add-module="$GITHUB_WORKSPACE" \ --add-module="$GITHUB_WORKSPACE/cvary" \ > /dev/null make -j"$(nproc)" - name: Assert config-load behaviour against the real module run: | set -euo pipefail cd "nginx-${NGINX_VERSION}" mkdir -p iprefix/logs cat > iprefix/interop.conf <&1) echo "$out" echo "$out" | grep -q 'test is successful' # G5: no gzip_vary advice of any kind survives — the header # does not depend on that directive any more. if echo "$out" | grep -qi 'gzip_vary'; then echo "✗ config load still mentions gzip_vary:" echo "$out" | grep -i 'gzip_vary' exit 1 fi echo "✓ clean load against the real module, no gzip_vary advice" - name: Assert runtime Vary contract (compression_vary on and off) # THE point of G5. Both locations have "gzip_vary off". Before # G5, /off/ (compression_vary off — its DEFAULT) served a zstd # body with NO Vary at all: a shared cache would store it and # hand it to a client that cannot decode zstd. That was the # residual risk a warning was asked to cover. Now the module # emits the field itself, so BOTH locations must carry exactly # one Vary: Accept-Encoding — never zero (the cache-poisoning # bug) and never two (compression_vary flattens what it finds, # and a doubled field trips strict intermediaries). run: | set -euo pipefail cd "nginx-${NGINX_VERSION}" objs/nginx -p "$PWD/iprefix" -c interop.conf trap 'objs/nginx -p "$PWD/iprefix" -c interop.conf -s stop' EXIT sleep 0.5 check_one_vary() { local loc="$1" label="$2" hdr n hdr=$(curl -s -D - -o /dev/null -H "Accept-Encoding: zstd" \ "http://127.0.0.1:18994${loc}x") echo "--- ${label} (${loc})" echo "$hdr" echo "$hdr" | grep -qi '^content-encoding: zstd' n=$(echo "$hdr" | grep -ci '^vary: accept-encoding' || true) if [ "$n" -ne 1 ]; then echo "✗ ${label}: expected exactly one Vary: Accept-Encoding, got $n" exit 1 fi echo "✓ ${label}: exactly one Vary: Accept-Encoding" } check_one_vary /on/ "gzip_vary off + compression_vary on" check_one_vary /off/ "gzip_vary off + compression_vary off" # ── ASAN + UBSAN build — catches the lifetime/UB bugs in this module's # history (per-request ctx, terminal-frame). Uses valgrind.suppress'd # knowledge; ASAN is faster than valgrind for CI. ─────────────────────── build-asan: name: Build ASAN+UBSAN runs-on: ${{ fromJSON(vars.POOL || '["ubuntu-latest"]') }} timeout-minutes: 30 needs: [resolve, release-build-fanout] env: # Resolved mainline nginx — feeds the cache keys, download and the # uploaded artifact path below. NGINX_VERSION: ${{ needs.resolve.outputs.nginx_version }} steps: - name: Checkout module uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Install dependencies run: | # apt profile the bare apt-get lines below rely on: loose timeouts # for the shared self-hosted box; Azure mirror only on the # ubuntu-latest fallback when vars.POOL is unset (PR #121) printf '%s\n' \ 'DPkg::Lock::Timeout "120";' \ 'Acquire::Retries "3";' \ 'Acquire::http::Timeout "20";' \ 'Acquire::https::Timeout "20";' \ | sudo tee /etc/apt/apt.conf.d/99-ci-net > /dev/null sudo apt-get update sudo apt-get install -y build-essential gnupg pkg-config libzstd-dev \ libpcre2-dev zlib1g-dev wget - name: Cache nginx source tarball id: nginx-tarball uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: nginx-${{ env.NGINX_VERSION }}.tar.gz key: nginx-src-${{ env.NGINX_VERSION }} - name: Download nginx source tarball if: steps.nginx-tarball.outputs.cache-hit != 'true' run: wget -q "https://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz" - name: Verify nginx tarball PGP signature run: ci/tools/verify-nginx-tarball.sh "nginx-${NGINX_VERSION}.tar.gz" - name: Configure & build nginx with ASAN + UBSAN env: # Keep the portable SHA-256 fallback covered, and covered where # it counts: this job's dcz suites then run the local hash # implementation under ASAN+UBSAN (sanitizers see nothing useful # inside libcrypto anyway), while the mainline build asserts the # EVP path. NGX_ZSTD_NO_LIBCRYPTO: 1 run: | tar -xzf "nginx-${NGINX_VERSION}.tar.gz" cd "nginx-${NGINX_VERSION}" # Recovery is required for the Test::Nginx lane below: upstream # nginx debug logging has known UBSan findings, and a fatal trap # kills its listener before the suite can finish. Reports remain # complete and are graded by module frames after both suites exit. SAN="-fsanitize=address,undefined -fsanitize-recover=undefined -fno-omit-frame-pointer -g3 -O1" ./configure \ --with-compat \ --with-debug \ --with-cc-opt="-DZSTD_STATIC_LINKING_ONLY $SAN $(pkg-config --cflags libzstd zlib libpcre2-8)" \ --with-ld-opt="$SAN $(pkg-config --libs libzstd zlib libpcre2-8)" \ --with-http_sub_module \ --with-http_auth_request_module \ --with-http_gzip_static_module \ --add-module="$GITHUB_WORKSPACE" make -j"$(nproc)" if grep -q 'NGX_HTTP_ZSTD_HAVE_LIBCRYPTO' objs/ngx_auto_config.h; then echo "libcrypto leaked into the portable-fallback build"; exit 1 fi echo "✓ ASAN+UBSAN nginx binary built (portable SHA-256 in use)" - name: Upload ASAN nginx binary uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: nginx-binary-asan path: nginx-${{ env.NGINX_VERSION }}/objs/nginx retention-days: 1 # ── Full test suite — runs against the mainline binary ──────────────────── tests: # Job name is FIXED -- see the `build` job for the full reasoning. It was # `Tests (nginx ${{ needs.resolve.outputs.nginx_version }} mainline)`, # which renamed the check on every nginx mainline release and so kept # this job out of the required-checks set. The version is reported by the # "Report resolved version" step below instead. Do NOT put the `needs` # expression back in the title. name: Tests runs-on: ${{ fromJSON(vars.POOL || '["ubuntu-latest"]') }} timeout-minutes: 20 needs: [resolve, build] env: # Resolved mainline nginx — names the build artifact to download. NGINX_VERSION: ${{ needs.resolve.outputs.nginx_version }} # Every Test::Nginx job on this self-hosted runner needs its own listen # port: the 1984 default collides across concurrent jobs (and with # ci-deep.yml's matrix), and the loser dies with "Address already in # use" at whatever test it happened to reach. Spaced by 10 because # Test::Nginx reserves port+1..+3 for its stream servers. TEST_NGINX_SERVER_PORT: "2014" steps: # Fixed job title -- surface the resolved version here instead. - name: Report resolved version run: | echo "Testing against nginx $NGINX_VERSION mainline" echo "### Tests: nginx $NGINX_VERSION mainline" \ >> "$GITHUB_STEP_SUMMARY" - name: Checkout module uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Cache apt packages uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: # User-owned dir, not the system /var/cache/apt/archives: apt runs # under sudo, so the post-step tar (unprivileged runner user) cannot # read the root-owned archives/lock or archives/partial (c2). # # Glob the .deb files rather than the directory: apt recreates # `partial/` (0700 _apt:root) and `lock` (root:root) INSIDE whatever # Dir::Cache::archives points at, so caching the dir itself still # gives tar two unreadable entries and it exits 2 -> "Failed to # save" (a warning, so CI stays green while nothing is ever cached). # Only the .deb files are worth caching anyway. path: ${{ runner.temp }}/apt-archives/*.deb key: apt-tests-${{ runner.os }}-libperl-cpanminus-libzstd1 restore-keys: apt-tests-${{ runner.os }}- - name: Install dependencies run: | # apt profile the bare apt-get lines below rely on: loose timeouts # for the shared self-hosted box; Azure mirror only on the # ubuntu-latest fallback when vars.POOL is unset (PR #121) mkdir -p "$RUNNER_TEMP/apt-archives" printf '%s\n' \ 'DPkg::Lock::Timeout "120";' \ 'Acquire::Retries "3";' \ 'Acquire::http::Timeout "20";' \ 'Acquire::https::Timeout "20";' \ "Dir::Cache::archives \"$RUNNER_TEMP/apt-archives\";" \ | sudo tee /etc/apt/apt.conf.d/99-ci-net > /dev/null sudo apt-get update sudo apt-get install -y \ libperl-dev \ cpanminus \ libzstd1 \ python3 \ python3-requests \ zstd - name: Download nginx binary (mainline, from build matrix) uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: # The build matrix uploads server-binary--; the # full Perl/Python suite runs against nginx mainline specifically. name: server-binary-nginx-${{ env.NGINX_VERSION }} path: nginx-bin - name: Set up nginx binary run: | chmod +x nginx-bin/nginx echo "TEST_NGINX_BINARY=$GITHUB_WORKSPACE/nginx-bin/nginx" >> "$GITHUB_ENV" echo "✓ nginx binary ready: $(nginx-bin/nginx -v 2>&1)" - name: Cache Perl modules uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ~/perl5 key: perl-modules-${{ runner.os }}-Test-Nginx-Socket-${{ env.TEST_NGINX_SOCKET_VERSION }} - name: Install Test::Nginx::Socket run: | ci/tools/install-test-nginx-socket.sh "$HOME/perl5" - name: Run filter module tests env: RUN_ID: ${{ github.run_id }} JOB_NAME: ${{ github.job }} run: | export PERL5LIB="$HOME/perl5/lib/perl5:${PERL5LIB:-}" export TEST_NGINX_SERVROOT="/tmp/nginx-servroot-filter-${RUN_ID}-${JOB_NAME}" mkdir -p "$TEST_NGINX_SERVROOT" cd "$GITHUB_WORKSPACE" perl ci/t/00-filter.t 2>&1 | sed 's/^Error:/Err:/' | tee ci/t/filter-tests.log if [ "${PIPESTATUS[0]}" -ne 0 ]; then echo "❌ Filter tests FAILED"; exit 1; fi echo "✓ Filter module: All tests passed" - name: Run config-load warning tests env: RUN_ID: ${{ github.run_id }} JOB_NAME: ${{ github.job }} run: | export PERL5LIB="$HOME/perl5/lib/perl5:${PERL5LIB:-}" # Strict-path fixtures validate every ancestor. Keep their servroot # under the runner's private home rather than the group-writable # checkout work area or world-writable /tmp so the # intended dot/symlink component is the first failing condition. STRICT_SERVROOT="$HOME/.nginx-servroot-confwarn-${RUN_ID}-${GITHUB_RUN_ATTEMPT}-${JOB_NAME}" export TEST_NGINX_SERVROOT="$STRICT_SERVROOT" trap 'rm -rf -- "$STRICT_SERVROOT"' EXIT rm -rf "$TEST_NGINX_SERVROOT" mkdir -p "$TEST_NGINX_SERVROOT" cd "$GITHUB_WORKSPACE" # Startup-time warnings (e.g. zstd_bypass_vary without zstd_bypass) # are only observable on the first request cycle, so they live in # their own repeat_each(1) file. perl ci/t/02-conf-warn.t 2>&1 | sed 's/^Error:/Err:/' | tee ci/t/conf-warn-tests.log if [ "${PIPESTATUS[0]}" -ne 0 ]; then echo "❌ Config-warning tests FAILED"; exit 1; fi echo "✓ Config-load warning tests passed" - name: Run static module tests env: RUN_ID: ${{ github.run_id }} JOB_NAME: ${{ github.job }} run: | export PERL5LIB="$HOME/perl5/lib/perl5:${PERL5LIB:-}" export TEST_NGINX_SERVROOT="$GITHUB_WORKSPACE/ci/t/servroot-static-${RUN_ID}-${JOB_NAME}" mkdir -p "$TEST_NGINX_SERVROOT" touch -d @1541504307 "$GITHUB_WORKSPACE/ci/t/suite/test" "$GITHUB_WORKSPACE/ci/t/suite/test.zst" cd "$GITHUB_WORKSPACE" perl ci/t/01-static.t 2>&1 | sed 's/^Error:/Err:/' | tee ci/t/static-tests.log if [ "${PIPESTATUS[0]}" -ne 0 ]; then echo "❌ Static tests FAILED"; exit 1; fi echo "✓ Static module: All tests passed" - name: Run dcz negotiation tests env: RUN_ID: ${{ github.run_id }} JOB_NAME: ${{ github.job }} run: | export PERL5LIB="$HOME/perl5/lib/perl5:${PERL5LIB:-}" export TEST_NGINX_SERVROOT="/tmp/nginx-servroot-dcz-${RUN_ID}-${JOB_NAME}" rm -rf "$TEST_NGINX_SERVROOT" mkdir -p "$TEST_NGINX_SERVROOT" cd "$GITHUB_WORKSPACE" perl ci/t/03-dcz.t 2>&1 | sed 's/^Error:/Err:/' | tee ci/t/dcz-tests.log if [ "${PIPESTATUS[0]}" -ne 0 ]; then echo "❌ dcz tests FAILED"; exit 1; fi echo "✓ dcz negotiation tests passed" - name: Run truncation smoke test run: | cd "$GITHUB_WORKSPACE" python3 ci/tools/test_encoding.py --nginx-binary "$TEST_NGINX_BINARY" --port 18080 echo "✓ Truncation smoke test passed" - name: Run Vary smoke test run: | cd "$GITHUB_WORKSPACE" python3 ci/tools/test_encoding.py --nginx-binary "$TEST_NGINX_BINARY" \ --port 18081 --gzip-vary on --expect-vary echo "✓ Vary smoke test passed" - name: Run boundary-size smoke test run: | cd "$GITHUB_WORKSPACE" python3 ci/tools/test_encoding.py --nginx-binary "$TEST_NGINX_BINARY" \ --port 18082 --fixture-lines 1900 echo "✓ Boundary-size smoke test passed" - name: Run repeated-request smoke test run: | cd "$GITHUB_WORKSPACE" python3 ci/tools/test_encoding.py --nginx-binary "$TEST_NGINX_BINARY" \ --port 18083 --request-count 2 echo "✓ Repeated-request smoke test passed" - name: Run concurrent-request smoke test run: | cd "$GITHUB_WORKSPACE" python3 ci/tools/test_encoding.py --nginx-binary "$TEST_NGINX_BINARY" \ --port 18084 --concurrent-requests 2 echo "✓ Concurrent-request smoke test passed" - name: Run terminal-frame regression test run: | cd "$GITHUB_WORKSPACE" python3 ci/tools/test_terminal_frame.py --nginx-binary "$TEST_NGINX_BINARY" --port 18085 echo "✓ Terminal-frame regression test passed" - name: Run $zstd_ratio dynamic-cacheable regression test run: | cd "$GITHUB_WORKSPACE" python3 ci/tools/test_var_dynamic_cacheable.py --nginx-binary "$TEST_NGINX_BINARY" --port 18142 echo "✓ Dynamic-cacheable variable regression test passed" - name: Run proxy_buffering-off truncation regression (bug B) run: | cd "$GITHUB_WORKSPACE" # Reproduces the production trigger: zstd over an unbuffered # (proxy_buffering off) chunked, no-Content-Length upstream. # Decodes and byte-compares the full body across the # ZSTD_CStreamInSize buffer-boundary size matrix. Fails on a # module that forwards a zero-size buffer ("zero size buf in # writer" -> aborted/truncated response). python3 ci/tools/test_proxy_unbuffered_truncation.py \ --nginx-binary "$TEST_NGINX_BINARY" \ --port 18086 --backend-port 18087 echo "✓ proxy_buffering-off truncation regression passed" # Deterministic drivers for the two timing-dependent paths # (buffer-cap nomem recovery, content-less flush) that the # coverage floor previously measured only when CI load # produced the right races — see ci/tools/test_slow_drain.py. python3 ci/tools/test_slow_drain.py \ --nginx-binary "$TEST_NGINX_BINARY" \ --port 18100 --backend-port 18101 echo "✓ slow-drain backpressure + data-less flush passed" # ci/t/00-filter.t TEST 92 pins the sub_filter/CCtx-reset # contract but its Test::Nginx tcp_reply array-send coalesces # the three backend chunks into one read locally, so it never # exercises the mid-stream sync-carrier path the bug needs # (see memory/labs/http-zstd/issues.md, 2026-08-13). This tool # drives the same scenario over a raw socket backend with a # real sleep between each send() call, and self-checks that # the reads stayed uncoalesced before trusting its own result. python3 ci/tools/test_sub_filter_cctx_reset.py \ --nginx-binary "$TEST_NGINX_BINARY" \ --port 18102 --backend-port 18103 echo "✓ sub_filter/CCtx-reset mid-stream regression passed" - name: Run broad compression correctness matrix timeout-minutes: 12 run: | cd "$GITHUB_WORKSPACE" # ~504 cells: {proxy-buffered, proxy-unbuffered, static} # x {incompressible, highly-compressible, css} # x {1, CStreamInSize +/-1, 200000, 1MiB} # x {zstd, gzip+zstd, gzip-only, identity} x repeat. # Every cell decodes per Content-Encoding and byte-compares # to origin, or asserts correct fallback. Catches the whole # truncation / zero-size-buf / livelock bug class across the # circumstances it has historically recurred in (a step-level # timeout makes a CPU-spin livelock a hard FAIL, not a hang). python3 ci/tools/test_compression_matrix.py \ --nginx-binary "$TEST_NGINX_BINARY" \ --port 18096 --backend-port 18097 echo "✓ compression correctness matrix passed" - name: Run per-request CCtx isolation regression timeout-minutes: 6 run: | cd "$GITHUB_WORKSPACE" # 774b4a5 class: a shared/not-reset ZSTD_CCtx silently bleeds # one request's compressor state into another (well-formed # but wrong body). Fire many distinct concurrent requests # through one worker over reused keepalive connections; each # response must decode byte-exact to its OWN origin. # --require-multi-slot additionally asserts, from the debug # witnesses, that MORE THAN ONE ring slot was actually lent. # Without it the byte-exactness oracle is satisfied by a build # where every concurrent claimant falls through to a # per-request context -- which is what the single-slot cache # did, so the test would pass without ever exercising the # concurrent slot borrowing the ring introduces. Only on this # non-sanitized job: UBSAN cannot log at debug at all (see the # --log-level warn note on the sanitizer jobs below). python3 ci/tools/test_concurrent_cctx_isolation.py \ --nginx-binary "$TEST_NGINX_BINARY" \ --port 18098 --backend-port 18099 \ --concurrency 16 --rounds 6 --require-multi-slot echo "✓ per-request CCtx isolation regression passed" - name: Run worker CCtx cache regression timeout-minutes: 6 run: | cd "$GITHUB_WORKSPACE" # The filter lends one worker-lifetime CCtx to one request at a # time instead of creating one per response. Two properties, and # the correctness suite above only covers the second: the cache # must actually ENGAGE (asserted from the debug witnesses -- a # cache that never hits passes every other test in this file), # and a reused context must carry no residue into the next # response. Also pins the profile partition: the retained # workspace is driven by zstd_comp_level, zstd_long AND # zstd_window_log and never shrinks, so a location differing in # any of the three must not reuse the cached context. python3 ci/tools/test_cctx_reuse.py \ --nginx-binary "$TEST_NGINX_BINARY" \ --port 18116 echo "✓ worker CCtx cache regression passed" - name: Run reload-under-load correctness regression timeout-minutes: 6 run: | cd "$GITHUB_WORKSPACE" # 2c538e0 CDict-cycle class: SIGHUP reload while requests are # in flight (zstd_dict_file set). Every response — old-cycle, # straddling, new-cycle — must still decode byte-exact; master # must survive; error log must stay clean. Complements # ci/tools/test_reload_leak.sh (which checks leaks, not response # correctness during reload). python3 ci/tools/test_reload_under_load.py \ --nginx-binary "$TEST_NGINX_BINARY" \ --port 18100 --backend-port 18101 \ --reloads 6 --workers 4 --duration 18 echo "✓ reload-under-load correctness regression passed" - name: Run zstd_long (LDM) directive regression timeout-minutes: 6 run: | cd "$GITHUB_WORKSPACE" # 8edb7bb zstd_long class: the directive had ZERO coverage. # A 16 MiB body whose duplicate region repeats beyond the # level-6 default window must (a) decode byte-exact and # (b) compress ~2:1 smaller with zstd_long on than off — # proving ZSTD_c_enableLongDistanceMatching actually reached # libzstd and engaged, and the directive neither corrupts nor # truncates the stream (the bug-B failure mode). python3 ci/tools/test_zstd_long_ldm.py \ --nginx-binary "$TEST_NGINX_BINARY" \ --port 18102 --backend-port 18103 echo "✓ zstd_long (LDM) directive regression passed" - name: Run window-cap + dictionary effect test timeout-minutes: 6 run: | cd "$GITHUB_WORKSPACE" # Frame-inspection effect test: with zstd_dict_file AND # zstd_window_log 15 set, the emitted frame must declare a # window <= 2^15 and reference the trained dictID (audit # C2/R1 CDict-path regression); decode requires the dict and # round-trips byte-exact. python3 ci/tools/test_window_cap.py \ --nginx-binary "$TEST_NGINX_BINARY" \ --port 18104 --backend-port 18105 echo "✓ window-cap + dictionary effect test passed" - name: Run dcz budget + CCtx ring-key regression timeout-minutes: 6 run: | cd "$GITHUB_WORKSPACE" # A dcz (RFC 9842) request does not compress at zstd_window_log: # it derives its own window from the negotiated dictionary, up to # 2^23. Nothing coupled that to either memory control before. # Asserts three things the fix has to hold at once: the dcz # request lands in its OWN CCtx ring slot (keyed on the effective # window, so it cannot raise a plain slot's permanently-retained # workspace), zstd_max_cctx_memory clamps the dcz window to the # budget-derived cap, and with NEITHER directive set the dcz # window is unchanged -- the clamp is opt-in and must not move # wire bytes for operators who never asked for a memory bound. # Needs --with-debug: the ring witnesses are debug log lines. python3 ci/tools/test_dcz_budget_ring.py \ --nginx-binary "$TEST_NGINX_BINARY" \ --port 18160 --backend-port 18162 echo "✓ dcz budget + CCtx ring-key regression passed" # Config-load regression: "zstd_long on" + "zstd_max_cctx_memory" # made nginx -t die with SIGFPE, because libzstd's estimator # divides by an LDM sub-parameter that is only derived lazily # during compression setup. ci/t/00-filter.t cannot cover this: # its "--- must_die" cannot distinguish a crash from a # diagnostic. This asserts on the exit status and its sign. python3 ci/tools/test_ldm_budget_config.py \ --nginx-binary "$TEST_NGINX_BINARY" --port 18124 echo "✓ zstd_long + zstd_max_cctx_memory config validation passed" # Implicit per-request CCtx memory advisory. zstd_max_cctx_memory # is optional, so the estimator used to be skipped entirely for # the common no-budget config and an expensive profile # (zstd_comp_level 22, zstd_long on) committed hundreds of MB per # concurrent response silently. The matrix asserts the advisory # fires on the large profiles, stays silent on the ordinary ones # and on an explicit budget/acknowledgement, and -- the part # Test::Nginx cannot express -- that it never turns a working # nginx -t into a failing one. python3 ci/tools/test_cctx_advisory_policy.py \ --nginx-binary "$TEST_NGINX_BINARY" --port 18130 echo "✓ CCtx memory advisory policy matrix passed" # Aggregate "zstd_buffers number size" bound: ngx_conf_set_bufs_slot() # (nginx core) never checks the product, and neither does the # merge that applies an inherited or defaulted value. Overflow # is a hard error; a representable-but-large total is advisory # only. ci/t cannot express "PASS and warns"; see the tool. python3 ci/tools/test_bufs_bound_policy.py \ --nginx-binary "$TEST_NGINX_BINARY" --port 18141 echo "✓ zstd_buffers aggregate bound policy matrix passed" # zstd_bypass_vary config-load table: rejection (comma/semicolon # /quote/bare wildcard), acceptance (non-solitary wildcard), # verbatim retention and http-vs-location override. None of this # is expressible via ci/t's --- must_die (it needs both a PASS # assertion and an exact Vary-header byte comparison); see the # tool's docstring. python3 ci/tools/test_bypass_vary_config_policy.py \ --nginx-binary "$TEST_NGINX_BINARY" --port 18150 echo "✓ zstd_bypass_vary config-load policy matrix passed" - name: Run RFC 9842 dcz end-to-end test timeout-minutes: 6 run: | cd "$GITHUB_WORKSPACE" # dcz negotiation matrix plus the wire contract ci/t/03-dcz.t # cannot check: the 40-byte frame header (skippable magic + # dictionary SHA-256), decode via `zstd -d -D` capped at the # RFC's 8 MB client-window guarantee, byte-compare to origin, # and proof the dictionary engaged (dcz body < plain zstd). python3 ci/tools/test_dcz.py --nginx-binary "$TEST_NGINX_BINARY" \ --port 18106 --tls-port 18117 --insecure-port 18118 # Shared-cache partitioning of the dcz cross-origin decision: # Sec-Fetch-Site selects the representation, so a real # proxy_cache in front of the module must not serve the dcz # variant to a cross-site request (or suppress it for a # same-origin one). Runs the {same-origin, cross-site, # absent} x {both fill orders} matrix end to end. python3 ci/tools/test_dcz_cache_partition.py \ --nginx-binary "$TEST_NGINX_BINARY" --port 18120 --origin-port 18121 echo "✓ RFC 9842 dcz end-to-end test passed" # dcz 40-byte prefix inlining: one compressor-buffer allocation # per response, not a separate dedicated prefix buffer, and the # wire body still decodes byte-exact. Needs --with-debug (same as # the ring-key test above): the allocation-count oracle greps # debug log lines ("zstd get_buf: allocated buffer" / # "zstd dcz: 40-byte frame header queued"). This tool previously # existed in ci/tools/ but was never wired into any CI job. python3 ci/tools/test_dcz_prefix_alloc.py \ --nginx-binary "$TEST_NGINX_BINARY" --port 18311 echo "✓ dcz prefix single-allocation regression passed" - name: Test summary if: success() run: echo "✓ All Perl + Python end-to-end tests passed" - name: Upload test logs on failure if: failure() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: test-logs path: | ${{ github.workspace }}/ci/t/filter-tests.log ${{ github.workspace }}/ci/t/static-tests.log ${{ github.workspace }}/ci/t/dcz-tests.log retention-days: 7 # ── Smoke tests under ASAN+UBSAN — fails on any memory/UB error ─────────── tests-asan: name: Tests ASAN+UBSAN runs-on: ${{ fromJSON(vars.POOL || '["ubuntu-latest"]') }} timeout-minutes: 20 needs: build-asan env: # Own Test::Nginx listen port — see the tests job for why. TEST_NGINX_SERVER_PORT: "2024" steps: - name: Checkout module uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Install dependencies run: | # apt profile the bare apt-get lines below rely on: loose timeouts # for the shared self-hosted box; Azure mirror only on the # ubuntu-latest fallback when vars.POOL is unset (PR #121) printf '%s\n' \ 'DPkg::Lock::Timeout "120";' \ 'Acquire::Retries "3";' \ 'Acquire::http::Timeout "20";' \ 'Acquire::https::Timeout "20";' \ | sudo tee /etc/apt/apt.conf.d/99-ci-net > /dev/null sudo apt-get update sudo apt-get install -y python3 python3-requests zstd libzstd1 \ libperl-dev cpanminus - name: Download ASAN nginx binary uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: nginx-binary-asan path: nginx-bin - name: Set up nginx binary run: | chmod +x nginx-bin/nginx echo "TEST_NGINX_BINARY=$GITHUB_WORKSPACE/nginx-bin/nginx" >> "$GITHUB_ENV" - name: Cache Perl modules uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ~/perl5 key: perl-modules-${{ runner.os }}-Test-Nginx-Socket-${{ env.TEST_NGINX_SOCKET_VERSION }} - name: Install Test::Nginx::Socket run: | ci/tools/install-test-nginx-socket.sh "$HOME/perl5" - name: Run Perl suites under ASAN+UBSAN # Put the static-file / magic-probe / conditional / HEAD paths (covered # only by these Perl suites, audit CI7) through ASan/UBSan. Functional # pass/fail is also gated here: a bailout or incomplete TAP stream is # not sanitizer coverage. Upstream nginx core has known UBSan reports # on debug-log paths, so keep recovery enabled, parse complete reports, # and fail when any report contains a zstd-module frame. detect_leaks=0 # because nginx does not free every allocation at exit by design. env: ASAN_OPTIONS: detect_leaks=0:halt_on_error=0:abort_on_error=0:exitcode=0:log_path=/tmp/asan-zstd UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=0:log_path=/tmp/ubsan-zstd RUN_ID: ${{ github.run_id }} JOB_NAME: ${{ github.job }} run: | export PERL5LIB="$HOME/perl5/lib/perl5:${PERL5LIB:-}" ci/tools/test_sanitizer_suite_verdict.sh rm -f /tmp/asan-zstd.* /tmp/ubsan-zstd.* cd "$GITHUB_WORKSPACE" export TEST_NGINX_SERVROOT="/tmp/nginx-servroot-asan-${RUN_ID}-${JOB_NAME}" mkdir -p "$TEST_NGINX_SERVROOT" filter_status=0 perl ci/t/00-filter.t > ci/t/filter-asan.log 2>&1 || filter_status=$? touch -d @1541504307 "$GITHUB_WORKSPACE/ci/t/suite/test" "$GITHUB_WORKSPACE/ci/t/suite/test.zst" export TEST_NGINX_SERVROOT="$GITHUB_WORKSPACE/ci/t/servroot-static-${RUN_ID}-${JOB_NAME}" mkdir -p "$TEST_NGINX_SERVROOT" static_status=0 perl ci/t/01-static.t > ci/t/static-asan.log 2>&1 || static_status=$? python3 ci/tools/sanitizer_suite_verdict.py \ --suite "00-filter.t:ci/t/filter-asan.log:$filter_status" \ --suite "01-static.t:ci/t/static-asan.log:$static_status" \ --sanitizer-log '/tmp/asan-zstd.*' \ --sanitizer-log '/tmp/ubsan-zstd.*' \ --sanitizer-log 'ci/t/filter-asan.log' \ --sanitizer-log 'ci/t/static-asan.log' - name: Run smoke tests under ASAN+UBSAN # Leak gating here is NOT free: every harness below starts nginx and # stops it with SIGTERM, and each one accepts -15 as a clean exit # (e.g. ci/tools/test_encoding.py's `returncode not in (0, -15)`). LSan # reports at worker exit and signals a leak through the exit STATUS, # so a leak was invisible twice over -- the harness waved the status # through, and this step never looked at LSan's output at all. Both # halves are fixed here: reports go to files, and the step greps them # after the harnesses run. # # log_path is what makes that possible: without it LSan writes to # stderr, which the harnesses capture and discard. detect_leaks stays # 1 -- the point is to gate on leaks, not to silence them the way the # Perl-suite step above does. env: ASAN_OPTIONS: detect_leaks=1:abort_on_error=1:strict_string_checks=1:halt_on_error=1:log_path=/tmp/lsan-smoke # ABSOLUTE path: LSan resolves this against the CWD of the process # it fires in -- an nginx worker started by a harness, whose CWD is # not the workspace. A relative path there fails to open and LSan # aborts, so the suppressions must not depend on where nginx ran. # verbosity=1 supplies a per-process exit-hook witness. LLVM's # ptrace preflight is only a heuristic and can false-warn before the # real stop-the-world check; ci/tools/lsan_log_verdict.sh separates # that warning from an actual attach failure. LSAN_OPTIONS: >- suppressions=${{ github.workspace }}/ci/suppressions/lsan.suppress:print_suppressions=0:verbosity=1 UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1 run: | cd "$GITHUB_WORKSPACE" rm -f /tmp/lsan-smoke.* python3 ci/tools/test_encoding.py --nginx-binary "$TEST_NGINX_BINARY" --port 18090 python3 ci/tools/test_encoding.py --nginx-binary "$TEST_NGINX_BINARY" --port 18091 --fixture-lines 1900 python3 ci/tools/test_encoding.py --nginx-binary "$TEST_NGINX_BINARY" --port 18092 --concurrent-requests 2 python3 ci/tools/test_terminal_frame.py --nginx-binary "$TEST_NGINX_BINARY" --port 18093 python3 ci/tools/test_proxy_unbuffered_truncation.py \ --nginx-binary "$TEST_NGINX_BINARY" --port 18094 --backend-port 18095 # Backpressure pause/resume under sanitizers: the nomem and # empty-flush paths share lifetime history with the # truncation class, so drive them deliberately here too. # --log-level warn: a UBSAN nginx (-fno-sanitize-recover) # fatally traps core's own debug logging (the "%V?%V" lines # pass NULL for empty r->args into ngx_sprintf_str's nonnull # arg), so the sanitized binary cannot log at debug at all. # The tool still forces both paths and byte-checks the # streams; the log-witness assertions run in the tests and # coverage jobs, whose binaries are not sanitized. python3 ci/tools/test_slow_drain.py \ --nginx-binary "$TEST_NGINX_BINARY" --port 18110 --backend-port 18111 \ --log-level warn # See the tests job's "sub_filter/CCtx-reset" step for why # this exists. --log-level warn: same UBSAN debug-log trap as # test_slow_drain.py above -- the body round-trip assertion # still runs, only the invocation-count self-check is skipped. python3 ci/tools/test_sub_filter_cctx_reset.py \ --nginx-binary "$TEST_NGINX_BINARY" --port 18114 --backend-port 18115 \ --log-level warn # Matrix under ASAN/UBSAN at repeat=1 (252 cells) — the # truncation/zero-size-buf/livelock class has UB+lifetime # history, so exercise the full circumstance matrix with # sanitizers, not just the happy path. python3 ci/tools/test_compression_matrix.py \ --nginx-binary "$TEST_NGINX_BINARY" --port 18096 \ --backend-port 18097 --repeat 1 # CCtx isolation under ASAN/UBSAN: a shared/not-reset context # is a lifetime bug — exactly what ASAN is for. Lighter # concurrency/rounds since the sanitized binary is slower. python3 ci/tools/test_concurrent_cctx_isolation.py \ --nginx-binary "$TEST_NGINX_BINARY" --port 18098 \ --backend-port 18099 --concurrency 8 --rounds 3 # dcz under ASAN/UBSAN: ZSTD_CCtx_refPrefix references # config-pool memory from the request path and the 40-byte # frame header is hand-assembled — both lifetime/OOB classes. python3 ci/tools/test_dcz.py --nginx-binary "$TEST_NGINX_BINARY" \ --port 18089 --tls-port 18109 --insecure-port 18113 # Shared-cache partitioning of the dcz cross-origin decision: # Sec-Fetch-Site selects the representation, so a real # proxy_cache in front of the module must not serve the dcz # variant to a cross-site request (or suppress it for a # same-origin one). Runs the {same-origin, cross-site, # absent} x {both fill orders} matrix end to end. python3 ci/tools/test_dcz_cache_partition.py \ --nginx-binary "$TEST_NGINX_BINARY" --port 18122 --origin-port 18123 # Baseline loaded-and-blocking control (PR2 step 22): the ASAN # binary is a static --add-module build, the shape this check # needs to prove the compiled-in `zstd on;` default is OFF. # Every other case in this job asserts response CORRECTNESS, # none of them individually depend on compression having # happened at all -- this is the control that would catch a # module that failed to load or a decision-flip in the enable # default, which every other green case here would miss. python3 ci/tools/test_baseline_loaded.py \ --nginx-binary "$TEST_NGINX_BINARY" \ --port 18107 --backend-port 18108 --off-port 18112 bash ci/tools/lsan_log_verdict.sh '/tmp/lsan-smoke.*' echo "✓ All smoke tests clean under ASAN+UBSAN" - name: zstd_dict_file reload leak check (CDict lifecycle) env: ASAN_OPTIONS: detect_leaks=1:abort_on_error=1:halt_on_error=1 UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1 run: | cd "$GITHUB_WORKSPACE" bash ci/tools/test_reload_leak.sh "$TEST_NGINX_BINARY" 5 myguard-labs-nginx-zstd-module-3486b5b/.github/workflows/bump.yml000066400000000000000000000117271525232545500251310ustar00rootroot00000000000000name: Bump versions # Weekly: check the upstream release feeds for newer releases than what's # pinned here -- nginx-stable/angie in ci-deep.yml's build-flavors matrix, # the harness jobs' static mainline pin (HARNESS_NGINX_VERSION in ci-deep.yml # and harness-fault-arms.yml), and the Windows build's sources in # ci/tools/windows-pins.sh. The matrix's own mainline entry is resolved # dynamically at CI run time (build-test.yml's `resolve` job / ci-deep.yml's # per-job resolve steps, through ci/tools/nginx-releases.sh) and has no # static pin; ci/tools/bump-versions.sh's header lists exactly what is and # is not bumped. # # ci/tools/bump-versions.sh does the actual work (also runnable locally with # --dry-run to preview). Opens a PR rather than pushing straight to master -- # master carries a required-pull-request ruleset with no bypass actor, so a # direct push is always rejected (audit sha e289021 F4); the bot instead # pushes a branch and opens a PR via BUMP_PR_TOKEN (a classic PAT -- the # default GITHUB_TOKEN cannot create PRs on myguard-labs repos). Normal # review/required-checks still gate the merge. # # BUMP_PR_TOKEN authenticates BOTH halves: gh (via GH_TOKEN) for the PR, and # git (via GIT_ASKPASS) for the branch push. persist-credentials is false, so # the push has no ambient credential of its own -- without the askpass helper # it dies on "could not read Username for 'https://github.com'". # # Action pins (keep in sync with build-test.yml): # actions/checkout@v5.1.0 -> fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 on: schedule: # Weekly, Monday 04:00 UTC. - cron: "0 4 * * 1" workflow_dispatch: {} concurrency: group: bump-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true permissions: contents: read jobs: bump: name: Bump version pins runs-on: ${{ fromJSON(vars.POOL || '["ubuntu-latest"]') }} timeout-minutes: 15 steps: - name: Checkout uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false fetch-depth: 0 # This scheduled workflow also falls back to ubuntu-latest for forks. - name: Install dependencies run: | sudo apt-get update sudo apt-get install -y curl gh git gnupg python3 - name: Run bump script id: bump env: # The angie release lookup hits api.github.com; unauthenticated the # shared runner IP is rate-limited to 403s. GH_TOKEN: ${{ secrets.BUMP_PR_TOKEN }} run: bash ci/tools/bump-versions.sh - name: Open PR if anything changed env: GH_TOKEN: ${{ secrets.BUMP_PR_TOKEN }} BUMP_PR_TOKEN: ${{ secrets.BUMP_PR_TOKEN }} run: | set -euo pipefail if [ -z "$(git status --porcelain)" ]; then echo "nothing changed, skipping PR" exit 0 fi branch="bump/versions-$(date -u +%Y%m%d)" git config user.name "myguard-bump-bot" git config user.email "actions@users.noreply.github.com" git checkout -b "$branch" git add -A git commit -m "chore: bump version pins Automated weekly version check (ci/tools/bump-versions.sh)." # persist-credentials is false, so the checkout leaves no git # credential behind and a plain `git push` prompts for a username # (fatal: could not read Username, run 29722422889). GH_TOKEN # authenticates gh but not git, so hand git its own credential via # the askpass helper -- keeps the token out of argv and out of # .git/config, unlike an inline x-access-token@ push URL. git config credential.helper '' askpass="$(mktemp)" cat > "$askpass" <<'ASKPASS' #!/bin/sh case "$1" in Username*) echo x-access-token ;; *) printf '%s' "$BUMP_PR_TOKEN" ;; esac ASKPASS chmod 700 "$askpass" # Force-push: the branch name is date-based, so a same-day re-run # (a manual dispatch, or a retry after a failure) collides with the # branch the earlier run created and is otherwise rejected # non-fast-forward. The branch is bot-owned and disposable. GIT_ASKPASS="$askpass" git push --force-with-lease origin "$branch" rm -f "$askpass" # An open PR for this branch may already exist from that earlier # run; the push above has refreshed it, so there is nothing to do. if [ -n "$(gh pr list --head "$branch" --state open --json number -q '.[0].number')" ]; then echo "PR for $branch already open, refreshed by the push" exit 0 fi gh pr create \ --title "chore: bump version pins ($(date -u +%Y-%m-%d))" \ --body "Automated weekly version check (ci/tools/bump-versions.sh). Review the diff and let required CI checks gate the merge." \ --base master \ --head "$branch" myguard-labs-nginx-zstd-module-3486b5b/.github/workflows/ci-deep.yml000066400000000000000000001070371525232545500254740ustar00rootroot00000000000000name: CI Deep # Extended safety net: weekly + on-demand, never on pull_request or push. # CodeQL, fuzzing, full coverage, native sanitizer soak, and Valgrind belong # here because recent PR runs measured them at roughly 4-10 minutes each before # queueing. The PR gate keeps the faster assertion-bearing build, sanitizer, # compatibility, scanner, Windows/arm64, and testkit checks. # # A manual dispatch can run a SUBSET via the `jobs` input (default `all`); # the weekly cron passes no inputs and therefore always runs everything. # Use a subset when the full set would monopolise the shared pool for the # hours the fuzz and helgrind legs need. # # Four explicit self-hosted chains avoid relying on GitHub's unspecified ready # queue order: # lanes 1-2: one long fuzz target each; # lane 3: memcheck -> coverage -> CodeQL -> serialized flavor builds # -> scanners -> native ASan/UBSan soak; # lane 4: all testkit scenarios under Memcheck -> Helgrind. # This is longest-processing-time-first for the measured workloads and caps # peak demand at four even if the backing runner pool later grows. # * memcheck AND helgrind run under ci/tools/soak.sh, which already passes # --suppressions=ci/suppressions/valgrind.suppress and --error-exitcode=99 # for BOTH tools # (see ci/tools/soak.sh) — the drift-fix baseline for every module. # # build-flavors exists because build-test.yml (the fast PR gate) only builds # ONE pinned nginx version (mainline) — the module's README documents Angie # support and tracks nginx stable too, but nothing in CI actually built angie # or nginx stable until this job. It is here (weekly), not in the PR gate, # because each leg is a from-scratch configure+compile+Test::Nginx run and # three of them would blow the PR gate's budget. Uses ci/tools/ci-build.sh, which # supports flavor+version args and sha256-verifies the tarball it downloads. # # Action pins (keep in sync with build-test.yml header): # actions/checkout@v5.1.0 -> fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # actions/upload-artifact@v6.0.0 -> b7c566a772e6b6bfb58ed0dc250532a479d7789f env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true NGINX_VERSION: "" HARNESS_NGINX_VERSION: "1.31.5" # Keep in sync with build-test.yml's own copy of this pin -- see that # file's env: block for the full rationale (unpinned cpanm resolves to # whatever CPAN serves at run time; a warm cache can also retain a # different historical release). 0.32 is what every install site in # this repo already resolves to (verified against a recent successful # CI log) and is also CPAN's current latest release. TEST_NGINX_SOCKET_VERSION: "0.32" TEST_NGINX_SOCKET_DIST: "AGENT/Test-Nginx-0.32.tar.gz" on: schedule: # Weekly Sunday 03:37 UTC, away from top-of-hour scheduler pressure. - cron: "37 3 * * 0" workflow_dispatch: inputs: fuzz_seconds: description: "Per-target fuzz duration (seconds)" required: false default: "3600" jobs: # Which jobs a manual dispatch should run. The full set occupies most # of the shared self-hosted pool for hours (fuzz alone is capped at # 600min), so an on-demand run usually wants a subset -- e.g. after a # suppression-file fix, only the memcheck soak is meaningful. The # scheduled weekly run supplies no inputs, so `all` is the default and # cron behaviour is unchanged. description: "Jobs to run" required: false default: all type: choice options: - all - memcheck+coverage - build-flavors - scanners # A long soak is often the ONLY evidence for a finding: fuzz is capped at # 600 minutes, memcheck and helgrind at 240. Keying the group on the job # selection (and on the run id for the scheduled run) keeps a targeted # dispatch -- e.g. `jobs=scanners`, a ~2 minute job, the documented common # case above -- from cancelling an in-flight weekly cron run on the same # ref and destroying hours of soak with no artifact. Two dispatches of the # SAME selection still coalesce, which is the only case where cancelling is # the intended behaviour. concurrency: group: >- ci-deep-${{ github.workflow }}-${{ github.ref }}-${{ github.event_name == 'schedule' && format('schedule-{0}', github.run_id) || format('dispatch-{0}', inputs.jobs || 'all') }} cancel-in-progress: true permissions: contents: read jobs: build-flavors: name: Build & Test::Nginx (${{ matrix.flavor }} ${{ matrix.version }}) needs: codeql if: >- ${{ !cancelled() && contains(fromJSON('["all","build-flavors"]'), inputs.jobs || 'all') }} runs-on: ${{ fromJSON(vars.POOL || '["ubuntu-latest"]') }} timeout-minutes: 30 env: # Job-level band -- see ci/tools/max-port.sh. Each serialized matrix leg # keeps its own port via matrix.port below; this declares the job's OWN # territory to the port-band policy checker, which only understands one # band per job. The legs' # individual ports below all sit inside this band's width and are well # clear of the 1984 Test::Nginx default, so a job that forgets to wire # TEST_NGINX_PORT fails loudly on that default instead of silently # landing inside this job's own territory. TEST_BASE_PORT: 19300 strategy: fail-fast: false # This is lane 3's compatibility tail. Serializing the three cells keeps # the complete deep workflow at four self-hosted jobs maximum. max-parallel: 1 matrix: include: # Keep distinct ports even though max-parallel is one: manual reruns # and a future topology change must not fall back to port 1984. # Spaced by 10 because Test::Nginx reserves port+1..+3 for its # stream servers. Kept distinct from build-test.yml's ports too, # since both workflows can run at once on this runner. - flavor: nginx version: "1.31.2" label: mainline port: 19300 - flavor: nginx version: "1.30.4" label: stable port: 19310 - flavor: angie version: "1.12.1" label: angie port: 19320 steps: - name: Checkout module uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Install dependencies run: | # apt profile the bare apt-get lines below rely on: loose timeouts # for the shared self-hosted box; Azure mirror only on the # ubuntu-latest fallback when vars.POOL is unset (PR #121) printf '%s\n' \ 'DPkg::Lock::Timeout "120";' \ 'Acquire::Retries "3";' \ 'Acquire::http::Timeout "20";' \ 'Acquire::https::Timeout "20";' \ | sudo tee /etc/apt/apt.conf.d/99-ci-net > /dev/null sudo apt-get update sudo apt-get install -y \ build-essential curl gnupg libpcre2-dev libzstd-dev pkg-config \ wget zstd libperl-dev cpanminus ccache mold - name: Hash module build inputs id: build-inputs run: echo "hash=$(bash ci/tools/build-input-hash.sh)" >> "$GITHUB_OUTPUT" - name: Cache ccache objects uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ~/.ccache key: ccache-ci-deep-${{ runner.os }}-${{ matrix.flavor }}-${{ matrix.version }}-${{ steps.build-inputs.outputs.hash }} restore-keys: | ccache-ci-deep-${{ runner.os }}-${{ matrix.flavor }}-${{ matrix.version }}- ccache-ci-deep-${{ runner.os }}- # Hybrid restore: this runner is self-hosted and persistent, so # .build/ already survives between runs on disk (ci-build.sh's own # "reuse unless NO_CACHE=1" check) -- this is the FALLBACK for when it # does not (ephemeral runner, first run on a fresh box, or the runner # pool changed). Exact-match only, never a restore-keys ladder: a # PARTIAL build-tree hit (some objects rebuilt at a newer source # revision, others stale) is worse than a clean rebuild, since a # mismatched .o silently links into a binary that matches nothing. - name: Cache build tree uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: .build/${{ matrix.flavor }}-${{ matrix.version }} key: build-tree-${{ runner.os }}-${{ matrix.flavor }}-${{ matrix.version }}-${{ steps.build-inputs.outputs.hash }} - name: Build server and dynamic modules # No CCACHE_DIR override: ccache's own default (~/.ccache) already # matches the "Cache ccache objects" step's path above. env: # Routed through env: rather than spliced into run: directly -- # both values come from this file's own hardcoded matrix.include # (never attacker input), but env: indirection is the pattern # zizmor's template-injection audit checks for regardless of # provenance, and the Discord steps below use the same pattern. MATRIX_FLAVOR: ${{ matrix.flavor }} MATRIX_VERSION: ${{ matrix.version }} run: bash ci/tools/ci-build.sh "$MATRIX_FLAVOR" "$MATRIX_VERSION" - name: Verify built modules env: MATRIX_FLAVOR: ${{ matrix.flavor }} MATRIX_VERSION: ${{ matrix.version }} run: | build=".build/$MATRIX_FLAVOR-$MATRIX_VERSION/objs" test -f "$build/$MATRIX_FLAVOR" test -f "$build/ngx_http_zstd_filter_module.so" test -f "$build/ngx_http_zstd_static_module.so" echo "TEST_NGINX_BINARY=$GITHUB_WORKSPACE/$build/$MATRIX_FLAVOR" >> "$GITHUB_ENV" - name: Cache Perl modules uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ~/perl5 key: perl-modules-${{ runner.os }}-Test-Nginx-Socket-${{ env.TEST_NGINX_SOCKET_VERSION }} - name: Install Test::Nginx::Socket run: | ci/tools/install-test-nginx-socket.sh "$HOME/perl5" - name: Verify test port band # Before the first `prove` -- see ci/tools/max-port.sh. Each serialized # matrix leg retains its own port inside this job's declared # 19300-width-64 band, so a future parallelism change cannot introduce # a collision or leave the stable/Angie legs unverified. env: MATRIX_PORT: ${{ matrix.port }} run: ci/tools/max-port.sh "$MATRIX_PORT" 10 - name: Run ci/t/ (filter + static + conf-warn + dcz) env: # See build-test.yml's identical setting for why this must be 20, # not the ~2s Test::Nginx::Socket default. TEST_NGINX_TIMEOUT: "20" # Per-leg listen port — see the matrix comment above. Test::Nginx # prefers TEST_NGINX_SERVER_PORT over TEST_NGINX_PORT when both are # set (see perldoc Test::Nginx::Socket), so the actual bind is the # per-leg matrix.port; TEST_NGINX_PORT carries the job's declared # TEST_BASE_PORT purely so the port-band policy checker (which only # understands TEST_NGINX_PORT) can see this job is wired at all. TEST_NGINX_SERVER_PORT: "${{ matrix.port }}" TEST_NGINX_PORT: "${{ env.TEST_BASE_PORT }}" MATRIX_FLAVOR: ${{ matrix.flavor }} MATRIX_VERSION: ${{ matrix.version }} run: | export PERL5LIB="$HOME/perl5/lib/perl5:${PERL5LIB:-}" cd "$GITHUB_WORKSPACE" # ci/t/01-static.t asserts on fixture mtimes (Last-Modified/If-Modified-Since); # pin them to a fixed timestamp, same as build-test.yml's tests job. touch -d @1541504307 ci/t/suite/test ci/t/suite/test.zst STRICT_SERVROOT="$HOME/.nginx-servroot-$MATRIX_FLAVOR-$MATRIX_VERSION-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" export TEST_NGINX_SERVROOT="$STRICT_SERVROOT" trap 'rm -rf -- "$STRICT_SERVROOT"' EXIT rm -rf -- "$TEST_NGINX_SERVROOT" mkdir -p "$TEST_NGINX_SERVROOT" prove -v ci/t/00-filter.t ci/t/02-conf-warn.t prove -v ci/t/03-dcz.t # 01-static.t serves fixtures via "root ../suite", resolved against # the servroot's PARENT, so the servroot must be a sibling of # ci/t/suite — i.e. inside ci/t/. A /tmp servroot makes every static # case 404. Same split as build-test.yml's tests job. export TEST_NGINX_SERVROOT="$GITHUB_WORKSPACE/ci/t/servroot-static" mkdir -p "$TEST_NGINX_SERVROOT" prove -v ci/t/01-static.t fuzz: name: Fuzz (${{ matrix.label }}, long) if: ${{ (inputs.jobs || 'all') == 'all' }} runs-on: ${{ fromJSON(vars.POOL || '["ubuntu-latest"]') }} timeout-minutes: ${{ vars.POOL && 350 || 330 }} strategy: fail-fast: false max-parallel: 2 matrix: include: - label: Accept-Encoding binary: fuzz_accept_encoding corpus: corpus minimized: corpus-min dict_arg: -dict=fuzz.dict artifact_prefix: crash- - label: dcz binary: fuzz_dcz corpus: corpus_dcz minimized: corpus-dcz-min dict_arg: "" artifact_prefix: crash-dcz- steps: - name: Checkout module uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Install clang and libFuzzer run: | printf '%s\n' \ 'DPkg::Lock::Timeout "120";' \ 'Acquire::Retries "3";' \ 'Acquire::http::Timeout "20";' \ 'Acquire::https::Timeout "20";' \ | sudo tee /etc/apt/apt.conf.d/99-ci-net > /dev/null sudo apt-get update sudo apt-get install -y clang curl python3 - name: Select fuzz duration env: RAW_FUZZ_SECONDS: ${{ github.event.inputs.fuzz_seconds }} GH_EVENT_NAME: ${{ github.event_name }} run: | if [ "$GH_EVENT_NAME" = "workflow_dispatch" ]; then case "$RAW_FUZZ_SECONDS" in ''|*[!0-9]*) echo "::error::fuzz_seconds must be a non-negative integer" exit 1 ;; esac if [ "$RAW_FUZZ_SECONDS" -gt 14400 ]; then echo "::error::fuzz_seconds exceeds the 14400s budget" exit 1 fi printf 'FUZZ_SECS=%s\n' "$RAW_FUZZ_SECONDS" >> "$GITHUB_ENV" else printf 'FUZZ_SECS=3600\n' >> "$GITHUB_ENV" fi - name: Build fuzz targets run: >- bash ci/fuzz/build.sh "$GITHUB_WORKSPACE/ci/fuzz/fuzz_accept_encoding" "$GITHUB_WORKSPACE/ci/fuzz/fuzz_dcz" - name: Fuzz selected target env: FUZZ_BINARY: ${{ matrix.binary }} FUZZ_CORPUS: ${{ matrix.corpus }} FUZZ_DICT_ARG: ${{ matrix.dict_arg }} FUZZ_ARTIFACT_PREFIX: ${{ matrix.artifact_prefix }} run: | set -euo pipefail cd ci/fuzz extra=() [ -z "$FUZZ_DICT_ARG" ] || extra+=("$FUZZ_DICT_ARG") "./$FUZZ_BINARY" \ -max_total_time="$FUZZ_SECS" \ -max_len=65536 \ "${extra[@]}" \ -print_final_stats=1 \ -artifact_prefix="$FUZZ_ARTIFACT_PREFIX" \ "$FUZZ_CORPUS/" - name: Minimize corpus if: success() env: FUZZ_BINARY: ${{ matrix.binary }} FUZZ_CORPUS: ${{ matrix.corpus }} MIN_CORPUS: ${{ matrix.minimized }} run: | set -euo pipefail cd ci/fuzz mkdir -p "$MIN_CORPUS" "./$FUZZ_BINARY" -merge=1 "$MIN_CORPUS/" "$FUZZ_CORPUS/" - name: Upload crash artifacts if: failure() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: fuzz-deep-${{ matrix.binary }}-crashes path: ${{ github.workspace }}/ci/fuzz/${{ matrix.artifact_prefix }}* retention-days: 7 - name: Upload minimized corpus if: success() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: fuzz-deep-${{ matrix.binary }}-corpus path: ${{ github.workspace }}/ci/fuzz/${{ matrix.minimized }} retention-days: 7 - name: Notify Discord on failure if: failure() && github.event_name == 'schedule' env: DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_BOT_TOKEN }} DISCORD_CHANNEL_BUILDS: ${{ secrets.DISCORD_CHANNEL_BUILDS }} FUZZ_LABEL: ${{ matrix.label }} GH_REPO: ${{ github.repository }} GH_SERVER_URL: ${{ github.server_url }} GH_RUN_ID: ${{ github.run_id }} run: | [ -n "$DISCORD_BOT_TOKEN" ] || exit 0 msg="🔴 $GH_REPO — CI Deep / fuzz $FUZZ_LABEL FAILED. $GH_SERVER_URL/$GH_REPO/actions/runs/$GH_RUN_ID" payload=$(python3 -c 'import json,sys; print(json.dumps({"content": sys.argv[1]}))' "$msg") curl -fsS -X POST -H "Authorization: Bot $DISCORD_BOT_TOKEN" \ -H "Content-Type: application/json" \ "https://discord.com/api/v10/channels/$DISCORD_CHANNEL_BUILDS/messages" \ -d "$payload" || true memcheck: name: Valgrind Memcheck soak if: ${{ contains(fromJSON('["all","memcheck+coverage"]'), inputs.jobs || 'all') }} runs-on: ${{ fromJSON(vars.POOL || '["ubuntu-latest"]') }} timeout-minutes: 240 steps: - name: Checkout module uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Install bootstrap dependencies run: | sudo apt-get update sudo apt-get install -y ca-certificates curl python3 - name: Resolve latest mainline nginx env: GH_TOKEN: ${{ github.token }} run: | set -euo pipefail ver=$(bash "$GITHUB_WORKSPACE/ci/tools/nginx-releases.sh" mainline) [ -n "$ver" ] || { echo "::error::cannot resolve nginx version"; exit 1; } echo "NGINX_VERSION=$ver" >> "$GITHUB_ENV" - name: Install dependencies run: | # apt profile the bare apt-get lines below rely on: loose timeouts # for the shared self-hosted box; Azure mirror only on the # ubuntu-latest fallback when vars.POOL is unset (PR #121) printf '%s\n' \ 'DPkg::Lock::Timeout "120";' \ 'Acquire::Retries "3";' \ 'Acquire::http::Timeout "20";' \ 'Acquire::https::Timeout "20";' \ | sudo tee /etc/apt/apt.conf.d/99-ci-net > /dev/null sudo apt-get update sudo apt-get install -y \ build-essential curl gnupg libpcre2-dev libzstd-dev openssl pkg-config \ python3 valgrind wget zlib1g-dev zstd - name: Build debug nginx + zstd module run: | set -euo pipefail # PGP-verified against the vendored keys every run -- a runner- # persisted tarball from a prior job is exactly as untrusted as a # fresh download until re-checked (same "cache hit is not a trust # boundary" rule as ci-build.sh, audit sha e289021 F3). "$GITHUB_WORKSPACE/ci/tools/fetch-verified-nginx.sh" "$NGINX_VERSION" cd "nginx-${NGINX_VERSION}" ./configure \ --with-compat --with-debug --with-threads --with-file-aio \ --with-http_ssl_module --with-http_v2_module \ --with-http_gzip_static_module --with-http_realip_module \ --with-cc-opt="-DZSTD_STATIC_LINKING_ONLY -DNGX_DEBUG_PALLOC=1 -g3 -O0 -fno-omit-frame-pointer -funwind-tables $(pkg-config --cflags libzstd zlib libpcre2-8)" \ --with-ld-opt="$(pkg-config --libs libzstd zlib libpcre2-8)" \ --add-module="$GITHUB_WORKSPACE" make -j"$(nproc)" - name: Memcheck soak (300s @ 4 conc, WITH suppressions) env: USE_VALGRIND: "1" run: | bash ci/tools/soak.sh \ "$GITHUB_WORKSPACE/nginx-${NGINX_VERSION}/objs/nginx" 300 4 - name: Notify Discord on failure if: failure() && github.event_name == 'schedule' env: DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_BOT_TOKEN }} DISCORD_CHANNEL_BUILDS: ${{ secrets.DISCORD_CHANNEL_BUILDS }} GH_REPO: ${{ github.repository }} GH_SERVER_URL: ${{ github.server_url }} GH_RUN_ID: ${{ github.run_id }} run: | [ -n "$DISCORD_BOT_TOKEN" ] || exit 0 msg="🔴 $GH_REPO — CI Deep / memcheck FAILED. $GH_SERVER_URL/$GH_REPO/actions/runs/$GH_RUN_ID" payload=$(python3 -c 'import json,sys; print(json.dumps({"content": sys.argv[1]}))' "$msg") curl -fsS -X POST -H "Authorization: Bot $DISCORD_BOT_TOKEN" \ -H "Content-Type: application/json" \ "https://discord.com/api/v10/channels/$DISCORD_CHANNEL_BUILDS/messages" \ -d "$payload" || true harness-memcheck: name: Valgrind Memcheck (harness scenarios) # Runs all six TEST_HARNESS consumer scenarios with the whole nginx under # Valgrind Memcheck. This is the only oracle in the suite that can see a # leak on a libzstd ERROR path: # # * the ASan legs run detect_leaks=0 (the testkit forces it in # prober_heap_env, because nginx never frees its configuration pool), # so LSan reports nothing at all there; # * LSan-at-exit in general is structurally blind to a chain link leaked # into r->pool, because nginx frees the request pool on teardown; # * a happy-path soak never arms the fault/counter sites. # # Arming a codec fault under memcheck is what closes that gap. Kept in # ci-deep (weekly) rather than the PR lane because memcheck runs the # worker 20-50x slower; this matches the testkit README's own guidance # that whole-server memcheck is a scheduled job, not a PR gate. if: ${{ contains(fromJSON('["all","memcheck+coverage"]'), inputs.jobs || 'all') }} runs-on: ${{ fromJSON(vars.POOL || '["ubuntu-latest"]') }} timeout-minutes: 240 steps: - name: Checkout module (with harness submodule) uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false submodules: recursive - name: Install dependencies run: | # builder02 apt profile: loose timeouts — the box runs jobs # concurrently and never talks to the Azure mirror (PR #121) printf '%s\n' \ 'DPkg::Lock::Timeout "120";' \ 'Acquire::Retries "3";' \ 'Acquire::http::Timeout "20";' \ 'Acquire::https::Timeout "20";' \ | sudo tee /etc/apt/apt.conf.d/99-ci-net > /dev/null sudo apt-get update sudo apt-get install -y \ build-essential curl gnupg libnghttp2-dev libpcre2-dev libssl-dev \ libzstd-dev openssl pkg-config python3 valgrind wget zlib1g-dev zstd - name: Build harness nginx tree run: | set -euo pipefail "$GITHUB_WORKSPACE/ci/tools/fetch-verified-nginx.sh" "$HARNESS_NGINX_VERSION" cd "$GITHUB_WORKSPACE/nginx-$HARNESS_NGINX_VERSION" # Same spelling as harness-fault-arms.yml's build step, deliberately: # BOTH switches, set on the configure invocation itself. TEST_HARNESS=1 # alone compiles the probe sources and still yields a probe-free .so # -- measured in this repo (packaged=0 probe symbols, TEST_HARNESS=1 # alone=0, both=18) -- and the symbol gate below is what stops that # trap returning silently. # # ${CFLAGS:-}, never a bare $CFLAGS: this step runs under # `set -euo pipefail` and CFLAGS is unset on a clean runner, which # killed this job's sibling the first time the pattern shipped # (red run 32908007375). # # -O0 -g3 because every frame here is read by memcheck; an optimised # build reports inlined frames that do not map back to source lines. TEST_HARNESS=1 \ CFLAGS="${CFLAGS:-} -DNGX_TEST_HARNESS" \ ./configure \ --with-compat --with-debug \ --with-cc-opt="-DZSTD_STATIC_LINKING_ONLY -g3 -O0 -fno-omit-frame-pointer $(pkg-config --cflags libzstd zlib libpcre2-8)" \ --with-ld-opt="$(pkg-config --libs libzstd zlib libpcre2-8)" \ --add-dynamic-module="$GITHUB_WORKSPACE" # Plain `make`, NOT `make modules`: valgrind runs the SERVER, so this # job needs objs/nginx itself, which `make modules` does not build. make -j"$(nproc)" - name: "Gate: the built .so must carry probe symbols" run: | set -euo pipefail SO="$GITHUB_WORKSPACE/nginx-$HARNESS_NGINX_VERSION/objs/ngx_http_zstd_filter_module.so" COUNT=$(nm "$SO" | grep -c ' ngx_test_probe_' || true) echo "ngx_test_probe_* symbols: $COUNT" if [ "$COUNT" -eq 0 ]; then echo "::error::built .so carries zero ngx_test_probe_* symbols" \ "-- the harness switches did not take effect, so no fault could" \ "be armed and this whole job would pass by testing nothing" exit 1 fi - name: Build harness C tools (prober, fakesrv, etc.) run: bash ci/t/harness/ci/prober/build.sh - name: Memcheck all module testkit scenarios run: >- ci/tools/run-testkit-scenarios.sh --build "$GITHUB_WORKSPACE/nginx-$HARNESS_NGINX_VERSION" --version "$HARNESS_NGINX_VERSION" --runner "valgrind --error-exitcode=99 --leak-check=full --errors-for-leak-kinds=definite --track-origins=yes --suppressions=$GITHUB_WORKSPACE/ci/t/harness/ci/prober/valgrind.supp" --timeout-scale 40 --log-dir "$GITHUB_WORKSPACE" - name: Upload memcheck TAP logs if: always() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: harness-memcheck-tap-logs path: "*-tap.log" retention-days: 7 if-no-files-found: ignore - name: Notify Discord on failure if: failure() && github.event_name == 'schedule' env: DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_BOT_TOKEN }} DISCORD_CHANNEL_BUILDS: ${{ secrets.DISCORD_CHANNEL_BUILDS }} GH_REPO: ${{ github.repository }} GH_SERVER_URL: ${{ github.server_url }} GH_RUN_ID: ${{ github.run_id }} run: | [ -n "$DISCORD_BOT_TOKEN" ] || exit 0 msg="🔴 $GH_REPO — CI Deep / harness-memcheck FAILED. $GH_SERVER_URL/$GH_REPO/actions/runs/$GH_RUN_ID" payload=$(python3 -c 'import json,sys; print(json.dumps({"content": sys.argv[1]}))' "$msg") curl -fsS -X POST -H "Authorization: Bot $DISCORD_BOT_TOKEN" \ -H "Content-Type: application/json" \ "https://discord.com/api/v10/channels/$DISCORD_CHANNEL_BUILDS/messages" \ -d "$payload" || true helgrind: name: Valgrind Helgrind soak needs: harness-memcheck if: ${{ !cancelled() && (inputs.jobs || 'all') == 'all' }} runs-on: ${{ fromJSON(vars.POOL || '["ubuntu-latest"]') }} timeout-minutes: 240 steps: - name: Checkout module uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Install bootstrap dependencies run: | sudo apt-get update sudo apt-get install -y ca-certificates curl python3 - name: Resolve latest mainline nginx env: GH_TOKEN: ${{ github.token }} run: | set -euo pipefail ver=$(bash "$GITHUB_WORKSPACE/ci/tools/nginx-releases.sh" mainline) [ -n "$ver" ] || { echo "::error::cannot resolve nginx version"; exit 1; } echo "NGINX_VERSION=$ver" >> "$GITHUB_ENV" - name: Install dependencies run: | # apt profile the bare apt-get lines below rely on: loose timeouts # for the shared self-hosted box; Azure mirror only on the # ubuntu-latest fallback when vars.POOL is unset (PR #121) printf '%s\n' \ 'DPkg::Lock::Timeout "120";' \ 'Acquire::Retries "3";' \ 'Acquire::http::Timeout "20";' \ 'Acquire::https::Timeout "20";' \ | sudo tee /etc/apt/apt.conf.d/99-ci-net > /dev/null sudo apt-get update sudo apt-get install -y \ build-essential curl gnupg libpcre2-dev libzstd-dev openssl pkg-config \ python3 valgrind wget zlib1g-dev zstd - name: Build debug nginx + zstd module run: | set -euo pipefail # PGP-verified against the vendored keys every run -- a runner- # persisted tarball from a prior job is exactly as untrusted as a # fresh download until re-checked (same "cache hit is not a trust # boundary" rule as ci-build.sh, audit sha e289021 F3). "$GITHUB_WORKSPACE/ci/tools/fetch-verified-nginx.sh" "$NGINX_VERSION" cd "nginx-${NGINX_VERSION}" ./configure \ --with-compat --with-debug --with-threads --with-file-aio \ --with-http_ssl_module --with-http_v2_module \ --with-http_gzip_static_module --with-http_realip_module \ --with-cc-opt="-DZSTD_STATIC_LINKING_ONLY -DNGX_DEBUG_PALLOC=1 -g3 -O0 -fno-omit-frame-pointer -funwind-tables $(pkg-config --cflags libzstd zlib libpcre2-8)" \ --with-ld-opt="$(pkg-config --libs libzstd zlib libpcre2-8)" \ --add-module="$GITHUB_WORKSPACE" make -j"$(nproc)" - name: Helgrind soak (300s @ 4 conc, --error-exitcode=99, WITH suppressions) env: USE_HELGRIND: "1" run: | bash ci/tools/soak.sh \ "$GITHUB_WORKSPACE/nginx-${NGINX_VERSION}/objs/nginx" 300 4 - name: Notify Discord on failure if: failure() && github.event_name == 'schedule' env: DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_BOT_TOKEN }} DISCORD_CHANNEL_BUILDS: ${{ secrets.DISCORD_CHANNEL_BUILDS }} GH_REPO: ${{ github.repository }} GH_SERVER_URL: ${{ github.server_url }} GH_RUN_ID: ${{ github.run_id }} run: | [ -n "$DISCORD_BOT_TOKEN" ] || exit 0 msg="🔴 $GH_REPO — CI Deep / helgrind FAILED. $GH_SERVER_URL/$GH_REPO/actions/runs/$GH_RUN_ID" payload=$(python3 -c 'import json,sys; print(json.dumps({"content": sys.argv[1]}))' "$msg") curl -fsS -X POST -H "Authorization: Bot $DISCORD_BOT_TOKEN" \ -H "Content-Type: application/json" \ "https://discord.com/api/v10/channels/$DISCORD_CHANNEL_BUILDS/messages" \ -d "$payload" || true scanners: name: Security scanners needs: build-flavors if: >- ${{ !cancelled() && contains(fromJSON('["all","scanners"]'), inputs.jobs || 'all') }} # actionlint #711 lacks $/; generator parity is superrepo-scoped. uses: ./.github/workflows/security-scanners.yml # zizmor: ignore[self-repository] permissions: contents: read # Coverage is a REPORT, never a gate (step 26) -- the cheapest way to move # the number is a test that touches a line and asserts nothing, so a floor # buys the metric and sells the thing it proxied for. Weekly cadence # (this file, never PR-time) because ci/tools/coverage.sh does a full # from-scratch coverage-instrumented build plus the whole ci/t/ suite. coverage: name: Coverage report (src/) needs: memcheck if: >- ${{ !cancelled() && contains(fromJSON('["all","memcheck+coverage"]'), inputs.jobs || 'all') }} runs-on: ${{ fromJSON(vars.POOL || '["ubuntu-latest"]') }} # The coverage build compiles the probe in and runs all six harness # scenarios plus runtime drivers after Test::Nginx. A job capped below its # own workload can never pass, and the resulting "cancelled" reads as # flakiness rather than as the cap it is. timeout-minutes: 45 env: # This job's own Test::Nginx port band -- see ci/tools/max-port.sh. # Distinct from build-flavors' 19300 (width 64) and build-test.yml's # 19200 so a collision needs three concurrent runners occupied, not # two. coverage.sh runs `prove` internally without a caller-supplied # port, so it takes Test::Nginx's compiled-in default (1984) unless # TEST_NGINX_PORT steers it here. TEST_BASE_PORT: 19330 TEST_NGINX_PORT: 19330 steps: - name: Checkout module uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false # coverage.sh runs the ci/t/harness-scenarios/* scenarios by default # (COVERAGE_HARNESS=1) so the libzstd FAULT paths are counted rather # than reported as dead lines -- no ci/t/ test can reach them, since # libzstd cannot be made to fail from outside the process. Those # scenarios need the harness submodule; without it coverage.sh warns # and silently reports the smaller, pre-harness number. submodules: recursive - name: Install dependencies run: | # builder02 apt profile: loose timeouts — the box runs jobs # concurrently and never talks to the Azure mirror (PR #121) printf '%s\n' \ 'DPkg::Lock::Timeout "120";' \ 'Acquire::Retries "3";' \ 'Acquire::http::Timeout "20";' \ 'Acquire::https::Timeout "20";' \ | sudo tee /etc/apt/apt.conf.d/99-ci-net > /dev/null sudo apt-get update sudo apt-get install -y \ build-essential cpanminus curl gcovr gnupg libnghttp2-dev libpcre2-dev \ libperl-dev libssl-dev libzstd-dev openssl pkg-config \ python3 python3-requests strace wget zlib1g-dev zstd - name: Cache Perl modules uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ~/perl5 key: perl-modules-${{ runner.os }}-Test-Nginx-Socket-${{ env.TEST_NGINX_SOCKET_VERSION }} - name: Install Test::Nginx::Socket run: | ci/tools/install-test-nginx-socket.sh "$HOME/perl5" - name: Verify test port band # Before the first `prove` inside coverage.sh -- see ci/tools/max-port.sh. run: ci/tools/max-port.sh "$TEST_BASE_PORT" - name: Coverage build + report env: # Preserve the pre-expansion line-coverage floor while the broader # 86.6% baseline leaves headroom for compiler/version drift. COVERAGE_FAIL_UNDER: "85" run: bash ci/tools/coverage.sh nginx - name: Upload coverage report if: always() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: coverage-report path: .build/coverage-report/ retention-days: 7 codeql: name: CodeQL needs: coverage if: ${{ !cancelled() && (inputs.jobs || 'all') == 'all' }} # actionlint #711 lacks $/; generator parity is superrepo-scoped. uses: ./.github/workflows/codeql.yml # zizmor: ignore[self-repository] permissions: contents: read security-events: write # upload CodeQL SARIF to code scanning asan-soak: name: A/UBSan soak needs: scanners if: ${{ !cancelled() && (inputs.jobs || 'all') == 'all' }} # actionlint #711 lacks $/; generator parity is superrepo-scoped. uses: ./.github/workflows/asan.yml # zizmor: ignore[self-repository] permissions: contents: read myguard-labs-nginx-zstd-module-3486b5b/.github/workflows/ci.yml000066400000000000000000000060171525232545500245550ustar00rootroot00000000000000--- name: CI # Single PR gate entry point. On merged master, only the testkit harness runs # to retain its required-check promotion signal; the already-tested build, # lint, Arch package, scanner, and Windows jobs do not run twice. Extended # CodeQL, fuzz, soak, and coverage work is scheduled by ci-deep.yml. # # Four self-hosted Linux lanes are budgeted explicitly: # 1. Build & Test: resolve/validation, then four dependency chains inside # build-test.yml (mainline, sanitizer, old-libzstd, vary -> linkage). # 2. Security scanners. # 3. Harness Fault Arms (all six nginx-module-testkit consumer scenarios). # 4. The second initial Build & Test job (resolve and validation run together). # # Resolve normally frees its lane in seconds and makes only mainline ready. A # short hosted barrier then releases the other three chains after mainline has # first claim on that lane; they fill lanes as validation, scanners, and # testkit finish. Lint, Arch packaging, Windows, and arm64 use GitHub-hosted # runners and do not consume these four lanes. on: pull_request: branches: [master, main] push: branches: [master] workflow_dispatch: {} concurrency: group: ci-orchestrator-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true permissions: contents: read jobs: lint: name: Lint if: github.event_name != 'push' # actionlint #711 lacks $/; generator parity is superrepo-scoped. uses: ./.github/workflows/lint.yml # zizmor: ignore[self-repository] permissions: contents: read build-test: name: Build & Test if: github.event_name != 'push' # actionlint #711 lacks $/; generator parity is superrepo-scoped. uses: ./.github/workflows/build-test.yml # zizmor: ignore[self-repository] permissions: contents: read arch-package: name: Arch package if: github.event_name != 'push' # actionlint #711 lacks $/; generator parity is superrepo-scoped. uses: ./.github/workflows/arch-package.yml # zizmor: ignore[self-repository] permissions: contents: read security-scanners: # This exact case is part of the existing status-check context. name: Security scanners if: github.event_name != 'push' # actionlint #711 lacks $/; generator parity is superrepo-scoped. uses: ./.github/workflows/security-scanners.yml # zizmor: ignore[self-repository] permissions: contents: read harness-fault-arms: # Kept in the fast gate because it reaches module-internal fault/resource # paths that Test::Nginx and ordinary sanitizers cannot observe. name: Harness Fault Arms # actionlint #711 lacks $/; generator parity is superrepo-scoped. uses: ./.github/workflows/harness-fault-arms.yml # zizmor: ignore[self-repository] permissions: contents: read windows-build: name: Windows build if: github.event_name != 'push' # actionlint #711 lacks $/; generator parity is superrepo-scoped. uses: ./.github/workflows/windows-build.yml # zizmor: ignore[self-repository] permissions: contents: read myguard-labs-nginx-zstd-module-3486b5b/.github/workflows/codeql.yml000066400000000000000000000256531525232545500254400ustar00rootroot00000000000000name: CodeQL # Reusable CodeQL workflow. CI Deep owns the weekly schedule; this file keeps a # manual entry point for focused reruns. Third-party actions are pinned to # commit SHAs; see the build-test.yml header. # nginx version is resolved at run time (see "Resolve latest mainline nginx" # step) rather than pinned, so this analysis tracks current mainline. env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true NGINX_VERSION: "" on: workflow_dispatch: {} # Called by ci-deep.yml. workflow_call: {} concurrency: group: codeql-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true permissions: contents: read jobs: codeql: name: CodeQL runs-on: ${{ fromJSON(vars.POOL || '["ubuntu-latest"]') }} timeout-minutes: 30 permissions: contents: read # build the CodeQL database from the tree security-events: write # upload SARIF to the Security tab; the only write needed steps: - name: Checkout module uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false # Required before resolving nginx on an ephemeral fork runner. - name: Install bootstrap dependencies run: | sudo apt-get update sudo apt-get install -y ca-certificates curl python3 - name: Resolve latest mainline nginx env: GH_TOKEN: ${{ github.token }} run: | set -euo pipefail ver=$(bash "$GITHUB_WORKSPACE/ci/tools/nginx-releases.sh" mainline) if [ -z "$ver" ]; then echo "::error::could not resolve the mainline nginx release (see the log)" exit 1 fi echo "Resolved nginx mainline: $ver" echo "NGINX_VERSION=$ver" >> "$GITHUB_ENV" - name: Install build dependencies # A slow-rather-than-dead mirror can still crawl past every # Acquire timeout; cap the step so a stall fails HERE, named, # with the log intact, instead of eating the job cap. timeout-minutes: 5 run: | # ubuntu-latest resolves apt through azure.archive.ubuntu.com, # whose outage shape is ~2 minutes of default Acquire timeout # per index file before the runner's mirror list falls back # (#117); the bare apt-get lines below rely on this file. printf '%s\n' \ 'DPkg::Lock::Timeout "120";' \ 'Acquire::Retries "1";' \ 'Acquire::http::Timeout "10";' \ 'Acquire::https::Timeout "10";' \ | sudo tee /etc/apt/apt.conf.d/99-ci-net > /dev/null sudo apt-get update sudo apt-get install -y build-essential gnupg pkg-config libzstd-dev \ libpcre2-dev python3 unzip wget zlib1g-dev - name: Cache nginx source tarball id: nginx-tarball uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: nginx-${{ env.NGINX_VERSION }}.tar.gz key: nginx-src-${{ env.NGINX_VERSION }} - name: Download nginx source tarball if: steps.nginx-tarball.outputs.cache-hit != 'true' run: wget -q "https://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz" - name: Verify nginx tarball PGP signature run: | set -euo pipefail wget -q "https://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz.asc" \ -O "nginx-${NGINX_VERSION}.tar.gz.asc" gnupghome="$(mktemp -d)" export GNUPGHOME="$gnupghome" chmod 700 "$gnupghome" for keyfile in ci/tools/keys/*.key; do gpg --quiet --import "$keyfile" 2>/dev/null done gpg --quiet --verify "nginx-${NGINX_VERSION}.tar.gz.asc" "nginx-${NGINX_VERSION}.tar.gz" rm -rf "$gnupghome" - name: Initialize CodeQL uses: github/codeql-action/init@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4 with: languages: c-cpp queries: security-extended # No `paths:`/`paths-ignore:` here, deliberately. GitHub documents # them as applying only to interpreted languages, or to a compiled # language analyzed WITHOUT building it: # # "You can use this option when you run the CodeQL actions on an # interpreted language (Python, Ruby, and JavaScript/TypeScript) # or when you analyze a compiled language without building the # code (currently supported for C/C++, C#, Java and Rust)." # # This job does a full traced `make` of C/C++, so both keys are # silently inert -- a `paths-ignore: nginx-*/**` here was carried # for months and filtered nothing (nginx-core alerts kept landing # in the Security tab on every mainline bump). # # The documented alternative, "specify appropriate build steps", # does not apply either: --add-module compiles the module objects # INTO the nginx binary, so there is no module-only compile, and # attempting one yields the "no source code seen during build" # empty-database failure. nginx core is therefore unavoidably # extracted, and scoping happens after analysis, on the SARIF -- # see the filter step below. - name: Build module sources for CodeQL run: | tar -xzf "nginx-${NGINX_VERSION}.tar.gz" cd "nginx-${NGINX_VERSION}" # --add-module (static) compiles the zstd module objects INTO the # nginx binary, so a full `make` is what the CodeQL tracer sees. # `make modules` only builds --add-dynamic-module .so targets and # leaves the static add-module untouched, which is why CodeQL # reported "no source code seen during build". Do not redirect # configure to /dev/null — a silent configure failure would also # produce an empty CodeQL database. ./configure \ --with-compat \ --with-debug \ --with-cc-opt="-DZSTD_STATIC_LINKING_ONLY $(pkg-config --cflags libzstd zlib libpcre2-8)" \ --with-ld-opt="$(pkg-config --libs libzstd zlib libpcre2-8)" \ --add-module="$GITHUB_WORKSPACE" make -j"$(nproc)" - name: Perform CodeQL analysis id: analyze uses: github/codeql-action/analyze@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4 with: category: "/language:c-cpp" output: sarif-results # 'failure-only' is the documented setting "for users # post-processing the SARIF file before uploading it to Code # Scanning": results are not published here, but debugging # information still reaches Code Scanning if the run fails. # Prefer it over 'never', which the action documents as # complicating debugging for external users. upload: failure-only # `analyze` finalizes the database and then tears the cluster # down ("Cleaned up database cluster directory"), so by the time # a following step runs, /src is already gone -- the # assertion below cannot see it under the default cleanup. Keep # the database on disk so the extraction scope stays checkable. cleanup-level: none - name: Drop nginx-core results from SARIF env: SARIF: sarif-results/cpp.sarif run: | set -euo pipefail if [ ! -f "$SARIF" ]; then echo "::error::expected CodeQL SARIF at $SARIF" ls -la sarif-results || true exit 1 fi python3 ci/tools/filter-codeql-sarif.py "$SARIF" - name: Assert database saw module sources, not just nginx core env: DB_LOCATIONS: ${{ steps.analyze.outputs.db-locations }} run: | set -euo pipefail # db-locations is a JSON map {"cpp": "/abs/path"} — use the # action's own reported path rather than guessing RUNNER_TEMP # layout, which is an implementation detail not part of its # documented contract. db="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1])["cpp"])' "$DB_LOCATIONS")" if [ -z "$db" ] || [ ! -d "$db" ]; then echo "::error::could not resolve the CodeQL cpp database path from db-locations: $DB_LOCATIONS" exit 1 fi # CodeQL stores extracted sources either as a src/ tree or as # src.zip, depending on version and cleanup level. Accept both: # unpack the zip to a scratch dir so the same find/grep logic # below works either way. src="$db/src" if [ ! -d "$src" ]; then if [ -f "$db/src.zip" ]; then src="$RUNNER_TEMP/codeql-src" rm -rf "$src" mkdir -p "$src" unzip -qq "$db/src.zip" -d "$src" echo "Unpacked src.zip for inspection" else echo "::error::CodeQL database has neither src/ nor src.zip under $db" ls -la "$db" || true exit 1 fi fi echo "Extracted source tree: $src" # grep -c exits 1 when it counts zero, and under `set -e` that # would abort here -- swallowing the ::error:: below, which is # the entire point of the assertion. Keep the `|| true`. module_hits="$(find "$src" -path '*/nginx-*' -prune -o \ \( -path '*/filter/*' -o -path '*/static/*' -o -name '*.c' -o -name '*.h' \) \ -type f -print | grep -vcE '/nginx-[0-9]' || true)" module_hits="${module_hits:-0}" nginx_core_only="$(find "$src" -path '*/nginx-*/src/*' -type f -print | wc -l)" echo "module source files in database: $module_hits" echo "nginx-core src/ files in database: $nginx_core_only" if [ "$module_hits" -eq 0 ]; then echo "::error::CodeQL database contains zero module source files" \ "(filter/, static/, *.c, *.h) — the extracted DB regressed to" \ "nginx-core-only or empty, same failure class as the original" \ "'no source code seen during build' report." exit 1 fi echo "✓ database includes module sources ($module_hits files)" # Publishing happens only AFTER the extraction-scope assertion above. # The assertion is what proves the database actually contained module # sources; uploading first would let a run whose extraction silently # collapsed to nginx-core-only publish an empty, filtered report that # Code Scanning accepts as an authoritative "no module findings". - name: Upload filtered SARIF uses: github/codeql-action/upload-sarif@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4 with: sarif_file: sarif-results/cpp.sarif category: "/language:c-cpp" # Surface a rejected SARIF as a failure here rather than a # silently absent update in the Security tab. wait-for-processing: true myguard-labs-nginx-zstd-module-3486b5b/.github/workflows/fuzzing.yml000066400000000000000000000146111525232545500256550ustar00rootroot00000000000000name: Fuzzing # Targeted manual fuzz regression: 120s libFuzzer runs on the Accept-Encoding # and dcz targets, plus replay of both recorded crash corpora. The long runs # live in ci-deep.yml (weekly cron + workflow_dispatch). # # Action pins (keep in sync with build-test.yml header): # actions/checkout@v5.1.0 -> fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # actions/upload-artifact@v6.0.0 -> b7c566a772e6b6bfb58ed0dc250532a479d7789f env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true NGINX_VERSION: "" on: workflow_dispatch: {} concurrency: group: fuzzing-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true permissions: contents: read jobs: fuzz-regression: name: Fuzz regression (120s) runs-on: ${{ fromJSON(vars.POOL || '["ubuntu-latest"]') }} timeout-minutes: 30 steps: - name: Checkout module uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Install clang + libFuzzer run: | # apt profile the bare apt-get lines below rely on: loose timeouts # for the shared self-hosted box; Azure mirror only on the # ubuntu-latest fallback when vars.POOL is unset (PR #121) printf '%s\n' \ 'DPkg::Lock::Timeout "120";' \ 'Acquire::Retries "3";' \ 'Acquire::http::Timeout "20";' \ 'Acquire::https::Timeout "20";' \ | sudo tee /etc/apt/apt.conf.d/99-ci-net > /dev/null sudo apt-get update sudo apt-get install -y clang - name: Build fuzz targets run: | bash ci/fuzz/build.sh \ "$GITHUB_WORKSPACE/ci/fuzz/fuzz_accept_encoding" \ "$GITHUB_WORKSPACE/ci/fuzz/fuzz_dcz" # Replay every triaged past crash FIRST -- fast (each is one input) # and deterministic, so a reintroduced bug fails in seconds instead # of waiting on the 120s fresh budget to happen to rediscover it. # The replay guard below reports the number of matching crash-* cases # explicitly and fails when a non-empty corpus contains an entry outside # that naming contract. Empty corpora and README metadata are still # allowed, but a renamed regression must not pass as a no-op. - name: Replay recorded crash regressions run: | set -euo pipefail cd ci/fuzz case_list="$(mktemp)" trap 'rm -f "$case_list"' EXIT find regressions -mindepth 1 -maxdepth 1 -print0 > "$case_list" cases=() total=0 while IFS= read -r -d '' case_path; do total=$((total + 1)) case_name="${case_path##*/}" if [[ "$case_name" == README.md ]]; then continue elif [[ "$case_name" == crash-* ]]; then cases+=("$case_path") else echo "::error::recorded regression does not match crash-* naming contract: $case_path" exit 1 fi done < "$case_list" if [ "${#cases[@]}" -eq 0 ]; then echo "no crash-* regression inputs found in ci/fuzz/regressions/; inspected $total corpus entries" else echo "replaying ${#cases[@]} recorded crash(es); inspected $total corpus entries" ./fuzz_accept_encoding "${cases[@]}" echo "✓ all recorded regressions still pass" fi # Same contract as the step above, for the dcz target. Kept as its # own directory because the two targets take different input # languages: an Accept-Encoding byte string is not an RFC 8941 # byte sequence, so replaying one corpus through the other target # proves nothing and would only mask a rename. - name: Replay recorded dcz crash regressions run: | set -euo pipefail cd ci/fuzz case_list="$(mktemp)" trap 'rm -f "$case_list"' EXIT find regressions_dcz -mindepth 1 -maxdepth 1 -print0 > "$case_list" cases=() total=0 while IFS= read -r -d '' case_path; do total=$((total + 1)) case_name="${case_path##*/}" if [[ "$case_name" == README.md ]]; then continue elif [[ "$case_name" == crash-* ]]; then cases+=("$case_path") else echo "::error::recorded dcz regression does not match crash-* naming contract: $case_path" exit 1 fi done < "$case_list" if [ "${#cases[@]}" -eq 0 ]; then echo "no crash-* dcz regression inputs found in ci/fuzz/regressions_dcz/; inspected $total corpus entries" else echo "replaying ${#cases[@]} recorded dcz crash(es); inspected $total corpus entries" ./fuzz_dcz "${cases[@]}" echo "✓ all recorded dcz regressions still pass" fi - name: Run fuzzer (120s regression) run: | set -euo pipefail cd ci/fuzz for size in 4095 4096 4097 65535 65536; do head -c "$size" /dev/zero >"corpus/boundary-$size" done set +e ./fuzz_accept_encoding \ -max_total_time=120 \ -max_len=65536 \ -dict=fuzz.dict \ -print_final_stats=1 \ -artifact_prefix=crash- \ corpus/ rc=$? set -e [ $rc -eq 0 ] || { echo "❌ fuzzer crash (exit $rc)"; exit 1; } echo "✓ no crashes in 120s" - name: Run dcz fuzzer (120s regression) run: | set -euo pipefail cd ci/fuzz for size in 4095 4096 4097 65535 65536; do head -c "$size" /dev/zero >"corpus_dcz/boundary-$size" done set +e ./fuzz_dcz \ -max_total_time=120 \ -max_len=65536 \ -print_final_stats=1 \ -artifact_prefix=crash-dcz- \ corpus_dcz/ rc=$? set -e [ $rc -eq 0 ] || { echo "❌ dcz fuzzer crash (exit $rc)"; exit 1; } echo "✓ no crashes in 120s" - name: Upload crash artifacts if: failure() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: fuzz-fast-crashes path: | ${{ github.workspace }}/ci/fuzz/crash-* ${{ github.workspace }}/ci/fuzz/crash-dcz-* retention-days: 7 myguard-labs-nginx-zstd-module-3486b5b/.github/workflows/harness-fault-arms.yml000066400000000000000000000104661525232545500277010ustar00rootroot00000000000000--- name: Harness Fault Arms # Fast nginx-module-testkit consumer gate. One harness-enabled nginx build runs # all six module-specific scenarios through ci/tools/run-testkit-scenarios.sh: # codec and pool-allocation faults, allocation neutrality, codec call counts, # and dictionary/no-dictionary setParameter call counts. The shared runner owns # expected-log allowances and non-vacuity/property witnesses, so PR, coverage, # and deep Memcheck surfaces cannot silently select different scenario sets. # # The upstream testkit's generic consumer-zstd fixture cannot replace these: # its consumers/ tree is gitignored, so it SKIPs against this module in testkit # CI. This repository builds the real module with TEST_HARNESS=1 plus # -DNGX_TEST_HARNESS and rejects a probe-free binary before running anything. # # The fixed job name is a check-name contract. It remains visible on every PR # and is selected by ci.yml on merged master so the separately documented # green-week promotion criterion has a stable branch signal. It is not # required yet. env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true HARNESS_NGINX_VERSION: "1.31.5" on: workflow_call: {} workflow_dispatch: {} concurrency: group: harness-fault-arms-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true permissions: contents: read jobs: harness-fault-arms: name: Harness Fault Arms runs-on: ${{ fromJSON(vars.POOL || '["ubuntu-latest"]') }} timeout-minutes: 20 steps: - name: Checkout module (with harness submodule) uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false submodules: recursive - name: Cache apt packages uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ${{ runner.temp }}/apt-archives/*.deb key: apt-harness-${{ runner.os }}-build-essential-libzstd-libpcre2-zlib-openssl-nghttp2 restore-keys: apt-harness-${{ runner.os }}- - name: Install dependencies run: | mkdir -p "$RUNNER_TEMP/apt-archives" printf '%s\n' \ 'DPkg::Lock::Timeout "120";' \ 'Acquire::Retries "3";' \ 'Acquire::http::Timeout "20";' \ 'Acquire::https::Timeout "20";' \ "Dir::Cache::archives \"$RUNNER_TEMP/apt-archives\";" \ | sudo tee /etc/apt/apt.conf.d/99-ci-net > /dev/null sudo apt-get update sudo apt-get install -y \ build-essential curl gnupg libnghttp2-dev libpcre2-dev libssl-dev \ libzstd-dev pkg-config wget zlib1g-dev - name: Fetch verified nginx source run: | set -euo pipefail bash ci/tools/fetch-verified-nginx.sh "$HARNESS_NGINX_VERSION" echo "NGINX_SRC_DIR=$GITHUB_WORKSPACE/nginx-$HARNESS_NGINX_VERSION" >> "$GITHUB_ENV" - name: Configure and build harness nginx run: | set -euo pipefail cd "$NGINX_SRC_DIR" TEST_HARNESS=1 \ CFLAGS="${CFLAGS:-} -DNGX_TEST_HARNESS -Wextra -Werror" \ ./configure \ --prefix=/etc/nginx \ --with-http_ssl_module \ --with-cc-opt="-DZSTD_STATIC_LINKING_ONLY" \ --add-dynamic-module="$GITHUB_WORKSPACE" make -j"$(nproc)" - name: Prove the module carries testkit probes run: | set -euo pipefail SO="$NGINX_SRC_DIR/objs/ngx_http_zstd_filter_module.so" COUNT=$(nm "$SO" | grep -c ' ngx_test_probe_' || true) echo "ngx_test_probe_* symbols: $COUNT" if [ "$COUNT" -eq 0 ]; then echo "::error::harness module carries zero ngx_test_probe_* symbols" exit 1 fi - name: Build nginx-module-testkit tools run: bash ci/t/harness/ci/prober/build.sh - name: Run all module testkit scenarios run: >- ci/tools/run-testkit-scenarios.sh --build "$NGINX_SRC_DIR" --version "$HARNESS_NGINX_VERSION" --log-dir "$GITHUB_WORKSPACE" - name: Upload TAP logs if: always() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: harness-scenario-tap-logs path: "*-tap.log" retention-days: 7 if-no-files-found: ignore myguard-labs-nginx-zstd-module-3486b5b/.github/workflows/lint.yml000066400000000000000000000137621525232545500251350ustar00rootroot00000000000000name: Lint # The same ci/linter/ scripts a developer runs locally, run again on the PR -- # so a clone that never enabled the hook (`git config core.hooksPath .githooks`) # cannot land nginx-convention, shell, Perl, Python or workflow-syntax # regressions. Local and remote run the SAME entry point on purpose: a CI-only # reimplementation drifts from the hook and the two stop agreeing. # # LINT_ONLY leaves out the "c" checker: its flawfinder and semgrep passes over # src/ are already the job of security-scanners.yml, at the same thresholds # (ci/linter/lint-c.sh mirrors that workflow). Running them twice per PR buys # nothing but queue time. The two that workflow does NOT run are covered # elsewhere rather than dropped: clang-tidy is CI-only there (it needs a # configured nginx tree), and cppcheck is a local-hook extra. # # runs-on: ubuntu-latest, NOT the self-hosted pool. This job needs no nginx # build, so it costs nothing against this repo's self-hosted capacity while # giving the fastest feedback in the suite. env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true on: # Triggered by ci.yml, like every other member of the suite. No # push/pull_request trigger here: two entry points would run it twice per PR. workflow_call: {} # NOTE on ${{ github.workflow }} in this key: for a workflow reached by # workflow_call it evaluates to the CALLER's name ("CI"), not "Lint" -- the # whole suite is a single run (verified: one run row per head sha). So that # term is identical across every member and distinguishes nothing; the literal # "lint-" prefix is what keeps this group distinct from its siblings. Keep the # prefix unique per member. Same applies to arch-package/asan/valgrind/ # fuzzing/codeql/security-scanners/windows-build/build-test. concurrency: group: lint-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true permissions: contents: read jobs: lint: name: Linters runs-on: ubuntu-latest timeout-minutes: 15 steps: - name: Checkout module uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Install linters # One installer for CI and for a fresh clone; a second apt list here # would be the thing that goes stale. run: ci/linter/install-linters.sh - name: Verify the toolchain is complete # A linter missing from PATH makes its check exit 2, which run-all.sh # reports as a failure rather than skipping -- this step just surfaces # WHICH one before the findings scroll past. run: ci/linter/install-linters.sh --check - name: Selftest the gate # Before trusting the next step's verdict, prove the gate can still say # no: a selector that matches nothing must exit 2, not report clean. # The step below narrows the run with LINT_ONLY, so a silent selector # is exactly the failure that would make this whole job vacuous. run: ci/linter/selftest.sh - name: Run linters env: # Every checker this module ships except "c" -- adding a # lint-*.sh without adding its name here leaves it running # locally and never in CI, which is the quieter half of a gate # that is not really a gate. This module has no lint-sync-stamp.sh # or lint-prompt-steps.* (those track the skeleton's own adoption # prompt, not this module), so they are not listed. LINT_ONLY: nginx sh python perl yaml spelling ci-runners ci-ports ci-cadence ci-topology ci-secrets ci-provenance docs-drift run: ci/linter/run-all.sh # ast-grep and gitleaks live in .pre-commit-config.yaml rather than in a # ci/linter/lint-*.sh, so run-all.sh above does not reach them. Without # these two steps they would be the only gates in this repo that a clone # which never ran `git config core.hooksPath .githooks` can bypass -- # exactly the failure mode this whole job exists to close. Both are # invoked the same way the hook invokes them, for the same reason # local and remote share run-all.sh: a CI-only reimplementation drifts. - name: ast-grep (structural sinks, gate on promoted rules) # -c is load-bearing: ast-grep resolves sgconfig.yml by walking UP from # the cwd, so dropping it binds to nothing here and reports green. # Only the promoted rules are errors; everything else stays advisory. run: | # -z/-0 so a path with a space or newline cannot split into two args. # xargs -r: no C files is a clean tree, not an invocation with none # (ast-grep with zero paths would scan the whole repo instead). git ls-files -z 'src/*.c' 'src/*.h' 'ci/*.c' 'ci/*.h' \ | xargs -0 -r ast-grep scan -c ci/ast-grep/sgconfig.yml \ --error=c-alloc-mul-overflow \ --error=c-memcpy-sizeof-pointer \ --error=c-sizeof-wrong-operand \ --error=c-unbounded-string-copy \ --error=c-format-string \ --error=c-shell-exec - name: gitleaks (whole tracked tree) # The hook scans the staged patch; there is no staged patch in CI, so # this scans the checked-out tree instead -- `detect --no-git` is the # whole-tree form, and it catches a secret that reached master through # a clone with no hook installed. # # -c .gitleaks.toml is required, not cosmetic: --no-git walks the # FILESYSTEM, so it descends into the gitignored .build/ nginx tree and # reports upstream nginx's QUIC test constants. The config allowlists # that path and keeps the default ruleset. Exit is 1 on a finding. # # Verify with a NON-EXAMPLE secret -- AWS's own AKIAIOSFODNN7EXAMPLE is # allowlisted upstream and reports clean, which reads like a dead gate. # See the probe recipe in .pre-commit-config.yaml above the hook. run: gitleaks detect --no-git -c .gitleaks.toml --no-banner --redact -v --source . myguard-labs-nginx-zstd-module-3486b5b/.github/workflows/security-scanners.yml000066400000000000000000000147741525232545500276540ustar00rootroot00000000000000name: Security scanners # Static/security scanners gate on PRs and run again in the weekly deep chain: # flawfinder, clang-tidy (cert-* / clang-analyzer-security.*), and semgrep over # the module sources, with reports uploaded as build artifacts. # # Action pins (keep in sync with build-test.yml header): # actions/checkout@v5.1.0 -> fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # actions/upload-artifact@v6.0.0 -> b7c566a772e6b6bfb58ed0dc250532a479d7789f env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true on: workflow_dispatch: {} # Called by ci.yml, the single PR entry point (step 15). workflow_call: {} concurrency: group: security-scanners-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true permissions: contents: read jobs: scanners: name: Security scanners runs-on: ${{ fromJSON(vars.POOL || '["ubuntu-latest"]') }} timeout-minutes: 20 steps: - name: Checkout module uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Install dependencies run: | # apt profile the bare apt-get lines below rely on: loose timeouts # for the shared self-hosted box; Azure mirror only on the # ubuntu-latest fallback when vars.POOL is unset (PR #121) printf '%s\n' \ 'DPkg::Lock::Timeout "120";' \ 'Acquire::Retries "3";' \ 'Acquire::http::Timeout "20";' \ 'Acquire::https::Timeout "20";' \ | sudo tee /etc/apt/apt.conf.d/99-ci-net > /dev/null sudo apt-get update sudo apt-get install -y \ build-essential clang-tidy curl flawfinder gnupg libzstd-dev \ nginx-dev pkg-config python3 python3-pip pipx wget # semgrep: isolate via pipx (PEP 668 blocks a bare pip3 install into the # system Python on the Ubuntu 24.04 self-hosted runner). Fall back to a # user pip install with --break-system-packages if pipx is unavailable. if command -v pipx >/dev/null; then pipx install --quiet semgrep==1.173.0 echo "$HOME/.local/bin" >> "$GITHUB_PATH" else pip3 install --quiet --user --break-system-packages semgrep==1.173.0 echo "$HOME/.local/bin" >> "$GITHUB_PATH" fi - name: Configure nginx src tree (clang-tidy headers) env: GH_TOKEN: ${{ github.token }} run: | set -euo pipefail ver=$(bash "$GITHUB_WORKSPACE/ci/tools/nginx-releases.sh" mainline) wget -q "https://nginx.org/download/nginx-${ver}.tar.gz" wget -q "https://nginx.org/download/nginx-${ver}.tar.gz.asc" -O "nginx-${ver}.tar.gz.asc" gnupghome="$(mktemp -d)" export GNUPGHOME="$gnupghome" chmod 700 "$gnupghome" for keyfile in "$GITHUB_WORKSPACE"/ci/tools/keys/*.key; do gpg --quiet --import "$keyfile" 2>/dev/null done gpg --quiet --verify "nginx-${ver}.tar.gz.asc" "nginx-${ver}.tar.gz" rm -rf "$gnupghome" tar -xzf "nginx-${ver}.tar.gz"; cd "nginx-${ver}" CC=gcc ./configure --with-compat --add-dynamic-module="$GITHUB_WORKSPACE" >/dev/null echo "NGINX_SRC_TREE=$PWD" >> "$GITHUB_ENV" - name: flawfinder # Threshold mirrors ci/linter/lint-c.sh exactly: gate at >=4 (below 4 # is risky-API grep noise; gating on it trains everyone to # --no-verify). --error-level makes flawfinder itself exit non-zero # at that level -- minlevel alone only controls what gets PRINTED, # so a bare --minlevel=1 with no --error-level never fails this step # regardless of what it finds. Move a threshold here, move it in # lint-c.sh in the SAME commit. run: | set -o pipefail source "$GITHUB_WORKSPACE/ci/linter/lib.sh" mapfile_checked files find "$GITHUB_WORKSPACE/src" "$GITHUB_WORKSPACE/ci" \ -type f \( -name '*.c' -o -name '*.h' \) -print [ "${#files[@]}" -gt 0 ] || { echo 'no C scanner targets found' >&2; exit 1; } flawfinder --minlevel=4 --error-level=4 "${files[@]}" \ 2>&1 | tee flawfinder.log - name: clang-tidy (cert-*, security.*) run: | set -euo pipefail N="$NGINX_SRC_TREE" INCLUDES="-I$N/src/core -I$N/src/event -I$N/src/event/modules -I$N/src/event/quic \ -I$N/src/os/unix -I$N/objs -I$N/src/http -I$N/src/http/modules -I$N/src/http/v2 -I/usr/include" status=0 for f in \ "$GITHUB_WORKSPACE/src/ngx_http_zstd_filter_module.c" \ "$GITHUB_WORKSPACE/src/ngx_http_zstd_static_module.c"; do # shellcheck disable=SC2086,SC2046 clang-tidy "$f" \ --checks='-*,cert-*,clang-analyzer-security.*,-clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling' \ --warnings-as-errors='*' \ -- -DZSTD_STATIC_LINKING_ONLY $INCLUDES $(pkg-config --cflags libzstd) \ 2>&1 | tee -a clang-tidy.log || status=1 done exit "$status" - name: semgrep # Same target and threshold as ci/linter/lint-c.sh: src/ (the module's # real source directory -- filter/ and static/ are a stale legacy # layout this workflow was still pointed at), gate >=WARNING, and # --jobs=1 --metrics=off (step 28: --jobs=1 is a CORRECTNESS flag on # this host's shared RLIMIT_MEMLOCK, not a speed one; --metrics=off # drops the semgrep.dev telemetry POST). `|| true` previously meant # this step could never fail on a real finding. run: | set -o pipefail source "$GITHUB_WORKSPACE/ci/linter/lib.sh" mapfile_checked files find "$GITHUB_WORKSPACE/src" "$GITHUB_WORKSPACE/ci" \ -type f \( -name '*.c' -o -name '*.h' \) -print [ "${#files[@]}" -gt 0 ] || { echo 'no C scanner targets found' >&2; exit 1; } semgrep scan --config p/c --config p/security-audit \ --severity=WARNING --severity=ERROR --error \ --jobs=1 --metrics=off \ "${files[@]}" 2>&1 | tee semgrep.log - name: Upload reports if: always() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: security-reports path: | ${{ github.workspace }}/flawfinder.log ${{ github.workspace }}/clang-tidy.log ${{ github.workspace }}/semgrep.log retention-days: 7 myguard-labs-nginx-zstd-module-3486b5b/.github/workflows/valgrind.yml000066400000000000000000000075021525232545500257700ustar00rootroot00000000000000name: Valgrind # Targeted manual Valgrind memcheck-lite soak — 60s against a debug # nginx build so PR feedback stays quick. The full memcheck + helgrind soak # lives in ci-deep.yml (weekly cron + workflow_dispatch). # # Action pins (keep in sync with build-test.yml header): # actions/checkout@v5.1.0 -> fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true NGINX_VERSION: "" on: workflow_dispatch: {} # Called by ci.yml, the single PR entry point (step 15). workflow_call: {} concurrency: group: valgrind-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true permissions: contents: read jobs: memcheck-lite: name: Memcheck lite (60s soak) runs-on: ${{ fromJSON(vars.POOL || '["ubuntu-latest"]') }} timeout-minutes: 45 steps: - name: Checkout module uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false # Required before resolving nginx on an ephemeral fork runner. - name: Install bootstrap dependencies run: | sudo apt-get update sudo apt-get install -y ca-certificates curl python3 - name: Resolve latest mainline nginx env: GH_TOKEN: ${{ github.token }} run: | set -euo pipefail ver=$(bash "$GITHUB_WORKSPACE/ci/tools/nginx-releases.sh" mainline) [ -n "$ver" ] || { echo "::error::cannot resolve nginx version"; exit 1; } echo "NGINX_VERSION=$ver" >> "$GITHUB_ENV" - name: Install dependencies run: | # apt profile the bare apt-get lines below rely on: loose timeouts # for the shared self-hosted box; Azure mirror only on the # ubuntu-latest fallback when vars.POOL is unset (PR #121) printf '%s\n' \ 'DPkg::Lock::Timeout "120";' \ 'Acquire::Retries "3";' \ 'Acquire::http::Timeout "20";' \ 'Acquire::https::Timeout "20";' \ | sudo tee /etc/apt/apt.conf.d/99-ci-net > /dev/null sudo apt-get update sudo apt-get install -y \ build-essential curl gnupg libpcre2-dev libzstd-dev openssl pkg-config \ valgrind wget zlib1g-dev zstd - name: Build debug nginx + zstd module run: | set -euo pipefail test -f "nginx-${NGINX_VERSION}.tar.gz" || \ wget -q -O "nginx-${NGINX_VERSION}.tar.gz" \ "https://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz" wget -q "https://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz.asc" \ -O "nginx-${NGINX_VERSION}.tar.gz.asc" gnupghome="$(mktemp -d)" export GNUPGHOME="$gnupghome" chmod 700 "$gnupghome" for keyfile in ci/tools/keys/*.key; do gpg --quiet --import "$keyfile" 2>/dev/null done gpg --quiet --verify "nginx-${NGINX_VERSION}.tar.gz.asc" "nginx-${NGINX_VERSION}.tar.gz" rm -rf "$gnupghome" tar -xzf "nginx-${NGINX_VERSION}.tar.gz" cd "nginx-${NGINX_VERSION}" ./configure \ --with-compat --with-debug --with-threads --with-file-aio \ --with-http_ssl_module --with-http_v2_module \ --with-http_gzip_static_module --with-http_realip_module \ --with-cc-opt="-DZSTD_STATIC_LINKING_ONLY -DNGX_DEBUG_PALLOC=1 -g3 -O0 -fno-omit-frame-pointer -funwind-tables $(pkg-config --cflags libzstd zlib libpcre2-8)" \ --with-ld-opt="$(pkg-config --libs libzstd zlib libpcre2-8)" \ --add-module="$GITHUB_WORKSPACE" make -j"$(nproc)" - name: Memcheck soak (60s @ 4 conc) env: USE_VALGRIND: "1" run: | bash ci/tools/soak.sh \ "$GITHUB_WORKSPACE/nginx-${NGINX_VERSION}/objs/nginx" \ 60 4 myguard-labs-nginx-zstd-module-3486b5b/.github/workflows/windows-build.yml000066400000000000000000000442151525232545500267530ustar00rootroot00000000000000--- # yamllint disable rule:line-length rule:truthy name: Windows build # MSVC and MinGW exercise different nginx win32 build paths. MSVC supports # static addons only; MinGW's GNU linker also supports dynamic PE modules. on: workflow_dispatch: {} # Called by ci.yml, the single PR entry point (step 15). workflow_call: {} concurrency: group: windows-build-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true permissions: contents: read env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true # The source pins (nginx, pcre2, zlib: version + sha256) are NOT here: each # job's first step reads them from ci/tools/windows-pins.sh, the one place # they live (shared with ci/tools/build-windows.sh, bumped by # ci/tools/bump-versions.sh). jobs: msvc: name: MSVC x64 static runs-on: windows-latest timeout-minutes: 30 steps: - name: Checkout module uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Load the pinned sources # Only the six values this workflow consumes are exported; the pins # file also carries OpenSSL/nasm/zstd for build-windows.sh. shell: bash run: | set -euo pipefail # shellcheck disable=SC1091 . ci/tools/windows-pins.sh for v in VER_NGINX SHA_NGINX VER_PCRE2 SHA_PCRE2 VER_ZLIB SHA_ZLIB; do [ -n "${!v:-}" ] || { echo "::error::ci/tools/windows-pins.sh does not set $v"; exit 1; } done { echo "NGINX_VERSION=$VER_NGINX" echo "NGINX_SHA256=$SHA_NGINX" echo "PCRE2_VERSION=$VER_PCRE2" echo "PCRE2_SHA256=$SHA_PCRE2" echo "ZLIB_VERSION=$VER_ZLIB" echo "ZLIB_SHA256=$SHA_ZLIB" } >> "$GITHUB_ENV" - name: Install static libzstd shell: pwsh run: | $ErrorActionPreference = 'Stop' & "$env:VCPKG_INSTALLATION_ROOT\vcpkg.exe" install zstd:x64-windows-static if ($LASTEXITCODE -ne 0) { throw "vcpkg failed with $LASTEXITCODE" } $lib = "$env:VCPKG_INSTALLATION_ROOT\installed\x64-windows-static\lib" if (-not (Test-Path "$lib\zstd_static.lib")) { if (-not (Test-Path "$lib\zstd.lib")) { throw "vcpkg zstd archive not found" } Copy-Item "$lib\zstd.lib" "$lib\zstd_static.lib" } - name: Fetch and verify sources shell: pwsh run: | $ErrorActionPreference = 'Stop' $sources = @( @{ file = "nginx-$env:NGINX_VERSION.tar.gz"; url = "https://nginx.org/download/nginx-$env:NGINX_VERSION.tar.gz"; sha = $env:NGINX_SHA256 }, @{ file = "pcre2-$env:PCRE2_VERSION.tar.gz"; url = "https://github.com/PCRE2Project/pcre2/releases/download/pcre2-$env:PCRE2_VERSION/pcre2-$env:PCRE2_VERSION.tar.gz"; sha = $env:PCRE2_SHA256 }, @{ file = "zlib-$env:ZLIB_VERSION.tar.gz"; url = "https://github.com/madler/zlib/releases/download/v$env:ZLIB_VERSION/zlib-$env:ZLIB_VERSION.tar.gz"; sha = $env:ZLIB_SHA256 } ) foreach ($s in $sources) { Invoke-WebRequest -Uri $s.url -OutFile $s.file -ConnectionTimeoutSeconds 30 -OperationTimeoutSeconds 60 $actual = (Get-FileHash $s.file -Algorithm SHA256).Hash.ToLower() if ($actual -ne $s.sha) { throw "sha256 mismatch for $($s.file): $actual" } tar -xzf $s.file if ($LASTEXITCODE -ne 0) { throw "tar failed for $($s.file)" } } - name: Locate MSVC shell: pwsh run: | $ErrorActionPreference = 'Stop' $vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" $root = & $vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath $vcvars = "$root\VC\Auxiliary\Build\vcvars64.bat" if (-not (Test-Path $vcvars)) { throw "vcvars64.bat not found" } "VCVARS=$vcvars" >> $env:GITHUB_ENV - name: Configure # Configure is a bash script, but every compiler probe needs the # environment installed by vcvars64.bat in this same cmd process. shell: cmd # zizmor: ignore[misfeature] working-directory: nginx-${{ env.NGINX_VERSION }} run: | call "%VCVARS%" || exit /b 1 bash "%GITHUB_WORKSPACE:\=/%/.github/scripts/configure-msvc.sh" || exit /b 1 - name: Pre-build PCRE2 and zlib shell: cmd # zizmor: ignore[misfeature] run: | call "%VCVARS%" || exit /b 1 cd pcre2-%PCRE2_VERSION% || exit /b 1 copy /y src\config.h.generic src\config.h || exit /b 1 copy /y src\pcre2.h.generic src\pcre2.h || exit /b 1 copy /y src\pcre2_chartables.c.dist src\pcre2_chartables.c || exit /b 1 cl -c -O2 -MT -DHAVE_CONFIG_H -DPCRE2_CODE_UNIT_WIDTH=8 -DPCRE2_STATIC -Isrc ^ src\pcre2_auto_possess.c src\pcre2_chartables.c src\pcre2_compile.c ^ src\pcre2_config.c src\pcre2_context.c src\pcre2_convert.c ^ src\pcre2_dfa_match.c src\pcre2_error.c src\pcre2_extuni.c ^ src\pcre2_find_bracket.c src\pcre2_maketables.c src\pcre2_match.c ^ src\pcre2_match_data.c src\pcre2_newline.c src\pcre2_ord2utf.c ^ src\pcre2_pattern_info.c src\pcre2_script_run.c src\pcre2_serialize.c ^ src\pcre2_string_utils.c src\pcre2_study.c src\pcre2_substitute.c ^ src\pcre2_substring.c src\pcre2_tables.c src\pcre2_ucd.c ^ src\pcre2_valid_utf.c src\pcre2_xclass.c || exit /b 1 link -lib -out:pcre2-8.lib *.obj || exit /b 1 cd ..\zlib-%ZLIB_VERSION% || exit /b 1 cl -c -O2 -MT adler32.c crc32.c deflate.c infback.c inffast.c inflate.c inftrees.c trees.c zutil.c compress.c uncompr.c gzclose.c gzlib.c gzread.c gzwrite.c || exit /b 1 link -lib -out:zlib.lib *.obj || exit /b 1 - name: Build shell: cmd # zizmor: ignore[misfeature] working-directory: nginx-${{ env.NGINX_VERSION }} run: | call "%VCVARS%" || exit /b 1 nmake -f objs/Makefile || exit /b 1 - name: Verify static modules and directives shell: pwsh working-directory: nginx-${{ env.NGINX_VERSION }} run: | $ErrorActionPreference = 'Stop' $table = Get-Content objs\ngx_modules.c -Raw foreach ($name in 'ngx_http_zstd_filter_module', 'ngx_http_zstd_static_module') { if ($table -notmatch $name) { throw "$name absent from ngx_modules.c" } } New-Item -ItemType Directory -Force conf, logs | Out-Null function Test-Directive([string]$directive, [bool]$valid) { @('events {}', 'http {', "server { listen 8080; $directive }", '}') | Set-Content -Encoding ascii conf/nginx.conf & .\objs\nginx.exe -t -p . -c conf/nginx.conf 2>$null if (($LASTEXITCODE -eq 0) -ne $valid) { throw "unexpected verdict for: $directive" } } Test-Directive 'zstd on; zstd_static on;' $true Test-Directive 'zstd_not_a_directive on;' $false exit 0 - name: Run static request smoke shell: pwsh working-directory: nginx-${{ env.NGINX_VERSION }} run: | $ErrorActionPreference = 'Stop' python -m pip install --disable-pip-version-check --no-input zstandard==0.25.0 New-Item -ItemType Directory -Force conf, logs, html | Out-Null @' import pathlib, zstandard root = pathlib.Path("html") plain = b"windows static zstd smoke\n" * 256 root.joinpath("plain.txt").write_bytes(plain) root.joinpath("static.txt").write_bytes(plain) root.joinpath("static.txt.zst").write_bytes(zstandard.ZstdCompressor().compress(plain)) '@ | Set-Content -Encoding ascii make-smoke-files.py python make-smoke-files.py @' events {} http { server { listen 127.0.0.1:18080; root html; zstd on; zstd_types text/plain; zstd_static on; } } '@ | Set-Content -Encoding ascii conf/nginx.conf @' events {} http { server { listen 127.0.0.1:18080; zstd_not_a_directive on; } } '@ | Set-Content -Encoding ascii conf/invalid.conf & .\objs\nginx.exe -t -p . -c conf/invalid.conf 2>$null if ($LASTEXITCODE -eq 0) { throw 'invalid zstd directive was accepted' } & .\objs\nginx.exe -t -p . -c conf/nginx.conf if ($LASTEXITCODE -ne 0) { throw "nginx -t failed with $LASTEXITCODE" } try { $nginx = Start-Process -FilePath .\objs\nginx.exe -ArgumentList @('-p', '.', '-c', 'conf/nginx.conf') -PassThru Start-Sleep -Milliseconds 500 if ($nginx.HasExited) { throw "nginx exited during startup with $($nginx.ExitCode)" } @' import http.client, time, zstandard expected = b"windows static zstd smoke\n" * 256 def request(path, accept=True): headers = {"Accept-Encoding": "zstd"} if accept else {} for _ in range(40): try: conn = http.client.HTTPConnection("127.0.0.1", 18080, timeout=2) conn.request("GET", path, headers=headers) resp = conn.getresponse() body = resp.read() return resp.status, dict(resp.getheaders()), body except OSError: time.sleep(0.25) finally: try: conn.close() except Exception: pass raise SystemExit("nginx did not answer") status, headers, body = request("/plain.txt") if status != 200 or headers.get("Content-Encoding") != "zstd": raise SystemExit(f"dynamic zstd response missing: {status} {headers!r}") if zstandard.ZstdDecompressor().decompress(body) != expected: raise SystemExit("dynamic zstd response decoded to unexpected bytes") status, headers, body = request("/static.txt") if status != 200 or headers.get("Content-Encoding") != "zstd": raise SystemExit(f"static zstd response missing: {status} {headers!r}") if zstandard.ZstdDecompressor().decompress(body) != expected: raise SystemExit("static zstd response decoded to unexpected bytes") status, _, _ = request("/missing.txt", accept=False) if status != 404: raise SystemExit(f"missing static file returned {status}, not 404") '@ | Set-Content -Encoding ascii smoke-request.py python smoke-request.py } finally { & .\objs\nginx.exe -s quit -p . -c conf/nginx.conf 2>$null if ($nginx -and -not $nginx.WaitForExit(5000)) { Stop-Process -Id $nginx.Id -Force } } mingw: name: MinGW-w64 x64 dynamic runs-on: windows-latest timeout-minutes: 30 steps: - name: Checkout module uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Load the pinned sources # Only the six values this workflow consumes are exported; the pins # file also carries OpenSSL/nasm/zstd for build-windows.sh. shell: bash run: | set -euo pipefail # shellcheck disable=SC1091 . ci/tools/windows-pins.sh for v in VER_NGINX SHA_NGINX VER_PCRE2 SHA_PCRE2 VER_ZLIB SHA_ZLIB; do [ -n "${!v:-}" ] || { echo "::error::ci/tools/windows-pins.sh does not set $v"; exit 1; } done { echo "NGINX_VERSION=$VER_NGINX" echo "NGINX_SHA256=$SHA_NGINX" echo "PCRE2_VERSION=$VER_PCRE2" echo "PCRE2_SHA256=$SHA_PCRE2" echo "ZLIB_VERSION=$VER_ZLIB" echo "ZLIB_SHA256=$SHA_ZLIB" } >> "$GITHUB_ENV" - name: Set up MSYS2 uses: msys2/setup-msys2@66cd2cce69caa17b53920067426061ca1de3a884 # v2.32.0 with: msystem: MINGW64 update: true install: >- base-devel mingw-w64-x86_64-gcc mingw-w64-x86_64-pcre2 mingw-w64-x86_64-zlib mingw-w64-x86_64-zstd - name: Fetch, configure and build shell: msys2 {0} run: | set -euo pipefail fetch() { local url=$1 sha=$2 file=${1##*/} curl -fsSLO --connect-timeout 30 --max-time 120 "$url" printf '%s %s\n' "$sha" "$file" | sha256sum -c - tar -xzf "$file" } fetch "https://nginx.org/download/nginx-$NGINX_VERSION.tar.gz" "$NGINX_SHA256" fetch "https://github.com/PCRE2Project/pcre2/releases/download/pcre2-$PCRE2_VERSION/pcre2-$PCRE2_VERSION.tar.gz" "$PCRE2_SHA256" fetch "https://github.com/madler/zlib/releases/download/v$ZLIB_VERSION/zlib-$ZLIB_VERSION.tar.gz" "$ZLIB_SHA256" module_dir=$(cygpath -m "$GITHUB_WORKSPACE") cd "nginx-$NGINX_VERSION" ZSTD_INC=/mingw64/include ZSTD_LIB=/mingw64/lib \ ./configure \ --with-cc=gcc \ --with-pcre="../pcre2-$PCRE2_VERSION" \ --with-zlib="../zlib-$ZLIB_VERSION" \ --add-dynamic-module="$module_dir" # A parallel build can start the dynamic-module link before nginx's # MinGW import library exists. The generated makefile does not expose # that library as a separate target, so preserve its declared order. make -j1 - name: Verify dynamic modules and directives shell: msys2 {0} run: | set -euo pipefail cd "nginx-$NGINX_VERSION" test -f objs/ngx_http_zstd_filter_module.so test -f objs/ngx_http_zstd_static_module.so mkdir -p conf logs cat > conf/nginx.conf <<'EOF' load_module objs/ngx_http_zstd_filter_module.so; load_module objs/ngx_http_zstd_static_module.so; events {} http { server { listen 8080; zstd on; zstd_static on; } } EOF ./objs/nginx.exe -t -p . -c conf/nginx.conf sed -i 's/zstd on/zstd_not_a_directive on/' conf/nginx.conf if ./objs/nginx.exe -t -p . -c conf/nginx.conf; then echo 'bogus directive was accepted' >&2 exit 1 fi - name: Run dynamic request smoke shell: msys2 {0} run: | set -euo pipefail cd "nginx-$NGINX_VERSION" pacman --noconfirm -S --needed mingw-w64-x86_64-python-zstandard mkdir -p conf logs html python - <<'PY' import pathlib, zstandard root = pathlib.Path("html") plain = b"windows dynamic zstd smoke\n" * 256 root.joinpath("plain.txt").write_bytes(plain) root.joinpath("static.txt").write_bytes(plain) root.joinpath("static.txt.zst").write_bytes(zstandard.ZstdCompressor().compress(plain)) PY cat > conf/nginx.conf <<'EOF' load_module objs/ngx_http_zstd_filter_module.so; load_module objs/ngx_http_zstd_static_module.so; events {} http { server { listen 127.0.0.1:18081; root html; zstd on; zstd_types text/plain; zstd_static on; } } EOF cat > conf/invalid.conf <<'EOF' load_module objs/ngx_http_zstd_filter_module.so; load_module objs/ngx_http_zstd_static_module.so; events {} http { server { listen 127.0.0.1:18081; zstd_not_a_directive on; } } EOF if ./objs/nginx.exe -t -p . -c conf/invalid.conf 2>/dev/null; then echo 'bogus directive was accepted' >&2 exit 1 fi ./objs/nginx.exe -t -p . -c conf/nginx.conf ./objs/nginx.exe -p . -c conf/nginx.conf & nginx_pid=$! sleep 0.5 if ! kill -0 "$nginx_pid" 2>/dev/null; then echo "nginx exited during startup" >&2 exit 1 fi trap './objs/nginx.exe -s quit -p . -c conf/nginx.conf >/dev/null 2>&1 || true' EXIT python - <<'PY' import http.client, time, zstandard expected = b"windows dynamic zstd smoke\n" * 256 def request(path, accept=True): headers = {"Accept-Encoding": "zstd"} if accept else {} for _ in range(40): try: conn = http.client.HTTPConnection("127.0.0.1", 18081, timeout=2) conn.request("GET", path, headers=headers) resp = conn.getresponse() body = resp.read() return resp.status, dict(resp.getheaders()), body except OSError: time.sleep(0.25) finally: try: conn.close() except Exception: pass raise SystemExit("nginx did not answer") status, headers, body = request("/plain.txt") if status != 200 or headers.get("Content-Encoding") != "zstd": raise SystemExit(f"dynamic zstd response missing: {status} {headers!r}") if zstandard.ZstdDecompressor().decompress(body) != expected: raise SystemExit("dynamic zstd response decoded to unexpected bytes") status, headers, body = request("/static.txt") if status != 200 or headers.get("Content-Encoding") != "zstd": raise SystemExit(f"static zstd response missing: {status} {headers!r}") if zstandard.ZstdDecompressor().decompress(body) != expected: raise SystemExit("static zstd response decoded to unexpected bytes") status, _, _ = request("/missing.txt", accept=False) if status != 404: raise SystemExit(f"missing static file returned {status}, not 404") PY myguard-labs-nginx-zstd-module-3486b5b/.gitignore000066400000000000000000000032561525232545500220340ustar00rootroot00000000000000# C build artifacts *.o *.d *.a *.so *.so.* *.lo *.la # Compiled nginx modules *.so # nginx build tree (when building out-of-tree inside this repo) objs/ # tools/ci-build.sh persistent build tree (nginx/angie source + compiled # binaries, one dir per flavor-version) .build/ # Local analysis and task-runner caches .grind-wt/ .mypy_cache/ .pytest_cache/ .ruff_cache/ # Test runner working directories (nginx test::nginx servroot) ci/t/servroot/ ci/t/servroot-*/ # Test::Nginx defaults to t/servroot/ relative to CWD when # TEST_NGINX_SERVROOT is unset (e.g. `prove ci/t/foo.t` run from the repo # root). CI always sets TEST_NGINX_SERVROOT explicitly, so this is narrowly # scoped to the generated servroot dir rather than ignoring all of `t/`, # which could later mask a real source directory. /t/servroot/ /t/servroot-*/ # Python bytecode cache (test tooling under tools/) __pycache__/ *.pyc # Editor and IDE .vscode/ .idea/ *.swp *.swo *~ # macOS .DS_Store # CI / local build leftovers *.log *.pid # Fuzzing build artifacts (seed corpus IS tracked; everything else is not) ci/fuzz/fuzz_accept_encoding ci/fuzz/generated_parser.inc ci/fuzz/fuzz_dcz ci/fuzz/generated_dcz.inc ci/fuzz/crash-* ci/fuzz/corpus-min/ ci/fuzz/corpus-dcz-min/ ci/fuzz/*.profraw # Local fuzz runs dump discovered units into corpus/; keep only the # curated NN_name seeds under version control. ci/fuzz/corpus/* !ci/fuzz/corpus/[0-9][0-9]_* # Unit test build artifacts (source + run.sh IS tracked) ci/tests/unit/test_accept_encoding ci/tests/unit/test_accept_encoding_chain ci/tests/unit/test_accept_encoding_legacy ci/tests/unit/test_static_probe ci/tests/unit/test_version_policy ci/tests/unit/*.gcda ci/tests/unit/*.gcno myguard-labs-nginx-zstd-module-3486b5b/.gitleaks.toml000066400000000000000000000020571525232545500226200ustar00rootroot00000000000000# gitleaks configuration for nginx-zstd-module. # # Extends the upstream default ruleset rather than replacing it -- dropping # `useDefault` here would leave the scan with NO rules and report a clean tree # for any secret at all, which is worse than not running gitleaks. [extend] useDefault = true [allowlist] description = "Paths this repository does not own" # .build/ is the throwaway nginx source tree the CI jobs unpack to compile # against (gitignored, see .gitignore). `gitleaks detect --no-git` walks the # filesystem rather than the index, so without this it reads that vendored # upstream tree and reports nginx's own QUIC test constants as findings -- # upstream nginx's code, not this module's secrets, and not fixable here. # # Scoped by PATH, not by fingerprint: the nginx version in the directory name # changes on every bump, so a fingerprint list would silently stop matching and # the noise would return. The pre-commit hook is unaffected either way -- it # scans the staged patch, which can never contain a gitignored file. paths = ['''^\.build/'''] myguard-labs-nginx-zstd-module-3486b5b/.gitmodules000066400000000000000000000001601525232545500222100ustar00rootroot00000000000000[submodule "ci/t/harness"] path = ci/t/harness url = https://github.com/myguard-labs/nginx-module-testkit.git myguard-labs-nginx-zstd-module-3486b5b/.markdownlint.yaml000066400000000000000000000041701525232545500235130ustar00rootroot00000000000000# markdownlint config for nginx-zstd-module # Used by ci/linter and editor integrations. # # Philosophy: enable rules that catch real defects; disable rules that fire on # legitimate prose patterns. Every disabled rule carries a comment explaining # what it would catch versus what it fights. # MD013 — line-length: disable # The prose blocks are hand-wrapped documentation. Tables and log-output # snippets legitimately run past 80 columns without breaking readability. # A hard line-length gate trades 120 findings for mechanical churn. MD013: false # MD004 — ul-style: disable # The rule enforces ONE bullet style across the whole tree, but this repo has # two established per-file conventions and neither is wrong. Measured over the # tracked .md files: README.md is asterisk (79 `*` vs 2 `-`), while every other # file is dash (69 `-` vs 0 `*`) — CONTRIBUTING.md, SECURITY.md, # ci/adoption-findings.md, ci/fuzz/README.md, ci/linter/README.md. # Any single `style:` setting therefore reddens one group or the other and # forces a mechanical rewrite of a file that is already internally consistent. # Per-file consistency is the property worth having, and MD004 cannot express # it, so the rule is off rather than the files rewritten. MD004: false # MD060 — table-column-style: disable # Pipe alignment in tables is cosmetic. The default expects spaces around pipes # that make maintenance tedious and add no semantic value. MD060: false # MD028 — no-blanks-blockquote: disable # README.md deliberately uses blank-line-separated blockquotes as consecutive # callouts. This pattern aids readability in rendered HTML. MD028: false # MD034 — no-bare-urls: disable # This rule fires on EMAIL ADDRESSES (github@myguard.nl, zchao1995@gmail.com), # not URLs. That is a false positive class in markdownlint itself — a match # inside `<...@...>` pattern at any nesting level gets flagged as a bare URL # when it is an email reference. MD034: false # MD041 — first-line-h1: disable # README.md opens with CI status badges before the H1 heading. This pattern is # conventional on GitHub and aids discoverability without breaking structure. MD041: false myguard-labs-nginx-zstd-module-3486b5b/.perlcriticrc000066400000000000000000000022631525232545500225270ustar00rootroot00000000000000# perlcritic config -- used by ci/linter/lint-perl.sh and by editor plugins. # # The Perl in this repo is a Test::Nginx::Socket suite (ci/t/*.t), not library # code. Four default policies at severity >=4 describe a .pm and fire on every # well-formed .t file; leaving them on lands the gate red on arrival, which # only teaches everyone to --no-verify. Each exclusion below is about the file # SHAPE, not about tolerating a real defect: # # RequireExplicitPackage a .t is a script, not a module; wrapping the suite # in a package would hide Test::Nginx's exported DSL. # RequireEndWithOne same reason -- a script has no module return value. # RequireUseStrict Test::Nginx::Socket enables strict and warnings in # RequireUseWarnings its import, so the policies see "code before # strictures" for pragmas that ARE in effect. # # Everything else at severity >=4 stays live. Add a policy here only with the # reason it cannot apply, not because it is inconvenient. severity = 4 [-Modules::RequireExplicitPackage] [-Modules::RequireEndWithOne] [-TestingAndDebugging::RequireUseStrict] [-TestingAndDebugging::RequireUseWarnings] myguard-labs-nginx-zstd-module-3486b5b/.pre-commit-config.yaml000066400000000000000000000460231525232545500243240ustar00rootroot00000000000000--- # yamllint disable rule:line-length # The exclude regex and the actionlint entry are single unsplittable # strings; wrapping either changes what they match. # Local pre-commit gates for http-zstd. # # GENERATED by tools/gen-precommit-config.py -- re-run it rather than # hand-editing paths, so every module's config stays derived from its real # layout instead of drifting toward a copied-around guess. # # Purpose: catch the cheap half of what CI catches, before a push burns a # remote round-trip. These hooks mirror the CI gates in .github/workflows/ # (flawfinder >=4, semgrep >=WARNING, cppcheck, ruff). If a threshold moves # there, move it here in the same commit, or local-green stops predicting # remote-green. # # The mirror is EXACT and was not always: flawfinder and # semgrep gated here while their four CI sites piped to `tee` with no # --error-level and an explicit `|| true`, so a finding blocked a local commit # and then sailed through CI. The fix went in the direction that KEEPS # coverage -- CI was gated up to these thresholds, not these relaxed down to # CI. If you find them diverged again, fix CI first. # # Detected in this repo: # C source dirs : src # Python dirs : ci # fuzz corpora : ci/fuzz/corpus, ci/fuzz/corpus_dcz, ci/fuzz/regressions # # What is NOT here, and why: # clang-tidy -- needs ngx_auto_config.h, which only exists after nginx is # configured. A hook cannot assume a built tree, and one that # silently skips when the tree is missing is a vacuous gate. # Stays CI-only on purpose. NOTE this reason does NOT extend # to cppcheck, which parses standalone and IS gated below. # clang-format -- no .clang-format in these repos, so there is no agreed style # to enforce; adding one would reformat existing code in an # unrelated diff. # codeql/asan/valgrind/fuzzing -- far too slow for a commit hook; CI-only. # # Fuzz harnesses (fuzz/, t/) are outside the lint scope: they deliberately do # unsafe things with untrusted input and would trip flawfinder forever. # # Install (once per clone): # pipx install pre-commit # git config core.hooksPath .githooks # # Do NOT run `pre-commit install` in this repo. It writes .git/hooks/pre-commit, # and core.hooksPath REPLACES .git/hooks/ entirely -- git consults one or the # other, never both. The result is an inert config that looks installed from # every angle: config present, install reported success, hook file on disk with # a plausible date, and nothing ever going red to reveal it. # .githooks/pre-commit invokes `pre-commit run --hook-stage pre-commit` itself, # which is what actually runs these hooks. A missing pre-commit binary is exit 2 # there (blocking), not a skip -- a gate that skips when its tool is absent # reports green while checking nothing. # # Run against everything without committing: # pre-commit run --all-files # # VERIFY BEFORE TRUSTING (a green hook proves nothing until seen red): # printf '#!/bin/bash\ncd /tmp/x\nrm -rf "${B}/"*\n' > _p.sh # pre-commit run shellcheck --files _p.sh # expect Failed # rm _p.sh # Note SC2086 is INFO and correctly does NOT trip shellcheck here -- probe with # a real warning-severity finding (SC2164/SC2115) or you will misread the gate. # Fuzz corpus files are byte-exact test inputs: a stripped space or an added # newline changes what is actually fuzzed. Never let a fixer touch them. # # .pem and .der are NOT in this list, deliberately. pre-commit applies the # top-level exclude before any per-hook rule, so a suffix named here is # invisible to EVERY hook -- including detect-private-key, whose entire job is # catching a .pem. Verified: an identical fake key reads "(no files to check) # Skipped" as probe.pem and "Private key found" as probe.txt. They are excluded # per-hook on the three fixers below instead, which is the only reason they were # ever in this list. exclude: '^ci/fuzz/corpus/|^ci/fuzz/corpus_dcz/|^ci/fuzz/regressions/|^ci/fuzz/regressions_dcz/|\.traefik-ref/|\.claude/worktrees/|\.build/|(^|/)objs/|(^|/)vendor/|(^|/)node_modules/|(^|/)\.venv/|\.(bin|gz|zip|png|ico|so|o|a)$' repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v6.0.0 hooks: # The three fixers rewrite files in place, so every byte-exact file has # to stay out of their reach. Per-hook, not top-level: a top-level # exclude would also blind detect-private-key and the linters. # # Beyond keys and DER blobs, two kinds of file here are CONTENT, not # source, and a "harmless" trailing newline changes what they mean: # fuzz corpora / regressions a corpus entry IS the input bytes. # Appending \n makes it a different input, so # the regression silently stops reproducing the # crash it was committed to pin. # *.patch dpkg-source/quilt match context at zero fuzz; # rewriting the tail makes the patch not apply. # Caught 2026-08-22 by a `pre-commit run --all-files`, which mutated one # of each. - id: trailing-whitespace exclude: '(^ci/fuzz/corpus/|^ci/fuzz/corpus_dcz/|^ci/fuzz/regressions/|^ci/fuzz/regressions_dcz/|^ci/t/.*\.t$|\.(pem|der|patch)$)' - id: end-of-file-fixer exclude: '(^ci/fuzz/corpus/|^ci/fuzz/corpus_dcz/|^ci/fuzz/regressions/|^ci/fuzz/regressions_dcz/|^ci/t/.*\.t$|\.(pem|der|patch)$)' - id: mixed-line-ending args: [--fix=lf] exclude: '(^ci/fuzz/corpus/|^ci/fuzz/corpus_dcz/|^ci/fuzz/regressions/|^ci/fuzz/regressions_dcz/|^ci/t/.*\.t$|\.(pem|der|patch)$)' - id: check-merge-conflict - id: check-yaml - id: check-added-large-files args: [--maxkb=512] # Test vectors are not secrets. A derived module's unit tests can carry # locked PEM fixtures (an elided MIGH... blob, a documented P-256 # vector); gating on those is a pure false positive, while the hook still # guards every real path. - id: detect-private-key # `(ci/)?` because the skeleton standard relocates the suite under ci/: # a root-anchored ^tests?/ stops matching the moment a module moves # tests/ to ci/tests/, and the hook then fires on the relocated PEM # fixtures and blocks the move commit itself. Hit in every module # adopting the ci/ layout. exclude: '^(ci/)?tests?/' - repo: local hooks: # Mirrors the CI "gitleaks" step. Different scope on purpose: a commit # hook can only see what is being committed, so it scans the staged # patch, while CI scans the whole history -- a secret that was committed # and later removed is still published, and only the history scan finds # it. Measured at zero findings before the gate went in. # # detect-private-key above is not a substitute: it matches PEM key # headers only, and misses every token-shaped secret (PATs, bot tokens, # cloud keys), which is what actually leaks out of these repos. # # VERIFY WITH A NON-EXAMPLE SECRET. The obvious probe -- AWS's own # AKIAIOSFODNN7EXAMPLE / wJalrXUtnFEMI... pair -- is allowlisted by the # default ruleset and reports "no leaks found", which reads exactly like # a broken gate. Probe with a plausible token instead: # Build the probe token from parts -- a literal one written out here is # itself a leak by this hook's own rules, and would land the gate red on # its own config file: # T=xoxb-2024827913265; T="$T-2024891053289-fJk2mQ8vNwLpXcRt7HzYbA31" # printf 'k = "%s"\n' "$T" > p.txt # git add p.txt && gitleaks git --staged --no-banner --redact -v # git reset -q p.txt && rm p.txt # expect: leaks found: 1 - id: gitleaks name: gitleaks (staged patch) entry: gitleaks git --staged --no-banner --redact language: system pass_filenames: false always_run: true # Mirrors the CI "flawfinder (gate on >=4)" step: fires on exactly what # CI blocks on. Noise below level 4 would train everyone to --no-verify. - id: flawfinder name: flawfinder (gate on >=4) entry: flawfinder --minlevel=4 --error-level=4 --quiet language: system files: '^(src|ci)/.*\.[ch]$' pass_filenames: true # Mirrors the CI "semgrep (gate on >=WARNING)" step, same configs and # same severity floor. WARNING not ERROR: the registry's ERROR tier has # no C rules that fire on this kind of code, so an ERROR gate could never # go red (verified 2026-07-30). # # require_serial IS LOAD-BEARING, do not remove it to "speed the hook up": # pre-commit otherwise shards the file list across one process per core # (32 here), and every semgrep-core allocates its own io_uring. That blows # past RLIMIT_MEMLOCK (ulimit -l, 8192 KB on this box) and semgrep-core # dies with # Unix_error: Cannot allocate memory io_uring_queue_init # surfacing as a bare "exit code 2" with no message. That failure is # indistinguishable from a real finding at a glance, so it would have # this gate permanently red for a reason that has nothing to do with the # code. One process, one ring, no contention. # # --quiet is also deliberately ABSENT: it suppressed exactly the error # text above and turned a diagnosable crash into a silent exit 2. # # --jobs=1 closes the SECOND source of the same crash. require_serial # above stops pre-commit forking one hook process per core; it does # nothing about semgrep's own internal parallelism, which forks # semgrep-core per core inside that single process and allocates an # io_uring for each. A target with few enough C files never reaches the # limit and looks fine, so this only surfaces as the tree grows -- which # is exactly when a green-to-red flip reads as a real finding. - id: semgrep name: semgrep (gate on >=WARNING) entry: semgrep scan --jobs=1 --config p/c --config p/security-audit --severity=WARNING --severity=ERROR --error language: system files: '^(src|ci)/.*\.[ch]$' pass_filenames: true require_serial: true # Mirrors the CI "Static analysis" step in build-test.yml: same enable # set, same suppressions, same -I. Any of those diverging makes local # green stop predicting remote green, which is this file's whole job. # # unusedStructMember is suppressed because cppcheck sees one TU at a # time here and cannot know a member is read from another file; -I is # the C source dir, so headers next to the sources resolve rather than # being reported as missing. # # -DNGX_PTR_SIZE=8: the breaker probe-word layout selects on this macro # and #errors when it is undefined. It normally arrives from nginx's # generated objs/ngx_auto_config.h, which cppcheck never sees, so it has # to be supplied -- an undefined macro in #if silently evaluates to 0 and # would pick the wrong layout for the whole analysis (H-1). # # Unlike clang-tidy this needs NO configured nginx tree, which is why it # can gate here at all. - id: cppcheck name: cppcheck (warning,performance,portability) entry: cppcheck --enable=warning,performance,portability --error-exitcode=1 --suppress=missingIncludeSystem --suppress=normalCheckLevelMaxBranches --suppress=unusedFunction --suppress=unusedStructMember -I src -DNGX_PTR_SIZE=8 language: system files: '^(src|ci)/.*\.[ch]$' pass_filenames: true # Structural sink sweep: this superrepo's own c-* / nginx-* rules plus # the C/C++ slice of CodeRabbit's ast-grep essentials pack (Apache-2.0, # CWE-tagged). Those are the rules CodeRabbit runs on a pull request, so # a hit here is a finding the bot would have raised -- caught before the # push rather than in review. # # -c ci/ast-grep/sgconfig.yml is LOAD-BEARING, do not drop it to let # ast-grep "find the config itself". It resolves sgconfig.yml by walking # UP from the working directory, so without -c this hook would silently # bind to /opt/myguard/sgconfig.yml while the clone sits there, and bind # to NOTHING in a standalone clone -- scanning against an empty ruleset # and reporting green. The rules are vendored into ci/ast-grep/ by # tools/vendor-astgrep-rules.py precisely so this gate is self-contained. # Refresh them with that script; `--check` makes staleness CI-able. # # ast-grep has NO severity-threshold flag (unlike semgrep --severity), and # exits non-zero only on `error`. The gate is therefore built by PROMOTING # named rules with --error=; everything else stays advisory -- printed, # never blocking. That distinction is the whole reason this can be gated # at all: measured across every module repo 2026-08-08, 318 of 368 # findings are note[nginx-unchecked-palloc], which in nginx C is # frequently legitimate (call sites propagate NULL deliberately). Gating # the note tier would land every repo red on arrival and train everyone # to --no-verify -- the same failure the shellcheck note below describes. # # A rule joins the promoted list only after its findings are measured and # confirmed real. Verified 2026-08-08 that the promoted set exits 1 on a # real finding, 0 on a clean file, and 0 on a note-only file. - id: ast-grep name: ast-grep (structural sinks, gate on promoted rules) entry: ast-grep scan -c ci/ast-grep/sgconfig.yml --error=c-alloc-mul-overflow --error=c-sizeof-wrong-operand --error=c-unbounded-string-copy --error=c-format-string --error=c-shell-exec language: system files: '^src/.*\.[ch]$' pass_filenames: true # Python is the runtime TEST HARNESS in these repos, so it carries the # same weight as C -- but nothing linted it before this hook. # # RUF059 (unused-unpacked-variable) is why this hook exists. A captured- # then-never-read variable is the known tell for the vacuous-assertion # class: a test computed `hits_before_confirm`, never read it, and its # only surviving assertion was one BOTH the pass and fail states satisfy. # CodeRabbit caught that by hand; a linter catches it for free. # # F,B are the correctness tiers (undefined names, unused imports, real # bug patterns). PLE and RET joined them once measured at zero backlog -- # PLE is pylint's error tier, so a finding there is a bug rather than a # preference. I (import sorting) joined on the same rule: measured across # all nine module repos it is 100% safe-autofixable (`ruff check --select # I --fix`), so it can never land red on arrival anywhere. # # The rest of the "cheap" tier was MEASURED and deliberately left out # 2026-08-02, because the label was wrong. Across cache-turbo: N 3, C4 8, # SIM 11, UP 13, PLW 15, RUF 31 -- but only 21 of those 84 are # safe-autofixable, leaving 63 hand-edits in a 15.6k-line test harness. # The two biggest residuals are pure preference on test scaffolding: # PLW1510 wants `check=False` spelled out at 13 call sites that already # inspect `r.returncode`, and SIM105 wants contextlib.suppress for 11 # try/except/pass blocks. That is churn across the file every runtime # test lives in, for no correctness gain, against a tree whose ledger # repeatedly warns that harness churn is how vacuous assertions get # introduced. Re-measure with --statistics before revisiting; do not # trust a remembered count. # # Style tiers are still deliberately NOT selected: PT018 and # E501 carry three-figure backlogs and would land this red on arrival, # training everyone to --no-verify -- the same failure the shellcheck # note below describes. Widen only with a commit that also fixes the # backlog AND moves the matching CI step. # # Count a tier with `ruff check --select --statistics`, never a # grep: ruff 0.16 moved the path onto a `--> path:line:col` line, so a # grep anchored on the old `path:line:col:` format reports 0 for every # tier and makes a widening look free. # # `language: system` resolves whatever ruff is on PATH, while CI pins an # exact version -- so local and remote can disagree if a release changes # a selected rule. That is tolerable only because every tier here is # verified against BOTH the pinned CI version and the local one before it # is selected; the I widening was checked clean on 0.15.20 and 0.16.1. # Verify the same way after any widening, or pin this entry to match. - id: ruff name: ruff (correctness tiers F,B,I,PLE,RET,RUF059) entry: ruff check --force-exclude --select F,B,I,PLE,RET,RUF059 language: system files: '^ci/.*\.py$' pass_filenames: true # CI workflows are code that breaks silently; actionlint catches bad if: # expressions, unknown contexts and shell errors inside run: blocks. # # -config-file: declares this repo's self-hosted runner labels explicitly # rather than relying on actionlint finding .github/actionlint.yaml by # cwd-guessing -- pre-commit hooks do not guarantee cwd the way a bare # `actionlint` invocation from a shell does. # # SHELLCHECK_OPTS applies to the shellcheck actionlint runs over `run:` # blocks. Without it that embedded pass reports at INFO while the # standalone shellcheck hook below gates at warning -- so the same SC2086 # would be ignored in a .sh file and blocking in a workflow. Same floor # in both places. - id: actionlint name: actionlint (workflow syntax) entry: env SHELLCHECK_OPTS=-Swarning actionlint -config-file .github/actionlint.yaml language: system files: ^\.github/workflows/.*\.ya?ml$ pass_filenames: true # -S warning: block on warning/error, not info. The info tier (e.g. # SC2015) fires on existing scripts and would land this red on arrival, # which just teaches everyone to --no-verify. # # NOTE some repos have pre-existing warning-level findings in build/test # scripts (SC2043 in a unit-test helper, SC2010 in a fuzz build script, # SC2089/2090 in a package build script, all seen in derived modules). # They do NOT block normal work, because a commit hook only ever sees the # files being committed -- they surface on `pre-commit run --all-files`. # They are real and worth fixing; they are deliberately NOT suppressed # here, so touching one of those scripts makes you fix it. - id: shellcheck name: shellcheck (gate on >=warning) entry: shellcheck -S warning language: system files: \.(sh|bash)$ pass_filenames: true myguard-labs-nginx-zstd-module-3486b5b/.vale-styles/000077500000000000000000000000001525232545500223645ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/.vale-styles/.gitkeep000066400000000000000000000000001525232545500240030ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/.vale.ini000066400000000000000000000012161525232545500215440ustar00rootroot00000000000000# Prose lens for this repository's Markdown. # # Deliberately built on Vale's BUILT-IN checks only (Vale.Terms, Vale.Spelling, # Vale.Repetition). No style package is vendored or downloaded, so the lens is # self-contained and runs identically in CI, in a fork, and under # review-lint.sh's `--no-global` invocation. # # Spelling is off: this is a systems-programming README full of identifiers # (zstd, dcz, refPrefix, CCtx, ngx_*), and a wordlist large enough to silence # them would suppress the real misspellings codespell already catches better. StylesPath = .vale-styles MinAlertLevel = warning [*.md] Vale.Repetition = error Vale.Terms = warning myguard-labs-nginx-zstd-module-3486b5b/.yamllint000066400000000000000000000021111525232545500216630ustar00rootroot00000000000000# yamllint config for nginx-skeleton-module -- used by ci/linter/lint-yaml.sh # and by any editor integration, so both agree. # # GitHub Actions workflows are not hand-written prose YAML; three default rules # fight their syntax rather than catching bugs: # document-start a workflow never starts with ---, and adding one everywhere # is churn with no reader benefit. # truthy `on:` is the trigger key, not the boolean `on`. check-keys # off keeps the rule live for VALUES (yes/no/on/off), which is # where a real YAML 1.1 surprise bites. # line-length inline `run:` shell and ${{ }} expressions legitimately run # long. 120 as a WARNING: visible, not blocking. lint-yaml.sh # deliberately does not pass --strict, so warnings do not fail # the gate -- errors (real syntax/structure problems) do. extends: default rules: document-start: disable truthy: check-keys: false comments: min-spaces-from-content: 1 line-length: max: 120 level: warning myguard-labs-nginx-zstd-module-3486b5b/CHANGES000066400000000000000000000443071525232545500210410ustar00rootroot00000000000000Changes with http-zstd Unreleased *) Change: a location that configures dcz dictionaries at "zstd_comp_level" 9 or above, or with "zstd_long on", now emits a config-load warning naming the profile and the size of the largest configured dictionary. A dcz response re-references its dictionary with ZSTD_CCtx_refPrefix() on every request, rebuilding its match tables at a cost set by dictionary size and level and independent of the response body (measured, 1 KB body: an 8 MB dictionary costs 0.75 ms at level 3 against 35 ms at level 9). The warning is advisory only -- no request-path behaviour, dcz wire output or configuration acceptance changes, and the common profile of dcz dictionaries at the default level stays silent. *) Bugfix: "$zstd_ratio" is now exact for the whole 64-bit range of byte counts. The previous form scaled the input count before dividing, so a large enough response overflowed the intermediate and reported a wrong ratio -- for bytes_in near UINT64_MAX it logged 0.000 where the true ratio was 0.999. The variable is now produced by exact integer long division that extracts each of the three fractional digits without an intermediate that can wrap, and is correct for expanding streams as well as shrinking ones. Log output only; no response bytes change. *) Tests: removed the standalone dcz CDict-vs-refPrefix equivalence probes. They were one-off investigation tools, not production-helper regression tests, and keeping them under ci/tools made the dcz test surface look stronger than it was. *) Feature: the request benchmark now has an opt-in dcz workload with canonical dictionary negotiation headers, deterministic hit/miss fixtures, and a response-encoding preflight. perf_stat_recipe.py drives the required 1, 4, and 16 configured-dictionary matrix and attaches counters only to nginx workers and compares them per measured successful request, while the existing ab_bench.py default remains the plain-zstd workload. *) Feature: "zstd_static_dict_bypass on" makes the precompressed static handler stand aside when a main request carries Available-Dictionary and explicitly accepts dcz, allowing the filter module to negotiate the dictionary response first. The opt-in applies to both "zstd_static on" and "always", inherits across http/server/location configuration, and emits the complete Accept-Encoding / Available-Dictionary / Sec-Fetch-Site Vary key before declining. The default remains off; when a later dcz policy or dictionary match fails, the skipped sidecar is not revisited and the filter falls back to ordinary zstd or identity without allowing a shared cache to cross that routing decision. *) Feature: "zstd_dcz_dict_trust_hashes on" opts out of verifying a supplied dictionary hash literal against the file's bytes: the literal is trusted verbatim as the negotiation key and the load-time SHA-256 is skipped. The verify-always default is unchanged. The hashing pass is the config-load cost at scale -- measured at 737 dictionary lines, nginx -t drops from 5.1s to 0.9s (user CPU 4.3s to 0.03s; sys identical, the read happens either way) -- so a content-addressed deployment whose tooling derives each literal from the file it ships can reclaim that on every reload, owning the stated risk that a stale or mistyped literal is advertised verbatim. Lines without a literal are hashed as always. The directive is order-enforced like zstd_dict_strict_path: declaring it after a literal-carrying zstd_dcz_dict_file is a config-load error. *) Bugfix: on 32-bit (ILP32) builds with large-file support, the pledged response length used to size the first output buffer was narrowed to size_t before ZSTD_compressBound(), truncating a declared length above 4 GiB modulo 2^32 -- a 4 GiB body pledged as 4 GiB + 389 bytes was sized for 389 bytes. The length is now checked for representability, and against libzstd's own width-dependent input ceiling, at full off_t width before any narrowing; a length that fails either check keeps the configured buffer size instead of shrinking. Lengths that are representable still take the first-buffer sizing unchanged, on every width. LP64 builds, where off_t and size_t are the same width, were never affected. *) Bugfix: per-request byte counters for $zstd_bytes_in and $zstd_bytes_out wrapped on 32-bit (ILP32) systems for responses exceeding 4 GiB. Both counters are now uint64_t. *) Security: "zstd_dict_strict_path on" enforced its trust policy only on the final path component and ignored file ownership. The open used O_NOFOLLOW on the whole path, which the kernel applies to the leaf alone -- every intermediate component was resolved normally, so "/srv/current/dict.bin" with "current" a symlink was followed silently. A local writer able to repoint an intermediate component therefore chose the bytes a privileged reload snapshotted into every worker, despite strict mode being on, and despite the release-symlink layout being the exact case the directive documents itself as refusing. Strict mode now resolves the path one component at a time with openat(O_NOFOLLOW|O_DIRECTORY) and opens the leaf relative to the verified parent descriptor, so no component is traversed through a symlink. It additionally requires an absolute path with no "." or ".." component, which the walk cannot otherwise verify. Separately, the trust check tested only S_IWGRP|S_IWOTH. A dictionary owned by an unprivileged account at an ordinary mode 0644 passed while a root master read it -- and that owner can rewrite the file at will, steering what the next privileged reload loads, which is precisely the less-privileged writer the directive exists to exclude. Strict mode now also requires the file to be owned by the effective uid of the config-parsing master, or by root. Both rules are confined to "zstd_dict_strict_path on". The default ("off") is byte-for-byte unchanged, so a release-symlink deployment keeps working unconfigured. On a platform without POSIX.1-2008 openat() (including Windows) strict mode now fails closed at config load rather than degrading silently to the leaf-only guarantee. *) Bugfix: the optional SHA-256 argument to "zstd_dcz_dict_file" was trusted verbatim as the dcz negotiation key and never checked against the dictionary it named. The load-time hashing pass was skipped entirely when the argument was present, so "zstd_dcz_dict_file new.dict " compressed against new.dict while advertising the old dictionary's hash; every client that had stored the old dictionary matched the advertisement, received a frame built from a dictionary it did not hold, and could not decode the body. A stale or mistyped but syntactically valid literal was accepted silently, and the checksum in the directive gave false confidence that the pair had been validated. The dictionary is now always hashed from the bytes actually read, and that hash is always the negotiation key. A supplied literal is compared against it and a mismatch aborts config load with an error naming the file, the supplied hash and the computed one, so "nginx -t" catches the desynchronization before any client sees it. The argument keeps its syntax and its meaning as a deploy-time assertion; only the unverified fast path is gone. Configurations whose literal matches the file are unaffected, and "$zstd_dcz_dicts_hashed" now counts every loaded dictionary rather than only those without a literal. *) Bugfix: a dcz (RFC 9842) request escaped "zstd_max_cctx_memory" and contaminated a worker CCtx ring slot. A dcz response does not compress at "zstd_window_log": it derives its own window from the negotiated dictionary, up to the RFC's 8 MB client guarantee. Two consequences, both reachable on a default configuration by any client that wins dcz negotiation. First, the ring. Each slot is keyed on the parameters that drive its retained workspace, because ZSTD_sizeof_CCtx() never shrinks on reset. The key used "zstd_window_log" rather than the window the request would actually use, so a dcz request borrowed a slot vetted for the (typically much smaller) configured window and raised that slot's retained workspace permanently; every later plain request matching the same key then reused a context whose floor was the dcz figure. The key now carries the EFFECTIVE window log, so dcz requests occupy their own slots and a slot's retained size again equals the figure config load vetted for it. Second, the budget. "zstd_max_cctx_memory" is vetted at config load against "zstd_window_log" only, so the dcz window walked straight through a bound the operator had asked for and config load had accepted (measured, libzstd 1.5.7 at level 3: 3663393 bytes at the default window versus 9954849 at window log 23, a 2.72x escape; 6.74x at level 1). When "zstd_max_cctx_memory" is set to a POSITIVE budget, the dcz window is now clamped to the largest window log whose estimated CCtx memory still fits the budget -- the same treatment "zstd_window_log" already got, since both are memory ceilings and a dictionary must not silently void one. The cap is computed once at config load, so the request path only compares. The clamp is OPT-IN and changes dcz wire bytes only for operators who explicitly configured a memory bound. With neither "zstd_window_log" nor "zstd_max_cctx_memory" set -- the default configuration -- the dcz window is exactly what it was before. *) Feature: "zstd_buffers number size" now bounds the aggregate number x size product at config load, in four tiers. The directive previously delegated entirely to nginx core's ngx_conf_set_bufs_slot(), which range-checks each argument but never the product, so a typo ("zstd_buffers 100000 100000;" instead of "100 100k;") or a value inherited unchanged from an outer block could request an overflowing, or merely enormous, or even multi-exabyte per-response output-chain pool -- the last of these previously passed with only a log line. An overflowing product is a hard config-load error naming both operands, unconditionally. A representable product above 8 MB is a warning (never a failure) naming the total, marked as per-response, and cross-referencing the "zstd_max_cctx_memory" advisory for the other half of the per-request memory budget. A representable product above 256 MB is now a hard config-load error unless the new "zstd_buffers_unsafe on;" directive (http, server, location, inherited) is also set, in which case it is accepted and still logged as a warning rather than silenced. The check covers an explicit value, one inherited from an outer block, and the module's own default alike. *) Feature: compression enabled without an explicit "zstd_max_cctx_memory" now estimates the per-request compressor working set at config load and warns when it exceeds 32 MB. zstd_max_cctx_memory is optional and rarely set, so the estimator was previously skipped for the common case: a later "zstd_comp_level 22" or "zstd_long on" committed hundreds of MB per concurrent response (level 22 estimates ~769 MB, zstd_long at the default level ~138 MB) even though the module already knew the figure at config load. The warning names the estimate, the level, and the retained worker bound -- each worker keeps up to four contexts, so worker RSS can reach four times the per-request figure, and that again per worker process. It is advisory only and never fails the configuration: a config that loads today keeps loading. Set "zstd_max_cctx_memory " to have a bound enforced at config load instead, or "zstd_max_cctx_memory 0" to acknowledge the profile and silence the warning. An explicit non-zero budget keeps its existing hard-failure behaviour unchanged. Requires the same -DZSTD_STATIC_LINKING_ONLY build as the directive; a build without the estimator API does not warn rather than implying a guarantee it cannot compute. *) Bugfix: "nginx -t" no longer dies with SIGFPE when "zstd_long on" is combined with "zstd_max_cctx_memory". The config-load budget check calls ZSTD_estimateCStreamSize_usingCCtxParams(), which divides by the LDM hash rate. libzstd derives its long-distance matching sub-parameters lazily during compression setup, so a ZSTD_CCtx_params that had only had enableLongDistanceMatching set still carried a zero divisor and the whole process died on a floating-point exception during configuration testing. Setting zstd_window_log did not avoid it. The four LDM sub-parameters are now seeded with the defaults libzstd documents, which both removes the crash and makes the estimate accurate for long mode: it is within 0.05% of a real streaming compressor's measured footprint, so the configured budget keeps its meaning instead of being skipped. Because enabling long mode raises the default window to 128 MB, "zstd_long on" now needs a budget of roughly that size, or an explicit zstd_window_log to bring it down. *) Security: dcz is now offered only in a secure context, as RFC 9842 section 8 requires. The negotiation gate checked the dictionary hash, the main request, the request headers and the coding weight but never consulted transport security, so a matching request over plain HTTP was served as Content-Encoding: dcz. Dictionary compression over cleartext gives a network attacker a length oracle over content the dictionary already describes. Requests on a non-TLS connection now fall back to plain zstd. Deployments that terminate TLS on a proxy in front of nginx see a cleartext connection here and must opt in explicitly with the new zstd_dcz_assume_secure_transport directive. The module never infers the client's scheme from X-Forwarded-Proto or any other request header: those are client-supplied on a directly reachable listener, so trusting one would let any client re-enable dcz over cleartext. Affects only configurations using zstd_dcz_dict_file, which is new in this release, so no released version is affected. *) Security: the dcz cross-origin gate could be bypassed by a duplicate request header. Sec-Fetch-Site and Available-Dictionary are not in nginx's known-header table, so nginx chains duplicate occurrences rather than rejecting them, and this module evaluated only the first. A request carrying "Sec-Fetch-Site: same-origin" ahead of the client's real "cross-site" therefore passed the RFC 9842 section 8.3 same-origin partitioning check, and the response was compressed against a dictionary that must not be used cross-site. Both headers are now counted and dcz falls back to plain zstd when either appears more than once. Affects only configurations using zstd_dcz_dict_file, which is new in this release and not present in 0.90.8 or earlier, so no released version is vulnerable. Reaching it also requires an intermediary that merges or forwards a client-supplied duplicate header, or a request-smuggling desync; a browser cannot trigger it alone. Found during this release's audit, not reported externally, so no CVE is assigned. *) Change: the repository adopted the myguard nginx-module skeleton standard. CI material moved under ci/ (t/, tests/unit/, tools/, fuzz/, linter/), the module sources moved to src/, and .github/ workflows/ci.yml became the single pull-request entry point that calls Lint, Build&Test, Security Scanners, Fuzzing, Valgrind, CodeQL, A/UBSan and Windows build as reusable workflows. Long runners moved to the scheduled ci-deep.yml lane. Adds a unit-test layer over the real Accept-Encoding decision function, a fuzz dictionary derived from live call sites with a drift gate, a coverage report scoped to src/, and a tracked pre-commit hook. No module behaviour changed; directives, defaults and the module binary are unaffected. *) Feature: RFC 9842 dcz dictionary compression. New repeatable zstd_dcz_dict_file directive (http/server/location) loads dictionaries hashed at config load; a request whose Available-Dictionary matches and whose Accept-Encoding lists dcz explicitly gets the response compressed against that dictionary (ZSTD_CCtx_refPrefix, window capped at the RFC's 8 MB client guarantee) as Content-Encoding: dcz with the 40-byte dcz frame header. Both response variants emit Vary: Available-Dictionary; cross-site requests (Sec-Fetch-Site) fall back to plain zstd. Covered by t/03-dcz.t, tools/test_dcz.py (also under ASAN and in coverage), and new dcz fuzz corpus entries. *) Added CI matrix building nginx mainline, nginx stable, and angie (monthly, in ci-deep.yml's build-flavors job), running the Perl Test::Nginx suite against each. *) Extended tools/ci-build.sh to accept a flavor argument (nginx|angie) and persist the built server + both dynamic modules under .build/, instead of only building pinned nginx mainline into a /tmp dir that got deleted on exit. *) Added tools/bump-versions.sh + weekly bump.yml workflow to keep the nginx-stable/angie pins current. myguard-labs-nginx-zstd-module-3486b5b/CONTRIBUTING.md000066400000000000000000000225571525232545500223020ustar00rootroot00000000000000# Contributing Thank you for contributing — genuinely. Small modules like this live or die on people who show up with a patch, a bug report, or an awkward question. You are welcome here. That said: this document is not decoration. Every rule below exists because someone (usually us) broke something in a way that took a weekend to clean up. Read it once before your first PR and you will save us both a round trip. Break the rules and CI will catch you anyway — the robots are patient, but they do not negotiate. If anything here is unclear, **ask**. Asking early is a sign you're doing it right, not that you're slow. Open an issue, open a draft PR, or mail **github@myguard.nl**. We answer, and we mentor — everyone who works on this code started by not knowing nginx internals either. ## TL;DR checklist - [ ] One feature or fix per PR — no stacked PRs, no drive-by refactors. - [ ] Code follows nginx style and matches the code around it. - [ ] Every new feature or bugfix ships a test in the same PR. - [ ] README updated in the same PR if behaviour changed. - [ ] All CI checks green. No skipping, no "it works on my machine". - [ ] Commit messages: imperative subject, body explains *why*. No AI co-author trailers. - [ ] The local hook is enabled (see below) — it runs the same checks the Lint job runs, so you find them in a second instead of a CI round-trip. ## Enable the local hook The repository ships its own pre-commit hook. It is **not** active in a fresh clone — git never enables hooks automatically — so turn it on once: ```sh git config core.hooksPath .githooks ``` `.githooks/pre-commit` then runs **two** gates on every commit, and fails on either: 1. `ci/linter/run-all.sh --staged` — the same `ci/linter/` checkers the **Lint** workflow runs (nginx-convention, shell, Python, Perl, YAML, spelling, and the CI-policy checkers). 2. the `.pre-commit-config.yaml` hooks — secret detection (gitleaks, detect-private-key), the C SAST gates (flawfinder, semgrep, cppcheck), ast-grep, ruff, actionlint and shellcheck. Neither list is a superset of the other, which is why the hook runs both. **`pre-commit` itself is a hard prerequisite.** When `.pre-commit-config.yaml` is present and the `pre-commit` binary is missing, the hook exits 2 and blocks the commit rather than warning — a gate that skips itself when its tool is absent reports green while checking nothing. The same applies to a missing linter: `run-all.sh` exits 2 and the commit is blocked. Install everything the hook calls — `pre-commit` included — with: ```sh ci/linter/install-linters.sh # apt + pipx + cpan + upstream binaries ci/linter/install-linters.sh --check # report what is present; non-zero if any # required tool is missing ``` `--check` exits non-zero when a tool is missing, so it is safe to use as a gate in your own scripts — a tool that is absent is never reported as clean. Verify the hook is actually live before trusting it — this gate was green and inert for days once, because `core.hooksPath` makes the copy `pre-commit install` writes into `.git/hooks/` unreachable while every surface a human checks still looks installed: ```sh printf 'probe \n\nno newline at eof' > _p.txt git add _p.txt && git commit -m probe -- _p.txt # expect BLOCKED git reset -q HEAD _p.txt; rm -f _p.txt ``` Both `trim trailing whitespace` and `fix end of files` must report Failed. If the commit lands, `.pre-commit-config.yaml` is not being consulted. Timing: the hook takes ~0.5s for a docs-only change and ~3.6s when a C file is staged (semgrep is most of that). To run the full set by hand without committing: ```sh ci/linter/run-all.sh # every checker over the tracked tree LINT_ONLY="sh python" ci/linter/run-all.sh # narrow to named checkers ``` One trap worth knowing: `run-all.sh` enumerates files with `git ls-files`, so a **new untracked file is invisible to it** and the run reads green while checking nothing. `git add -N` the file first. ## How CI works here Every PR runs bounded merge gates without monopolising the four self-hosted runner lanes: - **Lint** — runs the repository's deterministic source, workflow, security, documentation-drift, and four-lane topology checks. - **Build & Test** — builds the module against current nginx (and, where applicable, Angie) and runs the unit tests under **ASan/UBSan**. AddressSanitizer and UndefinedBehaviorSanitizer are compiler instrumentation that make memory bugs (use-after-free, buffer overflows, signed overflow) crash loudly at the exact line instead of corrupting memory silently. If ASan complains, the bug is real — fix it, don't suppress it. - **Security scanners** (`security-scanners.yml`) — flawfinder, clang-tidy (`cert-*`, `clang-analyzer-security.*`) and semgrep over the module sources. Static analysis: it reads the code without running it and flags dangerous patterns. - **Harness fault arms** (`harness-fault-arms.yml`) — runs all six shared nginx-module-testkit scenarios against an instrumented module build. Visible on every PR and rerun on merged `master`, but not yet a required check. - **Windows build** (`windows-build.yml`) — compiles the Win32 paths with MSVC and MinGW, which the Linux lanes cannot cover. Long work — two fuzz targets, full coverage, CodeQL, native sanitizer soak, compatibility builds, full Memcheck and Helgrind — runs weekly and on manual dispatch in `ci-deep.yml`. Its dependency chains keep exactly four self-hosted lanes busy without an uncontrolled ready-job burst. The expanded coverage report enforces an 85% line floor. That capacity model assumes the repository `POOL` variable selects the shared pool; if unset, jobs use their declared GitHub-hosted fallback. Your PR merges when **all** checks are green. If a gate fails and you believe the gate is wrong, say so in the PR — with evidence, not vibes. Before pushing a parser or fuzz-harness change, fuzz locally first: `ci/fuzz/build.sh` compiles the target under ASAN+UBSAN (`-fsanitize=address,undefined`), so a stale harness signature or a sanitizer-caught bug surfaces right there instead of in CI. (The module's own C sources are compiled `-Werror` — see [`build-test.yml`](.github/workflows/build-test.yml) — the fuzz harness itself is not.) ## Coding conventions - **nginx style.** This is an nginx module: follow the [nginx style guide](https://nginx.org/en/docs/dev/development_guide.html#code_style) — 4-space indents, `ngx_` types (`ngx_int_t`, `ngx_str_t`, …), K&R-ish bracing as used by nginx core, `/* comments */`. - **Match the surrounding code.** When in doubt, the file you are editing is the style guide. A patch that reads like the code around it is a patch we can review quickly. - **Memory comes from pools.** Allocate from the request/config pool (`ngx_palloc`) unless you have a documented reason not to. If you `malloc`, you own the cleanup handler. - **Handle every error path.** nginx runs for months; "can't happen" happens. Check return values, log with `ngx_log_error`, fail closed. - **Comments explain *why*, not *what*.** A surprising nginx-internals fact, a footgun, a rejected alternative — write it down at the call site or in the README. No undocumented behaviour ships. ## Tests Every function and every feature gets a test **in the same PR** that adds it. Not a follow-up PR. Not "later". Same PR. - New parser or handler → a unit or runtime test exercising it, including the ugly inputs (empty, oversized, malformed, truncated). - Bug fix → a regression test that **fails before the fix and passes after**. That's the proof the test actually tests something. - New input parser → a libFuzzer target in `ci/fuzz/`. Where tests live varies slightly per module (unit suites, `t/*.t` Test::Nginx files, runtime suites under `ci/tools/`) — look at the existing tests in this repo and put yours next to them. A PR that adds code without a test will not be merged, and yes, we check. ## Pull requests - **One feature or issue per PR.** The title says what it does. If a PR grows a second concern, split it. - **No stacked PRs.** Every PR branches from and targets the default branch independently. Stacks fall over the moment PR #1 gets review changes, and untangling them costs more than the stacking saved. - **Open an issue first** for anything non-trivial; the PR references it (`Closes #N`). - **Keep it reviewable in one sitting.** Small PRs merge fast; 2000-line PRs grow moss while we find an afternoon to do them justice. - **Update the README in the same PR** when behaviour, directives, or defaults change. The README must never lag the default branch. - The default branch is protected by convention: changes land via PR with green CI, not direct push. ## Commits - Imperative subject line ("add X", "fix Y"), ≤ 72 chars. - Body explains *why* — the design choice made and what was rejected. - No AI co-author trailers. None. ## Ask for help Stuck on nginx internals? Not sure where a test belongs? Fuzzer output looks like hieroglyphics? Ask. Open a draft PR with what you have and say what you're unsure about — a draft PR full of questions is a perfectly good contribution. We would much rather spend ten minutes pointing you in the right direction than review a week of effort aimed at the wrong wall. ## Contact Questions, security reports, or anything that doesn't fit an issue: **github@myguard.nl** myguard-labs-nginx-zstd-module-3486b5b/LICENSE000066400000000000000000000025051525232545500210450ustar00rootroot00000000000000BSD 2-Clause License Copyright (c) 2018, Alex Zhang Copyright (C) 2026 Thijs Eilander All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. myguard-labs-nginx-zstd-module-3486b5b/README.md000066400000000000000000003201651525232545500213240ustar00rootroot00000000000000[![CI](https://github.com/myguard-labs/nginx-zstd-module/actions/workflows/ci.yml/badge.svg)](https://github.com/myguard-labs/nginx-zstd-module/actions/workflows/ci.yml) [![Lint](https://github.com/myguard-labs/nginx-zstd-module/actions/workflows/lint.yml/badge.svg)](https://github.com/myguard-labs/nginx-zstd-module/actions/workflows/lint.yml) [![Build&Test](https://github.com/myguard-labs/nginx-zstd-module/actions/workflows/build-test.yml/badge.svg)](https://github.com/myguard-labs/nginx-zstd-module/actions/workflows/build-test.yml) [![Security Scanners](https://github.com/myguard-labs/nginx-zstd-module/actions/workflows/security-scanners.yml/badge.svg)](https://github.com/myguard-labs/nginx-zstd-module/actions/workflows/security-scanners.yml) [![CI Deep](https://github.com/myguard-labs/nginx-zstd-module/actions/workflows/ci-deep.yml/badge.svg)](https://github.com/myguard-labs/nginx-zstd-module/actions/workflows/ci-deep.yml) [![Windows build](https://github.com/myguard-labs/nginx-zstd-module/actions/workflows/windows-build.yml/badge.svg)](https://github.com/myguard-labs/nginx-zstd-module/actions/workflows/windows-build.yml) 📖 **Background reading:** - [zstd nginx module: what it does, bugs fixed](https://deb.myguard.nl/articles/zstd-nginx-module-bugs-fixed/) - [nginx zstd vs brotli vs zlib-ng — a compression comparison](https://deb.myguard.nl/articles/nginx-zstd-vs-brotli-vs-zlib-ng-compression/) # zstd-nginx-module An nginx module for [Zstandard (zstd)](https://facebook.github.io/zstd/) compression. Zstandard typically achieves substantially better compression ratios than gzip, making it a good choice for reducing transmitted response sizes. Throughput depends on the level and payload: at the low levels recommended for web traffic, zstd trades some encode speed for that ratio -- see the [measured benchmark table](#benchmarks) rather than assuming a general speed win. This is a hardened fork: every PR is exercised against **nginx mainline**, the filter/static suites and runtime regressions run under **ASAN/UBSAN**, flawfinder/semgrep/clang-tidy run on every change, and a weekly deep pass fuzzes both parser targets and additionally covers **nginx stable** and **[Angie](https://angie.software/)** (see [Testing & CI](#testing--ci)). # Table of Contents * [Status](#status) * [Synopsis](#synopsis) * [Set and forget](#set-and-forget) * [Installation](#installation) * [Directives](#directives) * [ngx_http_zstd_filter_module](#ngx_http_zstd_filter_module) * [zstd](#zstd) * [zstd_comp_level](#zstd_comp_level) * [zstd_min_length](#zstd_min_length) * [zstd_max_length](#zstd_max_length) * [zstd_types](#zstd_types) * [zstd_buffers](#zstd_buffers) * [zstd_buffers_unsafe](#zstd_buffers_unsafe) * [zstd_target_cblock_size](#zstd_target_cblock_size) * [zstd_window_log](#zstd_window_log) * [zstd_long](#zstd_long) * [zstd_max_cctx_memory](#zstd_max_cctx_memory) * [zstd_bypass](#zstd_bypass) * [zstd_bypass_vary](#zstd_bypass_vary) * [zstd_dict_file](#zstd_dict_file) * [zstd_dict_strict_path](#zstd_dict_strict_path) * [zstd_dict_file_unsafe](#zstd_dict_file_unsafe) * [zstd_dcz_dict_file](#zstd_dcz_dict_file) * [zstd_dcz_dict_trust_hashes](#zstd_dcz_dict_trust_hashes) * [zstd_dcz_assume_secure_transport](#zstd_dcz_assume_secure_transport) * [ngx_http_zstd_static_module](#ngx_http_zstd_static_module) * [zstd_static](#zstd_static) * [zstd_static_dict_bypass](#zstd_static_dict_bypass) * [Variables](#variables) * [$zstd_ratio](#zstd_ratio) * [$zstd_bytes_in](#zstd_bytes_in) * [$zstd_bytes_out](#zstd_bytes_out) * [$zstd_dcz_dicts_hashed](#zstd_dcz_dicts_hashed) * [Compatibility](#compatibility) * [Testing & CI](#testing--ci) * [Benchmarks](#benchmarks) * [Operations](#operations) * [Security](#security) * [Author](#author) * [License](#license) # Status Production-oriented. The module originates from the upstream [tokers/zstd-nginx-module](https://github.com/tokers/zstd-nginx-module) and has since had an extensive audit pass: a regression test for every known historical bug class, ASAN/UBSAN runtime checks, and continuous fuzzing of the request-parsing path (see [Testing & CI](#testing--ci)). Bug reports and pull requests are welcome. # Synopsis ```nginx http { # Compress text responses for clients that support zstd. # Defaults: level 3, web-content MIME types, and a 1024-byte minimum. zstd on; # Optional for zstd (it emits Vary: Accept-Encoding itself); set this # if gzip or brotli are also enabled so their variants are cached too. gzip_vary on; server { listen 80; server_name example.com; # Dynamic compression via filter module location /api/ { proxy_pass http://backend; } # Serve pre-compressed .zst files for static assets location /static/ { zstd_static on; root /var/www; } } } ``` For pre-compressed static files, generate them alongside the originals: ```bash # Compress all JS and CSS files in the static directory find /var/www/static -name "*.js" -o -name "*.css" | \ xargs -I{} zstd -3 -k {} # This creates file.js.zst next to file.js, etc. ``` # Set and forget If you just want sane production compression without reading every directive, paste this into the `http {}` block of `nginx.conf` and move on. It is tuned for typical web traffic (HTML/JSON/JS/CSS/SVG) and relies on the module's built-in defaults for everything not shown. ```nginx http { # --- zstd: set and forget --- zstd on; # level 3, 1 KiB minimum, common web types # Optional for zstd: the module emits Vary: Accept-Encoding itself, so # this is only needed if gzip/brotli are also enabled on the location. gzip_vary on; # Pre-compressed static assets (optional but free if you ship .zst) # zstd_static on; } ``` Why these values, and why nothing else is needed: * **`zstd_comp_level 3`** — the built-in default; for real web content this beats `gzip -6` on ratio at comparable or better speed (see [Benchmarks](#benchmarks)). Levels ≥ 9 cost CPU steeply for marginal gain; only raise it for infrequently-generated, cached responses. * **`zstd_min_length 1024`** — the built-in default; below about 1 KiB the zstd frame overhead and CPU cost usually outweigh the saving. * **`zstd_types` is intentionally not set.** Its built-in list covers HTML, plain text, CSS, JavaScript, JSON, XML/feed formats, SVG, and common structured JSON variants (see [`zstd_types`](#zstd_types)). * **`zstd_buffers` is intentionally not set.** The default is now `2 × ZSTD_CStreamOutSize()` — libzstd's own recommended streaming output unit (~128 KB each). This lets every compress call flush a full internal block without fragmentation. Only override it if you run thousands of concurrent connections on a memory-constrained box and need to trade some throughput for a lower per-request memory floor (see [`zstd_buffers`](#zstd_buffers)). * **`zstd_long`, `zstd_window_log`, `zstd_dict_file`, `zstd_target_cblock_size` are intentionally not set.** They are specialist levers (very large repetitive bodies, hard per-request memory caps, shared dictionaries). The defaults are correct for general traffic; reach for these only with a measured reason. That is the entire recommended baseline. Everything past this point in the README is reference detail and tuning for specific workloads — you do not need it to run the module well. # Installation ## Arch Linux The [`packaging/arch`](packaging/arch) directory contains an AUR-compatible `PKGBUILD` and a multi-stage Arch Linux `Dockerfile` with digest-pinned base images. Build and smoke test the package without changing the host system: ```bash cd packaging/arch ./build.sh sudo pacman -U dist/nginx-mod-zstd-*.pkg.tar.zst ``` The package is compiled against Arch's matching `nginx-src` package, installs both dynamic modules, and enables them through `/etc/nginx/modules.d/20-zstd.conf`. The container also compiles and smoke-tests the current checkout separately, so pull requests test their own module source while the exported package continues to use the pinned release archive. The container intentionally updates packages from Arch's live repositories. The exact nginx dependency protects the dynamic-module ABI by failing closed after an Arch nginx update. When that happens, update `_nginxver`, bump `pkgrel`, regenerate `.SRCINFO`, and refresh the container image digests. Build nginx with the module using `--add-dynamic-module`: ```bash ./configure --add-dynamic-module=/path/to/zstd-nginx-module make && make install ``` Then load the modules in `nginx.conf`: ```nginx load_module modules/ngx_http_zstd_filter_module.so; load_module modules/ngx_http_zstd_static_module.so; ``` **Notes:** * Both `ngx_http_zstd_filter_module` and `ngx_http_zstd_static_module` are compiled together. * If you are using a custom zstd installation, set `ZSTD_INC` (path to `zstd.h`) and `ZSTD_LIB` (path to the library) before running `configure`. If unset, the system-installed zstd is used. * **This plain `./configure` line does not enable `-DZSTD_STATIC_LINKING_ONLY`.** With no `ZSTD_INC`/`ZSTD_LIB` set, `auto/zstd` goes straight to auto-discovery (dynamic linking, then pkg-config), and neither of those paths ever defines the flag — it is only ever added when `ZSTD_INC` and `ZSTD_LIB` are both set *and* they resolve to a static `libzstd.a` (`zstd_static.lib` on MSVC) at that exact location. There is no configure flag that turns it on directly. A build produced by the command above therefore does **not** have the experimental memory-estimator API compiled in: `zstd_max_cctx_memory` will fail `nginx -t` with `"zstd_max_cctx_memory" requires the module to be built with -DZSTD_STATIC_LINKING_ONLY ...`, and the implicit no-budget advisory (see [Directives](#directives)) silently does not run. To get the flag, point `ZSTD_INC`/`ZSTD_LIB` at a static libzstd build before running `configure`; see [`zstd_max_cctx_memory`](#zstd_max_cctx_memory) and [Compatibility](#compatibility) for the full requirement. * **Windows:** MSVC builds the modules statically into `nginx.exe`; MinGW-w64 can also build them as dynamic `.so`-named PE DLLs. [`ci/tools/build-windows.sh`](ci/tools/build-windows.sh) assembles the MSVC build (and optionally ngx_brotli, the unified compression module, and headers-more; library builds follow the modules selected), with every source version and SHA-256 pinned in [`ci/tools/windows-pins.sh`](ci/tools/windows-pins.sh) and verified before any compile. The usual Windows-nginx caveats apply (effectively single-worker via `select()`, no HTTP/3 — local dev use, not production). The `zstd_static` frame probes (magic number, truncation, declared-window and skippable-frame-chain checks) run on Windows too: the probe needs an offset-explicit read that does not move the shared cached descriptor's file position, which is `pread(2)` on POSIX and `ngx_read_file()` — `ReadFile()` with an `OVERLAPPED` offset — on Win32. The verdict logic itself is one shared function on both platforms. [`.github/workflows/windows-build.yml`](.github/workflows/windows-build.yml) is the executable MinGW recipe and verifies both toolchains. * Dynamic modules (`.so`) require dynamic linking against `libzstd.so`. The build scripts auto-detect and prefer this. Ensure the zstd shared library is installed and available at runtime (`libzstd-dev` on Debian/Ubuntu, `libzstd-devel` on RHEL/Fedora). * On POSIX, when `ZSTD_LIB` is set to a non-standard path, the build embeds an RPATH pointing to that directory in the module `.so`. This means the module will load `libzstd.so` from that exact path at runtime. If the library is later moved (e.g. by a package upgrade), the module will fail to load. Use the system package and leave `ZSTD_LIB` unset to avoid this. # Compatibility | Component | Minimum | Recommended | CI-verified | |---|---|---|---| | **nginx** | 1.9.11 (first `--add-dynamic-module` release) | latest mainline / stable | latest mainline resolved by the PR gate; pinned mainline, stable, and Angie in CI Deep's weekly matrix | | **Angie** | 1.x | latest | **1.12.1** | | **libzstd** | **1.4.0** | **≥ 1.5.7** | 1.5.x (full suite) + **1.4.x** fallback-paths build | | **OS** | Linux/BSD/RHEL-family | — | Ubuntu (GitHub runners) | Notes on the libzstd floor — these are enforced in code, not assumed: * At startup, dynamically linked builds compare the loaded libzstd version with the headers used to compile the module and warn on any mismatch. Normal ABI-compatible skew remains supported. Startup is refused only when a configured compile-gated feature crosses its runtime floor: a negative `zstd_comp_level` below 1.4.0, or `zstd_target_cblock_size` below 1.5.6. * **< 1.4.0**: the streaming API the module uses (`ZSTD_compressStream2`) is unavailable; this is the hard minimum. Negative `zstd_comp_level` values are also unsupported and are clamped to `1` with a warning (guarded by `#if ZSTD_VERSION_NUMBER >= 10400`). * **< 1.5.6**: `zstd_target_cblock_size` has no effect — the directive is accepted as a warned no-op (apply path gated by `#if ZSTD_VERSION_NUMBER >= 10506`, with a config-load warning). Everything else works. This fallback path is exercised in CI by a dedicated "Build (libzstd 1.4.x — fallback paths)" job that links the module against a privately built libzstd 1.4.x and runs the decode-and-compare smoke test. That lane proves API fallback compatibility only — it is not a production-safety signal, and 1.4.5 is not the recommended version for production traffic. * **≥ 1.5.6**: every directive is fully functional. * **≥ 1.5.7 (recommended for production)**: fixes a rare upstream dictionary-compression corruption on 32-bit ARM after a long-lived `CCtx` is reused across a very large number of operations. This module reuses a worker-lifetime `CCtx`, supports dictionaries, and allows levels up to 22, so deployments combining dictionaries, 32-bit builds, and sustained traffic should run 1.5.7 or later; the 1.4.0 floor above is unaffected and unchanged. * **`zstd_max_cctx_memory`** additionally requires the module to be built with `-DZSTD_STATIC_LINKING_ONLY` so libzstd's experimental memory-estimator API is available. This project's own CI opts into that flag by pointing `ZSTD_INC`/`ZSTD_LIB` at a static libzstd (see [Installation](#installation)) — it is **not** the outcome of the plain `./configure --add-dynamic-module=...` command documented above, which auto-discovers a dynamic libzstd and never defines the flag. Without it, the directive is rejected at config load with a clear, actionable error rather than silently no-op'd. "CI-verified" means the PR or weekly deep workflow builds and runs the full test suite against that exact version (see [Testing & CI](#testing--ci)). Other versions within the stated ranges are expected to work but are not continuously exercised. # Directives ## ngx_http_zstd_filter_module This filter module compresses responses on the fly using zstd. It runs after the upstream or file handler generates the response, and before nginx sends it to the client. Compression is applied only when the client signals support via `Accept-Encoding: zstd`. The eligible status set is: every 2xx except the bodyless `204 No Content` and `205 Reset Content` and except `206 Partial Content` (a range response's `Content-Range` is computed against the selected representation, and a new content coding would invalidate it, so 206 passes through as core `gzip` does), plus `403`, `404` and `410` (which often carry compressible error bodies; core `gzip` compresses 403 and 404 today, and nginx/nginx#1466, approved for 1.31.6, adds 410). Every other status is passed through uncompressed. > **`Vary: Accept-Encoding` is emitted automatically — you do not need `gzip_vary on`.** Whenever a response's encoding depends on `Accept-Encoding`, both this module and `zstd_static on` emit the `Vary: Accept-Encoding` header themselves, so proxies and CDNs keep the compressed and uncompressed variants apart without any directive on your part. Emission is duplicate-safe: with `gzip_vary on` nginx emits the field and the module stays quiet, with `gzip_vary off` the module emits it — exactly one `Vary: Accept-Encoding` line either way. > > This used to be a deployment prerequisite backed only by a config-load warning: with the default `gzip_vary off`, nginx *suppresses* the header entirely, so a single overlooked location shipped zstd bodies that a shared cache would then hand to clients unable to decode them. Correctness now belongs to the module rather than to an operator directive, and the warning is gone with it. `gzip_vary on` remains perfectly fine to set (it also covers other encoders such as `gzip`); it is simply no longer load-bearing for zstd. > > `zstd_static always` is deliberately excluded when > [`zstd_static_dict_bypass`](#zstd_static_dict_bypass) is off: it then ignores > `Accept-Encoding` entirely, so its response is not a negotiated variant and > correctly carries no `Vary: Accept-Encoding`. The opt-in bypass is the stated > exception because it routes dictionary-aware requests by that field. > > Interoperating with [ngx_http_compression_vary_filter_module](https://github.com/HanadaLee/ngx_http_compression_vary_filter_module) is safe in every combination. That module emits and *flattens* `Vary` from `r->gzip_vary`, which this module still sets; because it folds the fields it finds rather than appending blindly, the header this module emits is merged rather than doubled. With `compression_vary off` (its default) this module's own emission still covers you — previously that exact combination produced no `Vary` at all, which is the residual risk the old summary warning existed to describe. CI asserts the single-`Vary` contract against the real module in both of its states. > **ETag behaviour:** When a response is compressed, nginx automatically weakens the `ETag` value (converting `"abc"` to `W/"abc"` if it was strong). This is correct per HTTP semantics — a compressed representation is a different entity variant — but it means strong ETag validation (`If-Match`) will not match across compressed and uncompressed responses. CDN edge nodes that cache both variants will see different ETags for each. > **Coexisting with `gzip` and `brotli`:** It is safe to enable `zstd`, the [`brotli`](https://github.com/google/ngx_brotli) filter, and the built-in [`gzip`](https://nginx.org/en/docs/http/ngx_http_gzip_module.html) filter on the same location with overlapping `*_types`. A response is only ever compressed once: nginx body filters run in a fixed chain, and the first encoder whose `Accept-Encoding` test passes wins, setting `Content-Encoding` so the later encoders skip the already-encoded body. Relative to the built-in `gzip`, `zstd` is always ordered to run **before** it (both in static and dynamic builds), so a client advertising `Accept-Encoding: gzip, zstd` receives `zstd`; clients that do not advertise `zstd` fall through to `gzip`. Each encoded variant is cached separately by proxies and CDNs because every encoder in that chain advertises `Vary: Accept-Encoding`; this module emits it itself, and `gzip`/`brotli` emit theirs via `gzip_vary`, so set `gzip_vary on` if you rely on those encoders too. > > **`zstd` vs `brotli` ordering (dynamic builds):** the fixed `before brotli` guarantee holds for **static** builds, where `filter/config` explicitly places `zstd` ahead of `ngx_http_brotli_filter_module` in the module array. For **dynamic** modules, `ngx_brotli` and this module share the same filter anchor and neither constrains itself relative to the other, so the body-filter chain is built in **reverse `load_module` order** — whichever is loaded **last** runs first and wins. To make `zstd` win a `br, zstd` negotiation, load it last: > > ```nginx > load_module modules/ngx_http_brotli_filter_module.so; > load_module modules/ngx_http_zstd_filter_module.so; # loaded last → runs first → wins > ``` > > Swap the two lines to prefer `brotli`. If you require a fixed winner regardless of operator load order, prefer a static build (or pick one of the two filters per location). > > **Selection policy — server preference, not client qvalue ranking.** When several encoders are enabled and the client lists more than one as acceptable, the winner is decided by **server-side filter order** (zstd runs first relative to gzip, and relative to brotli only when static builds or dynamic `load_module` order put zstd first), **not** by the client's relative `q` weights. So `Accept-Encoding: zstd;q=0.5, gzip;q=0.9` still yields `zstd`, even though the client ranked `gzip` higher. This is a deliberate server-preference policy. RFC 9110 §12.5.3 describes preferring the acceptable coding with the highest non-zero qvalue, which a single per-coding filter cannot implement (it cannot see the other codings' weights); honouring it would require a shared negotiation step ahead of the body filters. A coding whose effective explicit `q` value is `0` remains unacceptable — only the *relative* ranking between acceptable codings is server-decided. > **Repeated `Accept-Encoding` field lines are one comma-joined list.** A client may split its codings across several `Accept-Encoding` request header lines. [RFC 9110 §5.3](https://www.rfc-editor.org/rfc/rfc9110#section-5.3) makes a repeated list-valued field semantically identical to the single field whose value is those lines joined in order with commas, and this module negotiates accordingly — every line is evaluated, not just the first. So > > ```http > Accept-Encoding: gzip > Accept-Encoding: zstd > ``` > > is `gzip, zstd` and negotiates `zstd`, exactly as the one-line form would. The same applies to the [`dcz`](#zstd_dcz_dict_file) token. > > **Duplicate codings across lines retain order.** The comma-joined field is parsed in received order, so the **latest explicit token wins**: `zstd;q=0` followed by `zstd;q=1` accepts zstd, while the reverse order declines it. A `*` wildcard stays subordinate to an explicit token, so any line naming the coding explicitly still decides. --- ### zstd **Syntax:** `zstd on | off;` **Default:** `zstd off;` **Context:** `http, server, location, if in location` Enables or disables on-the-fly zstd compression for responses. **Example:** ```nginx http { zstd on; # enable everywhere gzip_vary on; # optional for zstd (emitted automatically); covers gzip/brotli too server { location /downloads/ { zstd off; # already-compressed archives: skip } } } ``` --- ### zstd_comp_level **Syntax:** `zstd_comp_level level;` **Default:** `zstd_comp_level 3;` **Context:** `http, server, location` Sets the zstd compression level. Accepted values depend on the installed zstd library version: | Range | Meaning | |---|---| | `1` to `ZSTD_maxCLevel()` (22) | Standard levels — higher = better ratio, slower | | `0` | Library default (`ZSTD_CLEVEL_DEFAULT`, currently level 3) | | `ZSTD_minCLevel()` (-131072) to `-1` | Fast/negative levels — lower ratio, minimal CPU cost (requires zstd ≥ 1.4.0) | **Choosing a level:** * `1` — Fastest compression; suitable for high-throughput APIs or when latency is critical. * `3` (default) — Good all-around balance of ratio and speed; the zstd library's own default. * `6`–`9` — Better ratios with moderate CPU cost; suitable for large, infrequently-changed responses. * Negative levels (`-1` to `-5`) — Ultra-fast, for cases where you want some compression with nearly zero overhead. For most web-serving workloads, levels `1`–`3` are recommended. Avoid high levels (> 9) in production unless responses are generated infrequently and cached. **Example:** ```nginx http { zstd on; zstd_comp_level 3; # balanced default for live traffic server { location /api/ { zstd_comp_level 1; # latency-sensitive: fastest level } location /reports/ { zstd_comp_level 12; # large, cached, infrequently generated } } } ``` > **Performance note:** when a response has a known exact > `Content-Length` (the common proxied/static case), the module passes > that size to zstd up front (`ZSTD_CCtx_setPledgedSrcSize`). zstd then > sizes its internals to the input and writes a more compact frame > header, giving a small speed/ratio improvement at no cost. This is > automatic, per request, and requires no configuration. Chunked / > unknown-length responses are unaffected (they stream as before). --- ### zstd_min_length **Syntax:** `zstd_min_length length;` **Default:** `zstd_min_length 1024;` **Context:** `http, server, location` Sets the minimum response size (in bytes) required for compression to apply. The size is taken from the `Content-Length` response header; responses without `Content-Length` are always eligible. > **Note:** The built-in default is `1024` bytes. Smaller responses often lose > their savings to zstd frame overhead while still consuming compression CPU. > **Caveat — chunked and proxied responses bypass this directive entirely.** > The threshold can only be applied when the body length is known up front, so > the check is skipped whenever `Content-Length` is absent — which is the normal > case for chunked transfer encoding and for most proxied upstreams. Such > responses are compressed regardless of how small they turn out to be, and a > body below the threshold can come out *larger* than the original: a 47-byte > chunked body under `zstd_min_length 1024` is returned as 56 bytes with > `Content-Encoding: zstd`. Enforcing the threshold on a streaming body would > require buffering the response until the threshold is decided, which changes > when headers may be sent; the directive deliberately does not do this. If a > small-response floor matters for an upstream, ensure the upstream sets > `Content-Length`, or disable compression for that location. **Example:** ```nginx zstd_min_length 1024; # skip compression for responses smaller than 1 KiB ``` --- ### zstd_max_length **Syntax:** `zstd_max_length length;` **Default:** `—` (no limit) **Context:** `http, server, location` Sets the maximum response size that will be compressed. The limit is enforced in two places: * **Before compression starts**, when the response advertises a `Content-Length` larger than the limit: the response is passed through uncompressed (no CPU spent). * **During compression**, the running input total is tracked unconditionally — regardless of whether the response has a declared `Content-Length` — and if it exceeds the limit the request is **aborted** (logged as `zstd: input exceeded zstd_max_length ...`). This covers a chunked/streaming response with no `Content-Length`, and also a known-length upstream that streams more bytes than it declared. Compression has already begun and the client is mid-stream, so the only safe action is to terminate the response — protecting the worker from an unbounded or runaway upstream is preferred over completing one oversized response. > **Behaviour once compression has started:** the case cannot be served uncompressed-instead (the `Content-Encoding: zstd` stream is already in flight), so exceeding the limit here ends the request rather than transparently passing through. Size the limit with headroom for the largest response you legitimately compress on that location. If you routinely serve very large streaming bodies (proxied video, big downloads), prefer simply not enabling `zstd` on those locations. By default there is no upper limit. You may want to set one if very large responses (e.g. multi-megabyte file downloads) should bypass compression to avoid holding the worker process busy. **Example:** ```nginx zstd_max_length 10m; # don't compress responses larger than 10 MB ``` --- ### zstd_types **Syntax:** `zstd_types mime-type ...;` **Default:** ```nginx zstd_types text/html text/plain text/css text/csv application/json application/x-ndjson application/json-seq application/javascript text/xml application/xml application/xml+rss text/javascript image/svg+xml application/atom+xml application/ld+json application/manifest+json application/problem+json application/rss+xml application/vnd.api+json application/xhtml+xml application/wasm text/wgsl; ``` **Context:** `http, server, location` When omitted, the default covers common textual web representations. If set explicitly, it follows nginx's usual type-directive behaviour: `text/html` is included along with the listed MIME types. Use `*` to match all MIME types. > **Note:** Only compressible content types (text, structured data, SVG, etc.) benefit from compression. Binary formats such as images (JPEG, PNG, WebP), audio, and video are already compressed and should be excluded. **Example for a typical web application:** ```nginx zstd_types text/plain text/css text/csv text/xml text/javascript application/json application/x-ndjson application/json-seq application/javascript application/xml application/xml+rss application/atom+xml application/ld+json application/manifest+json application/problem+json application/rss+xml application/vnd.api+json application/xhtml+xml application/wasm text/wgsl image/svg+xml; ``` --- ### zstd_buffers **Syntax:** `zstd_buffers number size;` **Default:** `zstd_buffers 2 ;` (the size is libzstd's recommended streaming output unit, ~128 KB) **Context:** `http, server, location` Configures the number and size of output buffers used during compression. The total buffer space is `number × size`. The default buffer **size** is `ZSTD_CStreamOutSize()` — the value libzstd documents as the minimum at which `ZSTD_compressStream2()` can flush a complete internal block in a single call. With any smaller buffer, zstd is forced to fragment a block across calls, costing extra compression round-trips and output-chain allocations per response. Earlier versions used a heuristic (`32 4k`, then `4 32k`) that approximated this; the module now asks libzstd for the exact value so it stays correct if the library changes it. The default **count** is `2`: one buffer being filled by the compressor while the other is in flight down the output chain. This sets the per-request filter-memory floor at roughly `2 × ZSTD_CStreamOutSize()` (~256 KB), up from the previous ~128 KB — the deliberate cost of never forcing zstd to flush mid-block. If that trade is wrong for your workload (many concurrent connections, memory-constrained), set `zstd_buffers` explicitly to a smaller value; configurations that set it are unaffected by this default. > **Bounded exception with dictionary responses:** The first output buffer in a response using `zstd_dcz_dict_file` is allocated at `size + 40` bytes to hold the RFC 9842 skippable-frame dictionary-identifier prefix inline. This is a bounded, one-time +40-byte overhead per dcz response, not a per-buffer or per-request scaling change. Increasing these values allows larger chunks to be accumulated before writing, potentially improving throughput at the cost of higher per-request memory usage. **Example:** ```nginx http { zstd on; # The built-in default (applied when the directive is omitted): # 2 buffers sized to ZSTD_CStreamOutSize() (~128 KB each on # libzstd 1.5.x), i.e. ~256 KB/request. The line below is that # default written explicitly — leaving zstd_buffers unset is # equivalent and recommended: zstd_buffers 2 128k; server { # Memory-constrained box with very high concurrency: # trade some throughput for a lower per-request floor. location /high-fanout/ { zstd_buffers 4 16k; # 64 KB/request instead of ~256 KB } } } ``` > The default **size** is whatever `ZSTD_CStreamOutSize()` returns for > the linked libzstd (~128 KB on 1.5.x); `2 128k` above is the > human-readable equivalent for that version. Prefer leaving the > directive unset so the size always tracks the library — only write it > explicitly when you are deliberately overriding it. #### Aggregate bound on `number × size` `number` and `size` are each range-checked on their own by nginx core, but the **product** was not checked at all until this policy was added — a typo (`zstd_buffers 100000 100000;` instead of `100 100k;`) or a value inherited unchanged from an outer block could request an overflowing or merely enormous per-response output-chain pool, multiplied by every concurrent response under load. Four tiers apply to the resolved value, in ascending severity: 1. **`≤ 8 MB`** — silent. This covers nginx's own `zstd_buffers 32 4k` default (128 KB) and this module's own `2 × ZSTD_CStreamOutSize()` default (~256 KB at level 6) with generous headroom. 2. **`> 8 MB`, `≤ 256 MB`** — a **warning**, not an error. A config that loads today keeps loading after an upgrade: ```text nginx: [warn] "zstd_buffers" (merged value) requests 1 x 8388609 bytes = ~8388609 bytes of output-chain memory PER RESPONSE; that is on top of the per-request compressor (CCtx) working set -- see the "zstd_max_cctx_memory" advisory above for that figure -- and both are multiplied by concurrent responses under load ``` 3. **`> 256 MB`, no acknowledgement** — a **hard config-load error**. Unlike the CCtx memory estimate (which depends on libzstd internals the operator never directly sees), `number` and `size` are two integers the operator typed out literally, so at 256 MB — two hundred times nginx's own default — a refusal by default is the safer reading of "this is almost certainly a mistake": ```text nginx: [emerg] "zstd_buffers" (merged value) requests 2147483647 x 1073741824 bytes = ~2305843008139952128 bytes of output-chain memory PER RESPONSE, above the 256 MB hard cap -- that is on top of the per-request compressor (CCtx) working set (see the "zstd_max_cctx_memory" advisory above) and both are multiplied by concurrent responses under load. Lower "zstd_buffers", or set "zstd_buffers_unsafe on;" to acknowledge this total is intentional ``` 4. **`> 256 MB`, with `zstd_buffers_unsafe on;`** — accepted, still logged as a **warning** (not silenced) so the acknowledgement remains visible in the log: ```text nginx: [warn] "zstd_buffers" (merged value) requests 2147483647 x 1073741824 bytes = ~2305843008139952128 bytes of output-chain memory PER RESPONSE, above the 256 MB hard cap; accepted because "zstd_buffers_unsafe on;" acknowledges it. ... ``` `number × size` **overflowing the platform's `size_t`** is refused unconditionally at every tier, including with `zstd_buffers_unsafe on;` set — there is no representable acknowledgement for a product that cannot exist: ```text nginx: [emerg] "zstd_buffers" (explicit directive) requests 9223372036854775807 buffers of 9223372036854775807 bytes each; that product overflows the platform's size_t and cannot be a config any operator meant to write ``` The total named in tiers 2–4 is **per response**; add it to the CCtx figure from the [`zstd_max_cctx_memory`](#zstd_max_cctx_memory) advisory to size the full per-request compressor + output-chain budget, then multiply by expected concurrency. `zstd_buffers_unsafe` is `http, server, location` scoped and inherits like any other directive here — set it once at the level that also sets `zstd_buffers`, or higher up if every location beneath needs the same acknowledgement. The check runs once the value is fully resolved (explicit, inherited from an outer block, or this module's own default) so all three sources are covered by the same bound; an explicit value that also happens to be the one that ends up in effect is reported once, not twice. --- ### zstd_buffers_unsafe **Syntax:** `zstd_buffers_unsafe on | off;` **Default:** `off` **Context:** `http, server, location` Acknowledges and accepts the total memory allocated by [`zstd_buffers`](#zstd_buffers) when the product of `number × size` exceeds 256 MB. By default, such configurations fail to load with a config-load error. Setting this directive to `on` skips the hard error and instead logs a warning (the warning is always emitted, even with `zstd_buffers_unsafe on;`, so the acknowledgement remains visible). This directive is intended for operators who have calculated their total concurrency, compressor memory, and output-chain memory budgets and deliberately chosen a buffer configuration above the safety threshold. If you are not sure whether you need this, leave it `off` — the hard error is the safer default. **Example:** ```nginx http { # An unusually large buffer pool, above the 256 MB threshold. # Only do this if you have explicitly calculated the total # per-request cost and verified that it fits your deployment. zstd_buffers 1000 10m; zstd_buffers_unsafe on; } ``` --- ### zstd_target_cblock_size **Syntax:** `zstd_target_cblock_size size;` **Default:** `—` (disabled, uses ZSTD library defaults) **Context:** `http, server, location` **Requires:** libzstd ≥ v1.5.6 Sets the target compressed block size for zstd frames. Controlling block size improves incremental response parsing, particularly in browsers where CSS/JavaScript in the response head must be available as soon as possible. > **Rationale:** When the zstd encoder produces large compressed blocks, the entire block must be decompressed before any content within it becomes available to the client. Smaller blocks allow incremental decompression and earlier access to critical resources. For example, CSS in `` can be parsed sooner if it lands in an early, smaller block. > **Compatibility:** This directive requires libzstd v1.5.6 or later. On older versions, the directive is accepted as a warned no-op. If not set (value 0 or unset), zstd uses its internal defaults, typically yielding blocks of 128–256 KB depending on the compression level and content. **Example:** ```nginx http { # Smaller blocks = faster incremental parsing, slightly lower compression ratio zstd_target_cblock_size 65536; # 64 KB blocks } ``` **Effect:** Lower values increase the number of blocks and may reduce compression ratio slightly, but improve streaming/incremental decompression. Common values: | Value | Use Case | |---|---| | Not set | Default behavior; good all-around balance | | `16384` (16 KB) | Very aggressive incremental parsing; reduces ratio notably | | `65536` (64 KB) | Moderate; CSS/JS in head typically available faster | | `262144` (256 KB) | Conservative; minimal ratio impact | --- ### zstd_window_log **Syntax:** `zstd_window_log exponent;` **Default:** `—` (disabled; zstd uses its level-derived default) **Context:** `http, server, location` Caps the zstd compression **window** at `2^exponent` bytes. zstd's per-request working memory is dominated by the window size (roughly the window plus match-table overhead), so without a cap a high compression level on large response bodies lets each concurrent request inflate the worker's resident memory unpredictably. Bounding `window_log` gives a hard, predictable per-request memory ceiling. Typical values are `20`–`24` (1–16 MB). Lower values reduce memory and the compression ratio on inputs larger than the window; on responses smaller than the window there is no ratio impact. Unset (or `0`) keeps zstd's default window for the configured level. > **Note:** This bounds compressor memory, not the amount of response > body buffered by nginx (that is governed by `zstd_buffers`). For a hard > limit on how much input is ever fed to the compressor regardless of > `Content-Length`, see also `zstd_max_length`. **Example:** ```nginx http { zstd on; zstd_comp_level 9; zstd_window_log 21; # cap the window at 2 MB per request } ``` --- ### zstd_long **Syntax:** `zstd_long on | off;` **Default:** `zstd_long off;` **Context:** `http, server, location` Enables zstd **long-distance matching** (`ZSTD_c_enableLongDistanceMatching`). zstd keeps a secondary long-range hash table that finds repeated sequences far beyond the regular match window, which can meaningfully improve the compression ratio on large, internally repetitive bodies — concatenated JSON, HTML with repeated boilerplate, log dumps, sitemaps. Off by default: the win only appears on inputs large enough to exceed the match window, and it costs a modest, bounded amount of extra per-request memory for the long-range table. Small responses should not pay that allocation, so enable it only on locations that serve large repetitive payloads. Interacts with [`zstd_window_log`](#zstd_window_log): an explicit `zstd_window_log` still takes precedence over the window zstd would otherwise derive when long mode is on, so the per-request memory ceiling remains under your control. **Example:** ```nginx location /api/bulk-export { zstd on; zstd_comp_level 12; zstd_long on; # large, highly repetitive JSON zstd_window_log 24; # keep the memory ceiling explicit } ``` --- ### zstd_max_cctx_memory **Syntax:** `zstd_max_cctx_memory size;` **Default:** `—` (no budget enforced; see [the advisory](#implicit-advisory-when-no-budget-is-set) below) **Context:** `http, server, location` **Requires:** module built with `-DZSTD_STATIC_LINKING_ONLY` against libzstd ≥ 1.4.0. This is **not** the result of the plain `./configure --add-dynamic-module=...` command in [Installation](#installation) — that auto-discovers a dynamic libzstd and never defines the flag. It is only set when `ZSTD_INC`/`ZSTD_LIB` point at a static libzstd build; see [Installation](#installation) and [Compatibility](#compatibility). Asserts at **config load** that the combined zstd parameters configured for the location (`zstd_comp_level`, `zstd_window_log`, `zstd_long`, `zstd_target_cblock_size`) do not need more than `size` bytes of per-request compressor working memory. If they would, nginx refuses to start with a clear, actionable error pointing at the smallest set of parameters to lower. The budget is checked against libzstd's own `ZSTD_estimateCStreamSize_usingCCtxParams()`, so the number is authoritative — it accounts for the level's *strategy tables* (chain/hash/search), the window, and LDM, not just the window. This matters because **lowering `zstd_window_log` alone does not bound memory for high levels**: level 22 at windowLog 20 still allocates ~640 MB, because the table size is driven by the level/strategy, not the window. ```nginx http { zstd on; zstd_comp_level 19; # would otherwise eat ~90 MB / request zstd_max_cctx_memory 256m; # accepted: level 19 fits in 256 MB } server { location /risky/ { zstd_comp_level 22; zstd_max_cctx_memory 64m; # REFUSED at config load: # "the configured zstd parameters need ~833 MB of per-request # compressor memory, which exceeds zstd_max_cctx_memory 64m; # lower zstd_comp_level (currently 22), lower zstd_window_log, # disable zstd_long, or raise the budget" } } ``` > **Interaction with `zstd_long`.** Enabling long-distance matching > raises libzstd's default window to 128 MB (`windowLog` 27) unless > `zstd_window_log` sets one explicitly, and adds the LDM hash table on > top. That is a large jump: level 3 costs ~3.6 MB per request without > `zstd_long` and ~144 MB with it. The budget check accounts for this — > it derives the same LDM sub-parameters libzstd would and estimates > against them, so `zstd_long on` with a budget below ~128 MB is > refused at config load rather than accepted and blown at runtime. > Pair `zstd_long on` with an explicit `zstd_window_log` to bring it > back down (level 3 with `zstd_window_log 20` costs ~2.7 MB). #### Implicit advisory when no budget is set Setting `zstd_max_cctx_memory` is optional, and most configurations never do. Compression enabled with **no** `zstd_max_cctx_memory` anywhere in the inheritance chain therefore still runs the same estimate at config load, and emits a **warning** — not an error — when the profile needs more than **32 MB** of per-request compressor memory: ```text nginx: [warn] the configured zstd parameters need ~144328225 bytes of per-request compressor memory (zstd_comp_level 3); each worker retains up to 4 such contexts, so worker RSS can reach ~577312900 bytes, and that again per worker process. Set "zstd_max_cctx_memory" to a budget to have this enforced at config load, or "zstd_max_cctx_memory 0" to acknowledge the profile and silence this warning ``` This exists because the expensive levers are one line away from a working config: a later `zstd_comp_level 22` or `zstd_long on` commits hundreds of MB per concurrent response, and without the advisory the module knew that number at config load and said nothing. The 32 MB threshold sits in the natural gap between `zstd_comp_level` 11 (~28.7 MB) and 12 (~52.7 MB), so the ordinary web-serving range is silent and only genuinely large profiles are named: | profile (level 3 unless stated) | per request | per worker (×4) | advisory | | --- | ---: | ---: | :---: | | `zstd_comp_level 1` | 1.3 MB | 5.2 MB | no | | default (`zstd_comp_level 3`) | 3.5 MB | 14.0 MB | no | | `zstd_comp_level 9` | 16.7 MB | 67.0 MB | no | | `zstd_comp_level 11` | 28.7 MB | 115.0 MB | no | | `zstd_comp_level 12` | 52.7 MB | 211.0 MB | **yes** | | `zstd_comp_level 19` | 89.5 MB | 358.0 MB | **yes** | | `zstd_comp_level 22` | 769.5 MB | 3078.0 MB | **yes** | | `zstd_long on` | 137.6 MB | 550.6 MB | **yes** | | `zstd_window_log 27` | 129.5 MB | 518.0 MB | **yes** | | `zstd_long on` + `zstd_window_log 20` | 2.6 MB | 10.3 MB | no | *(`ZSTD_estimateCStreamSize_usingCCtxParams()`, libzstd 1.5.7, streaming with unknown content size — the module's own path.)* Two ways to silence it, and they mean different things: * **`zstd_max_cctx_memory ;`** — you want the bound *enforced*. Exceeding it is a hard config-load error, as documented above. * **`zstd_max_cctx_memory 0;`** — you have read the number and accept it. Nothing is enforced and nothing is warned about. It never fails the configuration on its own: a config that loads today keeps loading after an upgrade. Note that the advisory requires the same `-DZSTD_STATIC_LINKING_ONLY` build as the directive; a build without the estimator API simply does not warn, rather than implying a guarantee it cannot compute. **Why a config-load assert and not a runtime cap.** The directive does **not** silently tune anything. A too-tight budget is a hard error so operators see the misconfiguration up front, instead of discovering it as a worker-RSS surprise under concurrency. Without `-DZSTD_STATIC_LINKING_ONLY` the estimator API is unavailable; in that case the directive is rejected at config load with the same kind of clear message, never silently no-op'd. > **Note:** This bounds **per-request** memory at one CCtx. The total > worker memory ceiling at the request limit is roughly > `worker_connections × zstd_max_cctx_memory` in the worst case > (every connection actively compressing). > **Retained memory between requests.** Each worker keeps a small ring > of compression contexts (4 slots) so consecutive responses reuse an > already-allocated workspace instead of building one per response. > A slot is keyed on the complete set of parameters that drive that > workspace — `zstd_comp_level`, `zstd_long` and the **effective** window > log — and is never re-parameterised, so a slot's retained size stays at > the figure `zstd_max_cctx_memory` vetted for that profile at config load. > "Effective" matters for one case: a [dcz](#zstd_dcz_dict_file) > request derives its own window from the negotiated dictionary rather than > from `zstd_window_log`, so it is keyed on that derived value and occupies > its own slot instead of raising a plain request's slot permanently. > An idle worker therefore holds up to 4 workspaces rather than > releasing them between requests. Budget that constant, not the number > of configured profiles: a busy slot is skipped before its profile is > compared, so concurrent requests on a single profile can seed several > slots with that same profile. Each retained workspace stays at its own > profile's vetted figure, so this is a floor well below the > `worker_connections × zstd_max_cctx_memory` ceiling above, not an > addition to it. --- ### zstd_bypass **Syntax:** `zstd_bypass string ...;` **Default:** `—` **Context:** `http, server, location` Disables on-the-fly compression for the current request when at least one of the given string parameters evaluates to a non-empty value that is not `"0"`. Each parameter is typically a variable (often driven by a `map`), so the decision is made per request rather than statically. ```nginx map $request_uri $no_zstd { default 0; ~^/wp-admin/ 1; # authenticated admin: reflects input + nonces ~^/wp-json/ 1; # REST: responses mix tokens with user data } server { zstd on; zstd_bypass $no_zstd; # skip those paths zstd_bypass $http_x_no_compression; # honour a client opt-out header } ``` > **Security note — BREACH:** `zstd_bypass` is the intended lever for > mitigating [BREACH](https://en.wikipedia.org/wiki/BREACH)-style > attacks, which exploit the *size* of a compressed HTTP body that > contains **both** a secret (CSRF token, session data) **and** > attacker-influenced reflected input. Use it to serve identity on the > specific endpoints where that combination occurs. > > Be honest about what this does and does not do: **no HTTP compressor > can be made BREACH-safe while still compressing** — the attack is > inherent to compression ratio as a side channel. `zstd_bypass` only > lets you *exclude* the at-risk responses. The effective, primary > BREACH defenses live in the application: per-request CSRF token > masking, separating secrets from reflected input, and > referer/origin checks. Treat `zstd_bypass` as a containment tool, not > a fix. (CRIME and POODLE are unrelated TLS-layer attacks and are not > addressed — or addressable — here; configure `ssl_protocols` > appropriately instead.) > **Why this module does not pad responses (the "anti-BREACH length > padding" question).** A frequently requested "fix" is to add random > padding to the compressed body so its length no longer reveals the > compression ratio. This module deliberately does **not** do that, and > will not, for concrete reasons: > > 1. **Random padding does not remove the signal — it adds noise the > attacker averages out.** BREACH is a guess-and-measure oracle: the > attacker replays the same request thousands of times, changing one > guessed byte at a time. A correct guess compresses ~1 byte smaller. > Random padding of variance σ adds zero-mean noise to each > measurement; averaging N samples shrinks the noise by √N while the > 1-byte signal stays put. The attacker simply requests more times. > Published BREACH follow-up work (e.g. *Rupture*) automates exactly > this statistical recovery against padded/noised responses. Padding > raises the request count, not the difficulty class — it buys the > *appearance* of a fix while the secret still leaks. > 2. **Padding that *would* defeat it is not "padding" anymore.** The > only length transform that actually closes the oracle is forcing > every response to a fixed size (or coarse power-of-two buckets) > *independent of content* — which throws away most of the > compression you enabled zstd for, on every response, to defend the > small subset that mixes a secret with reflected input. That is a > strictly worse trade than `zstd_bypass` on those endpoints (full > ratio everywhere else, identity exactly where it is unsafe). > 3. **It moves a security boundary into the wrong layer.** Whether a > response safely mixes secrets and attacker input is an > application-semantics decision (is this field a CSRF token? is that > substring reflected query input?). A compression filter cannot see > that distinction; a per-response byte transform here cannot make an > application-layer information-flow problem safe, and shipping one > would invite operators to *believe* it had. > > So the module gives you the one lever that is honest and effective — > `zstd_bypass`, to serve identity on the specific at-risk endpoints — > and points you at the real fixes (CSRF token masking, separating > secrets from reflected input, origin/referer checks). A built-in > padding knob would trade real bandwidth for false confidence, so it is > intentionally absent. See [`SECURITY.md`](SECURITY.md) for the > in-scope / out-of-scope statement. > **Cache safety with request-driven bypass.** When a `zstd_bypass` > predicate depends on a **request header or cookie** (e.g. > `zstd_bypass $http_x_no_compression`), the compressed-vs-identity > decision now varies on that header. A shared cache (proxy / CDN) that > does not key on it can store the identity response and serve it to a > normal client, or store the compressed response and serve it to a > bypass client. Either declare that variance with > [`zstd_bypass_vary`](#zstd_bypass_vary) **or** mark such responses > `Cache-Control: private` / `no-store`. Bypass predicates that depend > only on `$request_uri`/`$uri` (already part of the cache key) do not > need this. > > The module warns at config load when a `zstd_bypass` predicate reads a > `$http_*` or `$cookie_*` variable **directly** with no > `zstd_bypass_vary` configured in the same location — the same > misconfiguration named above, caught before it reaches a shared cache. > The check only recognizes the literal `$http_*`/`$cookie_*` spellings; > an indirect predicate (e.g. a `map` result variable derived from a > header or cookie) is not resolved and stays silent — pairing > `zstd_bypass_vary` with a map-derived predicate remains an explicit > operator responsibility. --- ### zstd_bypass_vary **Syntax:** `zstd_bypass_vary field-name;` **Default:** `—` **Context:** `http, server, location` Appends `field-name` to the response `Vary` header on every response for which this module actually reaches the `zstd_bypass` decision — both the compressed variant and the bypassed-identity variant — so a shared cache keys on the request header that drives a header/cookie-based [`zstd_bypass`](#zstd_bypass). A response declined earlier by an eligibility gate (ineligible status, already encoded, below `zstd_min_length`, above `zstd_max_length`, header-only, content type not in [`zstd_types`](#zstd_types), or `Cache-Control: no-transform`) never reaches that decision: it would be served identity regardless of the bypass predicate, so no bypass-driven variance exists to declare and this directive adds no `Vary` field to it. Use it whenever a bypass predicate reads a request header or cookie: ```nginx server { zstd on; zstd_bypass $http_x_no_compression; # client opt-out header zstd_bypass_vary X-No-Compression; # so caches key on it } ``` The module emits this as an additional `Vary` header line; caches union all `Vary` fields, so it coexists with the `Vary: Accept-Encoding` the module emits automatically. It does not itself decide anything — it only makes the existing bypass behaviour cacheable without poisoning. --- ### zstd_dict_file **Syntax:** `zstd_dict_file /path/to/dict;` **Default:** `—` **Context:** `http` Loads a pre-trained zstd dictionary for use during compression. Dictionaries can significantly improve compression ratios for small, structurally similar responses (e.g. JSON API responses). > **Requires explicit opt-in.** This directive emits an ordinary `Content-Encoding: zstd` response that was compressed with an external dictionary. That is **not** HTTP dictionary negotiation — [RFC 9842](https://www.rfc-editor.org/rfc/rfc9842) (Sept 2025) defines the `dcz` content coding and `Available-Dictionary` for that, implemented by [zstd_dcz_dict_file](#zstd_dcz_dict_file) below, which is what you want unless you control both ends. A generic client that only advertises `Accept-Encoding: zstd` **cannot decode** the result, and a shared cache keys it as an ordinary zstd variant. nginx therefore refuses to start with `zstd_dict_file` set unless you also set `zstd_dict_file_unsafe on;`, acknowledging that you control both ends and will key any shared cache accordingly. > **Warning:** The `Content-Encoding: zstd` token in HTTP does not include any mechanism for the client to discover or negotiate which dictionary the server is using. Only use this directive if you control both ends of the connection and can guarantee that both the server and client use the same dictionary (for example, by advertising it via a custom HTTP header). See [tokers/zstd-nginx-module#2](https://github.com/tokers/zstd-nginx-module/issues/2) for background. > **Parameter precedence with a dictionary.** A `ZSTD_CDict` bakes in the compression parameters it was built with, and libzstd's `ZSTD_CCtx_refCDict()` lets those supersede the per-request CCtx parameters. To keep `zstd_window_log` an effective cap with a dictionary loaded, the CDict is built with `ZSTD_createCDict_advanced()` seeding `windowLog` from `zstd_window_log` (on the static-linked builds this module ships), so the baked window matches the CCtx window and `zstd_max_cctx_memory`'s estimate — computed from the same `windowLog` — stays accurate. The CDict is built per distinct (`zstd_comp_level`, `zstd_window_log`) combination, so changing either in a child `location` rebuilds it rather than silently reusing the parent's. `zstd_long` applies via the CCtx (it is not a `ZSTD_compressionParameters` field, so `refCDict` does not override it). On a non-static-linked build the advanced builder is unavailable and the CDict falls back to a level-only digest; there `zstd_window_log` is not honored while a dictionary is loaded. **Example:** ```nginx http { # Loaded once per cycle; must be readable by the nginx user. # Train it with: zstd --train samples/*.json -o /etc/nginx/api.dict zstd_dict_file /etc/nginx/api.dict; zstd_dict_file_unsafe on; # required: acknowledges non-RFC-9842 mode zstd on; zstd_types application/json; server { location /api/ { # Tell a cooperating client which dictionary was used, # since HTTP cannot negotiate it (see warning above). add_header X-Zstd-Dict "api.dict-v1" always; } } } ``` --- ### zstd_dict_file_unsafe **Syntax:** `zstd_dict_file_unsafe on | off;` **Default:** `off` **Context:** `http` Acknowledges that responses compressed with [`zstd_dict_file`](#zstd_dict_file) are not RFC 9842 negotiated — they carry an ordinary `Content-Encoding: zstd` even though they were compressed with an external dictionary. This is an **unsafe** configuration for shared caches: a generic client that only advertises `Accept-Encoding: zstd` will receive a body compressed against a dictionary it does not possess and cannot decode. nginx refuses to start if `zstd_dict_file` is configured without `zstd_dict_file_unsafe on;`, acknowledging that you control both ends of the connection (client and server) and will ensure the shared cache keys the response appropriately (or is absent). Only use this directive when you control both the client and server and can guarantee that both use the same dictionary. If you need standards-based HTTP negotiation, use [`zstd_dcz_dict_file`](#zstd_dcz_dict_file) instead. **Example:** See [`zstd_dict_file`](#zstd_dict_file) for an example that includes this directive. --- ### zstd_dict_strict_path **Syntax:** `zstd_dict_strict_path on | off;` **Default:** `off` **Context:** `http` Both dictionary loaders ([`zstd_dict_file`](#zstd_dict_file) and [`zstd_dcz_dict_file`](#zstd_dcz_dict_file)) always reject a FIFO, socket, directory, device node, or empty file — a config-load error either way, `on` or `off`. This directive controls an *additional*, opt-in trust policy on top of that: with it `on`, both loaders also refuse a **symlink at any component of the path** (not only the final one), a target — or an ancestor **directory** — **writable by group or other**, and a target or ancestor directory **not owned by the loading principal or by root**. It additionally requires the path to be **absolute** and free of `.`/`..` components. > **Why it matters.** A root master reloads its configuration (`nginx -s reload` or a supervisor-driven SIGHUP) and re-reads every dictionary from disk at that moment. If the path is a symlink a less-privileged local writer can repoint, the file itself is group/world-writable, or an **ancestor directory** is group/world-writable or foreign-owned, that writer chooses or mutates the bytes the master snapshots into every worker on the next reload — without ever needing privilege to touch the running nginx process. `on` closes all three. The path is resolved **one component at a time** with `openat(O_NOFOLLOW|O_DIRECTORY)`, so an intermediate symlink (`/srv/current/dict.bin` with `current` a symlink) is refused rather than silently traversed — a whole-path `O_NOFOLLOW` guards only the final component and would follow `current` without complaint. Every directory fd the walk opens along the way — the root `/` included — is `fstat()`-checked against the same rule as the leaf **before** it is trusted as the base for the next component: owned by neither root nor the loading principal, or writable by group or other, and the whole path is refused, naming the offending component. There is **no sticky-bit exemption** for this ancestor check — a sticky world-writable directory (a `/tmp`-style layout) still lets an unprivileged user create a new entry in it, which is the steering this check exists to refuse, sticky bit or not; deploy dictionaries under a directory tree that is itself owned and writable only by the deploying principal, not merely a temp directory with a sane leaf file. The path must name a file: a trailing `/` (`zstd_dict_file /srv/dicts/zstd.dict/;`) is refused as naming a directory, matching `open(2)`'s `ENOTDIR`, so a configuration that relied on one stops loading under `on` — and since a rejected reload keeps the previous worker generation serving, check `nginx -t` rather than assuming the reload took. The leaf is then opened relative to the verified parent descriptor, and before a single byte is read its own mode is checked against `S_IWGRP|S_IWOTH` **and** its owner against the effective uid of the config-parsing master (root-owned files are also accepted, root not being less privileged than the loader). Owner-writability alone is not enough at either level: a dictionary — or an ancestor directory — owned by an unprivileged account at an entirely ordinary mode (`0644` for a file, `0755` for a directory) lets that owner rewrite or replace the bytes a later privileged reload snapshots, which is precisely the writer this directive exists to exclude. Every check runs against the **already-open file descriptor** — never by re-opening the path — so a rename or symlink swap after the check cannot smuggle in a different file (no TOCTOU window). > **Platform support.** The component walk needs POSIX.1-2008 `openat()`. Where it is unavailable (including Windows), `zstd_dict_strict_path on` **fails closed** with a config-load error rather than falling back to the weaker leaf-only guarantee — the directive never claims a protection the platform cannot deliver. `off` (the default) is unaffected everywhere. > **Default is `off`.** A release-symlink layout (`/srv/current -> /srv/releases//`, with the dictionary loaded from a path through `current`) is a common, legitimate deployment pattern, and it depends on exactly the symlink indirection `on` refuses — at the intermediate `current` component as much as at the leaf. Turning this on is a deliberate hardening step for operators who deploy dictionaries to an immutable, content-addressed path (e.g. a filename embedding the content hash) rather than through a movable symlink — do not enable it against an existing release-symlink deployment without first moving to that layout, or every reload will fail with `nginx -s reload` and the **old worker generation keeps serving** (a rejected reload never tears down the running cycle). **Example (hardened, content-addressed dictionary path):** ```nginx http { zstd_dict_strict_path on; zstd_dcz_dict_file /srv/dicts/app-a1b2c3d4.bin; } ``` --- ### zstd_dcz_dict_file **Syntax:** `zstd_dcz_dict_file /path/to/dictionary [sha256hex];` **Default:** `—` **Context:** `http`, `server`, `location` (repeatable) Standards-based dictionary compression per [RFC 9842](https://www.rfc-editor.org/rfc/rfc9842) (Compression Dictionary Transport). Each occurrence loads one dictionary — typically a **previous version of the resource** (delta-style) or a trained dictionary — and registers its SHA-256 as a negotiation key. When a request arrives whose `Available-Dictionary` header matches a loaded dictionary **and** whose `Accept-Encoding` lists `dcz` explicitly with a non-zero weight, the response is compressed against that dictionary and sent as `Content-Encoding: dcz`: a fixed 40-byte header (a zstd skippable frame carrying the dictionary's SHA-256) followed by an ordinary zstd frame. Chrome 130+ negotiates this automatically for same-origin resources. Every gate miss — a non-secure context ([see below](#zstd_dcz_assume_secure_transport)), unknown hash, no explicit `dcz` token (`*` deliberately does not match), an effective `dcz;q=0`, a malformed `Available-Dictionary`, or `Sec-Fetch-Site` other than `same-origin`/`none` — falls back to the plain `zstd` path. The `dcz` token is looked for across *every* `Accept-Encoding` field line using the same ordered duplicate-coding rule as plain zstd (see the negotiation notes under [`ngx_http_zstd_filter_module`](#ngx_http_zstd_filter_module)). Unlike `zstd_dict_file`, no opt-in flag is needed to make dcz *safe*: dcz is real HTTP negotiation, and only clients that advertised the dictionary ever receive it. One flag can still be required by your topology — RFC 9842 §8 restricts dcz to secure contexts, so a deployment where TLS is terminated *in front of* this nginx needs [zstd_dcz_assume_secure_transport](#zstd_dcz_assume_secure_transport). On a listener that terminates TLS itself, nothing extra is needed. The optional second argument supplies the dictionary's SHA-256 as 64 hex characters. By default it is **verified** against the bytes actually read at config load: it declares what the operator believes the file to be, and a mismatch fails the load with an error naming the file, the supplied hash and the computed one. Use it as a deploy-time guard that the file on this host is the one the config was generated for — a dictionary replaced or truncated behind the config is caught by `nginx -t` instead of reaching clients. Malformed values (wrong length, non-hex) are config-load errors, reported before the file is opened. A running cycle always serves from the bytes loaded at its config parse — a file changing on disk affects nothing until the next parse. What that next parse does is where the policies differ: with no literal, the computed key simply becomes the new file's hash, so clients still holding the old dictionary stop matching and fall back to plain `zstd` (safe); with a literal under the default, the mismatch **fails the load** — the deploy-time guard; with a literal under `trust_hashes on`, the stale literal keeps matching, which is the stated risk below. `zstd_dcz_dict_trust_hashes on;` (`http` only, default `off`) opts out of the verification: a supplied literal is then trusted **verbatim** as the negotiation key and the load-time SHA-256 is skipped entirely — lines without a literal are hashed as always. The hashing pass is the config-load cost at scale (measured at 737 dictionary lines: `nginx -t` drops from 5.1s to 0.9s, with the user-CPU column — the SHA-256 itself — going from 4.3s to 0.03s), so a content-addressed deployment whose tooling derives each literal from the file it ships can reclaim that on every `nginx -t` and reload. The stated trade: a stale or mistyped literal is advertised verbatim, and clients holding the advertised dictionary receive responses that may fail to decode, or silently decode to wrong content under a same-size stale dictionary. Only use it when the pipeline that writes the literal is the same one that placed the file. The directive must precede every `zstd_dcz_dict_file` carrying a literal; declaring it after one is a config-load error (the earlier literal was verified — correct, but the hashing the directive exists to skip was silently paid). The module handles the response side; **advertising** the dictionary to clients is one `add_header` on the dictionary resource (usually the resource itself, so today's file becomes the dictionary for tomorrow's): ```nginx http { zstd on; server { location /app/ { # Yesterday's release, now serving as the delta base. zstd_dcz_dict_file /var/www/releases/app-v41.js; # Today's file is itself the dictionary for the NEXT release: # a returning client sends Available-Dictionary: :sha256:. add_header Use-As-Dictionary 'match="/app/*.js"'; } } } ``` Operational notes: * **Caching.** Whenever dictionaries are configured, the module emits `Vary: Available-Dictionary, Sec-Fetch-Site` on **both** the dcz and the plain-zstd variants — which encoding a client receives depends on that request header for every client, and a shared cache that failed to key on it could serve an undecodable dcz body to a dictionary-less client. `Sec-Fetch-Site` is in that list for the same reason: dcz is refused for any value other than `same-origin`/`none`, so a shared cache that ignored it would hand the dcz representation to a cross-site request — bypassing the origin gate — or suppress dcz for a legitimate same-origin client, depending on which one filled the cache first. Security decision inputs stay in `Vary`. `Vary: Accept-Encoding` is emitted alongside automatically, on its own header line. * **Dictionary lifecycle.** Files are read once at config parse, and hashed — a supplied hash literal is verified against the bytes read, not substituted for hashing them, unless [`zstd_dcz_dict_trust_hashes on`](#zstd_dcz_dict_file) opts that line out of the pass entirely (`nginx -t` validates everything; empty or > 10 MB files and duplicate dictionary hashes are load errors — under `trust_hashes` a supplied hash is compared as declared, and with computed hashes "duplicate" means identical content). Rotate dictionaries by updating the config and reloading. A location that declares its own `zstd_dcz_dict_file` list replaces the inherited one wholesale. Hashing uses libcrypto's EVP SHA-256 when detected at build time — roughly an order of magnitude faster, which at hundreds of registered dictionaries turns seconds of `nginx -t`/reload time into a blip (`NGX_ZSTD_NO_LIBCRYPTO=1` in the configure environment opts out) — with a portable implementation built in as the fallback. * **Window and memory.** dcz frames never declare a window above 8 MB (the RFC's unconditional client guarantee), sized down to dictionary + expected content when smaller. Either memory ceiling below that still wins — a dictionary does not void a cap the operator asked for. An explicit [`zstd_window_log`](#zstd_window_log) clamps the dcz window directly, and [`zstd_max_cctx_memory`](#zstd_max_cctx_memory) set to a POSITIVE budget clamps it to the largest window log whose estimated CCtx memory still fits that budget (computed once at config load, so the request path only compares). Both are **opt-in**: with `zstd_window_log` unset and `zstd_max_cctx_memory` either unset or `0` — the default, or an explicit "accept the estimate, do not enforce it" — the dcz window is the dictionary-derived one, unchanged. Note that a budget tight enough to move the window costs ratio on dcz responses, which is the trade an explicit memory bound asks for. Dictionaries are referenced per request via `ZSTD_CCtx_refPrefix()` (RFC 9842 `type=raw` semantics); the per-request table build over the dictionary costs roughly milliseconds per MB of dictionary, so prefer focused dictionaries (the previous version of one resource) over giant blobs. The cost is set by dictionary size and compression level and is independent of the response body, so a small body pays it in full: measured on a 1 KB body, an 8 MB dictionary costs 0.75 ms at [`zstd_comp_level`](#zstd_comp_level) 3 against 35 ms at level 9 and 395 ms at level 19. Level 3 is flat in dictionary size because the default strategy does not build the expensive match tables; from level 9 it climbs steeply, and [`zstd_long`](#zstd_long) adds its own per-request table build. Configuring dictionaries at level 9 or above, or with `zstd_long on`, therefore emits a config-load **warning** naming the profile and the size of the largest configured dictionary — advisory only, nothing is refused. Neither lever is client-controlled (a client only selects among dictionaries you configured, via `Available-Dictionary`), so this is a cost of your own profile rather than something a request can provoke. * **Secure context.** RFC 9842 §8 restricts dictionary transport to secure contexts, so dcz is only negotiated over TLS. A plain-HTTP request that would otherwise match falls back to plain `zstd`. Behind a TLS-terminating proxy, see [zstd_dcz_assume_secure_transport](#zstd_dcz_assume_secure_transport). * **Cross-origin.** Requests with `Sec-Fetch-Site` other than `same-origin`/`none` fall back to plain zstd rather than attempting RFC 9842's CORS legs. That decision is declared in `Vary` (see Caching above) so a shared cache partitions on it. `Dictionary-ID` is not consumed — the hash is a complete key — so omit `id=` from `Use-As-Dictionary` (clients would echo it, but nothing here needs it). **Troubleshooting:** if `Vary: Available-Dictionary, Sec-Fetch-Site` appears but dcz never negotiates for hashes you know are right, run `nginx -T | grep dcz_dict_file` and check the loaded entries are your dictionary *files* — a deploy-generated list of directives must be pulled in with `include`, not passed to `zstd_dcz_dict_file` itself (which loyally loads the list file as a one-entry dictionary that matches nothing). Also check the error log for a rejected reload: a failed `nginx -t` leaves the previous configuration running. Verify a response end-to-end with the zstd CLI (the 40-byte header is a valid skippable frame, so no stripping is needed): ```bash curl -s -H 'Accept-Encoding: zstd, dcz' \ -H "Available-Dictionary: :$(openssl dgst -sha256 -binary app-v41.js | base64):" \ https://example.com/app/app-v42.js \ | zstd -d -D app-v41.js | diff - app-v42.js && echo "byte-exact" ``` --- ### zstd_dcz_dict_trust_hashes **Syntax:** `zstd_dcz_dict_trust_hashes on | off;` **Default:** `off` **Context:** `http` Opts out of verifying the SHA-256 of each dictionary file when a hash literal is supplied to [`zstd_dcz_dict_file`](#zstd_dcz_dict_file). By default, a supplied hash is verified against the file's actual content at config load, catching deployment errors where the file on disk does not match what the config expects. With `zstd_dcz_dict_trust_hashes on;`, the supplied literal is trusted **verbatim** as the negotiation key and the hashing pass is skipped entirely. This is an optimization for deployments where dictionaries are identified by content-addressed paths and the tooling that generates the config is the same tooling that places the files — a pipeline that can be trusted to derive each literal from the file it ships. It reclaims the config-load cost of hashing: measured on a production config with 737 dictionary lines, `nginx -t` runs 5.10s with the verify pass (4.26s user — the SHA-256 alone) against 0.87s with trusted literals (0.03s user). Lines without a literal are hashed as always; trust changes only what a supplied literal means. The stated trade: a stale or mistyped literal is advertised verbatim, and clients holding the advertised dictionary receive responses they may fail to decode — or, when the stale dictionary happens to be the same size, that silently decode to wrong content. Only enable this when the pipeline that writes the literal is the same one that placed the file. The directive must precede every [`zstd_dcz_dict_file`](#zstd_dcz_dict_file) that carries a literal; declaring it after one is a config-load error. **Example (content-addressed deployment with optimized hashing):** ```nginx http { zstd on; zstd_dcz_dict_trust_hashes on; # pipeline places files and generates hashes server { location /app/ { # Hash comes from the same tool that places the file; # the verify pass is skipped. zstd_dcz_dict_file /srv/dicts/app-v41-a1b2c3d4.bin a1b2c3d4e5f6789012345678abcdef0123456789abcdef0123456789abcdef01; } } } ``` --- ### zstd_dcz_assume_secure_transport **Syntax:** `zstd_dcz_assume_secure_transport on | off;` **Default:** `zstd_dcz_assume_secure_transport off;` **Context:** `http`, `server`, `location` Asserts that the client-facing hop is TLS even though this nginx sees plaintext, re-enabling [`dcz`](#zstd_dcz_dict_file) negotiation on a non-TLS connection. [RFC 9842](https://www.rfc-editor.org/rfc/rfc9842) §8 requires compression dictionary transport to be used only in a **secure context**. By default the module enforces that directly: `dcz` is negotiated only when the request arrived on a TLS connection, and a plain-HTTP request that would otherwise match falls back to plain `zstd`. That is the fail-closed direction — dictionary compression over cleartext hands a network attacker a length oracle over content the dictionary already describes. When TLS is terminated by a load balancer or CDN in front of this nginx, the connection nginx sees *is* plaintext and the check fires even though the client spoke HTTPS. Set this directive on in that deployment: ```nginx server { listen 8080; # behind a TLS terminator zstd on; zstd_dcz_assume_secure_transport on; # the client hop is HTTPS zstd_dcz_dict_file /var/www/releases/app-v41.js; } ``` The directive is an **operator acknowledgement, not a detection**. The module deliberately does not infer the client's scheme from `X-Forwarded-Proto`, `Forwarded` or any other request header: those are client-supplied on a directly reachable listener, so trusting one would let any client re-enable `dcz` over cleartext simply by sending it. Only enable this on a listener that is genuinely unreachable except through your TLS terminator — if the same listener can be hit directly over HTTP, enabling it puts those clients back in the situation §8 forbids. --- ## ngx_http_zstd_static_module This module serves pre-compressed `.zst` files in place of the originals, without running compression at request time. It is the zstd equivalent of nginx's `gzip_static` module. --- ### zstd_static **Syntax:** `zstd_static on | off | always;` **Default:** `zstd_static off;` **Context:** `http, server, location` Controls how pre-compressed `.zst` files are served. | Value | Behaviour | |---|---| | `off` | Disabled. Always serve the original file. | | `on` | Check whether the client supports zstd (`Accept-Encoding: zstd`). If yes and a `.zst` file exists, serve it. Otherwise fall back to the original. Emits `Vary: Accept-Encoding` automatically on both outcomes. | | `always` | Serve the `.zst` file if it exists regardless of `Accept-Encoding`, except for dictionary-aware requests when [`zstd_static_dict_bypass`](#zstd_static_dict_bypass) is enabled. Use this when you know all other clients support zstd (e.g. internal services). | When set to `on`, the module emits a `Vary: Accept-Encoding` response header itself as soon as a `.zst` sibling makes the URI depend on `Accept-Encoding` — including on the fallback path where the client does not accept zstd and the original file is served. Correct caching by proxies and CDNs therefore does not require [`gzip_vary`](https://nginx.org/en/docs/http/ngx_http_gzip_module.html#gzip_vary); setting `gzip_vary on` is compatible and never produces a duplicate header. With [`zstd_static_dict_bypass`](#zstd_static_dict_bypass) off, `zstd_static always` ignores `Accept-Encoding` and emits no `Vary`; the opt-in bypass is the exception described below. > **Warning (`always` mode):** When `zstd_static always` is set and > `zstd_static_dict_bypass` is off, this mode is non-conformant by design: > `.zst` files are served with `Content-Encoding: zstd` to every client, > including requests with no `Accept-Encoding`, `Accept-Encoding: identity`, > or `Accept-Encoding: zstd;q=0`. It suppresses `Vary: Accept-Encoding` and > performs no content-coding negotiation. With the > bypass on, matching dictionary-aware requests stand aside with the complete > cache key instead. `always` is safe only when every non-bypassed client is > guaranteed to decode zstd — for example, > internal service-to-service calls where you control both ends. > **Magic-number validation.** Before serving a `.zst`, the module reads the frame header prefix (up to 58 bytes: a canonical 40-byte dcz skippable prefix plus room for the following 18-byte maximum frame header, one offset-explicit read per distinct offset) and verifies each frame it walks starts with the zstd frame magic (`ZSTD_MAGICNUMBER` `0xFD2FB528`) or a skippable-frame magic (`ZSTD_MAGIC_SKIPPABLE_*`). The module walks a bounded chain of up to 4 leading skippable frames — enough for a dcz-style prefix plus an extra marker frame — before requiring the frame that follows to be a regular one; a 5th leading skippable frame is a hard decline rather than a longer search, so the walk cannot be turned into an unbounded scan. On any mismatch — a truncated download, mistaken rename (`cp foo.txt foo.zst`), non-zstd content, or an over-long skippable chain — the handler logs `zstd static: "..." is not a zstd frame (leading bytes 0x...)` (or the matching skippable-chain-length error) and **declines**; nginx then falls back to serving the uncompressed original, or returns 404 if no original is present. Probe verdicts are memoized per worker by exact path, file identity, size, and mtime in one bounded 64-entry cache. Malformed verdicts remain until eviction or an identity change. Valid verdicts are cached only when `open_file_cache` is enabled and expire no later than `open_file_cache_valid`, so even a same-inode, same-size rewrite inside one-second mtime granularity is revalidated on that bound. A replaced file or changed path, size, or mtime is probed immediately, and round-robin eviction permits an earlier re-probe. Without validation, the client could receive a body labelled `Content-Encoding: zstd` that it cannot decode. Each read is offset-explicit so it never disturbs the file position of the `open_file_cache` descriptor shared with other in-flight requests: `pread(2)` on POSIX, `ngx_read_file()` (a `ReadFile()` with an `OVERLAPPED` offset) on Win32. The verdict logic is a single shared function, so both platforms accept and reject exactly the same files. The probe is compiled out only on a POSIX build whose `configure` found no `pread(2)`, rather than degraded to a `read`+`lseek` pair that would corrupt those concurrent requests. > > **Declared-window validation.** The same probe parses the frame header (RFC 8878) and **declines any `.zst` whose first regular frame — the one reached after walking the bounded leading-skippable-frame chain above — declares a decompression window above 8 MB** — the limit browsers enforce for `Content-Encoding: zstd`; Firefox (`NS_ERROR_INVALID_CONTENT_ENCODING`) and Chromium (`ERR_CONTENT_DECODING_FAILED`) reject such frames before decoding a single byte. The scope is exactly that first regular frame: leading skippable frames within the bound are stepped over rather than inspected for a window, and in a (pathological, no common tooling emits one) concatenation of regular frames only the first is inspected — a regular frame's header does not declare its compressed length, so walking the sequence would mean decoding every block header in every frame. `zstd -t --memory=8MB` remains the complete pre-deploy check. This traps a nasty build-pipeline failure mode: streaming encoders that are not told the input size stamp the *compression level's* default window into every frame header, so a Node-based bundler can emit a 90 KB asset declaring a 128 MB window — the file decodes fine with the `zstd` CLI and serves byte-identically through nginx, yet fails in every browser. On decline the error log names the file and its declared window, and the request falls through to the zstd filter / `gzip_static` / identity, so the site keeps working while the build gets fixed (recompress with a window log ≤ 23; verify a build with `zstd -t --memory=8MB *.zst`). Single-segment frames are checked against their declared content size, which is their window. **Example:** ```nginx gzip_vary on; location /static/ { zstd_static on; root /var/www; } ``` Pre-compress files with a matching level to your workload: ```bash zstd -3 -k /var/www/static/app.js # creates app.js.zst alongside app.js ``` ### zstd_static_dict_bypass **Syntax:** `zstd_static_dict_bypass on | off;` **Default:** `zstd_static_dict_bypass off;` **Context:** `http, server, location` When enabled, `zstd_static` stands aside for a main request that carries any `Available-Dictionary` header and explicitly accepts `dcz` with a non-zero weight. This lets [`zstd_dcz_dict_file`](#zstd_dcz_dict_file) negotiate a dictionary response instead of having the content-phase static handler serve the plain `.zst` sidecar first. The check applies in both `zstd_static on` and `zstd_static always` modes. Before declining it emits the complete `Vary: Accept-Encoding, Available-Dictionary, Sec-Fetch-Site` cache key; the filter reuses those idempotent fields when it runs, while an identity fallback remains partitioned correctly when the filter is disabled or ineligible. The directive is deliberately off by default. It is only a routing hint: the static module does not validate the advertised digest, dictionary store match, secure context, or origin policy. If any later dcz gate rejects the request, the filter produces an ordinary zstd or identity response instead of returning to the skipped sidecar. Enable it only on locations whose dictionary-aware clients should prefer runtime dcz negotiation over a precompressed sidecar. --- # Variables ## $zstd_ratio The compression ratio achieved for the current response, expressed as the ratio of original size to compressed size (e.g. `3.42` means the compressed output is about 29% of the original). Only set when the filter module compressed the response. Useful in access logs: ```nginx log_format main '$remote_addr - $request - ratio: $zstd_ratio'; ``` --- ## $zstd_bytes_in The number of uncompressed (input) bytes the filter consumed for the current response. Only set once the filter has finished compressing the response (log phase); not found otherwise. See [`$zstd_bytes_out`](#zstd_bytes_out) below for a combined `log_format` example. ## $zstd_bytes_out The number of compressed (output) bytes the filter produced for the current response. Same availability as `$zstd_bytes_in`. Together these expose the **absolute** transfer saving, where `$zstd_ratio` only gives the ratio. By construction `$zstd_bytes_in / $zstd_bytes_out` equals `$zstd_ratio`: ```nginx log_format zstd '$request in=$zstd_bytes_in out=$zstd_bytes_out ' 'ratio=$zstd_ratio'; ``` ## $zstd_dcz_dicts_hashed How many dcz dictionaries were SHA-256-hashed at config load in the current configuration cycle. By default every registered [`zstd_dcz_dict_file`](#zstd_dcz_dict_file) entry is counted, including entries whose supplied hash is verified. With `zstd_dcz_dict_trust_hashes on`, supplied literals skip the pass and only entries without literals are counted; `0` therefore proves that every entry used a trusted literal. Constant for the lifetime of the configuration (reset on reload), so one probe request answers "did my deploy tooling's hashes actually take effect?": ```nginx add_header X-Dcz-Dicts-Hashed $zstd_dcz_dicts_hashed; ``` The regression suite asserts on this variable because the trusted-literal skip is otherwise unobservable when a supplied hash matches the file. --- # Testing & CI [`ci.yml`](.github/workflows/ci.yml) is the single PR gate entry point. It calls the bounded merge checks below; each member also keeps `workflow_dispatch:` so it can be run alone from the Actions tab. On merged `master`, CI selects only the testkit harness to preserve its promotion signal without duplicating the build, lint, Arch package, scanner, or Windows jobs. Long jobs are deliberately not called from it. **CI Deep** runs them weekly in four explicit self-hosted dependency chains; **Bump** opens version-bump PRs rather than gating them. The four-lane model applies when repository variable `POOL` selects the shared self-hosted pool; an unset variable uses each job's documented `ubuntu-latest` fallback instead. | Workflow | Cadence | What it does | |---|---|---| | **CI** ([`ci.yml`](.github/workflows/ci.yml)) | every PR + focused merged `master` signal + manual | Calls Lint, Build&Test, Arch Package, Security Scanners, Harness Fault Arms, and the hosted Windows build. Four Linux jobs are initially runnable; dependency chains refill each lane as it becomes free. Only Harness Fault Arms runs on merged `master`. | | **Lint** ([`lint.yml`](.github/workflows/lint.yml)) | PR via CI + manual | Runs local deterministic checks, including runner trust, port bands, cadence, provenance, and the enforced four-lane topology. | | **Build&Test** ([`build-test.yml`](.github/workflows/build-test.yml)) | PR via CI + manual | Builds nginx mainline with strict warnings, runs the full functional and runtime regression suites, tests libzstd 1.4.x fallbacks, linkage variants, and arm64. Its sanitizer lane runs the filter and static Test::Nginx suites plus the runtime regressions under ASAN/UBSAN, requiring complete TAP/clean exits and rejecting complete sanitizer reports that contain module frames. Its dependency graph forms four self-hosted lane chains. | | **Arch Package** ([`arch-package.yml`](.github/workflows/arch-package.yml)) | PR via CI + manual | Builds `nginx-mod-zstd` in a fresh Arch container, checks the released package with namcap, and smoke-tests both the package and current checkout with dynamic and precompressed Zstandard responses. Uploads the released package as a short-lived CI artifact. | | **Security Scanners** ([`security-scanners.yml`](.github/workflows/security-scanners.yml)) | PR via CI, weekly deep + manual | Runs flawfinder, clang-tidy, and semgrep over the module sources. | | **Harness Fault Arms** ([`harness-fault-arms.yml`](.github/workflows/harness-fault-arms.yml)) | PR and merged `master` via CI + manual, not required | Builds with the shared [`nginx-module-testkit`](https://github.com/myguard-labs/nginx-module-testkit) and runs all six fault, allocation, codec-count, and parameter-count scenarios through one non-vacuous scenario runner. | | **Windows build** ([`windows-build.yml`](.github/workflows/windows-build.yml)) | PR via CI + manual | Builds and smoke-checks MSVC x64 static and MinGW-w64 x64 dynamic modules. | | **CI Deep** ([`ci-deep.yml`](.github/workflows/ci-deep.yml)) | weekly + manual | Runs four explicit chains: two long fuzz targets; Memcheck then coverage, CodeQL, serialized nginx/stable/Angie builds, scanners, and native A/UBSan; and all six testkit scenarios under Memcheck followed by Helgrind. Coverage includes all Test::Nginx suites, broad assertion-bearing runtime drivers, and all testkit scenarios, with an 85% line floor. | | **Fuzzing** ([`fuzzing.yml`](.github/workflows/fuzzing.yml)) | manual | Provides targeted short runs. CI Deep runs its own long jobs for both parser targets. See [`ci/fuzz/README.md`](ci/fuzz/README.md). | | **Valgrind** ([`valgrind.yml`](.github/workflows/valgrind.yml)) | manual | Provides a targeted Memcheck-lite run. CI Deep runs its own full Memcheck and Helgrind soaks. | | **CodeQL** ([`codeql.yml`](.github/workflows/codeql.yml)) | manual/reusable; weekly via CI Deep | Runs GitHub's semantic C/C++ `security-extended` analysis. | | **A/UBSan** ([`asan.yml`](.github/workflows/asan.yml)) | manual/reusable; weekly via CI Deep | Runs a native mixed-load ASAN/UBSAN soak, complementing Build&Test's sanitizer suite. | | **Bump** ([`bump.yml`](.github/workflows/bump.yml)) | weekly + manual dispatch | Checks the GitHub release feeds (nginx, Angie, pcre2, zlib, OpenSSL, zstd) for newer releases than what's pinned — nginx-stable/Angie in CI Deep's `build-flavors` matrix, the harness jobs' mainline pin, and the Windows build's sources in [`ci/tools/windows-pins.sh`](ci/tools/windows-pins.sh) — and opens a PR (never pushes directly to protected `master`) with the updated pins + freshly-computed sha256 digests (nginx tarballs PGP-verified first). Does not gate merges itself — normal required checks review the PR. | The test suite includes a dedicated regression test for every known historical bug class: * infinite-loop / CPU-spin DoS on zero-length and sub-stream-size bodies; * the `proxy_buffering off` chunked-stream truncation ("bug B" — zero-size buffer forwarded to the writer), plus a ~504-cell compression-correctness matrix that decodes and byte-compares every transport × payload × encoding combination; * per-request `ZSTD_CCtx` isolation (one request's compressor state bleeding into another) and reload-under-load response correctness; * `$zstd_ratio` integer overflow on large bodies; * filter ordering vs `sub_filter`; negative compression levels; `zstd_types` parsing; `zstd_max_length` enforcement (known and chunked length); `zstd_window_log`; `zstd_long`/LDM; `zstd_bypass`; the pledged-source-size path; * `zstd_max_cctx_memory` rejects parameters that exceed the budget (config-load assertion); * `zstd_static` declines `.zst` files whose magic number is not a real zstd frame (defence-in-depth against truncated / mis-renamed files); * the multi-occurrence `Accept-Encoding` parser path (a header like `notzstd, zstd` must still negotiate zstd — covered by Perl tests *and* by continuous libFuzzer with ASAN/UBSAN over a NUL-free exact-size buffer); * the `zstd_dict_file` feature, long-URI `.zst` path handling, and the `ZSTD_CDict` config-reload leak. Run the suites locally: ```bash # Perl suites (needs Test::Nginx::Socket and a built nginx) TEST_NGINX_BINARY=/path/to/nginx prove \ ci/t/00-filter.t ci/t/01-static.t ci/t/02-conf-warn.t ci/t/03-dcz.t # Unit tests over the Accept-Encoding decision function ci/tests/unit/run.sh # End-to-end smoke tests python3 ci/tools/test_encoding.py --nginx-binary /path/to/nginx # Build and run the fuzzer (needs clang) bash ci/fuzz/build.sh && ./ci/fuzz/fuzz_accept_encoding -max_total_time=60 ci/fuzz/corpus/ ``` ## Repository layout Everything CI needs lives under `ci/`, so the repository root stays the module: sources, docs and packaging. ```text src/ the module itself ngx_http_zstd_filter_module.c the compression filter ngx_http_zstd_static_module.c the .zst static handler ngx_http_zstd_common.h shared helpers + the Accept-Encoding parser ci/ t/ Test::Nginx::Socket suites (00-filter, 01-static, …) tests/unit/ unit tests over the real decision TU tools/ soak.sh, test_encoding.py, benchmark.py, ci-build.sh, … fuzz/ libFuzzer target, corpus, dictionary, regressions linter/ the checker set run by the hook and the Lint workflow ast-grep/ structural rules .github/workflows/ ci.yml (the only pull_request entry point) + members .githooks/pre-commit the tracked local gate ``` The `Accept-Encoding` parser is sliced out of `src/ngx_http_zstd_common.h` into the fuzz and unit builds at build time by `ci/fuzz/extract_parser.sh`, so those targets always compile production code — there is no second copy to drift. ## Linting The same checkers run in three places: your commit (via `.githooks/pre-commit`), `ci/linter/run-all.sh` by hand, and the **Lint** workflow on every PR. ```sh git config core.hooksPath .githooks # enable the hook, once per clone ci/linter/install-linters.sh # install what the checkers need ci/linter/install-linters.sh --check # report presence; non-zero if any missing ci/linter/run-all.sh # run every checker over the tracked tree ``` Full description of each checker, what it is blind to, and how to narrow a run with `LINT_ONLY`: [`ci/linter/README.md`](ci/linter/README.md). Contributor setup and the `git ls-files` trap: [CONTRIBUTING.md](CONTRIBUTING.md). # Benchmarks Reproduce with `python3 ci/tools/benchmark.py` (drives the `zstd`/`gzip` CLIs linked against the same libzstd/zlib on the machine that runs it). Compression **ratio** for a given library/input/settings combination is stable across runs and machines, but not necessarily across libzstd versions — encoder heuristics change between releases while staying format-compatible, so expect the ratios below to shift somewhat on the recommended 1.5.7 rather than the 1.5.5 measured here. **Throughput** is not stable at all: it scales with CPU and varies run to run, so treat any MB/s figure as a rough range, not a point value. These numbers come from the CLI codecs directly: they exclude nginx's own filter, context, buffering, and flush overhead, so they are not a measurement of module throughput. Figures below: **libzstd 1.5.5**, single core, `--repeat 3`, best wall-time. For nginx request-path measurements, `ci/tools/ab_bench.py` keeps its existing plain-zstd workload by default and accepts `--workload dcz` for a focused RFC 9842 dictionary hit or miss. `ci/tools/perf_stat_recipe.py` runs the fixed 1, 4, and 16 configured-dictionary hit/miss matrix, records the response-encoding preflight and every successful measured request in JSON. Hardware counters attach only to the pinned nginx worker during measured release runs, excluding the Python harness, backend, wrk, startup, fixture creation, and cleanup; the comparison reports cache misses, references, instructions, and cycles per measured request. | Payload | Codec | Ratio | MB/s | |---|---|---:|---:| | HTML, 58 KB (test fixture) | gzip-6 | 15.5 | 29 | | | zstd-3 | 16.1 | 12 | | | zstd-19 | 17.1 | 0.8 | | JSON API, 256 KB | gzip-6 | 12.5 | 72 | | | zstd-1 | 43.8 | 52 | | | zstd-3 | 33.6 | 43 | | JS, 512 KB | gzip-6 | 12.7 | 108 | | | zstd-1 | 63.6 | 87 | | | zstd-3 | 40.8 | 78 | | Random 256 KB (incompressible) | gzip-6 | 1.00 | 31 | | | zstd-3 | 1.00 | 36 | Honest reading of these numbers: * On **small** payloads (the 58 KB HTML fixture), low-level zstd is roughly on par with `gzip -6` and a touch slower — gzip is well tuned for small text. zstd's advantage grows with payload size. * On **larger, structured** payloads zstd at a *low* level beats gzip decisively on **ratio** (e.g. ~44× vs ~12× on JSON, ~64× vs ~13× on JS). **Throughput is the opposite** in this table: gzip-6 CLI throughput is higher than zstd-1/zstd-3 on both JSON (72 MB/s vs 52 and 43) and JS (108 MB/s vs 87 and 78). Choose zstd here for the ratio and CPU-per-byte-stored win, not for raw CLI speed. For typical web traffic, `zstd_comp_level 1`–`3` is the sweet spot. * On the **incompressible** payload, zstd-3 is faster than gzip-6 (36 vs 31 MB/s) with both codecs at a 1.00 ratio — zstd detects incompressible input and gives up cheaply. * The synthetic JSON/JS generators are deliberately repetitive, so ratios there are inflated and *higher zstd levels show a lower ratio* — an artefact of trivially-redundant input, **not** representative of real assets. The HTML fixture (real-world content) shows the expected monotonic "higher level → better ratio, slower". * High levels (≥ 9) cost CPU steeply for marginal gain on web content — reserve them for infrequently-generated, cached responses. **How recent module changes affect these numbers.** The table above is driven by the `zstd`/`gzip` CLIs against the same libzstd, so it measures the *codec* — it is deliberately independent of nginx and does **not** move when the module's internals change. The compression **ratio** for a given level is therefore unchanged by any recent work. What changed is the module's per-response *overhead* inside nginx: * **Output buffers now default to `2 × ZSTD_CStreamOutSize()`** (previously a `4 × 32 KB` heuristic, and originally `32 × 4 KB`). Each `ZSTD_compressStream2()` call can now flush a complete internal block in one go instead of fragmenting it across calls, removing redundant compress round-trips and output-chain allocations per response. This shows up as lower CPU-per-response and less allocator churn under load — not as a different ratio or a different CLI MB/s figure. The trade is a higher per-request memory floor (~256 KB); see [`zstd_buffers`](#zstd_buffers). * **`$zstd_ratio` is computed by exact integer long division**, so it stays correct for byte counts near the 64-bit range and for streams that expand rather than shrink — a log-path cost only, no effect on the response itself. * **`zstd_long` (off by default)** can materially improve ratio on large, internally repetitive bodies that exceed the match window — but only when explicitly enabled, and the gain is workload-specific, so it is not reflected in the synthetic table above. Measure on your own assets before enabling. In short: the codec figures here are stable by design; the recent changes make the module *cheaper to run at the same ratio*, and add an opt-in ratio lever (`zstd_long`) for specific workloads. # Operations **Reloads (`nginx -s reload`).** Compression state is per request: a `ZSTD_CCtx` is created/reset per request and freed via an nginx pool cleanup. A graceful reload spins up new workers and drains old ones normally — in-flight responses on old workers finish on their existing context; new requests use new workers. There is no shared compression state to corrupt across a reload. The `zstd_dict_file` `ZSTD_CDict` is loaded once per cycle and freed on the old cycle's cleanup; a reload-leak regression for exactly this runs under ASAN in CI (`ci/tools/test_reload_leak.sh`). **`zstd_dict_file`.** Loaded at config load in the `http` context, into a `ZSTD_CDict` shared read-only by all workers (dictionary size capped at 10 MB). The dictionary must be readable by the nginx user at config load and reload. Changing it requires a reload. **Both ends must agree on the dictionary**: HTTP has no dictionary negotiation, so only use this where you control client and server (see the directive's warning). **Rollback.** The module adds no persistent state, on-disk format, or schema — it only transforms response bodies in memory. Rolling back is purely "load the previous `.so` / previous nginx binary and reload": 1. Keep the previously-known-good module `.so` (or full nginx binary). 2. To disable instantly without a binary change: set `zstd off;` (and `zstd_static off;`) and `nginx -s reload` — responses immediately serve identity; no client/cache corruption. The compressed and identity responses are different representations — the body bytes differ, not just the `Content-Encoding` field — and the automatic `Vary: Accept-Encoding` is what keeps each in its own cache variant, so a cache filled before the reload never hands a compressed body to a client that did not ask for one. 3. To revert the binary: restore the prior `.so`/binary, `nginx -t`, then `nginx -s reload`. No data migration, no irreversible step. A bad deploy is a one-line config change or a binary swap away from rolled back. **Pre-deploy soak.** `ci/tools/soak.sh ` drives sustained mixed load (tiny/medium/large/compressible payloads, zstd and non-zstd clients, the bypass path, a chunked upstream) and fails on any sanitizer report, leak, crash, `[alert]`/`[emerg]`, or corrupted response. Run it against an ASAN/UBSAN build (optionally `USE_VALGRIND=1`) before shipping a change. CI runs a 10-minute soak under ASAN+UBSAN on the weekly schedule (`Soak ASAN+UBSAN` job). # Security Compression of HTTP responses has a security dimension. See [`SECURITY.md`](SECURITY.md) for the vulnerability-disclosure process and the explicit in-scope / out-of-scope boundary (notably: BREACH containment is `zstd_bypass`, not a fix; CRIME/POODLE are TLS-layer and out of scope). The request parser is continuously fuzzed and the module is built and load-tested under ASAN/UBSAN. # Author Alex Zhang (张超) \, UPYUN Inc. Hardening, test suite, fuzzing and CI by Thijs Eilander and the [deb.myguard.nl](https://deb.myguard.nl/) maintainers. # License Licensed under the [BSD 2-Clause License](LICENSE). myguard-labs-nginx-zstd-module-3486b5b/SECURITY.md000066400000000000000000000064171525232545500216370ustar00rootroot00000000000000# Security Policy `zstd-nginx-module` runs inside the nginx/Angie worker process and operates on attacker-influenced input (the `Accept-Encoding` request header, response bodies of arbitrary upstreams). Security reports are taken seriously. ## Reporting a vulnerability **Do not open a public issue for security vulnerabilities.** Report privately via GitHub's [private vulnerability reporting](https://github.com/myguard-labs/nginx-zstd-module/security/advisories/new) ("Report a vulnerability" under the repository's Security tab). If that is unavailable, contact the maintainers at the address on the [deb.myguard.nl](https://deb.myguard.nl/) site. Please include: - Affected version / commit (`git rev-parse HEAD` or the release tag) - nginx or Angie version and libzstd version - A minimal reproduction: `nginx.conf` snippet + request, or a fuzz input / crash artifact - Observed vs. expected behaviour (crash, hang/CPU spin, memory growth, information disclosure, etc.) ## Scope In scope — issues in this module's code: - Memory safety: buffer overflow, use-after-free, uninitialised reads (the module is built and smoke-tested under ASAN/UBSAN) - Denial of service: worker crash, infinite loop / CPU spin, unbounded memory or input consumption - Request-parsing flaws in `ngx_http_zstd_accept_encoding()` (this function is continuously fuzzed) - Incorrect output that could mislead a client or cache (e.g. a corrupted compressed stream, missing `Vary`) Out of scope — not this module's layer: - **CRIME / POODLE** and other TLS-layer attacks — configure `ssl_protocols` / TLS appropriately; this module never sees TLS. - **BREACH** as a class — compression ratio is an inherent side channel; no HTTP compressor can be made BREACH-safe while compressing. The module provides `zstd_bypass` as a *containment lever* (exclude at-risk endpoints); the effective defence is application-layer (CSRF token masking, separating secrets from reflected input). A report that "compression leaks size" is not a vulnerability in this module. - Vulnerabilities in nginx, Angie, or libzstd themselves — report those upstream. - Misconfiguration (e.g. `zstd_static always` serving `.zst` to non-zstd clients) — documented behaviour, not a flaw. ## Handling - Acknowledgement target: within 7 days. - Triage and a fix or mitigation plan: as fast as severity warrants; worker-crash / RCE / DoS are prioritised. - Fixes land with a regression test (every historical security bug class in this module has a dedicated test) and, where applicable, a new fuzz seed. - Coordinated disclosure: a fix is prepared before public detail; the reporter is credited unless they prefer otherwise. ## Verifying a build Security-relevant CI runs on every change and is reproducible locally: ```bash # Memory safety # (build nginx with -fsanitize=address,undefined, then:) python3 ci/tools/test_encoding.py --nginx-binary /path/to/asan-nginx bash ci/tools/test_reload_leak.sh /path/to/asan-nginx # Fuzz the request parser bash ci/fuzz/build.sh && ./ci/fuzz/fuzz_accept_encoding -max_total_time=60 ci/fuzz/corpus/ ``` See [`.github/workflows/`](.github/workflows/) (build-test, codeql, security-scanners, fuzzing, valgrind, ci-deep, bump) and the [Testing & CI](README.md#testing--ci) section of the README. myguard-labs-nginx-zstd-module-3486b5b/auto/000077500000000000000000000000001525232545500210065ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/auto/zstd000066400000000000000000000246411525232545500217240ustar00rootroot00000000000000# auto/zstd — shared libzstd discovery for the zstd filter and static modules. # # Sourced exactly once from the top-level addon config before either module is # declared, so the static/dynamic/pkg-config probe sequence lives in one place # instead of being duplicated (and silently drifting) between filter/config and # static/config. Exports: # ngx_zstd_opt_I — compiler include FLAGS for (feature probes # and other flag contexts only) # ngx_zstd_incs — bare include DIRECTORY for ngx_module_incs: nginx's # auto/make prefixes every whitespace token of a # module's incs with the compiler's include option, so # flags placed there come out mangled ("-I -I/path # -I -DFOO"). gcc swallows that silently (the second # token becomes a bogus directory argument) and falls # back to system headers — masking version-mismatched # builds — while MSVC fails loudly with C1083, which # is how this was found. # ngx_zstd_opt_L — linker flags for libzstd # # The link flags flow ONLY through each module's ngx_module_libs (see # filter/config and static/config): nginx's auto/module routes those to # CORE_LIBS for a static (--add-module) build and to the module's own # link line for a dynamic (--add-dynamic-module) build. They must NOT be # appended to NGX_LD_OPT — that variable is global to the whole build, so # every OTHER dynamic module in the same nginx configure (e.g. an # ngx_brotli pair) would pick up a spurious DT_NEEDED on libzstd, which # surfaces as dependency bloat in distro packaging (this module's .so # requirements leaking onto unrelated module packages). ngx_feature_incs="#include " # ZSTD_compressStream2() (and ZSTD_CCtx_reset(), used unconditionally by the # module) establish a hard libzstd 1.4.0 floor. Enforce it at the feature # probe so an older library fails configure with the not-found error below # rather than failing the compile deep in the module sources. ngx_feature_test=" #if ZSTD_VERSION_NUMBER < 10400 #error libzstd 1.4.0 or newer is required #endif (void) ZSTD_compressStream2;" ngx_feature_libs= ngx_feature_run=no ngx_zstd_opt_I= ngx_zstd_incs= ngx_zstd_opt_L= if [ -n "$ZSTD_INC" ] || [ -n "$ZSTD_LIB" ]; then if [ "$NGX_CC_NAME" = "msvc" ] && command -v cygpath > /dev/null 2>&1; then # cl is a native Windows program: it reads an MSYS-style # /c/users/... path as \c\users\... on the current drive and # fails with C1083, and the MSYS shell does not rewrite paths # that nmake passes to cl out of the generated Makefile. # Normalize absolute POSIX spellings to the mixed C:/... form # so ZSTD_INC/ZSTD_LIB work however the operator typed them # (relative paths are left alone — both toolchains take those). case "$ZSTD_INC" in /*) ZSTD_INC=`cygpath -m "$ZSTD_INC"` ;; esac case "$ZSTD_LIB" in /*) ZSTD_LIB=`cygpath -m "$ZSTD_LIB"` ;; esac fi ngx_feature="ZStandard static library in $ZSTD_INC and $ZSTD_LIB" ngx_feature_path=$ZSTD_INC # The static archive's name is toolchain-specific: cmake's MSVC # build of libzstd emits zstd_static.lib (with ZSTD_BUILD_SHARED # =OFF), and cl/link accept it as a full path exactly like the # full-path libzstd.a below — -I and -D are cl-compatible too, so # this whole branch works under NGX_CC_NAME=msvc with only the # name swapped. (The gcc-spelled -L/-l dynamic retry further down # is not: cl drops those flags with a D9002 warning, so on msvc a # missing static archive falls through to the clear error message # rather than a silent half-link.) if [ "$NGX_CC_NAME" = "msvc" ]; then ngx_zstd_static_archive="zstd_static.lib" else ngx_zstd_static_archive="libzstd.a" fi # we try the static library first. The define is only a candidate here: # it rides ngx_zstd_opt_I for the probe, and is committed to CFLAGS (a # flag context every source file sees) ONLY once the static archive is # actually confirmed below. If it were added to CFLAGS unconditionally, # a subsequent dynamic-library fallback would compile against # ZSTDLIB_STATIC_API declarations while linking a shared object whose # ABI does not guarantee them -- see the dynamic branch's comment. ngx_zstd_opt_I="-I$ZSTD_INC -DZSTD_STATIC_LINKING_ONLY" ngx_zstd_incs="$ZSTD_INC" ngx_zstd_opt_L="$ZSTD_LIB/$ngx_zstd_static_archive" SAVED_CC_TEST_FLAGS=$CC_TEST_FLAGS CC_TEST_FLAGS="$ngx_zstd_opt_I $CC_TEST_FLAGS" SAVED_NGX_TEST_LD_OPT=$NGX_TEST_LD_OPT NGX_TEST_LD_OPT="$ngx_zstd_opt_L $NGX_TEST_LD_OPT" . auto/feature # restore CC_TEST_FLAGS=$SAVED_CC_TEST_FLAGS NGX_TEST_LD_OPT=$SAVED_NGX_TEST_LD_OPT if [ $ngx_found = yes ]; then # Static archive confirmed: commit the define to CFLAGS so every # source file compiles with the experimental API visible, matching # what was just probed and linked. CFLAGS="$CFLAGS -DZSTD_STATIC_LINKING_ONLY" else # then try the dynamic shared library. Do NOT carry # -DZSTD_STATIC_LINKING_ONLY into this probe or into CFLAGS: the # experimental (ZSTDLIB_STATIC_API) section has no stable ABI # guarantee, so a module built against it but linked dynamically # would import symbols (ZSTD_createCCtxParams and friends) that a # future libzstd.so is free to remove or reshape. ngx_zstd_opt_I is # reset to plain -I so downstream consumers (ngx_module_incs, the # version-probe below) see the dynamic build's real flags. ngx_feature="ZStandard dynamic library in $ZSTD_INC and $ZSTD_LIB" ngx_zstd_opt_I="-I$ZSTD_INC" if [ "${NGX_PLATFORM:-}" = win32 ]; then # MinGW produces PE/COFF modules. Its linker has no ELF runtime # search path to encode, and --enable-new-dtags/-rpath are either # rejected or ignored depending on the binutils version. The DLL # must be beside nginx.exe or otherwise on Windows' DLL search # path at runtime. ngx_zstd_opt_L="-L$ZSTD_LIB -lzstd" else ngx_zstd_opt_L="-L$ZSTD_LIB -lzstd -Wl,-rpath,$ZSTD_LIB" fi SAVED_CC_TEST_FLAGS=$CC_TEST_FLAGS CC_TEST_FLAGS="$ngx_zstd_opt_I $CC_TEST_FLAGS" SAVED_NGX_TEST_LD_OPT=$NGX_TEST_LD_OPT NGX_TEST_LD_OPT="$ngx_zstd_opt_L $NGX_TEST_LD_OPT" . auto/feature # restore CC_TEST_FLAGS=$SAVED_CC_TEST_FLAGS NGX_TEST_LD_OPT=$SAVED_NGX_TEST_LD_OPT if [ $ngx_found = no ]; then cat << END $0: error: the ngx_http_zstd modules require the ZStandard library, please be sure that "\$ZSTD_INC" and "\$ZSTD_LIB" are set correctly. END exit 1 fi fi else # auto-discovery: prefer dynamic linking (static libzstd.a lacks -fPIC # and cannot be linked into a shared .so dynamic module) ngx_feature="ZStandard dynamic library" ngx_zstd_opt_I= ngx_zstd_opt_L="-lzstd" SAVED_CC_TEST_FLAGS=$CC_TEST_FLAGS CC_TEST_FLAGS="$ngx_zstd_opt_I $CC_TEST_FLAGS" SAVED_NGX_TEST_LD_OPT=$NGX_TEST_LD_OPT NGX_TEST_LD_OPT="$ngx_zstd_opt_L $NGX_TEST_LD_OPT" . auto/feature # restore CC_TEST_FLAGS=$SAVED_CC_TEST_FLAGS NGX_TEST_LD_OPT=$SAVED_NGX_TEST_LD_OPT if [ $ngx_found = no ]; then # Dynamic auto-discovery failed. Try pkg-config (portable across # Linux, FreeBSD, OpenBSD, Gentoo, Rocky/RHEL) before giving up. # Note: -l:libzstd.a is GNU ld only and not used here. ngx_pkgconf= if command -v pkgconf > /dev/null 2>&1; then ngx_pkgconf=pkgconf elif command -v pkg-config > /dev/null 2>&1; then ngx_pkgconf=pkg-config fi if [ -n "$ngx_pkgconf" ] && $ngx_pkgconf --exists libzstd 2>/dev/null; then ngx_feature="ZStandard library via pkg-config" ngx_zstd_opt_I="$($ngx_pkgconf --cflags libzstd)" # bare directories for ngx_module_incs (usually empty: a # system libzstd needs no -I at all) ngx_zstd_incs="$($ngx_pkgconf --cflags-only-I libzstd \ | sed -e 's/-I *//g')" ngx_zstd_opt_L="$($ngx_pkgconf --libs libzstd)" SAVED_CC_TEST_FLAGS=$CC_TEST_FLAGS CC_TEST_FLAGS="$ngx_zstd_opt_I $CC_TEST_FLAGS" SAVED_NGX_TEST_LD_OPT=$NGX_TEST_LD_OPT NGX_TEST_LD_OPT="$ngx_zstd_opt_L $NGX_TEST_LD_OPT" . auto/feature # restore CC_TEST_FLAGS=$SAVED_CC_TEST_FLAGS NGX_TEST_LD_OPT=$SAVED_NGX_TEST_LD_OPT fi if [ $ngx_found = no ]; then cat << END $0: error: the ngx_http_zstd modules require the ZStandard library. Install libzstd-dev (Debian/Ubuntu), zstd-devel (RHEL/Rocky), dev-libs/zstd (Gentoo), or security/zstd (FreeBSD/OpenBSD ports), or set ZSTD_INC and ZSTD_LIB to the library location. END exit 1 fi fi fi # Advisory: report the detected zstd version when a directive's full # functionality needs a newer library than the 1.4.0 floor enforced above. # The module always uses ZSTD_CCtx_reset() + ZSTD_compressStream2() (both # 1.4.0); there is no ZSTD_initCStream_usingCDict() fallback path. The only # version-gated directive is zstd_target_cblock_size (libzstd >= 1.5.6). # # Ask the compiler rather than trying to parse preprocessor output. The # latter is compiler-specific (MSVC does not accept GCC's language-selection # and preprocessing flags), # while auto/feature already knows how to invoke the configured toolchain. ngx_feature="libzstd 1.5.6 or newer" ngx_feature_name= ngx_feature_run=no ngx_feature_incs="#include " ngx_feature_path= ngx_feature_test=" #if ZSTD_VERSION_NUMBER < 10506 #error libzstd older than 1.5.6 #endif (void) ZSTD_versionNumber()" ngx_feature_libs="$ngx_zstd_opt_L" SAVED_CC_TEST_FLAGS=$CC_TEST_FLAGS CC_TEST_FLAGS="$ngx_zstd_opt_I $CC_TEST_FLAGS" . auto/feature CC_TEST_FLAGS=$SAVED_CC_TEST_FLAGS if [ "$ngx_found" = no ]; then cat << END Note: libzstd older than 1.5.6 detected. "zstd_target_cblock_size" requires libzstd >= 1.5.6 and will have no effect at runtime on this build. Every other directive is fully functional. Upgrade libzstd to 1.5.6+ for that directive. END fi myguard-labs-nginx-zstd-module-3486b5b/ci/000077500000000000000000000000001525232545500204315ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/.adopted000066400000000000000000000020771525232545500220600ustar00rootroot00000000000000prompt_version: 241a7ac6f664bf98324f085489ed41a7f67eae56bf0ff500507ec84bb9f2f4eb skeleton_sha: 537f28ea996e6e540cebff486a9e8fff824b6918 adopted_date: 2026-08-22 steps_degraded: none # step 30 was recorded degraded at adoption: valgrind.suppress was the generic # circulated nginx file (28 unfilled blocks # naming ngx_http_lua_* and drizzle_state_connect, symbols this module never # links) and could mask this module's own errors. # # CLEARED at step 41 (2026-08-22). The defect narrowed to one block -- # Memcheck:Cond over obj:*, which muted the entire use-of-uninitialised-value # class in any object including ours. Measured: a program reading one # uninitialised int exits 99 without suppressions and 0 with that block alone. # Replaced with an empty file carrying the derivation rules; both memcheck and # helgrind accept it and the muted error is visible again. # Detail: memory/labs/http-zstd/issues.md, ci/adoption-findings.md section 5. # Still reported upstream in skeleton PR #41 -- the skeleton ships the same # file to every adopter. myguard-labs-nginx-zstd-module-3486b5b/ci/adoption-findings.md000066400000000000000000000741721525232545500244020ustar00rootroot00000000000000# Adoption findings (PR2, phase 5) Findings from steps 21-26 that are not simply "done" — either a mutation result worth recording, or something the module already had that diverges from what the phase asked for. Feeds step 33. ## Step 21 — unit tests over the real decision TU `ci/tests/unit/run.sh` + `test_accept_encoding.c`. Unlike the skeleton's scan core, the Accept-Encoding parser in `src/ngx_http_zstd_common.h` is pure, self-contained ASCII walking that needs only `ngx_strncasecmp()` / `ngx_strcasestrn()` — `ci/fuzz/ngx_shim.h` already supplies faithful copies of those, so this layer reuses the shim rather than linking a full nginx build tree's real `ngx_string.c`. That is a deliberate divergence from the skeleton's convention (which links the real upstream TU because its scan core calls `ngx_unescape_uri()`), not an oversight: this module's parser has no such dependency, and the shim's own header cites the exact upstream functions it copies. `run.sh` still regenerates `ci/fuzz/generated_parser.inc` from the shipped header via `extract_parser.sh` before every build, so the shipped parser is what gets linked, never a hand copy. 24 checks, all green: ```text $ bash ci/tests/unit/run.sh 2>&1 | tail -3 ok an unterminated quoted parameter value does not hang the walk and does not fabricate a matching zstd token 24/24 checks passed ``` ### Mutation pass: Accept-Encoding parser All 5 applied to `src/ngx_http_zstd_common.h`, observed, reverted — tree diffed clean against a pre-mutation backup after each. 1. **Wildcard/explicit precedence flipped** — swapped the order of the two `if` blocks at the end of `ngx_http_zstd_coding_weight()` so `star_q` is checked before `coding_q`. Command: manual `perl -0pi` swap, then `bash ci/tests/unit/run.sh`. Result: `22/24 checks passed` — `FAIL explicit 'zstd;q=0' overrides a permissive '*'`, `FAIL explicit 'zstd' overrides a restrictive '*;q=0'`. Reverted; `diff` against backup identical; re-run confirmed `24/24`. 2. **`scale /= 10` removed** from the fractional-digit loop in `ngx_http_zstd_eval_qvalue()` (line ~153) — every digit after the decimal point is now weighted 100 instead of decaying ×10 per digit, which also breaks the implicit 3-digit cap the `scale > 0` guard enforces. Result: `23/24 checks passed` — `FAIL a fourth q decimal digit makes the element non-matching`. (Originally expected this to move the 2-digit `q=0.05` case instead; it did not, because 2 digits at weight 100 each still lands >0 either way. Corrected the code comment in `test_accept_encoding.c` to describe the actually-observed failure rather than the originally-assumed one — see "a guess phrased like a command result gets acted on as fact".) 3. **Malformed-weight fallback to q=1** — inserted `if (q < 0) { q = 1000; }` before the `if (q >= 0)` block in `ngx_http_zstd_coding_weight()`, so a malformed `;q=` no longer makes the element non-matching. Result: `20/24 checks passed` — 4 checks fail (fourth-digit, missing value, bare `q`, end-of-buffer `q=`). 4. **`ngx_http_zstd_skip_quoted()` disabled** — `return p;` inserted before its own bounds check, unconditionally. Result: **the suite did not FAIL, it HUNG.** `case_skip_quoted_unterminated`'s unterminated `"..."` value makes the caller's element-skip loop (`while (*p != ',') { if (*p=='"') p = skip_quoted(p,end); else p++; }`) call the now-no-op `skip_quoted()` at the same `p` forever. Command: built to a distinct binary, ran under `timeout 5`. `$ timeout 5 /tmp/t2; echo exit=$?` → `exit=124`. This is exactly why `case_skip_quoted_unterminated` exists — a plain pass/fail assertion cannot distinguish "quote skip is a no-op" from "quote skip is correct but slow"; only a bounded run can. Recorded the observed hang, not the originally-assumed FAIL, in the test's own header comment. 5. **Off-by-one in the parameter-name scan** — `p < end` → `p <= end` in the name-scan loop inside `ngx_http_zstd_eval_qvalue()` (line 104). Ran under `clang -fsanitize=address,undefined` as well as plain `cc`. Result: `23/24 checks passed` — `FAIL 'q' with no '=value' makes the element non-matching, not an implicit q=1`. **No ASan trap fired** — the outer `while (p < end && *p == ';')` guard means the widened inner condition is only ever reached with `p == end` already inside the caller's bound, so this specific off-by-one is a logic bug, not a memory -safety one. Recorded rather than assumed: "off by one" does not automatically imply "overread", and asserting it would would have been an unverified claim. All 5 mutations caught (none SURVIVED per rejected-test item 8); each tripped a different, correctly-attributed check (no rejected-test item 4 shared-counter false-positive — verified per-mutation which named checks flipped, not just the aggregate count). ## Step 22 — live-server layer The module already has its own mature live-server driver convention: separate, purpose-named scripts under `ci/tools/` (`test_reload_under_load.py`, `test_concurrent_cctx_isolation.py`, `test_slow_drain.py`, `test_dcz.py`, `test_compression_matrix.py`, ...) rather than the skeleton's single `ci/tools/test_runtime.py` driver file. Per "adopt the convention, keep the content", this session kept the module's own multi-script convention rather than collapsing it into one `test_runtime.py` — the reviewed reason (step 22's "what belongs here" line) is about *placement* (driver vs `ci/t/`), and every existing script here already only does what the driver's three exemptions allow (concurrency, a signal at the master mid-traffic, or N separate own-connection requests), so nothing needed to move. **Gap found and closed: no baseline loaded-and-blocking control.** None of the existing driver scripts prove the module is actually acting — they assert response *correctness* under a stressor, which a module that failed to load, or whose `enable` predicate silently decided OFF, could still satisfy by accident (an unmodified passthrough is still byte-correct). Added `ci/tools/test_baseline_loaded.py`: two live requests against the same backend, one with `zstd on;` (must compress, decode byte-exact), one with NO zstd directive at all (must NOT be zstd-encoded — the compiled-in default). Wired into `build-test.yml`'s `tests-asan` job (`--port 18107 --backend-port 18108 --off-port 18112`, ports checked against every existing `--port`/`--backend-port` literal in the file to avoid a collision) because that job's binary is the one static (`--add-module`) build already produced in this pipeline (asan.yml's own build), and the "no directive" case needs a build where the module is compiled in but not turned on by config — a dynamic `load_module`-based build cannot express that distinction cleanly (no directive → unknown directive, config fails to parse, which the module- unloaded case already covers via a different binary). ### Mutation pass: Module loading behavior Ran locally against a hand-built static `--add-module` nginx (not part of the fast PR lane; the CI wiring above uses the pipeline's own asan.yml artifact instead). 1. **Module genuinely unloaded** — built the SAME nginx tree with no `--add-module` at all, ran the baseline script against it. `$ python3 ci/tools/test_baseline_loaded.py --nginx-binary /tmp/nginx-no-module` Result: `RuntimeError: nothing listening on 127.0.0.1:18110` (nginx fails to start at all — `zstd on;` is an unknown directive) — the script itself raises rather than silently passing. Confirms the baseline genuinely depends on the module being present. 2. **`enable` default flipped ON** — `src/ngx_http_zstd_filter_module.c` line 1919, `ngx_conf_merge_value(conf->enable, prev->enable, 0)` → `..., 1)`. Rebuilt (`objs/nginx`, distinct mtime/size verified against the pre-mutation binary each time — see "check artifact mtime before trusting a stack trace"), ran against a build with the mutation and a build without. Unmutated: `2/2 checks passed`, exit 0. Mutated: `FAIL baseline: response was zstd-encoded with NO zstd directive present -- compiled-in default is not OFF`, exit 1. **First attempt at this mutation test silently passed 2/2 against the mutated binary** — the script's own `--off-port` originally defaulted to `args.port + 1`, which collided with `--backend-port`'s default (18111 == 18110+1), so the "no directive" nginx failed to bind and the client request landed directly on the backend instead, which is trivially never zstd-encoded regardless of the mutation. Fixed by giving the off-case its own `--off-port` argument, distinct from both `--port` and `--backend-port`; re-ran and got the FAIL above. Left as a comment in the script at the call site so the next port choice does not repeat it. This is exactly the class of finding rejected-test item 4 (a shared resource that lets one check's assertion satisfy an unrelated one) generalizes to outside pure counters — recorded here rather than shipping the false-negative test. Reverted both source and binaries after; `diff` against the pre-mutation `.c` backup confirmed clean before committing. ## Step 25 note Out of my 21-26 slice, but investigated because it's adjacent to step 24's soak evidence and the finding is severe. **CONFIRMED, not just suspected: `valgrind.suppress` (then at the repo root; now `ci/suppressions/valgrind.suppress`) is a copied, generic file, not one derived from this module.** First pass (during step 22) searched for `valgrind.supp` (the skeleton's filename) and found nothing, and wrongly concluded no suppression file existed at all. It does — `ci/tools/soak.sh` then referenced `$(repo root)/valgrind.suppress` (note the different name, `.suppress` not `.supp`) — correcting that earlier miss here per "a guess phrased like a command result gets acted on as fact" / "a grep miss proves the spelling absent, not the gap real" (searched the wrong spelling, took the miss as proof of absence). Read the file (218 lines, 28 suppression blocks): **every single block is still named ``** (the valgrind `--gen-suppressions` template placeholder, never filled in), and at least two blocks reference symbols that do not exist anywhere in this build — `fun:ngx_http_lua_ndk_set_var_get` (this module does not vendor or depend on ngx_http_lua_module) and `fun:drizzle_state_connect` (a MySQL/Drizzle upstream module, also absent). This is the well-known generic nginx valgrind suppression file that circulates online, not a suppression set derived from `USE_VALGRIND=1 ci/tools/soak.sh `. It is exactly the "wrong tree" trap this phase's front matter names: a copied `valgrind.supp`/`.suppress` can suppress the module's OWN errors, and this one's `Memcheck:Leak` / `Memcheck:Cond` blocks are broad enough (bare `fun:ngx_alloc` / `fun:memcpy` frames with no caller chain reaching this module) that a real leak or uninitialized-read bug inside `ngx_http_zstd_*` could plausibly match one of them by accident, silently passing the gate. **Did not patch this.** Fixing it correctly requires a real `USE_VALGRIND=1 ci/tools/soak.sh 60 4` run against this module's own static build, per the file's own header instructions on how a suppression must be derived (or the skeleton's `valgrind.supp` header, which states the same discipline) — building nginx from source plus a 60s Valgrind soak is a build+soak long-runner, explicitly forbidden for a worker session to run locally, and blindly deleting or rewriting 28 suppression entries without that real run risks either reintroducing false leak noise (deleting the entries that ARE legitimate nginx-core lifetime allocations) or leaving the module under-suppressed against upstream noise that then reads as a module regression. **This is a MUST-FIX finding for whoever runs step 25/30 properly, ranked above the codeql.yml/ci-deep.yml items** (those two were checked and are already correctly target-scoped — see below): run the real soak, name every surviving suppression after the actual nginx-core call chain it matches (per the file's own documented convention), delete every block whose symbol does not appear in this build's call graph at all (the two confirmed above, and any others found the same way), and verify `valgrind --suppressions=ci/suppressions/valgrind.suppress ...`'s `ERROR SUMMARY` line shows nonzero suppressed count per surviving block (a suppression matching nothing is dead weight, same standard the skeleton's own header states). **codeql.yml and ci-deep.yml, checked in the same pass, are already correct** — no finding: `codeql.yml`'s `paths:` is `filter`, `static`, `*.c`, `*.h` with `paths-ignore: nginx-*/**` (module TU only, not copied from the skeleton), and `ci-deep.yml`'s matrix names this module's own real flavor/version triples (nginx 1.31.2, 1.30.4, angie 1.12.1), not the reference's. ## Step 23 — fuzz target, corpus and dictionary The fuzz target (`ci/fuzz/fuzz_accept_encoding.c`) already satisfied the hard requirement before this session: it links the real `ngx_http_zstd_accept_encoding()` / `ngx_http_zstd_eval_qvalue()` / `ngx_http_zstd_coding_weight()` via `extract_parser.sh` (never a reimplementation), carries a 9041-file curated corpus, and a `fuzz.dict` already built from the module's own real Accept-Encoding grammar (not the skeleton's tokens). It also already has an independent semantic-differential oracle (`ref_accepts()`), which is stronger than this phase's baseline ask. **Gap found and closed: `"dcz"` was missing from `fuzz.dict`.** The module has no single lookup table of coding names (unlike the skeleton's rule table) — the tokens are two literal call-site arguments to `ngx_http_zstd_coding_weight(ae, "TOKEN", ...)`, one in `src/ngx_http_zstd_filter_module.c` ("dcz") and one in `src/ngx_http_zstd_common.h`'s own wrapper ("zstd"). Added `ci/fuzz/gen_dict.sh`, which greps those exact call sites (both `.c` and `.h` — an earlier version of the script only scanned `.c` and missed "zstd", caught immediately by running `--check` against the tree; see the commands below) and replaces a marked block in `fuzz.dict` in place. Wired `gen_dict.sh --check` into `ci/linter/lint-c.sh`. ```text $ bash ci/fuzz/gen_dict.sh --check ✓ fuzz.dict coding tokens match src/*.c call sites: "dcz" "zstd" $ sed -i 's/"dcz"/"xyz"/' ci/fuzz/fuzz.dict && bash ci/fuzz/gen_dict.sh --check; echo exit=$? ✗ fuzz.dict's generated coding-token block is stale. Source call sites: "dcz" "zstd" Run: ci/fuzz/gen_dict.sh exit=1 # restored, re-checked clean $ bash ci/linter/lint-c.sh # with the same drift reintroduced fuzz.dict coding-token drift (ci/fuzz/gen_dict.sh --check) ✗ fuzz.dict's generated coding-token block is stale. # (rc=1, confirmed the linter gate actually fails on this) ``` Did not attempt to measure a coverage-vs-signature-reach delta the way the reference module's own step 23 evidence does (PROMPT.md cites a 23→35/645 table-literal reach figure) — this module's "table" is two literals, not 645, so that specific metric does not apply; noted here so nobody goes looking for a coverage number that was never produced. ## Step 24 — replay order and the ASan soak **Replay-then-fuzz order.** Added `ci/fuzz/regressions/` (was entirely missing) and a "Replay recorded crash regressions" step in `fuzzing.yml`, before the time-boxed fresh run, with a `nullglob` guard so an empty directory reports its case count rather than silently matching nothing forever. No fuzz-discovered crash existed to seed it with (a bounded local run — 693576 execs in 10s — found none against the current corpus/dict, which is itself a proof-of-clean, not a gap). Seeded one synthetic case, `crash-wildcard-precedence-flip` (`*, zstd;q=0`), from mutation 1 of step 21's mutation pass (wildcard/explicit precedence swap), which trips the fuzz target's own semantic-differential oracle: ```text # mutated ngx_http_zstd_coding_weight() (precedence swapped): $ ./fuzz_target_mutant /tmp/crash-wildcard-precedence-flip ==...== ERROR: libFuzzer: deadly signal ... LLVMFuzzerTestOneInput fuzz_accept_encoding.c:281 ... exit=77 # reverted to the real parser, rebuilt (distinct binary, verified same # source diffed clean against the pre-mutation backup before commit): $ ./fuzz_target_clean /tmp/crash-wildcard-precedence-flip Executed .../crash-wildcard-precedence-flip in 0 ms exit=0 ``` Then ran it through the actual wired CI step's own command (`shopt -s nullglob; cases=(regressions/crash-*); ./fuzz_accept_encoding "${cases[@]}"`) against the real build — 0ms, exit 0, matching the manual run above. This is the "deliberately reintroduced past bug caught in seconds" acceptance criterion, verified once then reverted, per the step's own instruction. **ASan soak reaches the module, with evidence.** `ci/tools/soak.sh`'s `worker()` requires `ok > 0` and `bad == 0` per worker, where `bad` is incremented both on a failed request AND on a response that comes back zstd-magic-prefixed but fails to `zstd -d` cleanly — so a green soak proves real compressed responses were served and decoded correctly under the sanitizer, not merely that nginx did not crash. `asan.yml`'s own comment overstated this as "asserts it saw >=1 block AND >=1 pass" (no such block/pass counter exists in soak.sh); corrected the comment to describe the actual mechanism instead of inventing an assertion that was not there — small, but exactly the kind of comment-vs-code drift the green-that-proves-nothing checklist warns about, in this case in the DOCUMENTATION rather than the check itself, so it was worth fixing on sight without expanding scope to touch soak.sh's logic (which was already correct). Did not run a local ASan soak — the build (~1-2 min) plus soak (60s+) crosses into long-runner territory for a worker session; the mechanism was verified by reading soak.sh's actual assertions, not by executing them locally. Flagged for whoever next runs `asan.yml` in CI to confirm the corrected comment matches the real log output. ## Stopped here Landed: steps 21, 22, 23, 24 fully, plus a confirmed (not just suspected) step 25 finding on the then-root `valgrind.suppress` — read and diagnosed, deliberately NOT patched (needs a real soak run, a long-runner). codeql.yml and ci-deep.yml (the other two of step 25's "three neighbours") were checked in the same pass and are already correct, no finding. Step 26 (coverage.sh + `coverage` ci-build.sh mode) not started this session — it needs its own build-tree work (a distinct instrumented build, not a flag on `debug`) that did not fit in the remaining budget at the standard this phase requires. See the worker return banner for exact scope remaining. ## Step 27 — the security-scanners gate was proven able to fail (supervisor, 2026-08-22) Verified rather than assumed, because the whole finding was that this gate could not fail before the fix. A `strcpy` into an 8-byte stack buffer appended to `src/ngx_http_zstd_filter_module.c`: ```sh set -o pipefail flawfinder --minlevel=4 --error-level=4 \ src/ngx_http_zstd_filter_module.c src/ngx_http_zstd_static_module.c -> exit 1 ``` Restored immediately; `git status src/` clean afterwards. Before this branch the same probe exited **0**: `--minlevel` only controls what is PRINTED, and without `--error-level` flawfinder always exits 0 however much it finds. The `| tee flawfinder.log` pipe would also have swallowed the status, so `set -o pipefail` on that step is load-bearing, not decoration. Also re-verified after all of the cycle's workflow edits: `ci.yml` is still the only workflow carrying a `pull_request:` trigger. Checked by parsing each workflow's `on:` key with a YAML loader, not by grepping for the string -- `on:` parses as the boolean True in YAML, so a naive grep is not the same test. ## Step 30 — the depth audit: does each gate reach the module? (2026-08-22) Five subjects, each answered from configuration plus step 26's coverage report. No long run started by this step. ### 1. The seam — VERIFIED, with a positive control `UNIT_ENTRY` = `ci/tests/unit/run.sh`; fuzz entry = `ci/fuzz/build.sh`. Neither compiles `src/*.c` directly: both `#include` `ci/fuzz/generated_parser.inc`, which `ci/fuzz/extract_parser.sh` slices verbatim out of the shipped `src/ngx_http_zstd_common.h`. Both entry points regenerate it at build time (`ci/tests/unit/run.sh:60`, `ci/fuzz/build.sh:19`), so there is no hand-maintained copy to drift. Proven, not assumed: - regenerating in a clean tree reproduces the committed `.inc` byte for byte (no drift); - renaming `ngx_http_zstd_eval_qvalue()` in `src/ngx_http_zstd_common.h` and re-running the extractor propagates the change into the generated TU (`PROBE` present) — so an edit to production code really does reach the unit and fuzz builds. Source restored, regeneration re-diffed clean. **What it now catches that it did not before:** a parser change that compiles in `src/` but was never exercised — previously possible only if someone remembered to re-copy the parser by hand. ### 2. ASan/UBSan — the soak config reaches the module `ci/tools/soak.sh` enables the module's own directives in more than one location: `zstd on`, `zstd_min_length`, `zstd_max_length`, `zstd_types`, `zstd_bypass $arg_nozstd`, `zstd_window_log 21`, and drives `/bypass` and `/bypass?nozstd=1` plus clients that do not advertise zstd. Step 26's coverage over `src/` reports 60.0% — the handler's lines execute, and the figure is not ~1%, so nginx core is not padding the denominator. **Catches now:** a use-after-free or overflow on the bypass and Vary/no-zstd-client paths, not only on the happy compress path. ### 3. Fuzzing — surface and dictionary match the parse surface `fuzz.dict` is derived from real call sites and gated against drift by `gen_dict.sh --check`, which passes: coding tokens `"dcz"` and `"zstd"` match the `ngx_http_zstd_coding_weight()` call sites across `src/*.c` **and** `src/*.h` (gen_dict.sh:43 globs both — 2 call sites in `ngx_http_zstd_filter_module.c`, 2 in `ngx_http_zstd_common.h`, so a `.c`-only scan would miss the header's). The parser's table literals are `"*"`, `"identity"`, `"q"` and `"zstd"`; all are reachable — the qvalue surface has 5 dedicated dict entries (`";q="`, `"q=0"`, `"q=1"`, `"q=0.0"`, `"q=1.0"`) and 597 of the 9280 corpus inputs carry a `q=`. Two regression inputs replay before each fresh run. **Catches now:** qvalue arithmetic and precedence defects, which the mutation pass in step 24 confirmed the target detects. ### 4. Coverage — measured over the module only `ci/tools/coverage.sh` reports with gcovr filtered to `src/`; unfiltered gcovr walks the whole configured nginx tree. Measured 60.0%, so the filter is real — a ~1% figure would have meant nginx core was in the denominator. ### 5. Valgrind / helgrind **Memcheck: was not trustworthy as a gate — FIXED at step 41 (2026-08-22).** The former root `valgrind.suppress` (now `ci/suppressions/valgrind.suppress`) was the generic circulated nginx file: all 28 blocks had unedited `` placeholders, naming `ngx_http_lua_*` and `drizzle_state_connect`, symbols this module never links. It is live config, not dead — `ci/tools/soak.sh:71,79` passes it to both memcheck and helgrind. The `fun:malloc` / `fun:memcpy` blocks turned out to be **anchored** to full nginx call stacks (`ngx_alloc` → `ngx_set_environment` → …) and could not reach this module's frames. The real defect was a single block: ```text { Memcheck:Cond obj:* } ``` `Memcheck:Cond` + `obj:*` matches an uninitialised-value conditional in any object at any depth, muting that whole class — including in `ngx_http_zstd_filter_module.so`. Measured on a four-line C program reading one uninitialised int: without suppressions valgrind exits 99 with 1 error; with that block alone it exits 0 with none. So `Memcheck lite (60s soak)` was green by construction for the Cond class, however long it ran. Fixed by replacing the file with an empty one (option 1 of the two the ledger recorded) carrying the derivation rules — suppress nothing, hide nothing. Verified that both memcheck and helgrind accept a comment-only suppressions file and that the previously-muted error becomes visible again. Sent upstream (skeleton PR #41) because the skeleton ships the same file to every adopter. **Helgrind: NOT applicable, settled with evidence.** The module has no shared cross-worker state. Counted over `src/*.c` and `src/*.h`: `ngx_shared_memory_add` 0, `shm_zone` 0, `ngx_slab` 0, `ngx_atomic` 0, `ngx_thread` 0, `pthread` 0. The only file-scope mutable state is the two filter-chain pointers at `src/ngx_http_zstd_filter_module.c:215-216`, assigned once in `ngx_http_zstd_filter_init()` (`:2358` and `:2361`) — the `postconfiguration` hook, which runs in the master process before fork. All other state is per-request `ctx` off the request pool. There is no cross-thread sharing for helgrind to reason about. ## Step 31 — re-audit the gates that drift (2026-08-22) Six explicit answers. `/proc/loadavg` was 1.4-2.5 (1-min) on a 6-slot box for every timing below. **Caching.** `ci/tools/ci-build.sh` is the single chokepoint; no workflow duplicates cache logic. Measured in PR2: `NO_CACHE=1` cold build of nginx 1.31.2 -> ccache 0/146 (0.00%); a second identical build -> 146/292 (50.00%) cumulative, i.e. **100% of the second run's own compiles hit**. Not 0%, so it is wired. `--with-cc="ccache gcc"` is load-bearing — nginx's configure ignores a bare `CC=`. **zizmor.** `zizmor --pedantic .github/workflows/` audits **11** workflows; `ls .github/workflows/*.yml` is **11**. Counts match, no member unaudited. "No findings to report (3 ignored, 11 suppressed)". All three `# zizmor: ignore[misfeature]` still name a reason that is still true — each sits on a `shell: cmd` step in `windows-build.yml` (lines 82, 89, 112) that needs the environment `vcvars64.bat` installs in that same cmd process. No suppression has outlived its subject. **actionlint and the fromJSON ternary.** Acknowledged, not used as evidence: actionlint is clean, and that says nothing about runner labels. The selector is `${{ fromJSON(vars.POOL || '["ubuntu-latest"]') }}` on all 10 build-test jobs — the approved form, no inline pool, no fork ternary. That is probe 2's finding, not actionlint's. **LINT_ONLY still matches the checkers that exist.** Normalized comparison, both directions: ```text on disk (ci/linter/lint-*.sh): c ci-cadence ci-ports ci-runners ci-secrets docs-drift nginx perl python sh spelling yaml in lint.yml LINT_ONLY: nginx sh python perl yaml spelling ci-runners ci-ports ci-cadence ci-secrets docs-drift on disk but NOT in LINT_ONLY: c <- deliberate, documented lint.yml:9-12 in LINT_ONLY but NOT on disk: (empty) ``` `c` is excluded because flawfinder/clang-tidy/semgrep over `src/` are `security-scanners.yml`'s job at the same thresholds (PR2 paired them). No orphan in either direction. **`run-all.sh` reads `git ls-files`** (`ci/linter/lib.sh:50`), so an untracked file is invisible and the run reads green while checking nothing. Staged before every timing and lint run in this pass. The oracle is the checker's file COUNT, not the exit code. **Hook timing — OVER the ~2s budget on a C change.** Measured, 3 runs each: ```text docs-only staged change: 0.51 / 0.52 / 0.56 s OK real C file staged: 3.59 / 3.59 s OVER ``` The first C measurement (0.49s) was a false pass: the file was `touch`ed but unmodified, so pre-commit skipped its hooks. Forcing a real content change gives 3.59s. Per-hook breakdown with a C file staged: ```text semgrep 2.17s <- 60% of the total, alone over budget cppcheck 0.63s flawfinder 0.13s shellcheck 0.08s ruff 0.08s ``` Not fixed here: the cheap levers (`--jobs=1 --metrics=off`) are already set, and dropping or narrowing semgrep is a gate-weakening change, not a speed fix. A C-file commit paying 3.6s is the honest cost of the current gate. Ledgered. ## Step 32 — re-measure the CI shape (2026-08-22) Numbers from `gh run list`/`gh run view` on run **32547163515** (PR2's final run, 2026-08-22), not estimates. **Total wall-clock 370s, 18 jobs.** Critical path is `resolve(6s) -> build(212s) -> tests(145s)` = 370s. Longest single legs: CodeQL 266s, MinGW 254s, build 212s, validation 215s, tests-asan 180s. **Exactly one `pull_request:` entry point holds.** Parsed with a YAML loader, not a grep — `on:` parses as boolean `True` in YAML, so a grep is a different test. `ci.yml` is the only one; no member carries `push:`. Step 14's demotion therefore still holds, and step 16's long-runner greps return `ci.yml` alone. **Every member is reached.** ci.yml calls 8 members (lint, build-test, security-scanners, codeql, fuzzing, valgrind, asan, windows-build); the set of workflows declaring `workflow_call` minus the set ci.yml calls is **empty**, so no member keeps a stale-green badge. **Peak concurrency 15 against 6 slots at t+21s** — measured, and worse than the 11 recorded at step 29. Two causes: 1. **The step 29 lane fix for `build-old-libzstd` was never actually in the tree.** `adoption-findings.md` recorded it as "landed (safe, single-file, verified)", but `build-test.yml:484` on master was still bare `needs: resolve`. A note claiming a change landed is not evidence it did — `git log -- ` showed no such commit. **Now genuinely applied** in this PR: `needs: [resolve, validation]` with `if: ${{ !cancelled() }}`, dropping this workflow's own fan-out from 7 to 6. Job `name:` deliberately unchanged, and the ruleset context `Build & Test / Build (libzstd 1.4.x — fallback paths)` re-checked against the live ruleset after the edit. 2. **The cross-workflow pile-on is unchanged and unfixable inside step 29.** codeql/security-scanners/fuzzing/valgrind/asan all start at t=0 alongside build-test's fan-out, because `needs:` cannot cross a reusable-workflow-call boundary: `needs:` on a `uses:` job waits for that member's ENTIRE workflow, so laning codeql behind build-test would serialize it behind the 370s critical path instead. The only real remedies are folding those single jobs into build-test.yml's job graph, or adding pool slots. Sent upstream as decision-class feedback (skeleton PR #41, finding 3). No check was deleted and no threshold widened. myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/000077500000000000000000000000001525232545500221535ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/README.md000066400000000000000000000026001525232545500234300ustar00rootroot00000000000000# Vendored ast-grep rules GENERATED by `tools/vendor-astgrep-rules.py` -- do not hand-edit. Re-run the generator instead, so every module stays derived from one source of truth. ## What is here `rules/own/` -- this superrepo's C and nginx structural rules. `rules/vendor/` -- the C/C++ slice of CodeRabbit's `essentials` ast-grep rule pack (Apache-2.0), CWE-tagged. These are the rules CodeRabbit itself runs on a pull request, so a local hit is a finding the bot would have raised. Only C/C++-language rules are vendored. The upstream pack spans 15 languages; the rest can never match a `.c` file. ## Why vendored rather than referenced The full pack lives at `/opt/myguard/.ast-grep`, outside this repo. ast-grep finds `sgconfig.yml` by walking up from the working directory, so a hook here would resolve the superrepo's config only while this clone sits inside `/opt/myguard`. Cloned standalone, ast-grep would find no config, scan against an empty ruleset, and exit 0 -- a gate that passes everything. ## Gating The pre-commit hook promotes a named subset to `error`; ast-grep exits non-zero only on `error` and has no severity-threshold flag. Everything else stays advisory -- printed, never blocking. The advisory tier is dominated by `nginx-unchecked-palloc`, which in nginx C is frequently legitimate. Run the full advisory sweep by hand: ast-grep scan -c ci/ast-grep/sgconfig.yml src/ myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/000077500000000000000000000000001525232545500233055ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/own/000077500000000000000000000000001525232545500241105ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/own/c-alloc-mul-overflow.yml000066400000000000000000000012631525232545500306030ustar00rootroot00000000000000id: c-alloc-mul-overflow language: c severity: warning message: >- Allocation size computed with a multiplication — malloc/ngx_palloc/ngx_pnalloc of `a * b`. If either factor is attacker-influenced the product can overflow size_t and under-allocate, then the following copy overflows the heap. Use a checked-multiply (nginx: ngx_palloc with a size validated against a bound, or reject early) or calloc's built-in overflow check. note: Dismiss if both factors are compile-time constants or provably bounded. rule: any: - pattern: malloc($A * $B) - pattern: ngx_palloc($POOL, $A * $B) - pattern: ngx_pnalloc($POOL, $A * $B) - pattern: ngx_pcalloc($POOL, $A * $B) myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/own/c-alloca-vla.yml000066400000000000000000000006611525232545500270710ustar00rootroot00000000000000id: c-alloca-vla language: c severity: warning message: >- alloca() — stack overflow / stack clash if the size is non-constant and attacker-influenced, and no failure path (alloca can't report OOM). Use a heap allocation (in nginx: ngx_pnalloc from the request pool) with a bounded size. note: Dismiss if the size is a small compile-time constant. rule: kind: call_expression has: field: function regex: ^alloca$ myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/own/c-assign-in-if.yml000066400000000000000000000010111525232545500273300ustar00rootroot00000000000000id: c-assign-in-if language: c severity: warning message: >- Assignment inside an `if`/`while` condition (`if (x = f())`) — almost always a typo for `==`, and silently tests the assigned value. If intentional, wrap in extra parens `if ((x = f()))` to signal intent and silence -Wparentheses. note: Fires on the bare form; the `((x = f()))` idiom nests deeper and won't match. rule: any: - kind: if_statement - kind: while_statement has: field: condition has: kind: assignment_expression myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/own/c-ctype-signed-char.yml000066400000000000000000000014711525232545500303640ustar00rootroot00000000000000id: c-ctype-signed-char language: c severity: warning message: >- ctype function (isdigit/isalpha/isspace/toupper/…) called without an explicit `(unsigned char)` cast. If the argument is a signed `char` >= 0x80 it becomes a negative int, which is undefined behavior for these functions. High-bit bytes in hostile input → UB / wrong classification. Cast: `isdigit((unsigned char)c)`. note: >- Candidate — fires on every ctype call; dismiss when the arg is already `(unsigned char)` or a known-ASCII literal. Reviewer confirms the cast. rule: kind: call_expression all: - has: field: function regex: "^(is(digit|alpha|alnum|space|upper|lower|punct|xdigit|\ cntrl|print|graph)|to(upper|lower))$" - not: has: kind: cast_expression stopBy: end myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/own/c-format-string.yml000066400000000000000000000013341525232545500276500ustar00rootroot00000000000000id: c-format-string language: c severity: warning message: >- Non-literal format string — printf/fprintf/syslog/ngx_*printf with a variable as the format argument. If that variable carries user input it is a format-string vuln (%n write, info leak). Use a literal "%s" and pass the data as an argument. note: >- Fires when the call contains no string literal at all (best-effort — the format arg position varies by function). Dismiss if the format is a trusted constant/enum-mapped literal. rule: kind: call_expression all: - has: field: function regex: ^(printf|fprintf|dprintf|vprintf|vfprintf|syslog|vsyslog)$ - not: has: kind: string_literal stopBy: end myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/own/c-memcpy-sizeof-pointer.yml000066400000000000000000000012371525232545500313230ustar00rootroot00000000000000id: c-memcpy-sizeof-pointer language: c severity: warning message: >- memcpy/memmove/ngx_memcpy length is sizeof(an identifier). If that identifier is a pointer, only 4/8 bytes are copied (or overread); use sizeof(*ptr) or the real byte length. ast-grep cannot distinguish pointers from fixed arrays or struct objects, so confirm the declaration before treating this as a defect. note: Advisory only because fixed-array and whole-struct copies are safe twins. rule: any: - pattern: memcpy($DST, $SRC, sizeof($PTR)) - pattern: ngx_memcpy($DST, $SRC, sizeof($PTR)) - pattern: memmove($DST, $SRC, sizeof($PTR)) constraints: PTR: kind: identifier myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/own/c-shell-exec.yml000066400000000000000000000007131525232545500271050ustar00rootroot00000000000000id: c-shell-exec language: c severity: warning message: >- Shell/command execution (system/popen/execl*/execvp). If any part of the argument is attacker-influenced this is command injection. Prefer execve with a fixed argv and no shell, and validate/allow-list inputs. note: Trace the command string back to input before dismissing. rule: kind: call_expression has: field: function regex: ^(system|popen|execl|execlp|execle|execvp|execvpe)$ myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/own/c-sizeof-wrong-operand.yml000066400000000000000000000012471525232545500311360ustar00rootroot00000000000000id: c-sizeof-wrong-operand language: c severity: warning message: >- ngx_cpymem/ngx_memcpy/memcpy length is `sizeof(x) - 1` where x is not a string literal — the `-1` idiom is for dropping the NUL of a literal; on an array or pointer it silently copies one byte too few (or, on a pointer, sizeof the pointer). Verify the length matches the real object size. note: Correct for `sizeof("literal") - 1`; this fires on non-literal operands. rule: any: - pattern: ngx_cpymem($DST, $SRC, sizeof($X) - 1) - pattern: ngx_memcpy($DST, $SRC, sizeof($X) - 1) - pattern: memcpy($DST, $SRC, sizeof($X) - 1) not: has: kind: string_literal stopBy: end myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/own/c-unbounded-string-copy.yml000066400000000000000000000010771525232545500313170ustar00rootroot00000000000000id: c-unbounded-string-copy language: c severity: warning message: >- Unbounded string copy (strcpy/strcat/sprintf/gets) — no length bound, buffer overflow if source is attacker-influenced. Prefer strlcpy/strlcat/snprintf, or nginx ngx_cpystrn / ngx_snprintf which return the end pointer and take a limit. note: Confirm the destination buffer size vs the max source length before dismissing. rule: any: - pattern: strcpy($DST, $SRC) - pattern: strcat($DST, $SRC) - pattern: sprintf($BUF, $$$) - pattern: gets($BUF) - pattern: stpcpy($DST, $SRC) myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/own/nginx-raw-malloc.yml000066400000000000000000000012511525232545500300110ustar00rootroot00000000000000id: nginx-raw-malloc language: c severity: info message: >- Raw malloc/calloc/realloc/free/strdup in nginx module code. nginx manages memory with pools (ngx_palloc / ngx_pnalloc / ngx_pcalloc, freed with the request/connection). Raw heap alloc here is a leak risk (no pool cleanup) and breaks the module's memory model. Use the request or cycle pool unless this is a genuinely long-lived / cross-request allocation. note: >- Legitimate for long-lived global state or where an explicit ngx_pool_cleanup is registered. Confirm lifetime before dismissing. rule: kind: call_expression has: field: function regex: ^(malloc|calloc|realloc|free|strdup|strndup)$ myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/own/nginx-unchecked-palloc.yml000066400000000000000000000013751525232545500311630ustar00rootroot00000000000000id: nginx-unchecked-palloc language: c severity: info message: >- Pool allocation site — verify the result is NULL-checked before use. ngx_palloc/ngx_pnalloc/ngx_pcalloc return NULL on failure; dereferencing the result without `if (p == NULL) return NGX_ERROR;` (or NGX_HTTP_INTERNAL_SERVER_ERROR) is a NULL-deref crash under memory pressure. note: >- Candidate only — this flags every pool-alloc call, it does NOT prove the guard is missing. The reviewer confirms whether a NULL check follows. Encoding the guard structurally is unreliable (declaration-init vs assignment, return/goto guards), so verification is left to the model pass. rule: kind: call_expression has: field: function regex: ^(ngx_palloc|ngx_pnalloc|ngx_pcalloc)$ myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/vendor/000077500000000000000000000000001525232545500246025ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/vendor/c/000077500000000000000000000000001525232545500250245ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/vendor/c/security/000077500000000000000000000000001525232545500266735ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/vendor/c/security/dont-call-system-c.yml000066400000000000000000000026321525232545500330400ustar00rootroot00000000000000id: dont-call-system-c language: c severity: warning message: >- Don't call `system`. It's a high-level wrapper that allows for stacking multiple commands. Always prefer a more restrictive API such as calling `execve` from the `exec` family. note: >- [CWE-78] Improper Neutralization of Special Elements used in an OS Command ('OS Command Injection'). [REFERENCES] - https://owasp.org/Top10/A03_2021-Injection ast-grep-essentials: true utils: PATTERN_SYSTEM_INSIDE_IF_STATEMENT: kind: call_expression all: - has: stopBy: neighbor kind: identifier regex: '^system$' - has: stopBy: neighbor kind: argument_list - inside: stopBy: end kind: parenthesized_expression inside: kind: if_statement PATTERN_SYSTEM: any: - kind: expression_statement - kind: return_statement - kind: field_declaration has: stopBy: neighbor kind: call_expression all: - has: stopBy: neighbor kind: identifier regex: '^system$' - has: stopBy: neighbor kind: argument_list rule: any: - matches: PATTERN_SYSTEM_INSIDE_IF_STATEMENT - matches: PATTERN_SYSTEM not: all: - has: stopBy: end kind: ERROR - inside: has: stopBy: end kind: ERROR file-access-before-action-c.yml000066400000000000000000000162541525232545500344600ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/vendor/c/securityid: file-access-before-action-c language: c severity: warning message: >- A check is done with `access` and then the file is later used. There is no guarantee that the status of the file has not changed since the call to `access` which may allow attackers to bypass permission checks. note: >- [CWE-367]: Time-of-check Time-of-use (TOCTOU) Race Condition [REFERENCES] - https://wiki.sei.cmu.edu/confluence/display/c/FIO45-C.+Avoid+TOCTOU+race+conditions+while+accessing+files ast-grep-essentials: true utils: u_1_pattern_1_identifier: kind: identifier regex: ^(fopen|freopen|remove|rename|access|open|stat|lstat|unlink|mkdir|rmdir|chdir)$ all: - precedes: kind: argument_list has: pattern: $SRC - inside: kind: call_expression not: inside: stopBy: end kind: parenthesized_expression inside: stopBy: end kind: compound_statement inside: kind: if_statement has: kind: parenthesized_expression has: stopBy: end any: - kind: binary_expression has: stopBy: end kind: parenthesized_expression has: kind: binary_expression all: - has: kind: call_expression nthChild: 1 all: - has: kind: identifier regex: ^(access|faccessat|faccessat2)$ precedes: kind: argument_list all: - has: nthChild: 1 pattern: $SRC - has: kind: identifier nthChild: 2 regex: ^(F_OK|R_OK|W_OK|X_OK)$ - has: kind: number_literal regex: ^(0)$ follows: regex: ^==$ - kind: binary_expression all: - has: nthChild: 1 kind: call_expression all: - has: nthChild: 1 kind: identifier regex: ^(access|faccessat|faccessat2)$ - has: nthChild: 2 kind: argument_list all: - has: nthChild: 1 pattern: $SRC - has: nthChild: 2 kind: identifier regex: ^(F_OK|R_OK|W_OK|X_OK)$ - has: nthChild: 2 kind: number_literal regex: ^(0)$ follows: regex: ^==$ identifier: any: - kind: identifier regex: ^(fopen|freopen|remove|rename|access|open|stat|lstat|unlink|mkdir|rmdir|chdir)$ u_3_pattern_3_field_expression: kind: field_expression has: nthChild: 1 stopBy: end matches: identifier all: - precedes: kind: argument_list has: pattern: $SRC - inside: kind: call_expression not: inside: stopBy: end kind: parenthesized_expression inside: stopBy: end kind: compound_statement inside: kind: if_statement has: kind: parenthesized_expression has: stopBy: end any: - kind: binary_expression has: stopBy: end kind: parenthesized_expression has: kind: binary_expression all: - has: kind: call_expression nthChild: 1 all: - has: kind: identifier regex: ^(access|faccessat|faccessat2)$ precedes: kind: argument_list all: - has: nthChild: 1 pattern: $SRC - has: kind: identifier nthChild: 2 regex: ^(F_OK|R_OK|W_OK|X_OK)$ - has: kind: number_literal regex: ^(0)$ follows: regex: ^==$ - kind: binary_expression all: - has: nthChild: 1 kind: call_expression all: - has: nthChild: 1 kind: identifier regex: ^(access|faccessat|faccessat2)$ - has: nthChild: 2 kind: argument_list all: - has: nthChild: 1 pattern: $SRC - has: nthChild: 2 kind: identifier regex: ^(F_OK|R_OK|W_OK|X_OK)$ - has: nthChild: 2 kind: number_literal regex: ^(0)$ follows: regex: ^==$ rule: any: - matches: u_1_pattern_1_identifier - matches: u_3_pattern_3_field_expression file-stat-before-action-c.yml000066400000000000000000000302611525232545500341640ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/vendor/c/securityid: file-stat-before-action-c language: c severity: warning message: >- A check is done with `stat` and then the file is used. There is no guarantee that the status of the file has not changed since the call to `stat` which may allow attackers to bypass permission checks. note: >- [CWE-367]: Time-of-check Time-of-use (TOCTOU) Race Condition [REFERENCES] - https://wiki.sei.cmu.edu/confluence/display/c/FIO45-C.+Avoid+TOCTOU+race+conditions+while+accessing+files ast-grep-essentials: true utils: u_1_pattern_1_identifier_nth1: kind: identifier regex: ^(fopen|freopen|remove|rename|access|open|stat|lstat|unlink|mkdir|rmdir|chdir)$ all: - not: inside: stopBy: end kind: parenthesized_expression nthChild: 1 inside: kind: if_statement - precedes: kind: argument_list has: pattern: $SRC - inside: kind: call_expression not: inside: kind: field_expression inside: stopBy: end kind: compound_statement inside: kind: if_statement has: kind: parenthesized_expression has: stopBy: end any: - kind: binary_expression has: stopBy: end kind: parenthesized_expression has: kind: binary_expression all: - has: kind: call_expression nthChild: 1 all: - has: kind: identifier regex: ^(stat|_stat|lstat|_lstat)$ precedes: kind: argument_list all: - has: nthChild: 1 pattern: $SRC - has: kind: number_literal regex: ^(0)$ - kind: binary_expression all: - has: nthChild: 1 kind: call_expression all: - has: nthChild: 1 kind: identifier regex: ^(stat|_stat|lstat|_lstat)$ - has: nthChild: 2 kind: argument_list all: - has: nthChild: 1 pattern: $SRC - has: nthChild: 2 kind: number_literal regex: ^(0)$ follows: regex: ^==$ u_2_pattern_1_identifier_nth2: kind: identifier regex: ^(fopen|freopen|remove|rename|access|open|stat|lstat|unlink|mkdir|rmdir|chdir)$ all: - not: inside: stopBy: end kind: parenthesized_expression nthChild: 1 inside: kind: if_statement - precedes: kind: argument_list has: pattern: $SRC - inside: kind: call_expression not: inside: kind: field_expression inside: stopBy: end kind: compound_statement inside: kind: if_statement has: kind: parenthesized_expression has: stopBy: end any: - kind: binary_expression has: stopBy: end kind: parenthesized_expression has: kind: binary_expression all: - has: kind: call_expression nthChild: 1 all: - has: kind: identifier regex: ^(fstatat|_fstatat)$ precedes: kind: argument_list all: - has: nthChild: 3 pattern: $SRC - has: kind: number_literal regex: ^(0)$ follows: regex: ^==$ - kind: binary_expression all: - has: nthChild: 1 kind: call_expression all: - has: nthChild: 1 kind: identifier regex: ^(fstatat|_fstatat)$ - has: nthChild: 2 kind: argument_list all: - has: nthChild: 2 pattern: $SRC - has: nthChild: 2 kind: number_literal regex: ^(0)$ follows: regex: ^==$ identifier: any: - kind: identifier regex: ^(fopen|freopen|remove|rename|access|open|stat|lstat|unlink|mkdir|rmdir|chdir)$ u_4_pattern_3_field_expression_identifier_nth1: kind: field_expression has: nthChild: 1 stopBy: end matches: identifier all: - not: inside: stopBy: end kind: parenthesized_expression nthChild: 1 inside: kind: if_statement - precedes: kind: argument_list has: pattern: $SRC - inside: kind: call_expression inside: stopBy: end kind: compound_statement inside: kind: if_statement has: kind: parenthesized_expression has: stopBy: end any: - kind: binary_expression has: stopBy: end kind: parenthesized_expression has: kind: binary_expression all: - has: kind: call_expression nthChild: 1 all: - has: kind: identifier regex: ^(stat|_stat|lstat|_lstat)$ precedes: kind: argument_list all: - has: nthChild: 1 pattern: $SRC - has: kind: number_literal regex: ^(0)$ follows: regex: ^==$ - kind: binary_expression all: - has: nthChild: 1 kind: call_expression all: - has: nthChild: 1 kind: identifier regex: ^(stat|_stat|lstat|_lstat)$ - has: nthChild: 2 kind: argument_list all: - has: nthChild: 1 pattern: $SRC - has: nthChild: 2 kind: number_literal regex: ^(0)$ follows: regex: ^==$ u_5_pattern_3_field_expression_identifier_nth2: kind: field_expression has: nthChild: 1 stopBy: end matches: identifier all: - not: inside: stopBy: end kind: parenthesized_expression nthChild: 1 inside: kind: if_statement - precedes: kind: argument_list has: pattern: $SRC - inside: kind: call_expression inside: stopBy: end kind: compound_statement inside: kind: if_statement has: kind: parenthesized_expression has: stopBy: end any: - kind: binary_expression has: stopBy: end kind: parenthesized_expression has: kind: binary_expression all: - has: kind: call_expression nthChild: 1 all: - has: kind: identifier regex: ^(fstatat|_fstatat)$ precedes: kind: argument_list all: - has: nthChild: 2 pattern: $SRC - has: kind: number_literal regex: ^(0)$ follows: regex: ^==$ - kind: binary_expression all: - has: nthChild: 1 kind: call_expression all: - has: nthChild: 1 kind: identifier regex: ^(fstatat|_fstatat)$ - has: nthChild: 2 kind: argument_list all: - has: nthChild: 2 pattern: $SRC - has: nthChild: 2 kind: number_literal regex: ^(0)$ follows: regex: ^==$ rule: any: - matches: u_1_pattern_1_identifier_nth1 - matches: u_2_pattern_1_identifier_nth2 - matches: u_4_pattern_3_field_expression_identifier_nth1 - matches: u_5_pattern_3_field_expression_identifier_nth2 myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/vendor/c/security/insecure-hash-c.yml000066400000000000000000000213441525232545500324000ustar00rootroot00000000000000id: insecure-hash-c language: c severity: warning message: >- This hashing algorithm is insecure. If this hash is used in a security context, such as password hashing, it should be converted to a stronger hashing algorithm. note: >- [CWE-328] Use of Weak Hash. [REFERENCES] - https://owasp.org/Top10/A02_2021-Cryptographic_Failures ast-grep-essentials: true utils: MATCH_PATTERN_ONE: kind: expression_statement has: stopBy: neighbor kind: call_expression all: - has: stopBy: neighbor kind: identifier regex: ^(EVP_md2|MD2|MD2_Final|MD2_Init|MD2_Update|MD2_options|EVP_md4|MD4|MD4_Final|MD4_Init|MD4_Transform|MD4_Update|EVP_md5|MD5|MD5_Final|MD5_Init|MD5_Transform|MD5_Update|EVP_sha1|SHA1_Final|SHA1_Init|SHA1_Transform|SHA1_Update)$ - has: stopBy: neighbor kind: argument_list u_2_match_pattern_two_evp_md_fetch: kind: expression_statement has: stopBy: neighbor kind: call_expression all: - has: stopBy: neighbor kind: identifier regex: ^(EVP_MD_fetch)$ - has: stopBy: neighbor kind: argument_list has: kind: string_literal all: - nthChild: position: 2 ofRule: not: kind: comment - has: kind: string_content regex: ^(MD2|MD4|MD5|SHA1|SHA-1)$ u_3_match_pattern_two_with_instance_evp_md_fetch: kind: expression_statement all: - has: stopBy: neighbor kind: call_expression all: - has: stopBy: neighbor kind: identifier regex: "^(EVP_MD_fetch)$" - has: stopBy: neighbor kind: argument_list has: stopBy: neighbor kind: identifier pattern: $Q nthChild: position: 2 ofRule: not: kind: comment - any: - follows: stopBy: end kind: declaration has: stopBy: end kind: init_declarator all: - has: stopBy: neighbor kind: pointer_declarator has: stopBy: neighbor kind: identifier pattern: $Q - has: stopBy: neighbor kind: string_literal has: stopBy: neighbor kind: string_content regex: ^(MD2|MD4|MD5|SHA1|SHA-1)$ - inside: stopBy: end follows: stopBy: end kind: declaration has: stopBy: end kind: init_declarator all: - has: stopBy: neighbor kind: pointer_declarator has: stopBy: neighbor kind: identifier pattern: $Q - has: stopBy: neighbor kind: string_literal has: stopBy: neighbor kind: string_content regex: ^(MD2|MD4|MD5|SHA1|SHA-1)$ MATCH_PATTERN_THREE: kind: expression_statement has: stopBy: neighbor kind: call_expression all: - has: stopBy: neighbor kind: identifier regex: ^(gcry_md_open|gcry_md_enable|gcry_md_read|gcry_md_extract)$ - has: stopBy: neighbor kind: argument_list has: stopBy: end kind: identifier regex: ^(GCRY_MD_MD2|GCRY_MD_MD4|GCRY_MD_MD5|GCRY_MD_SHA1)$ nthChild: position: 2 ofRule: not: kind: comment u_5_match_pattern_two_evp_get_digestbyname: kind: expression_statement has: stopBy: neighbor kind: call_expression all: - has: stopBy: neighbor kind: identifier regex: ^(EVP_get_digestbyname)$ - has: stopBy: neighbor kind: argument_list has: kind: string_literal all: - nthChild: position: 1 ofRule: not: kind: comment - has: kind: string_content regex: ^(MD2|MD4|MD5|SHA1|SHA-1)$ not: has: nthChild: position: 2 ofRule: not: kind: comment u_6_match_pattern_two_with_instance_evp_get_digestby: kind: expression_statement all: - has: stopBy: neighbor kind: call_expression all: - has: stopBy: neighbor kind: identifier regex: "^(EVP_get_digestbyname)$" - has: stopBy: neighbor kind: argument_list has: stopBy: neighbor kind: identifier pattern: $Q nthChild: position: 1 ofRule: not: kind: comment not: has: nthChild: position: 2 ofRule: not: kind: comment - any: - follows: stopBy: end kind: declaration has: stopBy: end kind: init_declarator all: - has: stopBy: neighbor kind: pointer_declarator has: stopBy: neighbor kind: identifier pattern: $Q - has: stopBy: neighbor kind: string_literal has: stopBy: neighbor kind: string_content regex: ^(MD2|MD4|MD5|SHA1|SHA-1)$ - inside: stopBy: end follows: stopBy: end kind: declaration has: stopBy: end kind: init_declarator all: - has: stopBy: neighbor kind: pointer_declarator has: stopBy: neighbor kind: identifier pattern: $Q - has: stopBy: neighbor kind: string_literal has: stopBy: neighbor kind: string_content regex: ^(MD2|MD4|MD5|SHA1|SHA-1)$ u_7_match_pattern_three_gcry_md_hash_buffers: kind: expression_statement has: stopBy: neighbor kind: call_expression all: - has: stopBy: neighbor kind: identifier regex: ^(gcry_md_hash_buffers|gcry_md_hash_buffer)$ - has: stopBy: neighbor kind: argument_list has: stopBy: end kind: identifier regex: ^(GCRY_MD_MD2|GCRY_MD_MD4|GCRY_MD_MD5|GCRY_MD_SHA1)$ nthChild: position: 1 ofRule: not: kind: comment rule: any: - kind: expression_statement any: - matches: MATCH_PATTERN_ONE - matches: u_2_match_pattern_two_evp_md_fetch - matches: u_3_match_pattern_two_with_instance_evp_md_fetch - matches: MATCH_PATTERN_THREE - matches: u_5_match_pattern_two_evp_get_digestbyname - matches: u_6_match_pattern_two_with_instance_evp_get_digestby - matches: u_7_match_pattern_three_gcry_md_hash_buffers not: all: - has: stopBy: end kind: ERROR - inside: stopBy: end kind: ERROR libxml2-audit-parser-c.yml000066400000000000000000000155651525232545500335420ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/vendor/c/securityid: libxml2-audit-parser-c language: c severity: warning message: >- The libxml2 library is used to parse XML. When auditing such code, make sure that either the document being parsed is trusted or that the parsing options are safe to consume untrusted documents. In such case make sure DTD or XInclude documents cannot be loaded and there is no network access. note: >- [CWE-611] Improper Restriction of XML External Entity Reference. [REFERENCES] - https://owasp.org/Top10/A05_2021-Security_Misconfiguration ast-grep-essentials: true utils: Pattern_having_three_child: kind: call_expression all: - has: kind: identifier regex: ^(xmlReadFile)$ - has: kind: argument_list all: - has: nthChild: position: 1 ofRule: not: kind: comment - has: nthChild: position: 2 ofRule: not: kind: comment - has: nthChild: position: 3 ofRule: not: kind: comment - not: has: nthChild: position: 4 ofRule: not: kind: comment Pattern_having_five_child: kind: call_expression all: - has: kind: identifier regex: ^(xmlParseInNodeContext|xmlReadMemory|xmlCtxtReadDoc|xmlCtxtReadFd)$ - has: kind: argument_list all: - has: nthChild: position: 1 ofRule: not: kind: comment - has: nthChild: position: 2 ofRule: not: kind: comment - has: nthChild: position: 3 ofRule: not: kind: comment - has: nthChild: position: 4 ofRule: not: kind: comment - has: nthChild: position: 5 ofRule: not: kind: comment - not: has: nthChild: position: 6 ofRule: not: kind: comment Pattern_having_four_child: kind: call_expression all: - has: kind: identifier regex: ^(xmlReadDoc|xmlReadFd|xmlCtxtReadFile)$ - has: kind: argument_list all: - has: nthChild: position: 1 ofRule: not: kind: comment - has: nthChild: position: 2 ofRule: not: kind: comment - has: nthChild: position: 3 ofRule: not: kind: comment - has: nthChild: position: 4 ofRule: not: kind: comment - not: has: nthChild: position: 5 ofRule: not: kind: comment Pattern_having_six_child: kind: call_expression all: - has: kind: identifier regex: ^(xmlReadIO|xmlCtxtReadMemory)$ - has: kind: argument_list all: - has: nthChild: position: 1 ofRule: not: kind: comment - has: nthChild: position: 2 ofRule: not: kind: comment - has: nthChild: position: 3 ofRule: not: kind: comment - has: nthChild: position: 4 ofRule: not: kind: comment - has: nthChild: position: 5 ofRule: not: kind: comment - has: nthChild: position: 6 ofRule: not: kind: comment - not: has: nthChild: position: 7 ofRule: not: kind: comment Pattern_having_seven_child: kind: call_expression all: - has: kind: identifier regex: ^(xmlCtxtReadIO)$ - has: kind: argument_list all: - has: nthChild: position: 1 ofRule: not: kind: comment - has: nthChild: position: 2 ofRule: not: kind: comment - has: nthChild: position: 3 ofRule: not: kind: comment - has: nthChild: position: 4 ofRule: not: kind: comment - has: nthChild: position: 5 ofRule: not: kind: comment - has: nthChild: position: 6 ofRule: not: kind: comment - has: nthChild: position: 7 ofRule: not: kind: comment - not: has: nthChild: position: 8 ofRule: not: kind: comment rule: kind: call_expression any: - matches: Pattern_having_five_child - matches: Pattern_having_four_child - matches: Pattern_having_six_child - matches: Pattern_having_seven_child - matches: Pattern_having_three_child not: all: - has: stopBy: end kind: ERROR - inside: stopBy: end kind: ERROR null-library-function-c.yml000066400000000000000000000363031525232545500340230ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/vendor/c/securityid: null-library-function-c language: C severity: warning message: >- The `$SOURCE` function returns NULL on error and this line dereferences the return value without checking for NULL. note: >- [CWE-476] NULL Pointer Dereference. [REFERENCES] - https://wiki.sei.cmu.edu/confluence/display/c/EXP34-C.+Do+not+dereference+null+pointers ast-grep-essentials: true rule: all: - not: has: stopBy: end kind: ERROR - any: - kind: subscript_expression # any: # - pattern: $SOURCE($$$)[$$$] # - pattern: ($SOURCE($$$))[$$$] all: - has: stopBy: end kind: call_expression all: - has: stopBy: end kind: identifier pattern: $SOURCE regex: ^fgets|::fgets|std::fgets|fopen|::fopen|std::fopen|getenv|::getenv|std::getenv|getgrent|::getgrent|std::getgrent|getgrgid|::getgrgid|std::getgrgid|getgrnam|::getgrnam|std::getgrnam|getlogin|::getlogin|std::getlogin|getpwent|::getpwent|std::getpwent|getpwnam|::getpwnam|std::getpwnam|getpwuid|::getpwuid|std::getpwuid|getpwuuid|::getpwuuid|std::getpwuuid|gets|::gets|std::gets|inet_ntop|::inet_ntop|std::inet_ntop|realpath|::realpath|std::realpath|tempnam|::tempnam|std::tempnam|tmpfile|::tmpfile|std::tmpfile|tmpnam|::tmpnam|std::tmpnam|memchr|::memchr|std::memchr|strcasestr_l|::strcasestr_l|std::strcasestr_l|strcasestr|::strcasestr|std::strcasestr|strchr|::strchr|std::strchr|strnstr|::strnstr|std::strnstr|strpbrk|::strpbrk|std::strpbrk|strrchr|::strrchr|std::strrchr|strstr|::strstr|std::strstr|strtok_r|::strtok_r|std::strtok_r|strtok|::strtok|std::strtok$ - has: stopBy: neighbor kind: argument_list - has: stopBy: end any: - kind: number_literal - kind: identifier - kind: call_expression all: - has: stopBy: neighbor kind: identifier pattern: $SINK regex: ^atof|::atof|std::atof|atoi|::atoi|std::atoi|atol_l|::atol_l|std::atol_l|atol|::atol|std::atol|atoll_l|::atoll_l|std::atoll_l|atoll|::atoll|std::atoll|getc|::getc|std::getc|fprintf|::fprintf|std::fprintf|fgetpos|::fgetpos|std::fgetpos|fseek|::fseek|std::fseek|fseeko|::fseeko|std::fseeko|fsetpos|::fsetpos|std::fsetpos|ftell|::ftell|std::ftell|ftello|::ftello|std::ftello|rewind|::rewind|std::rewind|strlen|::strlen|std::strlen|strtoimax|::strtoimax|std::strtoimax|strtod|::strtod|std::strtod|strtol|::strtol|std::strtol|strtoul|::strtoul|std::strtoul|strtoll|::strtoll|std::strtoll|strtoq|::strtoq|std::strtoq$ - has: stopBy: neighbor kind: argument_list has: stopBy: neighbor kind: call_expression nthChild: 1 all: - has: stopBy: neighbor kind: identifier pattern: $SOURCE regex: ^fgets|::fgets|std::fgets|fopen|::fopen|std::fopen|getenv|::getenv|std::getenv|getgrent|::getgrent|std::getgrent|getgrgid|::getgrgid|std::getgrgid|getgrnam|::getgrnam|std::getgrnam|getlogin|::getlogin|std::getlogin|getpwent|::getpwent|std::getpwent|getpwnam|::getpwnam|std::getpwnam|getpwuid|::getpwuid|std::getpwuid|getpwuuid|::getpwuuid|std::getpwuuid|gets|::gets|std::gets|inet_ntop|::inet_ntop|std::inet_ntop|realpath|::realpath|std::realpath|tempnam|::tempnam|std::tempnam|tmpfile|::tmpfile|std::tmpfile|tmpnam|::tmpnam|std::tmpnam|memchr|::memchr|std::memchr|strcasestr_l|::strcasestr_l|std::strcasestr_l|strcasestr|::strcasestr|std::strcasestr|strchr|::strchr|std::strchr|strnstr|::strnstr|std::strnstr|strpbrk|::strpbrk|std::strpbrk|strrchr|::strrchr|std::strrchr|strstr|::strstr|std::strstr|strtok_r|::strtok_r|std::strtok_r|strtok|::strtok|std::strtok$ - has: stopBy: neighbor kind: argument_list - kind: call_expression all: - has: stopBy: neighbor kind: identifier pattern: $SINK regex: ^atof|::atof|std::atof|atoi|::atoi|std::atoi|atol_l|::atol_l|std::atol_l|atol|::atol|std::atol|atoll_l|::atoll_l|std::atoll_l|atoll|::atoll|std::atoll|getc|::getc|std::getc|fprintf|::fprintf|std::fprintf|fgetpos|::fgetpos|std::fgetpos|fseek|::fseek|std::fseek|fseeko|::fseeko|std::fseeko|fsetpos|::fsetpos|std::fsetpos|ftell|::ftell|std::ftell|ftello|::ftello|std::ftello|rewind|::rewind|std::rewind|strlen|::strlen|std::strlen|strtoimax|::strtoimax|std::strtoimax|strtod|::strtod|std::strtod|strtol|::strtol|std::strtol|strtoul|::strtoul|std::strtoul|strtoll|::strtoll|std::strtoll|strtoq|::strtoq|std::strtoq$ - has: stopBy: neighbor kind: argument_list has: stopBy: neighbor kind: assignment_expression nthChild: 1 all: - has: stopBy: neighbor kind: identifier pattern: $VAR - has: stopBy: neighbor kind: call_expression all: - has: stopBy: neighbor kind: identifier pattern: $SOURCE regex: ^fgets|::fgets|std::fgets|fopen|::fopen|std::fopen|getenv|::getenv|std::getenv|getgrent|::getgrent|std::getgrent|getgrgid|::getgrgid|std::getgrgid|getgrnam|::getgrnam|std::getgrnam|getlogin|::getlogin|std::getlogin|getpwent|::getpwent|std::getpwent|getpwnam|::getpwnam|std::getpwnam|getpwuid|::getpwuid|std::getpwuid|getpwuuid|::getpwuuid|std::getpwuuid|gets|::gets|std::gets|inet_ntop|::inet_ntop|std::inet_ntop|realpath|::realpath|std::realpath|tempnam|::tempnam|std::tempnam|tmpfile|::tmpfile|std::tmpfile|tmpnam|::tmpnam|std::tmpnam|memchr|::memchr|std::memchr|strcasestr_l|::strcasestr_l|std::strcasestr_l|strcasestr|::strcasestr|std::strcasestr|strchr|::strchr|std::strchr|strnstr|::strnstr|std::strnstr|strpbrk|::strpbrk|std::strpbrk|strrchr|::strrchr|std::strrchr|strstr|::strstr|std::strstr|strtok_r|::strtok_r|std::strtok_r|strtok|::strtok|std::strtok$ - kind: call_expression all: - has: stopBy: neighbor kind: identifier pattern: $SINK - has: stopBy: neighbor kind: argument_list has: stopBy: neighbor kind: call_expression nthChild: 2 all: - has: stopBy: neighbor kind: identifier pattern: $SOURCE regex: ^fgets|::fgets|std::fgets|fopen|::fopen|std::fopen|getenv|::getenv|std::getenv|getgrent|::getgrent|std::getgrent|getgrgid|::getgrgid|std::getgrgid|getgrnam|::getgrnam|std::getgrnam|getlogin|::getlogin|std::getlogin|getpwent|::getpwent|std::getpwent|getpwnam|::getpwnam|std::getpwnam|getpwuid|::getpwuid|std::getpwuid|getpwuuid|::getpwuuid|std::getpwuuid|gets|::gets|std::gets|inet_ntop|::inet_ntop|std::inet_ntop|realpath|::realpath|std::realpath|tempnam|::tempnam|std::tempnam|tmpfile|::tmpfile|std::tmpfile|tmpnam|::tmpnam|std::tmpnam|memchr|::memchr|std::memchr|strcasestr_l|::strcasestr_l|std::strcasestr_l|strcasestr|::strcasestr|std::strcasestr|strchr|::strchr|std::strchr|strnstr|::strnstr|std::strnstr|strpbrk|::strpbrk|std::strpbrk|strrchr|::strrchr|std::strrchr|strstr|::strstr|std::strstr|strtok_r|::strtok_r|std::strtok_r|strtok|::strtok|std::strtok$ - has: stopBy: neighbor kind: argument_list - not: inside: stopBy: end kind: call_expression has: stopBy: neighbor kind: identifier pattern: $SINK - not: has: stopBy: end kind: call_expression all: - has: stopBy: neighbor kind: identifier pattern: $SOURCE - has: stopBy: neighbor kind: argument_list has: stopBy: end kind: call_expression has: stopBy: neighbor kind: identifier pattern: $SOURCE - kind: call_expression all: - has: stopBy: neighbor kind: identifier pattern: $SINK regex: ^bcopy|::bcopy|std::bcopy|memccpy|::memccpy|std::memccpy|memcpy|::memcpy|std::memcpy|memmove|::memmove|std::memmove|stpncpy|::stpncpy|std::stpncpy|strcat|::strcat|std::strcat|strcpy|::strcpy|std::strcpy|strcpy|::strcpy|std::strcpy|strlcat|::strlcat|std::strlcat|strlcpy|::strlcpy|std::strlcpy|strncat|::strncat|std::strncat|strpcpy|::strpcpy|std::strpcpy|wcpcpy|::wcpcpy|std::wcpcpy|wcpncpy|::wcpncpy|std::wcpncpy$ - has: stopBy: neighbor kind: argument_list has: stopBy: neighbor kind: assignment_expression pattern: $VAR = $SOURCE($$$) all: - has: stopBy: neighbor kind: identifier pattern: $VAR - has: stopBy: neighbor kind: call_expression all: - has: stopBy: neighbor kind: identifier pattern: $SOURCE regex: ^fgets|::fgets|std::fgets|fopen|::fopen|std::fopen|getenv|::getenv|std::getenv|getgrent|::getgrent|std::getgrent|getgrgid|::getgrgid|std::getgrgid|getgrnam|::getgrnam|std::getgrnam|getlogin|::getlogin|std::getlogin|getpwent|::getpwent|std::getpwent|getpwnam|::getpwnam|std::getpwnam|getpwuid|::getpwuid|std::getpwuid|getpwuuid|::getpwuuid|std::getpwuuid|gets|::gets|std::gets|inet_ntop|::inet_ntop|std::inet_ntop|realpath|::realpath|std::realpath|tempnam|::tempnam|std::tempnam|tmpfile|::tmpfile|std::tmpfile|tmpnam|::tmpnam|std::tmpnam|memchr|::memchr|std::memchr|strcasestr_l|::strcasestr_l|std::strcasestr_l|strcasestr|::strcasestr|std::strcasestr|strchr|::strchr|std::strchr|strnstr|::strnstr|std::strnstr|strpbrk|::strpbrk|std::strpbrk|strrchr|::strrchr|std::strrchr|strstr|::strstr|std::strstr|strtok_r|::strtok_r|std::strtok_r|strtok|::strtok|std::strtok$ - kind: call_expression # any: # - pattern: $SINK($$$, $SOURCE($$$)) # - pattern: $SINK($SOURCE($$$)) all: - has: stopBy: neighbor kind: identifier pattern: $SINK regex: ^fwrite|::fwrite|std::fwrite$ - has: stopBy: neighbor kind: argument_list has: stopBy: end kind: call_expression all: - has: stopBy: end kind: identifier pattern: $SOURCE regex: ^fgets|::fgets|std::fgets|fopen|::fopen|std::fopen|getenv|::getenv|std::getenv|getgrent|::getgrent|std::getgrent|getgrgid|::getgrgid|std::getgrgid|getgrnam|::getgrnam|std::getgrnam|getlogin|::getlogin|std::getlogin|getpwent|::getpwent|std::getpwent|getpwnam|::getpwnam|std::getpwnam|getpwuid|::getpwuid|std::getpwuid|getpwuuid|::getpwuuid|std::getpwuuid|gets|::gets|std::gets|inet_ntop|::inet_ntop|std::inet_ntop|realpath|::realpath|std::realpath|tempnam|::tempnam|std::tempnam|tmpfile|::tmpfile|std::tmpfile|tmpnam|::tmpnam|std::tmpnam|memchr|::memchr|std::memchr|strcasestr_l|::strcasestr_l|std::strcasestr_l|strcasestr|::strcasestr|std::strcasestr|strchr|::strchr|std::strchr|strnstr|::strnstr|std::strnstr|strpbrk|::strpbrk|std::strpbrk|strrchr|::strrchr|std::strrchr|strstr|::strstr|std::strstr|strtok_r|::strtok_r|std::strtok_r|strtok|::strtok|std::strtok$ - has: stopBy: neighbor kind: argument_list - kind: call_expression any: - pattern: $SINK($$$, $VAR = $SOURCE($$$)) - pattern: $SINK($VAR = $SOURCE($$$)) all: - has: stopBy: neighbor kind: identifier pattern: $SINK regex: ^fwrite|::fwrite|std::fwrite$ - has: stopBy: neighbor kind: argument_list has: stopBy: end kind: assignment_expression pattern: $VAR = $SOURCE($$$) all: - has: stopBy: neighbor kind: identifier - has: stopBy: neighbor kind: call_expression all: - has: stopBy: neighbor kind: identifier pattern: $SOURCE regex: ^fgets|::fgets|std::fgets|fopen|::fopen|std::fopen|getenv|::getenv|std::getenv|getgrent|::getgrent|std::getgrent|getgrgid|::getgrgid|std::getgrgid|getgrnam|::getgrnam|std::getgrnam|getlogin|::getlogin|std::getlogin|getpwent|::getpwent|std::getpwent|getpwnam|::getpwnam|std::getpwnam|getpwuid|::getpwuid|std::getpwuid|getpwuuid|::getpwuuid|std::getpwuuid|gets|::gets|std::gets|inet_ntop|::inet_ntop|std::inet_ntop|realpath|::realpath|std::realpath|tempnam|::tempnam|std::tempnam|tmpfile|::tmpfile|std::tmpfile|tmpnam|::tmpnam|std::tmpnam|memchr|::memchr|std::memchr|strcasestr_l|::strcasestr_l|std::strcasestr_l|strcasestr|::strcasestr|std::strcasestr|strchr|::strchr|std::strchr|strnstr|::strnstr|std::strnstr|strpbrk|::strpbrk|std::strpbrk|strrchr|::strrchr|std::strrchr|strstr|::strstr|std::strstr|strtok_r|::strtok_r|std::strtok_r|strtok|::strtok|std::strtok$ - has: stopBy: neighbor kind: argument_list - kind: pointer_expression has: stopBy: neighbor kind: call_expression all: - has: stopBy: end kind: identifier pattern: $SOURCE regex: ^fgets|::fgets|std::fgets|fopen|::fopen|std::fopen|getenv|::getenv|std::getenv|getgrent|::getgrent|std::getgrent|getgrgid|::getgrgid|std::getgrgid|getgrnam|::getgrnam|std::getgrnam|getlogin|::getlogin|std::getlogin|getpwent|::getpwent|std::getpwent|getpwnam|::getpwnam|std::getpwnam|getpwuid|::getpwuid|std::getpwuid|getpwuuid|::getpwuuid|std::getpwuuid|gets|::gets|std::gets|inet_ntop|::inet_ntop|std::inet_ntop|realpath|::realpath|std::realpath|tempnam|::tempnam|std::tempnam|tmpfile|::tmpfile|std::tmpfile|tmpnam|::tmpnam|std::tmpnam|memchr|::memchr|std::memchr|strcasestr_l|::strcasestr_l|std::strcasestr_l|strcasestr|::strcasestr|std::strcasestr|strchr|::strchr|std::strchr|strnstr|::strnstr|std::strnstr|strpbrk|::strpbrk|std::strpbrk|strrchr|::strrchr|std::strrchr|strstr|::strstr|std::strstr|strtok_r|::strtok_r|std::strtok_r|strtok|::strtok|std::strtok$ - has: stopBy: neighbor kind: argument_list - not: inside: stopBy: end any: - kind: subscript_expression # - kind: call_expression - not: has: stopBy: end any: - kind: assignment_expression - inside: stopBy: end kind: return_statement myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/vendor/c/security/sizeof-this-c.yml000066400000000000000000000063531525232545500321110ustar00rootroot00000000000000id: sizeof-this-c language: c severity: warning message: >- Do not use `sizeof(this)` to get the number of bytes of the object in memory. It returns the size of the pointer, not the size of the object. note: >- [CWE-467]: Use of sizeof() on a Pointer Type [REFERENCES] - https://wiki.sei.cmu.edu/confluence/display/c/ARR01-C.+Do+not+apply+the+sizeof+operator+to+a+pointer+when+taking+the+size+of+an+array ast-grep-essentials: true rule: not: has: stopBy: end any: - kind: ERROR - kind: pointer_expression - kind: sizeof_expression - kind: expression_statement any: - kind: macro_type_specifier all: - has: stopBy: end kind: identifier nthChild: 1 regex: ^sizeof$ - has: stopBy: end kind: type_descriptor nthChild: 2 not: has: nthChild: 2 has: kind: type_identifier pattern: $THIS - not: has: kind: function_declarator nthChild: 1 - kind: function_declarator all: - has: stopBy: end kind: field_identifier regex: ^sizeof$ nthChild: 1 - has: stopBy: end kind: parameter_list nthChild: 2 not: has: nthChild: 2 has: kind: parameter_declaration pattern: $THIS - not: has: kind: function_declarator nthChild: 1 # - not: # inside: # has: # nthChild: 1 - kind: parameter_declaration all: - has: kind: type_identifier nthChild: 1 regex: ^sizeof$ - any: - has: kind: abstract_function_declarator has: kind: parameter_list not: has: nthChild: 2 has: kind: parameter_declaration pattern: $THIS - has: kind: abstract_parenthesized_declarator not: has: stopBy: end nthChild: 2 has: stopBy: end kind: parameter_list has: kind: parameter_declaration pattern: $THIS - kind: sizeof_expression not: has: any: - nthChild: 2 - kind: parameter_declaration has: stopBy: end kind: identifier pattern: $THIS - kind: type_descriptor all: - has: kind: type_identifier regex: ^sizeof$ - has: stopBy: end kind: abstract_function_declarator has: kind: parameter_list not: has: stopBy: end nthChild: 2 has: kind: parameter_declaration pattern: $THIS constraints: THIS: regex: ^this$ myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/vendor/c/security/small-key-size-c.yml000066400000000000000000000030571525232545500325110ustar00rootroot00000000000000id: small-key-size-c language: c severity: warning message: >- $KEY_FUNCTION` is using a key size of only $KEY_BITS bits. This is less than the recommended key size of 2048 bits. note: >- [CWE-326]: Inadequate Encryption Strength [OWASP A02:2021]: Cryptographic Failures [OWASP A03:2017]: Sensitive Data Exposure [REFERENCES] https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-131Ar2.pdf https://owasp.org/Top10/A02_2021-Cryptographic_Failures ast-grep-essentials: true rule: kind: call_expression all: - has: stopBy: end kind: identifier regex: ^(DH_generate_parameters_ex|DSA_generate_parameters_ex|EVP_PKEY_CTX_set_dh_paramgen_prime_len|EVP_PKEY_CTX_set_dsa_paramgen_bits|EVP_PKEY_CTX_set_rsa_keygen_bits|RSA_generate_key_ex|RSA_generate_key_fips)$ - not: has: stopBy: end kind: field_identifier regex: ^(DH_generate_parameters_ex|DSA_generate_parameters_ex|EVP_PKEY_CTX_set_dh_paramgen_prime_len|EVP_PKEY_CTX_set_dsa_paramgen_bits|EVP_PKEY_CTX_set_rsa_keygen_bits|RSA_generate_key_ex|RSA_generate_key_fips)$ - has: stopBy: neighbor kind: argument_list has: stopBy: neighbor any: - kind: number_literal - kind: binary_expression - kind: unary_expression nthChild: 2 regex: ^([+-]*\(*[+-]*((0|[1-9][0-9]{0,2}|1[0-9]{3}|20[0-3][0-9]|204[0-7])(\.[0-9]+)?|((0|[1-9][0-9]{0,2}|1[0-9]{3}|20[0-3][0-9]|204[0-7])(\.[0-9]+)?\/[1-9][0-9]*)|(\.[0-9]+)|(\.[0-9]+\/[1-9][0-9]*))\)*)$ - not: has: stopBy: end kind: ERROR myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/vendor/c/security/world-writable-file-c.yml000066400000000000000000000241331525232545500335140ustar00rootroot00000000000000id: world-writable-file-c language: c severity: warning message: >- This call makes a world-writable file which allows any user on a machine to write to the file. This may allow attackers to influence the behaviour of this process by writing to the file. note: >- [CWE-732]: Incorrect Permission Assignment for Critical Resource [REFERENCES] - https://wiki.sei.cmu.edu/confluence/display/c/FIO06-C.+Create+files+with+appropriate+access+permissions ast-grep-essentials: true utils: follows_umask: follows: stopBy: end kind: expression_statement has: kind: call_expression nthChild: 1 all: - has: nthChild: 1 kind: identifier field: function regex: ^umask$ - has: nthChild: 2 kind: argument_list field: arguments u_2_and_2_equals_2_s_ixxxx: any: - kind: number_literal regex: ^-?([2367]|[0-9]*(0[2367]|1[014589]|2[2367]|3[014589]|4[2367]|5[014589]|6[2367]|7[014589]|8[2367]|9[014589]))$ - all: - any: - kind: binary_expression - kind: identifier - regex: (\s*S_I[A-Z]{4}\s*\|)*S_I[A-Z]{4} - regex: .*\bS_IWOTH\b.* rule: any: # chmod/fchmod/creat - any: - matches: u_2_and_2_equals_2_s_ixxxx - kind: identifier pattern: $MODE inside: stopBy: end follows: stopBy: end any: - kind: declaration all: - has: kind: init_declarator all: - has: kind: identifier field: declarator pattern: $MODE - has: nthChild: 2 matches: u_2_and_2_equals_2_s_ixxxx - kind: expression_statement any: - has: kind: assignment_expression all: - has: nthChild: 1 kind: identifier pattern: $MODE - has: nthChild: 2 matches: u_2_and_2_equals_2_s_ixxxx - has: kind: comma_expression has: kind: assignment_expression all: - has: nthChild: 1 kind: identifier pattern: $MODE - has: nthChild: 2 matches: u_2_and_2_equals_2_s_ixxxx nthChild: position: 2 ofRule: not: kind: comment inside: kind: argument_list nthChild: 2 not: has: nthChild: position: 3 ofRule: not: kind: comment follows: kind: identifier regex: ^(chmod|fchmod|creat)$ inside: kind: call_expression not: any: - matches: follows_umask - inside: stopBy: end matches: follows_umask # fchmodat - any: - matches: u_2_and_2_equals_2_s_ixxxx - kind: identifier pattern: $MODE inside: stopBy: end follows: stopBy: end any: - kind: declaration all: - has: kind: init_declarator all: - has: kind: identifier field: declarator pattern: $MODE - has: nthChild: 2 matches: u_2_and_2_equals_2_s_ixxxx - kind: expression_statement any: - has: kind: assignment_expression all: - has: nthChild: 1 kind: identifier pattern: $MODE - has: nthChild: 2 matches: u_2_and_2_equals_2_s_ixxxx - has: kind: comma_expression has: kind: assignment_expression all: - has: nthChild: 1 kind: identifier pattern: $MODE - has: nthChild: 2 matches: u_2_and_2_equals_2_s_ixxxx nthChild: position: 3 ofRule: not: kind: comment inside: kind: argument_list nthChild: 2 follows: kind: identifier regex: ^(fchmodat)$ inside: kind: call_expression not: any: - matches: follows_umask - inside: stopBy: end matches: follows_umask # open - any: - matches: u_2_and_2_equals_2_s_ixxxx - kind: identifier pattern: $MODE inside: stopBy: end follows: stopBy: end any: - kind: declaration all: - has: kind: init_declarator all: - has: kind: identifier field: declarator pattern: $MODE - has: nthChild: 2 matches: u_2_and_2_equals_2_s_ixxxx - kind: expression_statement any: - has: kind: assignment_expression all: - has: nthChild: 1 kind: identifier pattern: $MODE - has: nthChild: 2 matches: u_2_and_2_equals_2_s_ixxxx - has: kind: comma_expression has: kind: assignment_expression all: - has: nthChild: 1 kind: identifier pattern: $MODE - has: nthChild: 2 matches: u_2_and_2_equals_2_s_ixxxx nthChild: position: 3 ofRule: not: kind: comment inside: kind: argument_list nthChild: 2 not: has: nthChild: position: 4 ofRule: not: kind: comment follows: kind: identifier regex: ^(open)$ inside: kind: call_expression not: any: - matches: follows_umask - inside: stopBy: end matches: follows_umask # openat - any: - matches: u_2_and_2_equals_2_s_ixxxx - kind: identifier pattern: $MODE inside: stopBy: end follows: stopBy: end any: - kind: declaration all: - has: kind: init_declarator all: - has: kind: identifier field: declarator pattern: $MODE - has: nthChild: 2 matches: u_2_and_2_equals_2_s_ixxxx - kind: expression_statement any: - has: kind: assignment_expression all: - has: nthChild: 1 kind: identifier pattern: $MODE - has: nthChild: 2 matches: u_2_and_2_equals_2_s_ixxxx - has: kind: comma_expression has: kind: assignment_expression all: - has: nthChild: 1 kind: identifier pattern: $MODE - has: nthChild: 2 matches: u_2_and_2_equals_2_s_ixxxx nthChild: position: 4 ofRule: not: kind: comment inside: kind: argument_list nthChild: 2 not: has: nthChild: position: 5 ofRule: not: kind: comment follows: kind: identifier regex: ^(openat)$ inside: kind: call_expression not: any: - matches: follows_umask - inside: stopBy: end matches: follows_umask myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/vendor/cpp/000077500000000000000000000000001525232545500253645ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/vendor/cpp/security/000077500000000000000000000000001525232545500272335ustar00rootroot00000000000000dont-call-system-cpp.yml000066400000000000000000000026361525232545500336650ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/vendor/cpp/securityid: dont-call-system-cpp language: cpp severity: warning message: >- Don't call `system`. It's a high-level wrapper that allows for stacking multiple commands. Always prefer a more restrictive API such as calling `execve` from the `exec` family. note: >- [CWE-78] Improper Neutralization of Special Elements used in an OS Command ('OS Command Injection'). [REFERENCES] - https://owasp.org/Top10/A03_2021-Injection ast-grep-essentials: true utils: PATTERN_SYSTEM_INSIDE_IF_STATEMENT: kind: call_expression all: - has: stopBy: neighbor kind: identifier regex: '^system$' - has: stopBy: neighbor kind: argument_list - inside: stopBy: end kind: parenthesized_expression inside: kind: if_statement PATTERN_SYSTEM: any: - kind: expression_statement - kind: return_statement - kind: field_declaration has: stopBy: neighbor kind: call_expression all: - has: stopBy: neighbor kind: identifier regex: '^system$' - has: stopBy: neighbor kind: argument_list rule: any: - matches: PATTERN_SYSTEM_INSIDE_IF_STATEMENT - matches: PATTERN_SYSTEM not: all: - has: stopBy: end kind: ERROR - inside: has: stopBy: end kind: ERROR file-access-before-action-cpp.yml000066400000000000000000000267571525232545500353710ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/vendor/cpp/securityid: file-access-before-action-cpp language: cpp severity: warning message: >- A check is done with `access` and then the file is later used. There is no guarantee that the status of the file has not changed since the call to `access` which may allow attackers to bypass permission checks. note: >- [CWE-367]: Time-of-check Time-of-use (TOCTOU) Race Condition [REFERENCES] - https://wiki.sei.cmu.edu/confluence/display/c/FIO45-C.+Avoid+TOCTOU+race+conditions+while+accessing+files ast-grep-essentials: true utils: u_1_pattern_1_identifier: kind: identifier regex: ^(fopen|freopen|remove|rename|access|open|stat|lstat|unlink|mkdir|rmdir|chdir)$ all: - precedes: kind: argument_list has: pattern: $SRC - inside: kind: call_expression not: inside: stopBy: end kind: condition_clause inside: stopBy: end kind: compound_statement inside: kind: if_statement has: kind: condition_clause has: stopBy: end any: - kind: binary_expression has: stopBy: end kind: parenthesized_expression has: kind: binary_expression all: - has: kind: call_expression nthChild: 1 all: - has: kind: identifier regex: ^(access|faccessat|faccessat2)$ precedes: kind: argument_list all: - has: nthChild: 1 pattern: $SRC - has: kind: identifier nthChild: 2 regex: ^(F_OK|R_OK|W_OK|X_OK)$ - has: kind: number_literal regex: ^(0)$ follows: regex: ^==$ - kind: binary_expression all: - has: nthChild: 1 kind: call_expression all: - has: nthChild: 1 kind: identifier regex: ^(access|faccessat|faccessat2)$ - has: nthChild: 2 kind: argument_list all: - has: nthChild: 1 pattern: $SRC - has: nthChild: 2 kind: identifier regex: ^(F_OK|R_OK|W_OK|X_OK)$ - has: nthChild: 2 kind: number_literal regex: ^(0)$ follows: regex: ^==$ u_2_pattern_2_qualified_identifier: kind: qualified_identifier any: - regex: ^(folly::readFile|folly::writeFile|folly::writeFileAtomic|folly::writeFileAtomicNoThrow|folly::File)$ - regex: ^(boost::)?(filesystem::file_size|filesystem::create_directory|filesystem::create_directories|filesystem::remove|filesystem::remove_all|filesystem::rename|filesystem::copy_file|filesystem::copy|filesystem::copy_directory|filesystem::resize_file|filesystem::last_write_time|filesystem::permissions|filesystem::symlink_status|filesystem::create_symlink|filesystem::create_hard_link|filesystem::read_symlink)$ all: - precedes: kind: argument_list has: pattern: $SRC - inside: kind: call_expression not: inside: stopBy: end kind: condition_clause inside: stopBy: end kind: compound_statement inside: kind: if_statement has: kind: condition_clause has: stopBy: end any: - kind: binary_expression has: stopBy: end kind: parenthesized_expression has: kind: binary_expression all: - has: kind: call_expression nthChild: 1 all: - has: kind: identifier regex: ^(access|faccessat|faccessat2)$ precedes: kind: argument_list all: - has: nthChild: 1 pattern: $SRC - has: kind: identifier nthChild: 2 regex: ^(F_OK|R_OK|W_OK|X_OK)$ - has: kind: number_literal regex: ^(0)$ follows: regex: ^==$ - kind: binary_expression all: - has: nthChild: 1 kind: call_expression all: - has: nthChild: 1 kind: identifier regex: ^(access|faccessat|faccessat2)$ - has: nthChild: 2 kind: argument_list all: - has: nthChild: 1 pattern: $SRC - has: nthChild: 2 kind: identifier regex: ^(F_OK|R_OK|W_OK|X_OK)$ - has: nthChild: 2 kind: number_literal regex: ^(0)$ follows: regex: ^==$ identifier_and_qualified_identifier: any: - kind: identifier regex: ^(fopen|freopen|remove|rename|access|open|stat|lstat|unlink|mkdir|rmdir|chdir)$ - kind: qualified_identifier any: - regex: ^(folly::readFile|folly::writeFile|folly::writeFileAtomic|folly::writeFileAtomicNoThrow|folly::File)$ - regex: ^(boost::)?(filesystem::file_size|filesystem::create_directory|filesystem::create_directories|filesystem::remove|filesystem::remove_all|filesystem::rename|filesystem::copy_file|filesystem::copy|filesystem::copy_directory|filesystem::resize_file|filesystem::last_write_time|filesystem::permissions|filesystem::symlink_status|filesystem::create_symlink|filesystem::create_hard_link|filesystem::read_symlink)$ u_4_pattern_3_field_expression: kind: field_expression has: nthChild: 1 stopBy: end matches: identifier_and_qualified_identifier all: - precedes: kind: argument_list has: pattern: $SRC - inside: kind: call_expression not: inside: stopBy: end kind: condition_clause inside: stopBy: end kind: compound_statement inside: kind: if_statement has: kind: condition_clause has: stopBy: end any: - kind: binary_expression has: stopBy: end kind: parenthesized_expression has: kind: binary_expression all: - has: kind: call_expression nthChild: 1 all: - has: kind: identifier regex: ^(access|faccessat|faccessat2)$ precedes: kind: argument_list all: - has: nthChild: 1 pattern: $SRC - has: kind: identifier nthChild: 2 regex: ^(F_OK|R_OK|W_OK|X_OK)$ - has: kind: number_literal regex: ^(0)$ follows: regex: ^==$ - kind: binary_expression all: - has: nthChild: 1 kind: call_expression all: - has: nthChild: 1 kind: identifier regex: ^(access|faccessat|faccessat2)$ - has: nthChild: 2 kind: argument_list all: - has: nthChild: 1 pattern: $SRC - has: nthChild: 2 kind: identifier regex: ^(F_OK|R_OK|W_OK|X_OK)$ - has: nthChild: 2 kind: number_literal regex: ^(0)$ follows: regex: ^==$ rule: any: - matches: u_1_pattern_1_identifier - matches: u_2_pattern_2_qualified_identifier - matches: u_4_pattern_3_field_expression file-stat-before-action-cpp.yml000066400000000000000000000470371525232545500350750ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/vendor/cpp/securityid: file-stat-before-action-cpp language: cpp severity: warning message: >- A check is done with `stat` and then the file is used. There is no guarantee that the status of the file has not changed since the call to `stat` which may allow attackers to bypass permission checks. note: >- [CWE-367]: Time-of-check Time-of-use (TOCTOU) Race Condition [REFERENCES] - https://wiki.sei.cmu.edu/confluence/display/c/FIO45-C.+Avoid+TOCTOU+race+conditions+while+accessing+files ast-grep-essentials: true utils: u_1_pattern_1_identifier_nth1: kind: identifier regex: ^(fopen|freopen|remove|rename|access|open|stat|lstat|unlink|mkdir|rmdir|chdir)$ all: - precedes: kind: argument_list has: pattern: $SRC - inside: kind: call_expression all: - not: inside: stopBy: end kind: condition_clause - not: inside: kind: field_expression inside: stopBy: end kind: compound_statement inside: kind: if_statement has: kind: condition_clause has: stopBy: end any: - kind: binary_expression has: stopBy: end kind: parenthesized_expression has: kind: binary_expression all: - has: kind: call_expression nthChild: 1 all: - has: kind: identifier regex: ^(stat|_stat|lstat|_lstat)$ precedes: kind: argument_list all: - has: nthChild: 1 pattern: $SRC - has: kind: number_literal regex: ^(0)$ - kind: binary_expression all: - has: nthChild: 1 kind: call_expression all: - has: nthChild: 1 kind: identifier regex: ^(stat|_stat|lstat|_lstat)$ - has: nthChild: 2 kind: argument_list all: - has: nthChild: 1 pattern: $SRC - has: nthChild: 2 kind: number_literal regex: ^(0)$ follows: regex: ^==$ u_2_pattern_2_qualified_identifier_nth1: kind: qualified_identifier any: - regex: ^(folly::readFile|folly::writeFile|folly::writeFileAtomic|folly::writeFileAtomicNoThrow|folly::File)$ - regex: ^(boost::)?(filesystem::file_size|filesystem::create_directory|filesystem::create_directories|filesystem::remove|filesystem::remove_all|filesystem::rename|filesystem::copy_file|filesystem::copy|filesystem::copy_directory|filesystem::resize_file|filesystem::last_write_time|filesystem::permissions|filesystem::symlink_status|filesystem::create_symlink|filesystem::create_hard_link|filesystem::read_symlink)$ all: - precedes: kind: argument_list has: pattern: $SRC - inside: kind: call_expression all: - not: inside: stopBy: end kind: condition_clause - not: inside: kind: field_expression inside: stopBy: end kind: compound_statement inside: kind: if_statement has: kind: condition_clause has: stopBy: end any: - kind: binary_expression has: stopBy: end kind: parenthesized_expression has: kind: binary_expression all: - has: kind: call_expression nthChild: 1 all: - has: kind: identifier regex: ^(stat|_stat|lstat|_lstat)$ precedes: kind: argument_list all: - has: nthChild: 1 pattern: $SRC - has: kind: number_literal regex: ^(0)$ follows: regex: ^==$ - kind: binary_expression all: - has: nthChild: 1 kind: call_expression all: - has: nthChild: 1 kind: identifier regex: ^(stat|_stat|lstat|_lstat)$ - has: nthChild: 2 kind: argument_list all: - has: nthChild: 1 pattern: $SRC - has: nthChild: 2 kind: number_literal regex: ^(0)$ follows: regex: ^==$ u_3_pattern_1_identifier_nth2: kind: identifier regex: ^(fopen|freopen|remove|rename|access|open|stat|lstat|unlink|mkdir|rmdir|chdir)$ all: - precedes: kind: argument_list has: pattern: $SRC - inside: kind: call_expression all: - not: inside: stopBy: end kind: condition_clause - not: inside: kind: field_expression inside: stopBy: end kind: compound_statement inside: kind: if_statement has: kind: condition_clause has: stopBy: end any: - kind: binary_expression has: stopBy: end kind: parenthesized_expression has: kind: binary_expression all: - has: kind: call_expression nthChild: 1 all: - has: kind: identifier regex: ^(fstatat|_fstatat)$ precedes: kind: argument_list all: - has: nthChild: 3 pattern: $SRC - has: kind: number_literal regex: ^(0)$ follows: regex: ^==$ - kind: binary_expression all: - has: nthChild: 1 kind: call_expression all: - has: nthChild: 1 kind: identifier regex: ^(fstatat|_fstatat)$ - has: nthChild: 2 kind: argument_list all: - has: nthChild: 2 pattern: $SRC - has: nthChild: 2 kind: number_literal regex: ^(0)$ follows: regex: ^==$ u_4_pattern_2_qualified_identifier_nth2: kind: qualified_identifier any: - regex: ^(folly::readFile|folly::writeFile|folly::writeFileAtomic|folly::writeFileAtomicNoThrow|folly::File)$ - regex: ^(boost::)?(filesystem::file_size|filesystem::create_directory|filesystem::create_directories|filesystem::remove|filesystem::remove_all|filesystem::rename|filesystem::copy_file|filesystem::copy|filesystem::copy_directory|filesystem::resize_file|filesystem::last_write_time|filesystem::permissions|filesystem::symlink_status|filesystem::create_symlink|filesystem::create_hard_link|filesystem::read_symlink)$ all: - precedes: kind: argument_list has: pattern: $SRC - inside: kind: call_expression all: - not: inside: stopBy: end kind: condition_clause - not: inside: kind: field_expression inside: stopBy: end kind: compound_statement inside: kind: if_statement has: kind: condition_clause has: stopBy: end any: - kind: binary_expression has: stopBy: end kind: parenthesized_expression has: kind: binary_expression all: - has: kind: call_expression nthChild: 1 all: - has: kind: identifier regex: ^(fstatat|_fstatat)$ precedes: kind: argument_list all: - has: nthChild: 2 pattern: $SRC - has: kind: number_literal regex: ^(0)$ follows: regex: ^==$ - kind: binary_expression all: - has: nthChild: 1 kind: call_expression all: - has: nthChild: 1 kind: identifier regex: ^(fstatat|_fstatat)$ - has: nthChild: 2 kind: argument_list all: - has: nthChild: 2 pattern: $SRC - has: nthChild: 2 kind: number_literal regex: ^(0)$ follows: regex: ^==$ identifier_and_qualified_identifier: any: - kind: identifier regex: ^(fopen|freopen|remove|rename|access|open|stat|lstat|unlink|mkdir|rmdir|chdir)$ - kind: qualified_identifier any: - regex: ^(folly::readFile|folly::writeFile|folly::writeFileAtomic|folly::writeFileAtomicNoThrow|folly::File)$ - regex: ^(boost::)?(filesystem::file_size|filesystem::create_directory|filesystem::create_directories|filesystem::remove|filesystem::remove_all|filesystem::rename|filesystem::copy_file|filesystem::copy|filesystem::copy_directory|filesystem::resize_file|filesystem::last_write_time|filesystem::permissions|filesystem::symlink_status|filesystem::create_symlink|filesystem::create_hard_link|filesystem::read_symlink)$ u_6_pattern_3_field_expression_identifier_nth1: kind: field_expression has: nthChild: 1 stopBy: end matches: identifier_and_qualified_identifier all: - precedes: kind: argument_list has: pattern: $SRC - inside: kind: call_expression all: - not: inside: stopBy: end kind: condition_clause - not: inside: kind: field_expression inside: stopBy: end kind: compound_statement inside: kind: if_statement has: kind: condition_clause has: stopBy: end any: - kind: binary_expression has: stopBy: end kind: parenthesized_expression has: kind: binary_expression all: - has: kind: call_expression nthChild: 1 all: - has: kind: identifier regex: ^(stat|_stat|lstat|_lstat)$ precedes: kind: argument_list all: - has: nthChild: 1 pattern: $SRC - has: kind: number_literal regex: ^(0)$ follows: regex: ^==$ - kind: binary_expression all: - has: nthChild: 1 kind: call_expression all: - has: nthChild: 1 kind: identifier regex: ^(stat|_stat|lstat|_lstat)$ - has: nthChild: 2 kind: argument_list all: - has: nthChild: 1 pattern: $SRC - has: nthChild: 2 kind: number_literal regex: ^(0)$ follows: regex: ^==$ u_7_pattern_3_field_expression_identifier_nth2: kind: field_expression has: nthChild: 1 stopBy: end matches: identifier_and_qualified_identifier all: - precedes: kind: argument_list has: pattern: $SRC - inside: kind: call_expression all: - not: inside: stopBy: end kind: condition_clause - not: inside: kind: field_expression inside: stopBy: end kind: compound_statement inside: kind: if_statement has: kind: condition_clause has: stopBy: end any: - kind: binary_expression has: stopBy: end kind: parenthesized_expression has: kind: binary_expression all: - has: kind: call_expression nthChild: 1 all: - has: kind: identifier regex: ^(fstatat|_fstatat)$ precedes: kind: argument_list all: - has: nthChild: 2 pattern: $SRC - has: kind: number_literal regex: ^(0)$ follows: regex: ^==$ - kind: binary_expression all: - has: nthChild: 1 kind: call_expression all: - has: nthChild: 1 kind: identifier regex: ^(fstatat|_fstatat)$ - has: nthChild: 2 kind: argument_list all: - has: nthChild: 2 pattern: $SRC - has: nthChild: 2 kind: number_literal regex: ^(0)$ follows: regex: ^==$ rule: any: - matches: u_1_pattern_1_identifier_nth1 - matches: u_2_pattern_2_qualified_identifier_nth1 - matches: u_3_pattern_1_identifier_nth2 - matches: u_4_pattern_2_qualified_identifier_nth2 - matches: u_6_pattern_3_field_expression_identifier_nth1 - matches: u_7_pattern_3_field_expression_identifier_nth2 fix-format-security-error-cpp.yml000066400000000000000000000007571525232545500355400ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/vendor/cpp/securityid: fix-format-security-error-cpp language: cpp severity: warning message: The Format String exploit occurs when the submitted data of an input string is evaluated as a command by the application. ast-grep-essentials: true rule: pattern: $PRINTF($S, $VAR) constraints: PRINTF: # a format string function { regex: "^sprintf|fprintf$" } VAR: # not a literal string not: any: - { kind: string_literal } - { kind: concatenated_string } fix: $PRINTF($S, "%s", $VAR) myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/vendor/cpp/security/insecure-hash-cpp.yml000066400000000000000000000072131525232545500332770ustar00rootroot00000000000000id: insecure-hash-cpp language: cpp severity: warning message: >- This hashing algorithm is insecure. If this hash is used in a security context, such as password hashing, it should be converted to a stronger hashing algorithm. note: >- [CWE-328] Use of Weak Hash. [REFERENCES] - https://owasp.org/Top10/A02_2021-Cryptographic_Failures ast-grep-essentials: true utils: MATCH_PATTERN_ONE: kind: expression_statement has: stopBy: neighbor kind: call_expression all: - has: stopBy: neighbor kind: identifier regex: ^(EVP_md2|MD2|MD2_Final|MD2_Init|MD2_Update|MD2_options|EVP_md4|MD4|MD4_Final|MD4_Init|MD4_Transform|MD4_Update|EVP_md5|MD5|MD5_Final|MD5_Init|MD5_Transform|MD5_Update|EVP_sha1|SHA1_Final|SHA1_Init|SHA1_Transform|SHA1_Update)$ - has: stopBy: neighbor kind: argument_list MATCH_PATTERN_TWO: kind: expression_statement has: stopBy: neighbor kind: call_expression all: - has: stopBy: neighbor kind: identifier regex: "^(EVP_MD_fetch|EVP_get_digestbyname)$" - has: stopBy: neighbor kind: argument_list has: stopBy: end kind: string_content regex: ^(MD2|MD4|MD5|SHA1|SHA-1)$ MATCH_PATTERN_TWO_with_instance: kind: expression_statement all: - has: stopBy: neighbor kind: call_expression all: - has: stopBy: neighbor kind: identifier regex: "^(EVP_MD_fetch|EVP_get_digestbyname)$" - has: stopBy: neighbor kind: argument_list has: stopBy: neighbor kind: identifier pattern: $Q - follows: stopBy: end kind: declaration has: stopBy: end kind: init_declarator all: - has: stopBy: neighbor any: - kind: array_declarator has: stopBy: neighbor kind: identifier pattern: $Q - kind: pointer_declarator has: stopBy: neighbor kind: identifier pattern: $Q - kind: identifier pattern: $Q - has: stopBy: neighbor kind: string_literal has: stopBy: neighbor kind: string_content regex: ^(MD2|MD4|MD5|SHA1|SHA-1)$ MATCH_PATTERN_THREE: kind: expression_statement has: stopBy: neighbor kind: call_expression all: - has: stopBy: neighbor kind: identifier regex: "^(gcry_md_open|gcry_md_enable|gcry_md_read|gcry_md_extract|gcry_md_hash_buffers|gcry_md_hash_buffer)$" - has: stopBy: neighbor kind: argument_list has: stopBy: end kind: identifier regex: ^(GCRY_MD_MD2|GCRY_MD_MD4|GCRY_MD_MD5|GCRY_MD_SHA1)$ rule: any: - kind: expression_statement any: - matches: MATCH_PATTERN_ONE - matches: MATCH_PATTERN_TWO - matches: MATCH_PATTERN_TWO_with_instance - matches: MATCH_PATTERN_THREE not: all: - has: stopBy: end kind: ERROR - inside: stopBy: end kind: ERROR libxml2-audit-parser-cpp.yml000066400000000000000000000155711525232545500344370ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/vendor/cpp/securityid: libxml2-audit-parser-cpp language: Cpp severity: warning message: >- The libxml2 library is used to parse XML. When auditing such code, make sure that either the document being parsed is trusted or that the parsing options are safe to consume untrusted documents. In such case make sure DTD or XInclude documents cannot be loaded and there is no network access. note: >- [CWE-611] Improper Restriction of XML External Entity Reference. [REFERENCES] - https://owasp.org/Top10/A05_2021-Security_Misconfiguration ast-grep-essentials: true utils: Pattern_having_three_child: kind: call_expression all: - has: kind: identifier regex: ^(xmlReadFile)$ - has: kind: argument_list all: - has: nthChild: position: 1 ofRule: not: kind: comment - has: nthChild: position: 2 ofRule: not: kind: comment - has: nthChild: position: 3 ofRule: not: kind: comment - not: has: nthChild: position: 4 ofRule: not: kind: comment Pattern_having_five_child: kind: call_expression all: - has: kind: identifier regex: ^(xmlParseInNodeContext|xmlReadMemory|xmlCtxtReadDoc|xmlCtxtReadFd)$ - has: kind: argument_list all: - has: nthChild: position: 1 ofRule: not: kind: comment - has: nthChild: position: 2 ofRule: not: kind: comment - has: nthChild: position: 3 ofRule: not: kind: comment - has: nthChild: position: 4 ofRule: not: kind: comment - has: nthChild: position: 5 ofRule: not: kind: comment - not: has: nthChild: position: 6 ofRule: not: kind: comment Pattern_having_four_child: kind: call_expression all: - has: kind: identifier regex: ^(xmlReadDoc|xmlReadFd|xmlCtxtReadFile)$ - has: kind: argument_list all: - has: nthChild: position: 1 ofRule: not: kind: comment - has: nthChild: position: 2 ofRule: not: kind: comment - has: nthChild: position: 3 ofRule: not: kind: comment - has: nthChild: position: 4 ofRule: not: kind: comment - not: has: nthChild: position: 5 ofRule: not: kind: comment Pattern_having_six_child: kind: call_expression all: - has: kind: identifier regex: ^(xmlReadIO|xmlCtxtReadMemory)$ - has: kind: argument_list all: - has: nthChild: position: 1 ofRule: not: kind: comment - has: nthChild: position: 2 ofRule: not: kind: comment - has: nthChild: position: 3 ofRule: not: kind: comment - has: nthChild: position: 4 ofRule: not: kind: comment - has: nthChild: position: 5 ofRule: not: kind: comment - has: nthChild: position: 6 ofRule: not: kind: comment - not: has: nthChild: position: 7 ofRule: not: kind: comment Pattern_having_seven_child: kind: call_expression all: - has: kind: identifier regex: ^(xmlCtxtReadIO)$ - has: kind: argument_list all: - has: nthChild: position: 1 ofRule: not: kind: comment - has: nthChild: position: 2 ofRule: not: kind: comment - has: nthChild: position: 3 ofRule: not: kind: comment - has: nthChild: position: 4 ofRule: not: kind: comment - has: nthChild: position: 5 ofRule: not: kind: comment - has: nthChild: position: 6 ofRule: not: kind: comment - has: nthChild: position: 7 ofRule: not: kind: comment - not: has: nthChild: position: 8 ofRule: not: kind: comment rule: kind: call_expression any: - matches: Pattern_having_five_child - matches: Pattern_having_four_child - matches: Pattern_having_six_child - matches: Pattern_having_seven_child - matches: Pattern_having_three_child not: all: - has: stopBy: end kind: ERROR - inside: stopBy: end kind: ERROR missing-nul-cpp-string-memcpy-cpp.yml000066400000000000000000000314421525232545500363040ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/vendor/cpp/securityid: missing-nul-cpp-string-memcpy-copy-cpp language: cpp severity: warning message: >- The number of bytes copied from `$STR` does not include the NUL terminator. This can lead to an out-of-bounds read and information disclosure. One extra byte should be added to the length to ensure that the NUL terminator is copied. note: >- [CWE-125]: Out-of-bounds Read [REFERENCES] - https://wiki.sei.cmu.edu/confluence/display/c/STR31-C.+Guarantee+that+storage+for+strings+has+sufficient+space+for+character+data+and+the+null+terminator ast-grep-essentials: true rule: any: - kind: qualified_identifier - kind: identifier pattern: $MEMFUNC regex: ^(memcpy|wmemcpy|memmove|wmemmove|std::memcpy|std::wmemcpy|std::memmove|std::wmemmove)$ inside: stopBy: end any: - kind: call_expression all: - has: any: - kind: qualified_identifier - kind: identifier nthChild: 1 pattern: $MEMFUNC - has: kind: argument_list nthChild: 2 all: - has: pattern: $DEST nthChild: position: 1 ofRule: not: kind: comment - has: pattern: $STR.c_str() nthChild: position: 2 ofRule: not: kind: comment - has: pattern: strlen($STR.c_str()) nthChild: position: 3 ofRule: not: kind: comment - not: has: nthChild: position: 4 ofRule: not: kind: comment # - pattern: $MEMFUNC($DEST, $STR.c_str(), strlen($STR.c_str())) any: - inside: stopBy: end follows: stopBy: end any: - pattern: $TYPE $DEST[$DIM] = $$$; - pattern: $TYPE $DEST[$DIM]; - pattern: $TYPE *$DEST = $$$; - follows: stopBy: end any: - pattern: $TYPE $DEST[$DIM] = $$$; - pattern: $TYPE $DEST[$DIM]; - pattern: $TYPE *$DEST = $$$; - kind: call_expression all: - has: any: - kind: qualified_identifier - kind: identifier nthChild: 1 pattern: $MEMFUNC - has: kind: argument_list nthChild: 2 all: - has: pattern: $DEST nthChild: position: 1 ofRule: not: kind: comment - has: pattern: $STR.c_str() nthChild: position: 2 ofRule: not: kind: comment - has: pattern: $STR.size() nthChild: position: 3 ofRule: not: kind: comment - not: has: nthChild: position: 4 ofRule: not: kind: comment # pattern: $MEMFUNC($DEST, $STR.c_str(), $STR.size()) any: - inside: stopBy: end follows: stopBy: end any: - pattern: $TYPE $DEST[$DIM] = $$$; - pattern: $TYPE $DEST[$DIM]; - pattern: $TYPE *$DEST = $$$; - follows: stopBy: end any: - pattern: $TYPE $DEST[$DIM] = $$$; - pattern: $TYPE $DEST[$DIM]; - pattern: $TYPE *$DEST = $$$; - kind: call_expression all: - has: pattern: $MEMFUNC any: - kind: qualified_identifier - kind: identifier nthChild: 1 - has: kind: argument_list nthChild: 2 all: - has: pattern: $DEST nthChild: position: 1 ofRule: not: kind: comment - has: pattern: $STR.c_str() nthChild: position: 2 ofRule: not: kind: comment - has: pattern: $STR.length() nthChild: position: 3 ofRule: not: kind: comment - not: has: nthChild: position: 4 ofRule: not: kind: comment # pattern: $MEMFUNC($DEST, $STR.c_str(), $STR.length()) any: - inside: stopBy: end follows: stopBy: end any: - pattern: $TYPE $DEST[$DIM] = $$$; - pattern: $TYPE $DEST[$DIM]; - pattern: $TYPE *$DEST = $$$; - follows: stopBy: end any: - pattern: $TYPE $DEST[$DIM] = $$$; - pattern: $TYPE $DEST[$DIM]; - pattern: $TYPE *$DEST = $$$; - kind: call_expression all: - has: nthChild: 1 any: - kind: qualified_identifier - kind: identifier pattern: $MEMFUNC - has: nthChild: 2 kind: argument_list all: - has: pattern: $DEST nthChild: position: 1 ofRule: not: kind: comment - has: pattern: $STR.c_str() nthChild: position: 2 ofRule: not: kind: comment - has: pattern: $LEN nthChild: position: 3 ofRule: not: kind: comment - not: has: nthChild: position: 4 ofRule: not: kind: comment # pattern: $MEMFUNC($DEST, $STR.c_str(), $LEN) - all: - any: - follows: stopBy: end any: - pattern: $LEN = strlen($STR.c_str()); - pattern: $SET $LEN = strlen($STR.c_str()); - inside: stopBy: end follows: stopBy: end any: - pattern: $LEN = strlen($STR.c_str()); - pattern: $SET $LEN = strlen($STR.c_str()); - inside: stopBy: end follows: stopBy: end kind: declaration has: kind: init_declarator all: - has: kind: identifier pattern: $LEN - has: kind: call_expression pattern: from.size() - any: - inside: stopBy: end follows: stopBy: end any: - pattern: $TYPE $DEST[$DIM] = $$$; - pattern: $TYPE $DEST[$DIM]; - pattern: $TYPE *$DEST = $$$; - follows: stopBy: end any: - pattern: $TYPE $DEST[$DIM] = $$$; - pattern: $TYPE $DEST[$DIM]; - pattern: $TYPE *$DEST = $$$; - kind: call_expression all: - has: any: - kind: qualified_identifier - kind: identifier nthChild: 1 pattern: $MEMFUNC - has: kind: argument_list all: - has: pattern: $DEST nthChild: position: 1 ofRule: not: kind: comment - has: pattern: $STR.c_str() nthChild: position: 2 ofRule: not: kind: comment - has: pattern: $LEN nthChild: position: 3 ofRule: not: kind: comment - not: has: nthChild: position: 4 ofRule: not: kind: comment # pattern: $MEMFUNC($DEST, $STR.c_str(), $LEN) follows: stopBy: end any: - pattern: $LEN = $STR.size(); - pattern: $SET $LEN = $STR.size(); any: - inside: stopBy: end follows: stopBy: end any: - pattern: $TYPE $DEST[$DIM] = $$$; - pattern: $TYPE $DEST[$DIM]; - pattern: $TYPE *$DEST = $$$; - follows: stopBy: end any: - pattern: $TYPE $DEST[$DIM] = $$$; - pattern: $TYPE $DEST[$DIM]; - pattern: $TYPE *$DEST = $$$; - kind: call_expression all: - has: any: - kind: qualified_identifier - kind: identifier nthChild: 1 pattern: $MEMFUNC - has: kind: argument_list all: - has: pattern: $DEST nthChild: position: 1 ofRule: not: kind: comment - has: pattern: $STR.c_str() nthChild: position: 2 ofRule: not: kind: comment - has: pattern: $LEN nthChild: position: 3 ofRule: not: kind: comment - not: has: nthChild: position: 4 ofRule: not: kind: comment # pattern: $MEMFUNC($DEST, $STR.c_str(), $LEN) - any: - inside: stopBy: end follows: stopBy: end any: - pattern: $LEN = $STR.length(); - pattern: $SET $LEN = $STR.length(); - follows: stopBy: end any: - pattern: $LEN = $STR.length(); - pattern: $SET $LEN = $STR.length(); any: - inside: stopBy: end follows: stopBy: end any: - pattern: $TYPE $DEST[$DIM] = $$$; - pattern: $TYPE $DEST[$DIM]; - pattern: $TYPE *$DEST = $$$; - follows: stopBy: end any: - pattern: $TYPE $DEST[$DIM] = $$$; - pattern: $TYPE $DEST[$DIM]; - pattern: $TYPE *$DEST = $$$; null-library-function-cpp.yml000066400000000000000000000363071525232545500347270ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/vendor/cpp/securityid: null-library-function-cpp language: cpp severity: warning message: >- The `$SOURCE` function returns NULL on error and this line dereferences the return value without checking for NULL. note: >- [CWE-476] NULL Pointer Dereference. [REFERENCES] - https://wiki.sei.cmu.edu/confluence/display/c/EXP34-C.+Do+not+dereference+null+pointers ast-grep-essentials: true rule: all: - not: has: stopBy: end kind: ERROR - any: - kind: subscript_expression # any: # - pattern: $SOURCE($$$)[$$$] # - pattern: ($SOURCE($$$))[$$$] all: - has: stopBy: end kind: call_expression all: - has: stopBy: end kind: identifier pattern: $SOURCE regex: ^fgets|::fgets|std::fgets|fopen|::fopen|std::fopen|getenv|::getenv|std::getenv|getgrent|::getgrent|std::getgrent|getgrgid|::getgrgid|std::getgrgid|getgrnam|::getgrnam|std::getgrnam|getlogin|::getlogin|std::getlogin|getpwent|::getpwent|std::getpwent|getpwnam|::getpwnam|std::getpwnam|getpwuid|::getpwuid|std::getpwuid|getpwuuid|::getpwuuid|std::getpwuuid|gets|::gets|std::gets|inet_ntop|::inet_ntop|std::inet_ntop|realpath|::realpath|std::realpath|tempnam|::tempnam|std::tempnam|tmpfile|::tmpfile|std::tmpfile|tmpnam|::tmpnam|std::tmpnam|memchr|::memchr|std::memchr|strcasestr_l|::strcasestr_l|std::strcasestr_l|strcasestr|::strcasestr|std::strcasestr|strchr|::strchr|std::strchr|strnstr|::strnstr|std::strnstr|strpbrk|::strpbrk|std::strpbrk|strrchr|::strrchr|std::strrchr|strstr|::strstr|std::strstr|strtok_r|::strtok_r|std::strtok_r|strtok|::strtok|std::strtok$ - has: stopBy: neighbor kind: argument_list - has: stopBy: end any: - kind: number_literal - kind: identifier - kind: call_expression all: - has: stopBy: neighbor kind: identifier pattern: $SINK regex: ^atof|::atof|std::atof|atoi|::atoi|std::atoi|atol_l|::atol_l|std::atol_l|atol|::atol|std::atol|atoll_l|::atoll_l|std::atoll_l|atoll|::atoll|std::atoll|getc|::getc|std::getc|fprintf|::fprintf|std::fprintf|fgetpos|::fgetpos|std::fgetpos|fseek|::fseek|std::fseek|fseeko|::fseeko|std::fseeko|fsetpos|::fsetpos|std::fsetpos|ftell|::ftell|std::ftell|ftello|::ftello|std::ftello|rewind|::rewind|std::rewind|strlen|::strlen|std::strlen|strtoimax|::strtoimax|std::strtoimax|strtod|::strtod|std::strtod|strtol|::strtol|std::strtol|strtoul|::strtoul|std::strtoul|strtoll|::strtoll|std::strtoll|strtoq|::strtoq|std::strtoq$ - has: stopBy: neighbor kind: argument_list has: stopBy: neighbor kind: call_expression nthChild: 1 all: - has: stopBy: neighbor kind: identifier pattern: $SOURCE regex: ^fgets|::fgets|std::fgets|fopen|::fopen|std::fopen|getenv|::getenv|std::getenv|getgrent|::getgrent|std::getgrent|getgrgid|::getgrgid|std::getgrgid|getgrnam|::getgrnam|std::getgrnam|getlogin|::getlogin|std::getlogin|getpwent|::getpwent|std::getpwent|getpwnam|::getpwnam|std::getpwnam|getpwuid|::getpwuid|std::getpwuid|getpwuuid|::getpwuuid|std::getpwuuid|gets|::gets|std::gets|inet_ntop|::inet_ntop|std::inet_ntop|realpath|::realpath|std::realpath|tempnam|::tempnam|std::tempnam|tmpfile|::tmpfile|std::tmpfile|tmpnam|::tmpnam|std::tmpnam|memchr|::memchr|std::memchr|strcasestr_l|::strcasestr_l|std::strcasestr_l|strcasestr|::strcasestr|std::strcasestr|strchr|::strchr|std::strchr|strnstr|::strnstr|std::strnstr|strpbrk|::strpbrk|std::strpbrk|strrchr|::strrchr|std::strrchr|strstr|::strstr|std::strstr|strtok_r|::strtok_r|std::strtok_r|strtok|::strtok|std::strtok$ - has: stopBy: neighbor kind: argument_list - kind: call_expression all: - has: stopBy: neighbor kind: identifier pattern: $SINK regex: ^atof|::atof|std::atof|atoi|::atoi|std::atoi|atol_l|::atol_l|std::atol_l|atol|::atol|std::atol|atoll_l|::atoll_l|std::atoll_l|atoll|::atoll|std::atoll|getc|::getc|std::getc|fprintf|::fprintf|std::fprintf|fgetpos|::fgetpos|std::fgetpos|fseek|::fseek|std::fseek|fseeko|::fseeko|std::fseeko|fsetpos|::fsetpos|std::fsetpos|ftell|::ftell|std::ftell|ftello|::ftello|std::ftello|rewind|::rewind|std::rewind|strlen|::strlen|std::strlen|strtoimax|::strtoimax|std::strtoimax|strtod|::strtod|std::strtod|strtol|::strtol|std::strtol|strtoul|::strtoul|std::strtoul|strtoll|::strtoll|std::strtoll|strtoq|::strtoq|std::strtoq$ - has: stopBy: neighbor kind: argument_list has: stopBy: neighbor kind: assignment_expression nthChild: 1 all: - has: stopBy: neighbor kind: identifier pattern: $VAR - has: stopBy: neighbor kind: call_expression all: - has: stopBy: neighbor kind: identifier pattern: $SOURCE regex: ^fgets|::fgets|std::fgets|fopen|::fopen|std::fopen|getenv|::getenv|std::getenv|getgrent|::getgrent|std::getgrent|getgrgid|::getgrgid|std::getgrgid|getgrnam|::getgrnam|std::getgrnam|getlogin|::getlogin|std::getlogin|getpwent|::getpwent|std::getpwent|getpwnam|::getpwnam|std::getpwnam|getpwuid|::getpwuid|std::getpwuid|getpwuuid|::getpwuuid|std::getpwuuid|gets|::gets|std::gets|inet_ntop|::inet_ntop|std::inet_ntop|realpath|::realpath|std::realpath|tempnam|::tempnam|std::tempnam|tmpfile|::tmpfile|std::tmpfile|tmpnam|::tmpnam|std::tmpnam|memchr|::memchr|std::memchr|strcasestr_l|::strcasestr_l|std::strcasestr_l|strcasestr|::strcasestr|std::strcasestr|strchr|::strchr|std::strchr|strnstr|::strnstr|std::strnstr|strpbrk|::strpbrk|std::strpbrk|strrchr|::strrchr|std::strrchr|strstr|::strstr|std::strstr|strtok_r|::strtok_r|std::strtok_r|strtok|::strtok|std::strtok$ - kind: call_expression all: - has: stopBy: neighbor kind: identifier pattern: $SINK - has: stopBy: neighbor kind: argument_list has: stopBy: neighbor kind: call_expression nthChild: 2 all: - has: stopBy: neighbor kind: identifier pattern: $SOURCE regex: ^fgets|::fgets|std::fgets|fopen|::fopen|std::fopen|getenv|::getenv|std::getenv|getgrent|::getgrent|std::getgrent|getgrgid|::getgrgid|std::getgrgid|getgrnam|::getgrnam|std::getgrnam|getlogin|::getlogin|std::getlogin|getpwent|::getpwent|std::getpwent|getpwnam|::getpwnam|std::getpwnam|getpwuid|::getpwuid|std::getpwuid|getpwuuid|::getpwuuid|std::getpwuuid|gets|::gets|std::gets|inet_ntop|::inet_ntop|std::inet_ntop|realpath|::realpath|std::realpath|tempnam|::tempnam|std::tempnam|tmpfile|::tmpfile|std::tmpfile|tmpnam|::tmpnam|std::tmpnam|memchr|::memchr|std::memchr|strcasestr_l|::strcasestr_l|std::strcasestr_l|strcasestr|::strcasestr|std::strcasestr|strchr|::strchr|std::strchr|strnstr|::strnstr|std::strnstr|strpbrk|::strpbrk|std::strpbrk|strrchr|::strrchr|std::strrchr|strstr|::strstr|std::strstr|strtok_r|::strtok_r|std::strtok_r|strtok|::strtok|std::strtok$ - has: stopBy: neighbor kind: argument_list - not: inside: stopBy: end kind: call_expression has: stopBy: neighbor kind: identifier pattern: $SINK - not: has: stopBy: end kind: call_expression all: - has: stopBy: neighbor kind: identifier pattern: $SOURCE - has: stopBy: neighbor kind: argument_list has: stopBy: end kind: call_expression has: stopBy: neighbor kind: identifier pattern: $SOURCE - kind: call_expression all: - has: stopBy: neighbor kind: identifier pattern: $SINK regex: ^bcopy|::bcopy|std::bcopy|memccpy|::memccpy|std::memccpy|memcpy|::memcpy|std::memcpy|memmove|::memmove|std::memmove|stpncpy|::stpncpy|std::stpncpy|strcat|::strcat|std::strcat|strcpy|::strcpy|std::strcpy|strcpy|::strcpy|std::strcpy|strlcat|::strlcat|std::strlcat|strlcpy|::strlcpy|std::strlcpy|strncat|::strncat|std::strncat|strpcpy|::strpcpy|std::strpcpy|wcpcpy|::wcpcpy|std::wcpcpy|wcpncpy|::wcpncpy|std::wcpncpy$ - has: stopBy: neighbor kind: argument_list has: stopBy: neighbor kind: assignment_expression pattern: $VAR = $SOURCE($$$) all: - has: stopBy: neighbor kind: identifier pattern: $VAR - has: stopBy: neighbor kind: call_expression all: - has: stopBy: neighbor kind: identifier pattern: $SOURCE regex: ^fgets|::fgets|std::fgets|fopen|::fopen|std::fopen|getenv|::getenv|std::getenv|getgrent|::getgrent|std::getgrent|getgrgid|::getgrgid|std::getgrgid|getgrnam|::getgrnam|std::getgrnam|getlogin|::getlogin|std::getlogin|getpwent|::getpwent|std::getpwent|getpwnam|::getpwnam|std::getpwnam|getpwuid|::getpwuid|std::getpwuid|getpwuuid|::getpwuuid|std::getpwuuid|gets|::gets|std::gets|inet_ntop|::inet_ntop|std::inet_ntop|realpath|::realpath|std::realpath|tempnam|::tempnam|std::tempnam|tmpfile|::tmpfile|std::tmpfile|tmpnam|::tmpnam|std::tmpnam|memchr|::memchr|std::memchr|strcasestr_l|::strcasestr_l|std::strcasestr_l|strcasestr|::strcasestr|std::strcasestr|strchr|::strchr|std::strchr|strnstr|::strnstr|std::strnstr|strpbrk|::strpbrk|std::strpbrk|strrchr|::strrchr|std::strrchr|strstr|::strstr|std::strstr|strtok_r|::strtok_r|std::strtok_r|strtok|::strtok|std::strtok$ - kind: call_expression # any: # - pattern: $SINK($$$, $SOURCE($$$)) # - pattern: $SINK($SOURCE($$$)) all: - has: stopBy: neighbor kind: identifier pattern: $SINK regex: ^fwrite|::fwrite|std::fwrite$ - has: stopBy: neighbor kind: argument_list has: stopBy: end kind: call_expression all: - has: stopBy: end kind: identifier pattern: $SOURCE regex: ^fgets|::fgets|std::fgets|fopen|::fopen|std::fopen|getenv|::getenv|std::getenv|getgrent|::getgrent|std::getgrent|getgrgid|::getgrgid|std::getgrgid|getgrnam|::getgrnam|std::getgrnam|getlogin|::getlogin|std::getlogin|getpwent|::getpwent|std::getpwent|getpwnam|::getpwnam|std::getpwnam|getpwuid|::getpwuid|std::getpwuid|getpwuuid|::getpwuuid|std::getpwuuid|gets|::gets|std::gets|inet_ntop|::inet_ntop|std::inet_ntop|realpath|::realpath|std::realpath|tempnam|::tempnam|std::tempnam|tmpfile|::tmpfile|std::tmpfile|tmpnam|::tmpnam|std::tmpnam|memchr|::memchr|std::memchr|strcasestr_l|::strcasestr_l|std::strcasestr_l|strcasestr|::strcasestr|std::strcasestr|strchr|::strchr|std::strchr|strnstr|::strnstr|std::strnstr|strpbrk|::strpbrk|std::strpbrk|strrchr|::strrchr|std::strrchr|strstr|::strstr|std::strstr|strtok_r|::strtok_r|std::strtok_r|strtok|::strtok|std::strtok$ - has: stopBy: neighbor kind: argument_list - kind: call_expression any: - pattern: $SINK($$$, $VAR = $SOURCE($$$)) - pattern: $SINK($VAR = $SOURCE($$$)) all: - has: stopBy: neighbor kind: identifier pattern: $SINK regex: ^fwrite|::fwrite|std::fwrite$ - has: stopBy: neighbor kind: argument_list has: stopBy: end kind: assignment_expression pattern: $VAR = $SOURCE($$$) all: - has: stopBy: neighbor kind: identifier - has: stopBy: neighbor kind: call_expression all: - has: stopBy: neighbor kind: identifier pattern: $SOURCE regex: ^fgets|::fgets|std::fgets|fopen|::fopen|std::fopen|getenv|::getenv|std::getenv|getgrent|::getgrent|std::getgrent|getgrgid|::getgrgid|std::getgrgid|getgrnam|::getgrnam|std::getgrnam|getlogin|::getlogin|std::getlogin|getpwent|::getpwent|std::getpwent|getpwnam|::getpwnam|std::getpwnam|getpwuid|::getpwuid|std::getpwuid|getpwuuid|::getpwuuid|std::getpwuuid|gets|::gets|std::gets|inet_ntop|::inet_ntop|std::inet_ntop|realpath|::realpath|std::realpath|tempnam|::tempnam|std::tempnam|tmpfile|::tmpfile|std::tmpfile|tmpnam|::tmpnam|std::tmpnam|memchr|::memchr|std::memchr|strcasestr_l|::strcasestr_l|std::strcasestr_l|strcasestr|::strcasestr|std::strcasestr|strchr|::strchr|std::strchr|strnstr|::strnstr|std::strnstr|strpbrk|::strpbrk|std::strpbrk|strrchr|::strrchr|std::strrchr|strstr|::strstr|std::strstr|strtok_r|::strtok_r|std::strtok_r|strtok|::strtok|std::strtok$ - has: stopBy: neighbor kind: argument_list - kind: pointer_expression has: stopBy: neighbor kind: call_expression all: - has: stopBy: end kind: identifier pattern: $SOURCE regex: ^fgets|::fgets|std::fgets|fopen|::fopen|std::fopen|getenv|::getenv|std::getenv|getgrent|::getgrent|std::getgrent|getgrgid|::getgrgid|std::getgrgid|getgrnam|::getgrnam|std::getgrnam|getlogin|::getlogin|std::getlogin|getpwent|::getpwent|std::getpwent|getpwnam|::getpwnam|std::getpwnam|getpwuid|::getpwuid|std::getpwuid|getpwuuid|::getpwuuid|std::getpwuuid|gets|::gets|std::gets|inet_ntop|::inet_ntop|std::inet_ntop|realpath|::realpath|std::realpath|tempnam|::tempnam|std::tempnam|tmpfile|::tmpfile|std::tmpfile|tmpnam|::tmpnam|std::tmpnam|memchr|::memchr|std::memchr|strcasestr_l|::strcasestr_l|std::strcasestr_l|strcasestr|::strcasestr|std::strcasestr|strchr|::strchr|std::strchr|strnstr|::strnstr|std::strnstr|strpbrk|::strpbrk|std::strpbrk|strrchr|::strrchr|std::strrchr|strstr|::strstr|std::strstr|strtok_r|::strtok_r|std::strtok_r|strtok|::strtok|std::strtok$ - has: stopBy: neighbor kind: argument_list - not: inside: stopBy: end any: - kind: subscript_expression # - kind: call_expression - not: has: stopBy: end any: - kind: assignment_expression - inside: stopBy: end kind: return_statement myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/vendor/cpp/security/return-c-str-cpp.yml000066400000000000000000000106731525232545500331120ustar00rootroot00000000000000id: return-c-str-cpp language: cpp severity: warning message: >- "`$FUNC` returns a pointer to the memory owned by `$STR`. This pointer is invalid after `$STR` goes out of scope, which can trigger a use after free." note: >- [CWE-416] Use After Free [REFERENCES] - https://wiki.sei.cmu.edu/confluence/display/c/DCL30-C.+Declare+objects+with+appropriate+storage+durations - https://wiki.sei.cmu.edu/confluence/display/cplusplus/EXP54-CPP.+Do+not+access+an+object+outside+of+its+lifetime ast-grep-essentials: true rule: any: - pattern: return basic_string<$TYPE>($$$).$METHOD(); - pattern: return std::basic_string<$TYPE>($$$).$METHOD(); - pattern: return string($$$).$METHOD(); - pattern: return std::string($$$).$METHOD(); - pattern: return wstring($$$).$METHOD(); - pattern: return std::wstring($$$).$METHOD(); - pattern: return $STR.$METHOD(); any: - follows: stopBy: end all: - not: has: stopBy: end kind: storage_class_specifier - any: - kind: declaration not: pattern: $STR_VAL $STR = "$STRG"; - has: pattern: $STR_VAL - has: stopBy: end pattern: $STR - inside: stopBy: end follows: stopBy: end all: - not: has: stopBy: end kind: storage_class_specifier - any: - kind: declaration not: pattern: $STR_VAL $STR = "$STRG"; - has: pattern: $STR_VAL - has: pattern: $STR - inside: stopBy: end follows: stopBy: end all: - not: has: stopBy: end kind: storage_class_specifier - any: - kind: pointer_declarator not: has: stopBy: end pattern: $STR_VAL $STR = "$STRG"; has: kind: function_declarator all: - has: stopBy: end any: - kind: qualified_identifier - kind: type_identifier regex: ^(basic_string<.*>|std::basic_string<.*>|string|std::string|wstring|std::wstring|string(.*)|std::string(.*)|wstring(.*)|std::wstring(.*)|basic_string<.*>(.*)|std::basic_string<.*>(.*))$ - has: stopBy: end pattern: $STR - follows: stopBy: end all: - not: has: stopBy: end kind: storage_class_specifier - any: - kind: pointer_declarator has: kind: function_declarator all: - not: has: stopBy: end pattern: $STR_VAL $STR = "$STRG"; - has: stopBy: end any: - kind: qualified_identifier - kind: type_identifier regex: ^(basic_string<.*>|std::basic_string<.*>|string|std::string|wstring|std::wstring|string(.*)|std::string(.*)|wstring(.*)|std::wstring(.*)|basic_string<.*>(.*)|std::basic_string<.*>(.*))$ - has: stopBy: end pattern: $STR - pattern: return $STR_VAL.$METHOD(); not: all: - has: stopBy: end kind: ERROR - inside: stopBy: end kind: ERROR constraints: METHOD: regex: ^(c_str|data)$ STR_VAL: regex: ^(basic_string<.*>|std::basic_string<.*>|string|std::string|wstring|std::wstring|string(.*)|std::string(.*)|wstring(.*)|std::wstring(.*)|basic_string<.*>(.*)|std::basic_string<.*>(.*))$ myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/vendor/cpp/security/sizeof-this-cpp.yml000066400000000000000000000021141525232545500330000ustar00rootroot00000000000000id: sizeof-this-cpp language: cpp severity: warning message: >- Do not use `sizeof(this)` to get the number of bytes of the object in memory. It returns the size of the pointer, not the size of the object. note: >- [CWE-467]: Use of sizeof() on a Pointer Type [REFERENCES] - https://wiki.sei.cmu.edu/confluence/display/c/ARR01-C.+Do+not+apply+the+sizeof+operator+to+a+pointer+when+taking+the+size+of+an+array ast-grep-essentials: true utils: match_sizeof_this: kind: sizeof_expression has: kind: parenthesized_expression has: kind: this regex: "^this$" inside: stopBy: end kind: return_statement inside: kind: compound_statement follows: kind: function_declarator inside: kind: function_definition rule: kind: sizeof_expression all: - has: stopBy: end kind: this - not: has: stopBy: end any: - nthChild: 2 - kind: pointer_expression - kind: ERROR - kind: sizeof_expression myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/vendor/cpp/security/small-key-size-cpp.yml000066400000000000000000000030631525232545500334060ustar00rootroot00000000000000id: small-key-size-cpp language: cpp severity: warning message: >- $KEY_FUNCTION` is using a key size of only $KEY_BITS bits. This is less than the recommended key size of 2048 bits. note: >- [CWE-326]: Inadequate Encryption Strength [OWASP A02:2021]: Cryptographic Failures [OWASP A03:2017]: Sensitive Data Exposure [REFERENCES] https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-131Ar2.pdf https://owasp.org/Top10/A02_2021-Cryptographic_Failures ast-grep-essentials: true rule: kind: call_expression all: - has: stopBy: end kind: identifier regex: ^(DH_generate_parameters_ex|DSA_generate_parameters_ex|EVP_PKEY_CTX_set_dh_paramgen_prime_len|EVP_PKEY_CTX_set_dsa_paramgen_bits|EVP_PKEY_CTX_set_rsa_keygen_bits|RSA_generate_key_ex|RSA_generate_key_fips)$ - not: has: stopBy: end kind: field_identifier regex: ^(DH_generate_parameters_ex|DSA_generate_parameters_ex|EVP_PKEY_CTX_set_dh_paramgen_prime_len|EVP_PKEY_CTX_set_dsa_paramgen_bits|EVP_PKEY_CTX_set_rsa_keygen_bits|RSA_generate_key_ex|RSA_generate_key_fips)$ - has: stopBy: neighbor kind: argument_list has: stopBy: neighbor any: - kind: number_literal - kind: binary_expression - kind: unary_expression nthChild: 2 regex: ^([+-]*\(*[+-]*((0|[1-9][0-9]{0,2}|1[0-9]{3}|20[0-3][0-9]|204[0-7])(\.[0-9]+)?|((0|[1-9][0-9]{0,2}|1[0-9]{3}|20[0-3][0-9]|204[0-7])(\.[0-9]+)?\/[1-9][0-9]*)|(\.[0-9]+)|(\.[0-9]+\/[1-9][0-9]*))\)*)$ - not: has: stopBy: end kind: ERROR myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/vendor/cpp/security/std-return-data-cpp.yml000066400000000000000000000072071525232545500335620ustar00rootroot00000000000000id: std-return-data-cpp language: cpp severity: warning message: >- $FUNC` returns a pointer to the memory owned by `$VAR`. This pointer is invalid after `$VAR` goes out of scope, which can trigger a use after free. note: >- [CWE-416: Use After Free. [REFERENCES] - https://wiki.sei.cmu.edu/confluence/display/c/DCL30-C.+Declare+objects+with+appropriate+storage+durations ast-grep-essentials: true rule: kind: return_statement pattern: return $VAR.data(); all: - inside: stopBy: end kind: function_definition all: - has: nthChild: 1 pattern: $RETURN_TYPE - has: kind: pointer_declarator - any: - follows: stopBy: end all: - has: nthChild: 1 regex: ^(array<.*>|std::array<.*>|deque<.*>|std::deque<.*>|forward_list<.*>|std::forward_list<.*>|list<.*>|std::list<.*>|map<.*, .*>|std::map<.*, .*>|multimap<.*, .*>|std::multimap<.*, .*>|multiset<.*>|std::multiset<.*>|set<.*>|std::set<.*>|unordered_map<.*>|std::unordered_map<.*>|unordered_multimap<.*, .*>|std::unordered_multimap<.*, .*>|unordered_multiset<.*>|std::unordered_multiset<.*>|unordered_set<.*>|std::unordered_set<.*>|vector<.*>|std::vector<.*>)$ - has: stopBy: end # nthChild: 2 pattern: $VAR - not: inside: stopBy: end has: kind: storage_class_specifier - inside: stopBy: end kind: compound_statement - inside: stopBy: end follows: stopBy: end all: - has: nthChild: 1 regex: ^(array<.*>|std::array<.*>|deque<.*>|std::deque<.*>|forward_list<.*>|std::forward_list<.*>|list<.*>|std::list<.*>|map<.*, .*>|std::map<.*, .*>|multimap<.*, .*>|std::multimap<.*, .*>|multiset<.*>|std::multiset<.*>|set<.*>|std::set<.*>|unordered_map<.*>|std::unordered_map<.*>|unordered_multimap<.*, .*>|std::unordered_multimap<.*, .*>|unordered_multiset<.*>|std::unordered_multiset<.*>|unordered_set<.*>|std::unordered_set<.*>|vector<.*>|std::vector<.*>)$ - has: # nthChild: 2 stopBy: end pattern: $VAR - not: inside: stopBy: end has: kind: storage_class_specifier - inside: stopBy: end kind: compound_statement - inside: stopBy: end follows: stopBy: end kind: pointer_declarator all: - has: stopBy: end nthChild: 1 regex: ^(array<.*>|std::array<.*>|deque<.*>|std::deque<.*>|forward_list<.*>|std::forward_list<.*>|list<.*>|std::list<.*>|map<.*, .*>|std::map<.*, .*>|multimap<.*, .*>|std::multimap<.*, .*>|multiset<.*>|std::multiset<.*>|set<.*>|std::set<.*>|unordered_map<.*>|std::unordered_map<.*>|unordered_multimap<.*, .*>|std::unordered_multimap<.*, .*>|unordered_multiset<.*>|std::unordered_multiset<.*>|unordered_set<.*>|std::unordered_set<.*>|vector<.*>|std::vector<.*>)$ - has: # nthChild: 2 stopBy: end pattern: $VAR - not: inside: stopBy: end has: kind: storage_class_specifier std-vector-invalidation-cpp.yml000066400000000000000000000123451525232545500352350ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/vendor/cpp/securityid: std-vector-invalidation-cpp language: cpp severity: warning message: >- Modifying an `std::vector` while iterating over it could cause the container to reallocate, triggering memory corruption. note: >- [CWE-416: Use After Free. [REFERENCES] - https://wiki.sei.cmu.edu/confluence/display/c/MEM30-C.+Do+not+access+freed+memory - https://wiki.sei.cmu.edu/confluence/display/cplusplus/EXP54-CPP.+Do+not+access+an+object+outside+of+its+lifetime ast-grep-essentials: true rule: kind: call_expression all: - any: - pattern: $CONTAINER.erase($IT) all: - all: - not: follows: stopBy: end pattern: $CONTAINER.erase($IT) - not: precedes: stopBy: end pattern: $CONTAINER.erase($IT) - not: inside: stopBy: end kind: assignment_expression has: kind: identifier pattern: $IT nthChild: 1 - pattern: $CONTAINER.assign($$$) - pattern: $CONTAINER.clear($$$) - pattern: $CONTAINER.emplace_back($$$) - pattern: $CONTAINER.insert($$$) - pattern: $CONTAINER.resize($$$) - pattern: $CONTAINER.push_back($$$) - pattern: $CONTAINER.reserve($$$) - pattern: $CONTAINER.shrink_to_fit($$$) - pattern: $CONTAINER.resize($$$) - pattern: $CONTAINER.pop_back($$$) - not: inside: stopBy: end kind: for_statement has: stopBy: end any: - kind: break_statement - kind: continue_statement - kind: return_statement - kind: goto_statement - inside: stopBy: end kind: for_statement any: - all: - has: kind: declaration any: - pattern: std::vector<$TY>::$IT_TYPE $IT = $CONTAINER.begin() - all: - has: kind: dependent_type has: stopBy: end pattern: std::vector<$TY>::$IT_TYPE - has: stopBy: end kind: init_declarator all: - has: pattern: $IT - has: pattern: $CONTAINER.begin() - has: kind: binary_expression any: - pattern: $IT != $CONTAINER.end() - has: kind: update_expression any: - pattern: ++$IT - pattern: $IT++ - all: - has: kind: declaration any: - pattern: std::vector<$TY>::$IT_TYPE $IT = $CONTAINER.rbegin() - has: stopBy: end pattern: std::vector<$TY>::$IT_TYPE $IT = $CONTAINER.rbegin() - all: - has: kind: dependent_type has: stopBy: end pattern: std::vector<$TY>::$IT_TYPE - has: stopBy: end kind: init_declarator all: - has: pattern: $IT - has: pattern: $CONTAINER.rbegin() - has: kind: binary_expression any: - pattern: $IT != $CONTAINER.rend() - has: kind: update_expression any: - pattern: ++$IT - pattern: $IT++ - all: - has: kind: declaration any: - pattern: std::vector<$TY>::$IT_TYPE $IT = $CONTAINER.begin(), $IT_END = $CONTAINER.end() - pattern: std::vector<$TY>::$IT_TYPE $IT = $CONTAINER.rbegin(), $IT_END = $CONTAINER.rend() - has: stopBy: end any: - pattern: std::vector<$TY>::$IT_TYPE $IT = $CONTAINER.begin(), $IT_END = $CONTAINER.end() - pattern: std::vector<$TY>::$IT_TYPE $IT = $CONTAINER.rbegin(), $IT_END = $CONTAINER.rend() - has: kind: binary_expression any: - pattern: $IT != $IT_END - has: kind: update_expression any: - pattern: ++$IT - pattern: $IT++ - all: - not: has: stopBy: end kind: ERROR - not: inside: stopBy: end kind: ERROR string-view-temporary-string-cpp.yml000066400000000000000000000744651525232545500363010ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/vendor/cpp/securityid: string-view-temporary-string-cpp language: Cpp severity: warning message: >- This `std::string_view` is constructed from a temporary `std::string`. The `std::string` value is immeadiately destroyed after assignment and accessing data through the `std::string_view` will trigger a use-after-free. note: >- [CWE-416] Use After Free. [REFERENCES] - https://wiki.sei.cmu.edu/confluence/display/cplusplus/MEM50-CPP.+Do+not+access+freed+memory ast-grep-essentials: true utils: u_1_var_std_to_string: # $VAR = std::to_string(...); kind: expression_statement has: stopBy: neighbor kind: assignment_expression all: - has: stopBy: neighbor kind: identifier regex: ^(string_view|wstring_view)$ - has: stopBy: neighbor kind: call_expression all: - has: stopBy: neighbor kind: qualified_identifier regex: ^std::to_string$ - has: stopBy: neighbor kind: argument_list u_2_var_expr_substr: # $VAR = std::to_string(...); kind: expression_statement has: stopBy: neighbor kind: assignment_expression all: - has: stopBy: neighbor kind: identifier regex: ^(string_view|wstring_view)$ - has: stopBy: neighbor kind: call_expression all: - has: stopBy: neighbor kind: field_expression regex: ^(string.substr|wstring.substr)$ - has: stopBy: neighbor kind: argument_list u_3_var_expr: # $VAR = $EXPR + ... kind: expression_statement has: stopBy: neighbor kind: assignment_expression all: - has: stopBy: neighbor kind: identifier regex: ^(string_view|wstring_view)$ - has: stopBy: neighbor kind: binary_expression has: stopBy: neighbor kind: identifier regex: ^(wstring|string)$ nthChild: 1 u_4_var_expr: # $VAR = "..." + $EXPR kind: expression_statement has: stopBy: neighbor kind: assignment_expression all: - has: stopBy: neighbor kind: identifier regex: ^(string_view|wstring_view)$ - has: stopBy: end kind: binary_expression all: - has: stopBy: end kind: string_literal nthChild: 1 - has: stopBy: end kind: identifier regex: ^(string|wstring)$ u_5_var_instance_expr: # $VAR_instance = "..." + $EXPR kind: expression_statement all: - has: stopBy: neighbor kind: assignment_expression all: - has: stopBy: neighbor kind: identifier pattern: $VAR_INSTANCE - has: stopBy: neighbor kind: binary_expression all: - has: stopBy: neighbor kind: string_literal - has: stopBy: neighbor kind: identifier pattern: $EXPR_INSTANCE - any: - inside: stopBy: end follows: stopBy: end kind: declaration all: - has: stopBy: neighbor regex: ^(wstring|string|std::wstring|std::string|std::basic_string<.*>|basic_string<.*>)$ nthChild: 1 - has: stopBy: end kind: identifier pattern: $EXPR_INSTANCE - follows: stopBy: end any: - kind: field_declaration - kind: declaration all: - has: stopBy: neighbor regex: ^(wstring|string|std::wstring|std::string|std::basic_string<.*>|basic_string<.*>)$ nthChild: 1 - has: stopBy: end kind: identifier pattern: $EXPR_INSTANCE - follows: stopBy: end any: - kind: field_declaration - kind: declaration all: - has: stopBy: neighbor any: - kind: type_identifier - kind: qualified_identifier regex: ^(wstring|string|std::wstring|std::string|std::basic_string<.*>|basic_string<.*>)$ - has: stopBy: end kind: init_declarator has: stopBy: end kind: identifier pattern: $EXPR_INSTANCE - any: - inside: stopBy: end follows: stopBy: end any: - kind: field_declaration - kind: declaration all: - has: stopBy: neighbor regex: ^(string_view|basic_string_view<.*>|std::basic_string_view<.*>|std::string_view|std::wstring_view|wstring_view)$ nthChild: 1 - has: stopBy: end kind: identifier pattern: $VAR_INSTANCE - follows: stopBy: end any: - kind: field_declaration - kind: declaration all: - has: stopBy: neighbor regex: ^(string_view|basic_string_view<.*>|std::basic_string_view<.*>|std::string_view|std::wstring_view|wstring_view)$ nthChild: 1 - has: stopBy: end kind: identifier pattern: $VAR_INSTANCE u_6_var_instance_expr_instance: kind: expression_statement all: - has: stopBy: neighbor kind: assignment_expression all: - has: stopBy: neighbor kind: identifier pattern: $VAR_INSTANCE - has: stopBy: neighbor kind: binary_expression has: stopBy: end kind: identifier pattern: $EXPR_INSTANCE - any: - follows: stopBy: end any: - kind: field_declaration - kind: declaration all: - has: stopBy: neighbor regex: ^(wstring|string|std::wstring|std::string|std::basic_string<.*>|basic_string<.*>)$ nthChild: 1 - has: stopBy: end kind: identifier pattern: $EXPR_INSTANCE - follows: stopBy: end any: - kind: field_declaration - kind: declaration all: - has: stopBy: neighbor any: - kind: type_identifier - kind: qualified_identifier regex: ^(wstring|string|std::wstring|std::string|std::basic_string<.*>|basic_string<.*>)$ - has: stopBy: neighbor kind: init_declarator has: stopBy: end kind: identifier pattern: $EXPR_INSTANCE - inside: stopBy: end follows: stopBy: end any: - kind: field_declaration - kind: declaration all: - has: stopBy: end any: - kind: type_identifier - kind: qualified_identifier regex: ^(wstring|string|std::wstring|std::string|std::basic_string<.*>|basic_string<.*>)$ - has: stopBy: end pattern: $EXPR_INSTANCE - follows: stopBy: end any: - kind: field_declaration - kind: declaration all: - has: stopBy: neighbor regex: ^(string_view|basic_string_view<.*>|std::basic_string_view<.*>|std::string_view|std::wstring_view|wstring_view)$ nthChild: 1 - has: stopBy: end kind: identifier pattern: $VAR_INSTANCE u_7_var_instance_expr_instance_substr: # $VAR = std::to_string(...); kind: expression_statement all: - has: stopBy: neighbor kind: assignment_expression all: - has: stopBy: neighbor kind: identifier pattern: $VAR_INSTANCE - has: stopBy: neighbor kind: call_expression all: - has: stopBy: end kind: field_expression all: - any: - has: stopBy: end kind: identifier pattern: $EXPR_INSTANCE nthChild: 1 - has: stopBy: end any: - kind: type_identifier - kind: qualified_identifier regex: ^(wstring|string|std::wstring|std::string|std::basic_string<.*>|basic_string<.*>)$ - has: stopBy: neighbor kind: field_identifier regex: ^substr$ - has: stopBy: neighbor kind: argument_list - any: - follows: stopBy: end any: - kind: field_declaration - kind: declaration all: - has: stopBy: neighbor regex: ^(wstring|string|std::wstring|std::string|std::basic_string<.*>|basic_string<.*>)$ nthChild: 1 - has: stopBy: end kind: identifier pattern: $EXPR_INSTANCE - follows: stopBy: end any: - kind: field_declaration - kind: declaration all: - has: stopBy: neighbor any: - kind: type_identifier - kind: qualified_identifier regex: ^(wstring|string|std::wstring|std::string|std::basic_string<.*>|basic_string<.*>)$ - has: stopBy: neighbor kind: init_declarator has: stopBy: neighbor kind: identifier pattern: $EXPR_INSTANCE - inside: stopBy: end follows: stopBy: end any: - kind: field_declaration - kind: declaration all: - has: stopBy: end any: - kind: type_identifier - kind: qualified_identifier regex: ^(wstring|string|std::wstring|std::string|std::basic_string<.*>|basic_string<.*>)$ - has: stopBy: end pattern: $EXPR_INSTANCE - follows: stopBy: end any: - kind: field_declaration - kind: declaration all: - has: stopBy: neighbor regex: ^(string_view|basic_string_view<.*>|std::basic_string_view<.*>|std::string_view|std::wstring_view|wstring_view)$ nthChild: 1 - has: stopBy: end kind: identifier pattern: $VAR_INSTANCE u_8_var_instance_std_to_string: # $VAR = std::to_string(...); kind: expression_statement all: - has: stopBy: neighbor kind: assignment_expression all: - has: stopBy: neighbor kind: identifier pattern: $VAR_INSTANCE - has: stopBy: neighbor kind: call_expression all: - has: stopBy: end kind: qualified_identifier regex: ^std::to_string$ - has: stopBy: neighbor kind: argument_list - follows: stopBy: end any: - kind: field_declaration - kind: declaration all: - has: stopBy: neighbor regex: ^(string_view|basic_string_view<.*>|std::basic_string_view<.*>|std::string_view|std::wstring_view|wstring_view)$ nthChild: 1 - has: stopBy: neighbor kind: identifier pattern: $VAR_INSTANCE u_9_var_std_to_string: kind: call_expression all: - has: stopBy: neighbor regex: ^(basic_string_view<.*>|std::basic_string_view<.*>|string_view|std::string_view|wstring_view|std::wstring_view)$ - has: stopBy: neighbor kind: argument_list has: stopBy: neighbor kind: call_expression all: - has: stopBy: neighbor kind: qualified_identifier regex: ^std::to_string$ - has: stopBy: neighbor kind: argument_list u_10_var_std_to_string_as_declaration: kind: declaration all: - has: stopBy: neighbor regex: ^(basic_string_view<.*>|std::basic_string_view<.*>|string_view|std::string_view|wstring_view|std::wstring_view)$ - has: stopBy: end kind: call_expression all: - has: stopBy: neighbor kind: qualified_identifier regex: ^std::to_string$ - has: stopBy: neighbor kind: argument_list u_11_var_expr: kind: call_expression all: - has: stopBy: neighbor regex: ^(basic_string_view<.*>|std::basic_string_view<.*>|string_view|std::string_view|wstring_view|std::wstring_view)$ - has: stopBy: end kind: binary_expression has: stopBy: neighbor kind: identifier regex: ^(wstring|string)$ nthChild: 1 u_12_var_expr_instance: kind: call_expression all: - has: stopBy: neighbor regex: ^(std::basic_string_view<.*>|basic_string_view<.*>|string_view|std::string_view|wstring_view|std::wstring_view)$ - has: stopBy: end kind: argument_list has: stopBy: neighbor kind: binary_expression has: stopBy: end kind: identifier nthChild: 1 pattern: $EXPR_INSTANCE - any: - follows: stopBy: end any: - kind: field_declaration - kind: declaration all: - has: stopBy: neighbor regex: ^(wstring|string|std::wstring|std::string|std::basic_string<.*>|basic_string<.*>)$ nthChild: 1 - has: stopBy: end kind: identifier pattern: $EXPR_INSTANCE - follows: stopBy: end any: - kind: field_declaration - kind: declaration all: - has: stopBy: neighbor any: - kind: type_identifier - kind: qualified_identifier regex: ^(wstring|string|std::wstring|std::string|std::basic_string<.*>|basic_string<.*>)$ - has: stopBy: neighbor kind: init_declarator has: stopBy: neighbor kind: identifier pattern: $EXPR_INSTANCE - inside: stopBy: end follows: stopBy: end any: - kind: field_declaration - kind: declaration all: - has: stopBy: end any: - kind: type_identifier - kind: qualified_identifier regex: ^(wstring|string|std::wstring|std::string|std::basic_string<.*>|basic_string<.*>)$ - has: stopBy: end pattern: $EXPR_INSTANCE u_13_var_expr_instance: kind: call_expression all: - has: stopBy: neighbor regex: ^(basic_string_view<.*>|std::basic_string_view<.*>|string_view|std::string_view|wstring_view|std::wstring_view)$ - has: stopBy: end kind: argument_list has: stopBy: end kind: binary_expression all: - has: stopBy: neighbor kind: string_literal nthChild: 1 has: stopBy: neighbor kind: string_content - has: stopBy: end kind: identifier nthChild: 2 pattern: $EXPR_INSTANCE - any: - follows: stopBy: end any: - kind: field_declaration - kind: declaration all: - has: stopBy: neighbor regex: ^(wstring|string|std::wstring|std::string|std::basic_string<.*>|basic_string<.*>)$ nthChild: 1 - has: stopBy: neighbor kind: identifier pattern: $EXPR_INSTANCE - follows: stopBy: end any: - kind: field_declaration - kind: declaration all: - has: stopBy: neighbor any: - kind: type_identifier - kind: qualified_identifier regex: ^(wstring|string|std::wstring|std::string|std::basic_string<.*>|basic_string<.*>)$ - has: stopBy: neighbor kind: init_declarator has: stopBy: neighbor kind: identifier pattern: $EXPR_INSTANCE - inside: stopBy: end follows: stopBy: end any: - kind: field_declaration - kind: declaration all: - has: stopBy: end any: - kind: type_identifier - kind: qualified_identifier regex: ^(wstring|string|std::wstring|std::string|std::basic_string<.*>|basic_string<.*>)$ - has: stopBy: end pattern: $EXPR_INSTANCE u_14_var_expr: kind: call_expression all: - has: stopBy: neighbor regex: ^(basic_string_view<.*>|std::basic_string_view<.*>|string_view|std::string_view|wstring_view|std::wstring_view)$ - has: stopBy: end kind: argument_list has: stopBy: neighbor kind: binary_expression all: - has: stopBy: neighbor kind: string_literal nthChild: 1 has: stopBy: neighbor kind: string_content - has: stopBy: end kind: identifier nthChild: 2 regex: ^(wstring|string)$ u_15_var_expr_substr: kind: call_expression all: - has: stopBy: neighbor regex: ^(string_view|std::string_view|wstring_view|std::wstring_view)$ - has: stopBy: end kind: argument_list has: stopBy: neighbor kind: call_expression all: - has: stopBy: neighbor kind: field_expression regex: ^(string.substr|wstring.substr)$ - has: stopBy: neighbor kind: argument_list u_16_var_expr_instance_substr: kind: call_expression all: - has: stopBy: neighbor regex: ^(basic_string_view<.*>|string_view|std::string_view|wstring_view|std::wstring_view|std::basic_string_view<.*>)$ - has: stopBy: end kind: argument_list has: stopBy: neighbor kind: call_expression all: - has: stopBy: end kind: field_expression all: - any: - has: stopBy: end kind: identifier pattern: $EXPR_INSTANCE nthChild: 1 - has: stopBy: end any: - kind: type_identifier - kind: qualified_identifier regex: ^(wstring|string|std::wstring|std::string|std::basic_string<.*>|basic_string<.*>)$ - has: stopBy: neighbor kind: field_identifier regex: ^substr$ - has: stopBy: neighbor kind: argument_list - any: - follows: stopBy: end any: - kind: field_declaration - kind: declaration all: - has: stopBy: neighbor regex: ^(wstring|string|std::wstring|std::string|std::basic_string<.*>|basic_string<.*>)$ nthChild: 1 - has: stopBy: end kind: identifier pattern: $EXPR_INSTANCE - follows: stopBy: end any: - kind: field_declaration - kind: declaration all: - has: stopBy: neighbor any: - kind: type_identifier - kind: qualified_identifier regex: ^(wstring|string|std::wstring|std::string|std::basic_string<.*>|basic_string<.*>)$ - has: stopBy: neighbor kind: init_declarator has: stopBy: neighbor kind: identifier pattern: $EXPR_INSTANCE - inside: stopBy: end follows: stopBy: end any: - kind: field_declaration - kind: declaration all: - has: stopBy: end any: - kind: type_identifier - kind: qualified_identifier regex: ^(wstring|string|std::wstring|std::string|std::basic_string<.*>|basic_string<.*>)$ - has: stopBy: end pattern: $EXPR_INSTANCE u_17_var_instance_var_expr: # $VAR_instance $VAR = "..." + $EXPR kind: declaration all: - has: kind: type_identifier regex: ^(string_view|basic_string_view<.*>|std::basic_string_view<.*>|std::string_view|std::wstring_view|wstring_view)$ - has: stopBy: neighbor kind: init_declarator all: - has: stopBy: neighbor kind: identifier pattern: $VAR_INSTANCE - has: stopBy: neighbor kind: binary_expression all: - has: stopBy: neighbor kind: string_literal - has: stopBy: neighbor kind: identifier pattern: $EXPR_INSTANCE - any: - follows: stopBy: end any: - kind: field_declaration - kind: declaration all: - has: stopBy: neighbor regex: ^(wstring|string|std::wstring|std::string|std::basic_string<.*>|basic_string<.*>)$ nthChild: 1 - has: stopBy: neighbor kind: identifier pattern: $EXPR_INSTANCE - follows: stopBy: end any: - kind: field_declaration - kind: declaration all: - has: stopBy: neighbor any: - kind: type_identifier - kind: qualified_identifier regex: ^(wstring|string|std::wstring|std::string|std::basic_string<.*>|basic_string<.*>)$ - has: stopBy: neighbor kind: init_declarator has: stopBy: neighbor kind: identifier pattern: $EXPR_INSTANCE u_18_var_var_instance_expr: # $VAR_instance = "..." + $EXPR kind: declaration all: - has: nthChild: 1 regex: ^(basic_string_view<.*>|std::basic_string_view<.*>|string_view|std::string_view|wstring_view|std::wstring_view)$ - has: stopBy: neighbor kind: init_declarator all: - has: stopBy: neighbor kind: identifier - has: stopBy: neighbor kind: binary_expression all: - has: stopBy: neighbor kind: string_literal - has: stopBy: neighbor kind: identifier pattern: $EXPR_INSTANCE - any: - inside: stopBy: end follows: stopBy: end kind: declaration all: - has: stopBy: neighbor regex: ^(wstring|string|std::wstring|std::basic_string<.*>|basic_string<.*>)$ nthChild: 1 - has: stopBy: end kind: identifier pattern: $EXPR_INSTANCE - follows: stopBy: end any: - kind: field_declaration - kind: declaration all: - has: stopBy: neighbor regex: ^(wstring|string|std::wstring|std::string|std::basic_string<.*>|basic_string<.*>)$ nthChild: 1 - has: stopBy: end kind: identifier pattern: $EXPR_INSTANCE - follows: stopBy: end any: - kind: field_declaration - kind: declaration all: - has: stopBy: neighbor any: - kind: type_identifier - kind: qualified_identifier regex: ^(wstring|string|std::wstring|std::string|std::basic_string<.*>|basic_string<.*>)$ - has: stopBy: end kind: init_declarator has: stopBy: end kind: identifier pattern: $EXPR_INSTANCE rule: any: - kind: expression_statement any: - matches: u_1_var_std_to_string - matches: u_2_var_expr_substr - matches: u_3_var_expr - matches: u_4_var_expr - matches: u_5_var_instance_expr - matches: u_6_var_instance_expr_instance - matches: u_7_var_instance_expr_instance_substr - matches: u_7_var_instance_expr_instance_substr - matches: u_8_var_instance_std_to_string - kind: call_expression any: - matches: u_9_var_std_to_string - matches: u_11_var_expr - matches: u_12_var_expr_instance - matches: u_13_var_expr_instance - matches: u_14_var_expr - matches: u_15_var_expr_substr - matches: u_16_var_expr_instance_substr - kind: declaration any: - matches: u_10_var_std_to_string_as_declaration - matches: u_17_var_instance_var_expr - matches: u_18_var_var_instance_expr all: - not: has: stopBy: end kind: ERROR - not: inside: stopBy: end kind: ERROR world-writable-file-cpp.yml000066400000000000000000000241401525232545500343330ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/rules/vendor/cpp/securityid: world-writable-file-cpp language: cpp severity: warning message: >- This call makes a world-writable file which allows any user on a machine to write to the file. This may allow attackers to influence the behaviour of this process by writing to the file. note: >- [CWE-732]: Incorrect Permission Assignment for Critical Resource [REFERENCES] - https://wiki.sei.cmu.edu/confluence/display/c/FIO06-C.+Create+files+with+appropriate+access+permissions ast-grep-essentials: true utils: follows_umask: follows: stopBy: end kind: expression_statement has: kind: call_expression nthChild: 1 all: - has: nthChild: 1 kind: identifier field: function regex: ^umask$ - has: nthChild: 2 kind: argument_list field: arguments u_2_and_2_equals_2_s_ixxxx: any: - kind: number_literal regex: ^-?([2367]|[0-9]*(0[2367]|1[014589]|2[2367]|3[014589]|4[2367]|5[014589]|6[2367]|7[014589]|8[2367]|9[014589]))$ - all: - any: - kind: binary_expression - kind: identifier - regex: (\s*S_I[A-Z]{4}\s*\|)*S_I[A-Z]{4} - regex: .*\bS_IWOTH\b.* rule: any: # chmod/fchmod/creat - any: - matches: u_2_and_2_equals_2_s_ixxxx - kind: identifier pattern: $MODE inside: stopBy: end follows: stopBy: end any: - kind: declaration all: - has: kind: init_declarator all: - has: kind: identifier field: declarator pattern: $MODE - has: nthChild: 2 matches: u_2_and_2_equals_2_s_ixxxx - kind: expression_statement any: - has: kind: assignment_expression all: - has: nthChild: 1 kind: identifier pattern: $MODE - has: nthChild: 2 matches: u_2_and_2_equals_2_s_ixxxx - has: kind: comma_expression has: kind: assignment_expression all: - has: nthChild: 1 kind: identifier pattern: $MODE - has: nthChild: 2 matches: u_2_and_2_equals_2_s_ixxxx nthChild: position: 2 ofRule: not: kind: comment inside: kind: argument_list nthChild: 2 not: has: nthChild: position: 3 ofRule: not: kind: comment follows: kind: identifier regex: ^(chmod|fchmod|creat)$ inside: kind: call_expression not: any: - matches: follows_umask - inside: stopBy: end matches: follows_umask # fchmodat - any: - matches: u_2_and_2_equals_2_s_ixxxx - kind: identifier pattern: $MODE inside: stopBy: end follows: stopBy: end any: - kind: declaration all: - has: kind: init_declarator all: - has: kind: identifier field: declarator pattern: $MODE - has: nthChild: 2 matches: u_2_and_2_equals_2_s_ixxxx - kind: expression_statement any: - has: kind: assignment_expression all: - has: nthChild: 1 kind: identifier pattern: $MODE - has: nthChild: 2 matches: u_2_and_2_equals_2_s_ixxxx - has: kind: comma_expression has: kind: assignment_expression all: - has: nthChild: 1 kind: identifier pattern: $MODE - has: nthChild: 2 matches: u_2_and_2_equals_2_s_ixxxx nthChild: position: 3 ofRule: not: kind: comment inside: kind: argument_list nthChild: 2 follows: kind: identifier regex: ^(fchmodat)$ inside: kind: call_expression not: any: - matches: follows_umask - inside: stopBy: end matches: follows_umask # open - any: - matches: u_2_and_2_equals_2_s_ixxxx - kind: identifier pattern: $MODE inside: stopBy: end follows: stopBy: end any: - kind: declaration all: - has: kind: init_declarator all: - has: kind: identifier field: declarator pattern: $MODE - has: nthChild: 2 matches: u_2_and_2_equals_2_s_ixxxx - kind: expression_statement any: - has: kind: assignment_expression all: - has: nthChild: 1 kind: identifier pattern: $MODE - has: nthChild: 2 matches: u_2_and_2_equals_2_s_ixxxx - has: kind: comma_expression has: kind: assignment_expression all: - has: nthChild: 1 kind: identifier pattern: $MODE - has: nthChild: 2 matches: u_2_and_2_equals_2_s_ixxxx nthChild: position: 3 ofRule: not: kind: comment inside: kind: argument_list nthChild: 2 not: has: nthChild: position: 4 ofRule: not: kind: comment follows: kind: identifier regex: ^(open)$ inside: kind: call_expression not: any: - matches: follows_umask - inside: stopBy: end matches: follows_umask # openat - any: - matches: u_2_and_2_equals_2_s_ixxxx - kind: identifier pattern: $MODE inside: stopBy: end follows: stopBy: end any: - kind: declaration all: - has: kind: init_declarator all: - has: kind: identifier field: declarator pattern: $MODE - has: nthChild: 2 matches: u_2_and_2_equals_2_s_ixxxx - kind: expression_statement any: - has: kind: assignment_expression all: - has: nthChild: 1 kind: identifier pattern: $MODE - has: nthChild: 2 matches: u_2_and_2_equals_2_s_ixxxx - has: kind: comma_expression has: kind: assignment_expression all: - has: nthChild: 1 kind: identifier pattern: $MODE - has: nthChild: 2 matches: u_2_and_2_equals_2_s_ixxxx nthChild: position: 4 ofRule: not: kind: comment inside: kind: argument_list nthChild: 2 not: has: nthChild: position: 5 ofRule: not: kind: comment follows: kind: identifier regex: ^(openat)$ inside: kind: call_expression not: any: - matches: follows_umask - inside: stopBy: end matches: follows_umask myguard-labs-nginx-zstd-module-3486b5b/ci/ast-grep/sgconfig.yml000066400000000000000000000016241525232545500245000ustar00rootroot00000000000000# GENERATED by tools/vendor-astgrep-rules.py -- do not hand-edit. # # Self-contained ast-grep config for this module. Exists so the pre-commit # ast-grep hook gates identically in a standalone clone: ast-grep resolves # sgconfig.yml by walking UP from cwd, and outside /opt/myguard that walk finds # nothing, leaving the hook scanning against an empty ruleset and reporting # green while checking nothing. # # Refresh with: tools/vendor-astgrep-rules.py # Staleness check (CI-able): tools/vendor-astgrep-rules.py --check # # ruleDirs resolves relative to THIS FILE's directory, not to the working # directory ast-grep was invoked from -- so `rules` is correct even when the # hook calls `ast-grep scan -c ci/ast-grep/sgconfig.yml` from the repo root. # Verified 2026-08-08: spelling it `ci/ast-grep/rules` instead makes ast-grep # look for ci/ast-grep/ci/ast-grep/rules and fail. ruleDirs: - rules myguard-labs-nginx-zstd-module-3486b5b/ci/build-inputs.manifest000066400000000000000000000005151525232545500246010ustar00rootroot00000000000000# Paths are relative to the repository root. Directories include every regular # file below them. Keep this as the single source of truth for persistent build # trees and ccache keys. ci/build-inputs.manifest ci/tools/build-input-hash.sh ci/tools/ci-build.sh config auto/zstd filter/config filter/reorder-static.sh static/config src myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/000077500000000000000000000000001525232545500214275ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/README.md000066400000000000000000000070551525232545500227150ustar00rootroot00000000000000# Fuzzing Two coverage-guided libFuzzer targets, both slicing verbatim function bodies out of shipped production source at build time. ## fuzz_accept_encoding The RFC 7231 Accept-Encoding / q-value parser in [`../../src/ngx_http_zstd_common.h`](../../src/ngx_http_zstd_common.h): the entry point `ngx_http_zstd_accept_encoding()` and the `ngx_http_zstd_eval_qvalue()` helper it calls. Both are sliced into the fuzz target together. ### Why fuzz_accept_encoding It parses attacker-controlled header bytes in C, does pointer arithmetic against `ae->data`/`ae->len`, and runs the NUL-terminated `ngx_strcasestrn()` over the same buffer. That length-bounded vs. NUL-bounded mix, plus q-value edge cases, is the bug class the Perl suite cannot reach and that matches this module's historical bug profile (truncation, terminal-frame, lifetime). `extract_parser.sh` slices the verbatim bodies of both parser functions (`ngx_http_zstd_eval_qvalue` then `ngx_http_zstd_accept_encoding`, in definition order) out of the shipped header into `generated_parser.inc` at build time, and fails loudly if it cannot find either. ## fuzz_dcz The Available-Dictionary byte-sequence decoder, `ngx_http_zstd_dcz_decode_digest()`, in [`../../src/ngx_http_zstd_filter_module.c`](../../src/ngx_http_zstd_filter_module.c). It was extracted out of `ngx_http_zstd_dcz_negotiate()` specifically so this attacker-controlled-byte slice — the RFC 8941 byte-sequence framing gate plus `ngx_decode_base64()` — could be fuzzed without dragging in `ngx_http_request_t`. ### Why fuzz_dcz Same class as `fuzz_accept_encoding`: attacker-controlled header bytes, length-bounded parsing, a fixed-size destination buffer. `extract_dcz.sh` slices the verbatim body out of the shipped `.c` file into `generated_dcz.inc` at build time. The destination buffer must be `NGX_HTTP_ZSTD_DCZ_DECODE_BUF_LEN` (48) bytes, not `NGX_HTTP_ZSTD_SHA256_DIGEST_LEN` (32): an unpadded 44-character byte-sequence — still within the length gate — decodes to 33 bytes before the post-decode length check runs. This target caught that exact stack-buffer overflow during development (a too-small buffer introduced while extracting the helper, fixed before it shipped); `ngx_shim.h`'s `ngx_decode_base64()`/`ngx_decode_base64_internal()` are a faithful copy of `src/core/ngx_string.c` so the target exercises real base64 decode arithmetic, not a stand-in. ## No copy drift Neither target carries a hand-maintained copy of the function it fuzzes — both are sliced fresh from the shipped source on every build. `ngx_shim.h` supplies only the tiny nginx surface each needs (`ngx_str_t`, `ngx_tolower`, `ngx_strncasecmp`, `ngx_strcasestrn`, `ngx_decode_base64`), copied faithfully from upstream `src/core/ngx_string.{h,c}` with citations. ## Run locally ```bash bash ci/fuzz/build.sh # needs clang with libFuzzer cd ci/fuzz ./fuzz_accept_encoding -max_total_time=60 corpus/ ./fuzz_dcz -max_total_time=60 corpus_dcz/ ``` A crash drops a `crash-*` reproducer. Replay it with: ```bash ./fuzz_accept_encoding crash- ./fuzz_dcz crash- ``` ## CI [`fuzzing.yml`](../../.github/workflows/fuzzing.yml) provides a targeted manual run for both targets. The weekly [`ci-deep.yml`](../../.github/workflows/ci-deep.yml) campaign runs the two targets concurrently in separate lanes for the selected long duration. Fuzzing stays out of the bounded PR gate. ASAN+UBSAN are compiled in, so memory and undefined-behaviour bugs abort the run and fail the job. Each harness also traps if the function under test ever returns a value other than `NGX_OK`/`NGX_DECLINED`. myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/build.sh000077500000000000000000000020661525232545500230710ustar00rootroot00000000000000#!/usr/bin/env bash # # Build the Accept-Encoding and dcz-digest libFuzzer targets. # Usage: ci/fuzz/build.sh [accept-encoding-output-binary] [dcz-output-binary] # # Requires clang with libFuzzer (clang >= 6). CFLAGS/CC overridable for # OSS-Fuzz / ClusterFuzzLite, which pass their own sanitizer flags. set -euo pipefail FUZZ_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" OUT="${1:-$FUZZ_DIR/fuzz_accept_encoding}" DCZ_OUT="${2:-$FUZZ_DIR/fuzz_dcz}" CC="${CC:-clang}" # OSS-Fuzz sets $LIB_FUZZING_ENGINE and its own $CFLAGS; honour them. ENGINE="${LIB_FUZZING_ENGINE:--fsanitize=fuzzer}" CFLAGS="${CFLAGS:--g -O1 -fsanitize=address,undefined -fno-sanitize-recover=undefined}" bash "$FUZZ_DIR/extract_parser.sh" bash "$FUZZ_DIR/extract_dcz.sh" # shellcheck disable=SC2086 "$CC" $CFLAGS $ENGINE \ -I"$FUZZ_DIR" \ "$FUZZ_DIR/fuzz_accept_encoding.c" \ -o "$OUT" echo "✓ built fuzz target: $OUT" # shellcheck disable=SC2086 "$CC" $CFLAGS $ENGINE \ -I"$FUZZ_DIR" \ "$FUZZ_DIR/fuzz_dcz.c" \ -o "$DCZ_OUT" echo "✓ built fuzz target: $DCZ_OUT" myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/000077500000000000000000000000001525232545500227425ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/01_bare000066400000000000000000000000041525232545500240700ustar00rootroot00000000000000zstdmyguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/02_list000066400000000000000000000000121525232545500241320ustar00rootroot00000000000000gzip, zstdmyguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/03_q0000066400000000000000000000000101525232545500234760ustar00rootroot00000000000000zstd;q=0myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/04_q000000066400000000000000000000000121525232545500236410ustar00rootroot00000000000000zstd;q=0.0myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/05_qnonzero000066400000000000000000000000141525232545500250370ustar00rootroot00000000000000zstd;q=0.001myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/06_q1000066400000000000000000000000101525232545500235020ustar00rootroot00000000000000zstd;q=1myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/07_q1000000066400000000000000000000000141525232545500237270ustar00rootroot00000000000000zstd;q=1.000myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/08_q1_invalid000066400000000000000000000000121525232545500252140ustar00rootroot00000000000000zstd;q=1.5myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/09_space_semi000066400000000000000000000000131525232545500252770ustar00rootroot00000000000000zstd ;q=0.5myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/10_mixed000066400000000000000000000000241525232545500242670ustar00rootroot00000000000000br, zstd;q=0.9, gzipmyguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/11_case000066400000000000000000000000041525232545500240730ustar00rootroot00000000000000ZsTdmyguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/12_prefix000066400000000000000000000000051525232545500244570ustar00rootroot00000000000000xzstdmyguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/13_suffix000066400000000000000000000000051525232545500244670ustar00rootroot00000000000000zstdxmyguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/14_ws_qcap000066400000000000000000000000121525232545500246170ustar00rootroot00000000000000zstd;Q= 1myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/15_q_empty000066400000000000000000000000071525232545500246450ustar00rootroot00000000000000zstd;q=myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/16_q_dot_eol000066400000000000000000000000111525232545500251300ustar00rootroot00000000000000zstd;q=0.myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/20_wildcard_q1000066400000000000000000000000051525232545500253530ustar00rootroot00000000000000*;q=1myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/21_trailing_junk000066400000000000000000000000111525232545500260170ustar00rootroot00000000000000zstd;q=1xmyguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/22_explicit_over_wildcard000066400000000000000000000000171525232545500277130ustar00rootroot00000000000000zstd;q=0, *;q=1myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/23_bare_wildcard000066400000000000000000000000071525232545500257500ustar00rootroot00000000000000gzip, *myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/24_quoted_comma_phantom000066400000000000000000000000241525232545500273710ustar00rootroot00000000000000gzip;x="a, zstd";q=1myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/25_quoted_comma_real_zstd000066400000000000000000000000221525232545500277110ustar00rootroot00000000000000zstd;x="a,b";q=0.5myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/26_quoted_semicolon000066400000000000000000000000201525232545500265350ustar00rootroot00000000000000zstd;x="a;b";q=0myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/27_quoted_escape000066400000000000000000000000251525232545500260130ustar00rootroot00000000000000gzip;x="a\";zstd";q=1myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/28_dup_q000066400000000000000000000000141525232545500243010ustar00rootroot00000000000000zstd;q=1;q=1myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/29_dup_q_conflict000066400000000000000000000000141525232545500261630ustar00rootroot00000000000000zstd;q=0;q=1myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/30_quoted_name_phantom000066400000000000000000000000151525232545500272120ustar00rootroot00000000000000"a,zstd ";q=1myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/31_quoted_name_then_zstd000066400000000000000000000000101525232545500275420ustar00rootroot00000000000000"a",zstdmyguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/32_quoted_name_semicolon000066400000000000000000000000141525232545500275350ustar00rootroot00000000000000"a;zstd";q=1myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/33_dquote_suffix_name000066400000000000000000000000061525232545500270530ustar00rootroot00000000000000zstd"xmyguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/34_dquote_suffix_blob000066400000000000000000000000111525232545500270460ustar00rootroot00000000000000zstd"a,b"myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/35_dquote_suffix_inner_zstd000066400000000000000000000000141525232545500303130ustar00rootroot00000000000000zstd"a,zstd"myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/36_dquote_suffix_then_real000066400000000000000000000000131525232545500300750ustar00rootroot00000000000000gzip"x,zstdmyguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/37_ows_then_dquote000066400000000000000000000000071525232545500264020ustar00rootroot00000000000000zstd "xmyguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/38_ows_then_junk000066400000000000000000000000061525232545500260500ustar00rootroot00000000000000zstd xmyguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/39_tab_then_quoted000066400000000000000000000000151525232545500263410ustar00rootroot00000000000000zstd "a,zstd"myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/40_dcz_bare000066400000000000000000000000031525232545500247320ustar00rootroot00000000000000dczmyguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/41_zstd_dcz000066400000000000000000000000111525232545500250050ustar00rootroot00000000000000zstd, dczmyguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/42_dcz_q0000066400000000000000000000000151525232545500243460ustar00rootroot00000000000000zstd, dcz;q=0myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus/43_star_dcz000066400000000000000000000000141525232545500247770ustar00rootroot00000000000000*, dcz;q=0.5myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus_dcz/000077500000000000000000000000001525232545500236025ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus_dcz/bad_hash000066400000000000000000000000561525232545500252570ustar00rootroot00000000000000:AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE=:myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus_dcz/empty000066400000000000000000000000001525232545500246510ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus_dcz/invalid_chars000066400000000000000000000000251525232545500263300ustar00rootroot00000000000000:!!!!not-base64-@@@@:myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus_dcz/missing_leading_colon000066400000000000000000000000551525232545500300530ustar00rootroot00000000000000g7k9wqxI2ZoAVoHYOsS8IFfslAKIfEcMzpYkVhC7cNU=:myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus_dcz/missing_trailing_colon000066400000000000000000000000551525232545500302610ustar00rootroot00000000000000:g7k9wqxI2ZoAVoHYOsS8IFfslAKIfEcMzpYkVhC7cNU=myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus_dcz/no_colons000066400000000000000000000000541525232545500255150ustar00rootroot00000000000000g7k9wqxI2ZoAVoHYOsS8IFfslAKIfEcMzpYkVhC7cNU=myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus_dcz/single_colon000066400000000000000000000000011525232545500261670ustar00rootroot00000000000000:myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus_dcz/too_long000066400000000000000000000000721525232545500253440ustar00rootroot00000000000000:AgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg==:myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus_dcz/too_short000066400000000000000000000000161525232545500255420ustar00rootroot00000000000000:AwMDAwMDAwM=:myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus_dcz/two_colons000066400000000000000000000000021525232545500257030ustar00rootroot00000000000000::myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus_dcz/unpadded_32000066400000000000000000000000551525232545500256150ustar00rootroot00000000000000:g7k9wqxI2ZoAVoHYOsS8IFfslAKIfEcMzpYkVhC7cNU:myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/corpus_dcz/valid_hash000066400000000000000000000000561525232545500256300ustar00rootroot00000000000000:g7k9wqxI2ZoAVoHYOsS8IFfslAKIfEcMzpYkVhC7cNU=:myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/extract_dcz.sh000077500000000000000000000036351525232545500243070ustar00rootroot00000000000000#!/usr/bin/env bash # # Copyright (C) 2026 Thijs Eilander # SPDX-License-Identifier: BSD-2-Clause # # Slice the verbatim body of ngx_http_zstd_dcz_decode_digest() out of the # shipped ../../src/ngx_http_zstd_filter_module.c into # generated_dcz.inc. Same approach as extract_parser.sh (see that file's # header comment for the full rationale): the fuzz target stays locked to # production code, no hand-maintained copy to drift. # # ngx_http_zstd_dcz_decode_digest() is a pure extract out of # ngx_http_zstd_dcz_negotiate() -- the base64 length/prefix gate plus # ngx_decode_base64() -- with zero dependency on ngx_http_request_t or # config structures, which is what makes it fuzzable at all. set -euo pipefail FUZZ_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SRC="$FUZZ_DIR/../../src/ngx_http_zstd_filter_module.c" OUT="$FUZZ_DIR/generated_dcz.inc" if [ ! -f "$SRC" ]; then echo "✗ cannot find $SRC" >&2 exit 1 fi # Same extraction shape as extract_parser.sh: match the two-line # `static ngx_int_t` / function-name declaration style, then capture # through the matching closing brace at column 0. The leading sub() # strips a CR for CRLF checkouts, same reason as extract_parser.sh. awk ' { sub(/\r$/, "") } /^static (ngx_int_t|u_char \*)$/ { pending = 1; buf = $0 ORS; next } pending && /^ngx_http_zstd_dcz_decode_digest\(/ { capture = 1; pending = 0; print buf; print; next } pending { pending = 0; buf = "" } capture { print if ($0 == "}") { capture = 0 } } ' "$SRC" >"$OUT" if ! grep -q 'ngx_http_zstd_dcz_decode_digest' "$OUT" \ || [ "$(tail -n1 "$OUT")" != "}" ]; then echo "✗ failed to extract ngx_http_zstd_dcz_decode_digest() from $SRC" >&2 echo " (function signature/layout changed? update extract_dcz.sh)" >&2 rm -f "$OUT" exit 1 fi LINES=$(wc -l <"$OUT") echo "✓ extracted ngx_http_zstd_dcz_decode_digest() — $LINES lines -> $OUT" myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/extract_parser.sh000077500000000000000000000075401525232545500250220ustar00rootroot00000000000000#!/usr/bin/env bash # # Slice the verbatim bodies of the Accept-Encoding parser out of the # shipped ../../src/ngx_http_zstd_common.h into generated_parser.inc. That # is ngx_http_zstd_eval_qvalue() (the qvalue evaluator), its # ngx_http_zstd_parse_q_fraction() digit-walk helper, and its caller # ngx_http_zstd_coding_weight() and its chain/request-field callers, plus # ngx_http_zstd_accept_encoding(), in definition order so the .inc compiles # standalone. # # This keeps the fuzz target locked to production code: there is no # hand-maintained copy of the parser. If the function signature or body # changes upstream, the next fuzz build picks it up automatically. If a # function can no longer be found, we fail loudly rather than fuzz nothing. set -euo pipefail FUZZ_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" HEADER="$FUZZ_DIR/../../src/ngx_http_zstd_common.h" OUT="$FUZZ_DIR/generated_parser.inc" if [ ! -f "$HEADER" ]; then echo "✗ cannot find $HEADER" >&2 exit 1 fi # Extract each function from its return-type line through the matching # closing brace at column 0 (nginx style: definitions close with a bare # `}` in col 1). The functions have two distinct return-type lines # (`static u_char *` for the skip_quoted helper, `static ngx_int_t` for the # rest, either optionally carrying `ngx_inline`), so match on the following # definition line. ngx_inline is accepted because # ngx_http_zstd_accept_encoding() carries it: no module TU calls that # function any more (the request path evaluates the complete request field via # ngx_http_zstd_request_coding_weight()), so a plain `static` would trip # -Werror=unused-function in both TUs that include the header. ci/fuzz/ # ngx_shim.h defines ngx_inline for the sliced build. Capture them in # source order (skip_quoted, then parse_q_fraction, then eval_qvalue -- # which calls parse_q_fraction, so it must precede eval_qvalue in the # generated .inc -- then accept_encoding) so the generated .inc compiles # without forward declarations. # The leading sub() strips a CR so extraction also works from a Windows # checkout smudged to CRLF (core.autocrlf=true): the `$0 == "}"` # terminator and the anchored regexes below otherwise never match and # the build fails with the "header layout changed?" error misleadingly. awk ' { sub(/\r$/, "") } /^static (ngx_inline )?(ngx_int_t|u_char \*)$/ { pending = 1; buf = $0 ORS; next } pending && /^ngx_http_zstd_(skip_quoted|parse_q_fraction|eval_qvalue|coding_weight_ex|coding_weight|chain_coding_weight|request_coding_weight|accept_encoding|accepts)\(/ { capture = 1; pending = 0; print buf; print; next } pending { pending = 0; buf = "" } capture { print if ($0 == "}") { capture = 0 } } ' "$HEADER" >"$OUT" if ! grep -q 'ngx_http_zstd_chain_coding_weight' "$OUT" \ || ! grep -q 'ngx_http_zstd_request_coding_weight' "$OUT" \ || ! grep -q 'ngx_http_zstd_skip_quoted' "$OUT" \ || ! grep -q 'ngx_http_zstd_parse_q_fraction' "$OUT" \ || ! grep -q 'ngx_http_zstd_eval_qvalue' "$OUT" \ || ! grep -Eq '^ngx_http_zstd_coding_weight_ex\(' "$OUT" \ || ! grep -Eq '^ngx_http_zstd_coding_weight\(' "$OUT" \ || ! grep -q 'ngx_http_zstd_accept_encoding' "$OUT" \ || ! grep -q 'ngx_http_zstd_accepts' "$OUT" \ || [ "$(tail -n1 "$OUT")" != "}" ]; then echo "✗ failed to extract the Accept-Encoding parser from $HEADER" >&2 echo " (header layout changed? update extract_parser.sh)" >&2 rm -f "$OUT" exit 1 fi LINES=$(wc -l <"$OUT") echo "✓ extracted ngx_http_zstd_skip_quoted() + ngx_http_zstd_parse_q_fraction()" \ "+ ngx_http_zstd_eval_qvalue() + ngx_http_zstd_coding_weight_ex()" \ "+ ngx_http_zstd_coding_weight()" \ "+ ngx_http_zstd_chain_coding_weight()" \ "+ ngx_http_zstd_request_coding_weight() + ngx_http_zstd_accept_encoding()" \ "+ ngx_http_zstd_accepts() — $LINES lines -> $OUT" myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/fuzz.dict000066400000000000000000000021221525232545500232670ustar00rootroot00000000000000# libFuzzer dictionary for the Accept-Encoding parser. # Tokens the grammar actually uses — steers the fuzzer past the # structural boilerplate (codings, q-values, separators) far faster # than random bytes. # # The coding-name block below is GENERATED by ci/fuzz/gen_dict.sh from the # ngx_http_zstd_coding_weight(ae, "TOKEN", ...) call sites in src/*.c — do # not hand-edit between the markers, run the script instead. A hand-listed # dictionary of this module's own tokens goes stale silently (a merely # incomplete dictionary still produces a green crash-only fuzz run), and # ci/linter/lint-c.sh runs `gen_dict.sh --check` to catch the drift. # BEGIN generated coding tokens (ci/fuzz/gen_dict.sh) "dcz" "zstd" # END generated coding tokens # Other IANA content-codings a real client sends alongside zstd/dcz — not # this module's own tokens, but exercising the "not our coding" branches # (coding_q stays -1, falls through to the wildcard check) is part of the # same grammar surface. "gzip" "deflate" "br" "identity" "compress" "*" ";q=" "q=0" "q=1" "q=0.0" "q=1.0" ", " "," ";" "=" " " "\x09" myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/fuzz_accept_encoding.c000066400000000000000000000317301525232545500257620ustar00rootroot00000000000000/* * libFuzzer harness for ngx_http_zstd_accept_encoding() and the * ngx_http_zstd_eval_qvalue() helper it calls. * * Why this target: it parses attacker-controlled Accept-Encoding header * bytes in C and does pointer arithmetic against ae->data / ae->len. The * parser is now a fully length-bounded token walker (no NUL reliance), * so this harness deliberately feeds it a buffer sized to EXACTLY * ae->len with NO trailing NUL — ASAN then flags any single byte read at * or past ae->len. Together with RFC 7231 q-value parsing that is * exactly the bug class coverage-guided fuzzing finds and the Perl suite * misses, and it is a hard regression gate against re-introducing a * NUL-bounded scan over the len-bounded buffer. * * The real functions live in ../../src/ngx_http_zstd_common.h and depend on the * nginx tree. fuzz/extract_parser.sh slices just those functions into * generated_parser.inc at build time, so we always fuzz the SHIPPED code * with no copy drift. ngx_shim.h provides the tiny nginx surface it needs * with faithful upstream semantics. * * Build (see fuzz/build.sh): * clang -g -O1 -fsanitize=fuzzer,address,undefined \ * fuzz_accept_encoding.c -o fuzz_accept_encoding */ #include #include #include #include #include "ngx_shim.h" /* Generated by ci/fuzz/extract_parser.sh from ../../src/ngx_http_zstd_common.h. * Contains the verbatim bodies of ngx_http_zstd_eval_qvalue() and * ngx_http_zstd_accept_encoding(). */ #include "generated_parser.inc" /* * Independent reference oracle (CI5). * * The trap above only proves the production parser returns one of two * sentinels — it cannot catch a *wrong* answer (every wrong answer is still a * valid sentinel), so an RFC2-class semantic regression would pass unnoticed. * * This is a second, independently written decision procedure. It is * deliberately conservative: it returns -1 ("unsure") for any input outside a * small, unambiguous grammar (unexpected bytes, more than one parameter, a * non-"q" parameter, quoted strings). For the inputs it *is* confident about * it returns 1 (accept) / 0 (decline), and the harness asserts the production * parser agrees. Two implementations rarely share the same subtle bug, so a * divergence on a confident case is a real semantic defect — without false * traps on genuinely ambiguous edges. * * Returns 1 accept, 0 decline, -1 unsure. */ static int ref_accepts(const uint8_t *d, size_t n) { size_t i = 0; int zstd_q = -1; /* explicit zstd weight in milli, -1 absent */ int star_q = -1; /* "*" weight, -1 absent */ while (i < n) { size_t ts, te; int is_zstd, is_star, q; /* OWS / empty elements */ while (i < n && (d[i] == ' ' || d[i] == '\t' || d[i] == ',')) i++; if (i >= n) break; /* * A quoted-string can never be a valid coding name. The production * parser declines such an element: its name scan stops at the '"' * and the quote-aware element-skip swallows the whole quoted blob * (plus any trailing params) up to the next top-level ','. Mirror * that confidently so the differential GATES the name-position * phantom-token class (`"a,zstd "` must NOT yield zstd). Step over * any quoted-strings in this element; bail to "unsure" only on a * '\' escape or an unterminated quote (production's skip_quoted * handles those, but reproducing its exact leniency here risks a * wrong-confident answer, so decline to assert instead). The * element contributes no coding, so just advance past it. */ if (d[i] == '"') { while (i < n && d[i] != ',') { if (d[i] == '"') { i++; /* opening DQUOTE */ while (i < n && d[i] != '"') { if (d[i] == '\\') return -1; /* escapes → unsure */ i++; } if (i >= n) return -1; /* unterminated quote */ i++; /* closing DQUOTE */ } else { i++; } } if (i < n) i++; /* consume the ',' */ continue; /* non-matching element */ } /* token name: letters/digits or a lone '*' */ ts = i; while (i < n && ((d[i] >= 'A' && d[i] <= 'Z') || (d[i] >= 'a' && d[i] <= 'z') || (d[i] >= '0' && d[i] <= '9') || d[i] == '*' || d[i] == '-' || d[i] == '_')) { i++; } te = i; if (te == ts) { return -1; /* unexpected leading byte: give up */ } is_star = (te - ts == 1 && d[ts] == '*'); is_zstd = (te - ts == 4 && (d[ts] | 0x20) == 'z' && (d[ts+1] | 0x20) == 's' && (d[ts+2] | 0x20) == 't' && (d[ts+3] | 0x20) == 'd'); while (i < n && (d[i] == ' ' || d[i] == '\t')) i++; /* * Only ';', ',' or end may follow a coding name (RFC 9110 12.5.3: * `codings` is a token). Anything else -- `zstd"x`, `zstd "x`, * `zstd x` -- means the element offers no coding, and the * reference says so CONFIDENTLY rather than returning -1. An * unsure reference makes the differential assert nothing in * either direction, which is exactly how this class stayed * invisible through 15.9M runs. Skip to the element separator, * honouring quoting so a comma inside a quoted run does not * start a phantom element. */ if (i < n && d[i] != ';' && d[i] != ',') { int inq = 0; while (i < n && (inq || d[i] != ',')) { if (d[i] == '\\' && inq) { if (i + 1 >= n) return -1; /* dangling escape */ i += 2; continue; } if (d[i] == '"') inq = !inq; i++; } if (inq) return -1; /* unterminated quote */ if (i < n) i++; /* consume the ',' */ continue; /* contributes nothing */ } q = 1000; /* no weight → q=1 */ /* * Parameters: zero or more ';'-separated params. Understand a single * "q" weight, and for a non-"q" param confidently skip a token value * or a *simple* quoted-string value (no '\' escape, properly closed). * That is exactly enough to keep the reference confident on the * quoted-comma class — e.g. `gzip;x="a, zstd";q=1`, where the comma * is inside gzip's parameter, not an element separator — which the * old "q must be the first/only param" reference treated as unsure * and so could not catch. Anything outside this shape (an escape, an * unterminated quote, a repeated "q", trailing junk) returns -1 * (unsure) rather than risk a wrong-but-confident answer. */ { int q_seen = 0; while (i < n && d[i] == ';') { size_t ps, pe; int is_qp; i++; /* ';' */ while (i < n && (d[i] == ' ' || d[i] == '\t')) i++; ps = i; while (i < n && d[i] != '=' && d[i] != ';' && d[i] != ',' && d[i] != ' ' && d[i] != '\t' && d[i] != '"') { i++; } pe = i; /* empty parameter ("zstd;;q=1"): no such production in * RFC 9110, so the element is malformed. The parser * rejects it; stay confident and agree. */ if (pe == ps) return -1; is_qp = (pe - ps == 1 && (d[ps] == 'q' || d[ps] == 'Q')); while (i < n && (d[i] == ' ' || d[i] == '\t')) i++; if (i >= n || d[i] != '=') { if (is_qp) return -1; /* bare "q" is malformed */ continue; /* valueless non-q param */ } i++; /* '=' */ while (i < n && (d[i] == ' ' || d[i] == '\t')) i++; if (is_qp) { if (q_seen) return -1; /* repeated "q" */ q_seen = 1; if (i >= n) return -1; /* "q=" with no value */ if (d[i] == '0') { int scale = 100; i++; q = 0; if (i < n && d[i] == '.') { i++; while (i < n && d[i] >= '0' && d[i] <= '9' && scale > 0) { q += (d[i] - '0') * scale; scale /= 10; i++; } } } else if (d[i] == '1') { int k = 0; i++; q = 1000; if (i < n && d[i] == '.') { i++; while (i < n && d[i] == '0' && k < 3) { i++; k++; } } } else { return -1; } } else if (i < n && d[i] == '"') { i++; /* opening DQUOTE */ while (i < n && d[i] != '"') { if (d[i] == '\\') return -1; /* escapes → unsure */ i++; } if (i >= n) return -1; /* unterminated quote */ i++; /* closing DQUOTE */ } else { /* bare token value: a '"' here is not a well-formed * quoted-string opener (it would have to follow '=' * immediately) — bail to unsure rather than guess. */ while (i < n && d[i] != ';' && d[i] != ',' && d[i] != ' ' && d[i] != '\t' && d[i] != '"') { i++; } } /* after a value only OWS then ';' / ',' / end is valid; a * stray '"' (e.g. in a param name) lands here and is unsure */ while (i < n && (d[i] == ' ' || d[i] == '\t')) i++; if (i < n && d[i] != ';' && d[i] != ',') return -1; } } if (is_zstd) { zstd_q = q; } else if (is_star) { star_q = q; } /* element consumed: only a ',' separator or end may remain */ if (i < n) { if (d[i] != ',') return -1; /* trailing junk */ i++; } } if (zstd_q >= 0) return zstd_q > 0 ? 1 : 0; if (star_q >= 0) return star_q > 0 ? 1 : 0; return 0; } int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { /* * The parser is fully length-bounded, so do NOT append a trailing * NUL: allocate exactly `size` bytes (one byte when size == 0, since * malloc(0) is implementation-defined). ASAN then turns any read at * or past ae->len — i.e. any reintroduced NUL-bounded scan — into an * immediate, reproducible heap-buffer-overflow failure. */ size_t alloc = size ? size : 1; u_char *buf = (u_char *) malloc(alloc); if (buf == NULL) { return 0; } if (size) { memcpy(buf, data, size); } ngx_str_t ae; ae.data = buf; ae.len = size; ngx_int_t rc = ngx_http_zstd_accept_encoding(&ae); /* Result must be one of the two documented sentinels — anything else * means control flow fell through a path it should not have. */ if (rc != NGX_OK && rc != NGX_DECLINED) { __builtin_trap(); } /* * Semantic differential (CI5): when the independent reference is * confident, the production parser must agree. -1 = reference unsure * (ambiguous edge) → no assertion. */ { int ref = ref_accepts(data, size); if (ref >= 0 && (ref == 1) != (rc == NGX_OK)) { __builtin_trap(); } } /* * The generic walker behind the wrapper, exercised with the second * production token ("dcz", explicit-only — no wildcard) so the * parameterized name-match path also runs under ASAN on every input. * Weights are milli-q by contract: anything outside -1..1000 means * the walker fell through a path it should not have. */ { ngx_int_t dq = ngx_http_zstd_coding_weight(&ae, "dcz", sizeof("dcz") - 1, 0); if (dq < -1 || dq > 1000) { __builtin_trap(); } } free(buf); return 0; } myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/fuzz_dcz.c000066400000000000000000000071331525232545500234350ustar00rootroot00000000000000/* * Copyright (C) 2026 Thijs Eilander * SPDX-License-Identifier: BSD-2-Clause * * libFuzzer harness for ngx_http_zstd_dcz_decode_digest(). * * Why this target: it parses the attacker-controlled Available-Dictionary * request header -- an RFC 8941 byte-sequence gate (colon framing, a * 44-character length ceiling) followed by ngx_decode_base64() and a * fixed-32-byte destination buffer. That is exactly the bug class * coverage-guided fuzzing finds and the Perl suite (ci/t/03-dcz.t) does * not: off-by-one framing, a base64 length that rounds unexpectedly, or * a decoded length other than 32 landing past the destination buffer. * * The real function lives in ../../src/ngx_http_zstd_filter_module.c and * depends on the nginx tree only through ngx_str_t/ngx_int_t/NGX_OK/ * NGX_DECLINED and ngx_decode_base64() -- none of ngx_http_request_t. It * was extracted out of ngx_http_zstd_dcz_negotiate() specifically so it * could be fuzzed in isolation; see the extraction comment on the * function itself. ci/fuzz/extract_dcz.sh slices its verbatim body into * generated_dcz.inc at build time, so this always fuzzes the SHIPPED * code with no copy drift. ngx_shim.h provides the tiny nginx surface it * needs (ngx_decode_base64() included) with faithful upstream semantics. * * Build (see fuzz/build.sh): * clang -g -O1 -fsanitize=fuzzer,address,undefined \ * fuzz_dcz.c -o fuzz_dcz */ #include #include #include #include #include "ngx_shim.h" /* Generated by ci/fuzz/extract_dcz.sh from * ../../src/ngx_http_zstd_filter_module.c. Contains the verbatim body of * ngx_http_zstd_dcz_decode_digest(). */ #include "generated_dcz.inc" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { /* * No trailing NUL: ngx_http_zstd_dcz_decode_digest() and everything * it calls are fully length-bounded on raw.len, so ASAN turns any * read at or past that length into an immediate heap-buffer-overflow * failure -- the same discipline fuzz_accept_encoding.c uses. */ size_t alloc = size ? size : 1; u_char *buf = (u_char *) malloc(alloc); if (buf == NULL) { return 0; } if (size) { memcpy(buf, data, size); } ngx_str_t raw; raw.data = buf; raw.len = size; /* * Sized to NGX_HTTP_ZSTD_DCZ_DECODE_BUF_LEN (48), matching the real * caller's buffer -- NOT NGX_HTTP_ZSTD_SHA256_DIGEST_LEN (32). An * unpadded 44-character byte-sequence decodes to 33 bytes, one past * a tight 32-byte buffer, before the post-decode length check runs; * ASAN on a 32-byte buffer here would flag that as a harness bug * instead of exercising the real function's documented contract. * Poisoned so a decode path that writes fewer than * NGX_HTTP_ZSTD_SHA256_DIGEST_LEN bytes on a claimed-NGX_OK return * (a contract violation) leaves a visible non-zero tail rather than * silently reading back a zero-initialized buffer. */ u_char out[NGX_HTTP_ZSTD_DCZ_DECODE_BUF_LEN]; memset(out, 0xA5, sizeof(out)); ngx_int_t rc = ngx_http_zstd_dcz_decode_digest(raw, out); /* * Result must be one of the three documented sentinels: NGX_OK, * NGX_DECLINED (malformed byte-sequence framing, rejected before * decoding) or NGX_ERROR (well-formed framing, payload is not a * 32-byte base64 value). The caller distinguishes the two failure * values only to select a diagnostic; both are equally fail-closed. */ if (rc != NGX_OK && rc != NGX_DECLINED && rc != NGX_ERROR) { __builtin_trap(); } free(buf); return 0; } myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/gen_dict.sh000077500000000000000000000130651525232545500235470ustar00rootroot00000000000000#!/usr/bin/env bash # # Generate the coding-name entries of fuzz.dict from the module's real # parse surface, instead of a hand-maintained list that goes stale # silently: a merely incomplete dictionary still produces a green # crash-only fuzz run, so nothing else would ever catch the drift. # # This module has no single lookup table of coding names -- unlike a # trie-walk scanner, the tokens are two literal call-site arguments to # ngx_http_zstd_coding_weight(ae, "TOKEN", ...) in src/*.c and src/*.h (currently # "zstd" and "dcz"). Extract THOSE, rather than hand-copying them, so a # third coding name added to a future call site is picked up # automatically and a forgotten dictionary update fails --check instead # of silently shipping an incomplete dictionary. # # Usage: # ci/fuzz/gen_dict.sh regenerate fuzz.dict's coding-token lines # ci/fuzz/gen_dict.sh --check exit 1 if fuzz.dict's coding tokens have # drifted from the source (wired into # ci/linter/lint-c.sh) # # fuzz.dict also carries structural/grammar tokens (";q=", ",", etc.) that # are not coding names and are NOT touched by this script -- only the # quoted content-coding literals block, delimited by the markers below. set -euo pipefail DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT="$(cd "$DIR/../.." && pwd)" DICT="$DIR/fuzz.dict" # shellcheck source=ci/linter/lib.sh . "$ROOT/ci/linter/lib.sh" # The dictionary is byte-exact (its .gitattributes rule is -text), and # every consumer of this script compares LINES: a CRLF-contaminated # dict makes --check's marker match fail and report "stale tokens" -- # a misleading diagnosis for a line-ending problem -- while regeneration # would faithfully preserve the CR bytes on the lines it rewrites # around. Refuse CR up front, in both modes, with the real diagnosis. if [ -f "$DICT" ] && grep -q $'\r' "$DICT"; then echo "✗ gen_dict: $DICT contains CRLF line endings. The dictionary" \ "is byte-exact; restore the LF version (git checkout with its" \ "-text attribute honored) before regenerating or checking." >&2 exit 1 fi START_MARK="# BEGIN generated coding tokens (ci/fuzz/gen_dict.sh)" END_MARK="# END generated coding tokens" # Extract every coding-name literal passed to a coding-weight entry point # across src/*.c AND src/*.h: ngx_http_zstd_accept_encoding()'s own call # ("zstd") lives in the ngx_http_zstd_common.h wrapper, not in a .c file, so # both must be scanned. The functions' own DEFINITION lines (`const char # *coding`, no string literal) never match this pattern, so there is # nothing to exclude. # # Every entry point is matched: ngx_http_zstd_coding_weight() (one field # value), ngx_http_zstd_chain_coding_weight() (the linked field chain), and # ngx_http_zstd_request_coding_weight() (the complete request field). # Matching the `chain_` and `request_` forms is load-bearing, not defensive # tidiness: when the dcz call site moved to the request-field walker, a pattern # anchored on # the bare name stopped matching it and silently dropped "dcz" from the # dictionary, which is a fuzz-coverage loss that no test would have failed # on. Matching every form keeps a call site's token in the dictionary wherever # the negotiation is evaluated from. produce_tokens() { grep -hoE 'ngx_http_zstd_((chain|request)_)?coding_weight\([^,]+,[[:space:]]*"[A-Za-z0-9_-]+"' \ "$ROOT"/src/*.c "$ROOT"/src/*.h | grep -oE '"[A-Za-z0-9_-]+"$' | sort -u } mapfile_checked TOKENS produce_tokens if [ "${#TOKENS[@]}" -eq 0 ]; then echo "✗ gen_dict: no ngx_http_zstd_coding_weight(...) call sites found" \ "in src/*.c or src/*.h -- parser call convention changed? update gen_dict.sh" >&2 exit 1 fi GENERATED="$START_MARK"$'\n' for t in "${TOKENS[@]}"; do GENERATED+="$t"$'\n' done GENERATED+="$END_MARK" if [ "${1:-}" = "--check" ]; then if [ ! -f "$DICT" ]; then echo "✗ gen_dict --check: $DICT does not exist" >&2 exit 1 fi CURRENT="$(awk -v start="$START_MARK" -v end="$END_MARK" ' $0 == start { f=1 } f { print } $0 == end { if (f) exit } ' "$DICT")" if [ "$CURRENT" != "$GENERATED" ]; then echo "✗ fuzz.dict's generated coding-token block is stale." >&2 echo " Source call sites: ${TOKENS[*]}" >&2 echo " Run: ci/fuzz/gen_dict.sh" >&2 exit 1 fi echo "✓ fuzz.dict coding tokens match src/*.c + src/*.h call sites: ${TOKENS[*]}" exit 0 fi # BOTH markers, in order. Checking only START is not enough: awk below sets # skip=1 at START and only clears it at END, so a dict that lost its END marker # has every line after START swallowed and the mv writes the truncation back. # A later --check can still pass, because the generated block itself matches. if [ ! -f "$DICT" ]; then echo "✗ $DICT does not exist" >&2 exit 1 fi start_n="$(grep -cFx "$START_MARK" "$DICT" || true)" end_n="$(grep -cFx "$END_MARK" "$DICT" || true)" if [ "$start_n" -ne 1 ] || [ "$end_n" -ne 1 ]; then echo "✗ $DICT needs exactly one '$START_MARK' and one '$END_MARK' line" \ "(found $start_n and $end_n) -- add the marker block once by hand," \ "see the file header" >&2 exit 1 fi if [ "$(grep -nFx "$START_MARK" "$DICT" | cut -d: -f1)" -ge \ "$(grep -nFx "$END_MARK" "$DICT" | cut -d: -f1)" ]; then echo "✗ $DICT has '$END_MARK' before '$START_MARK'" >&2 exit 1 fi TMP="$(mktemp)" awk -v start="$START_MARK" -v end="$END_MARK" -v gen="$GENERATED" ' $0 == start { print gen; skip=1; next } $0 == end { skip=0; next } skip { next } { print } ' "$DICT" >"$TMP" mv "$TMP" "$DICT" echo "✓ regenerated fuzz.dict coding tokens from src/*.c + src/*.h: ${TOKENS[*]}" myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/ngx_shim.h000066400000000000000000000230061525232545500234150ustar00rootroot00000000000000/* * Minimal nginx shim for fuzzing ngx_http_zstd_accept_encoding(). * * The real ngx_http_zstd_common.h pulls in // * — the entire nginx tree. The Accept-Encoding parser only * touches a tiny, well-defined slice of that surface, so we reproduce just * that slice here with the EXACT upstream semantics. The fuzz target then * includes the real, shipped header unmodified, so we are fuzzing production * code — not a re-implementation. * * If nginx ever changes the semantics of ngx_strcasestrn / ngx_tolower this * shim must be updated to match; the comments below cite the upstream source * (src/core/ngx_string.{h,c}) these are copied from. */ #ifndef NGX_ZSTD_FUZZ_SHIM_H #define NGX_ZSTD_FUZZ_SHIM_H #include #include /* Guard against the real nginx headers being pulled in alongside the shim. */ #define NGX_HTTP_ZSTD_COMMON_H_SHIMMED 1 typedef intptr_t ngx_int_t; typedef uintptr_t ngx_uint_t; typedef unsigned char u_char; #define NGX_OK 0 #define NGX_ERROR -1 #define NGX_DECLINED -5 /* src/ngx_http_zstd_sha256.h */ #define NGX_HTTP_ZSTD_SHA256_DIGEST_LEN 32 /* src/ngx_http_zstd_filter_module.c: ngx_http_zstd_dcz_decode_digest()'s * output buffer size -- see that macro's definition for why it must * exceed NGX_HTTP_ZSTD_SHA256_DIGEST_LEN. */ #define NGX_HTTP_ZSTD_DCZ_DECODE_BUF_LEN 48 /* src/ngx_http_zstd_filter_module.c: padded base64 length of a 32-byte * digest, the length ceiling dcz_decode_digest() rejects above. */ #define NGX_HTTP_ZSTD_DCZ_DIGEST_B64_LEN 44 typedef struct { size_t len; u_char *data; } ngx_str_t; /* src/core/ngx_string.h: ngx_tolower(c) — ASCII-only, no locale. */ #define ngx_tolower(c) (u_char) ((c >= 'A' && c <= 'Z') ? (c | 0x20) : c) /* * src/core/ngx_config.h defines ngx_inline as the platform's inline * keyword (`inline` on every compiler this suite builds with). The sliced * parser uses it on ngx_http_zstd_accept_encoding(), which the module TUs * no longer reference -- inline is what keeps a plain `static` from * tripping -Werror=unused-function there. The extracted .inc carries the * keyword verbatim, so this layer has to spell it too. */ #define ngx_inline inline /* * ngx_table_elt_t, reduced to the two fields the chained-Accept-Encoding * walker and legacy request path touch. The legacy shape deliberately has * no ->next member, matching nginx 1.22.x. */ typedef struct ngx_table_elt_s ngx_table_elt_t; struct ngx_table_elt_s { ngx_str_t key; ngx_str_t value; #if !defined(NGX_ZSTD_LEGACY_SHIM) ngx_table_elt_t *next; #endif }; typedef struct ngx_list_part_s ngx_list_part_t; struct ngx_list_part_s { void *elts; ngx_uint_t nelts; ngx_list_part_t *next; }; typedef struct { ngx_list_part_t part; } ngx_list_t; typedef struct { ngx_table_elt_t *accept_encoding; ngx_list_t headers; } ngx_http_headers_in_t; typedef struct ngx_http_request_s ngx_http_request_t; struct ngx_http_request_s { ngx_http_request_t *main; ngx_http_headers_in_t headers_in; }; /* * The production header selects between (ae)->next and a NULL stub on * `nginx_version >= 1023000`, because ngx_table_elt_t.next does not * exist before 1.23.0. This layer has no nginx tree and no * nginx_version, so the default shim pins the modern chained shape. The * NGX_ZSTD_LEGACY_SHIM variant instead pins nginx 1.22.1: it removes ->next * and lets the request-field helper walk headers_in.headers. The dedicated * legacy unit test populates that list with repeated Accept-Encoding fields. */ #if defined(NGX_ZSTD_LEGACY_SHIM) #define nginx_version 1022001 #else #define nginx_version 1023000 #endif /* * The chain step, mirrored from src/ngx_http_zstd_common.h. The macro is * defined OUTSIDE any function body there, so extract_parser.sh -- which * slices function bodies -- does not carry it into the .inc and this * layer has to supply it. Kept byte-identical to the production * nginx_version >= 1023000 arm; the pre-1.23 arm is the NULL stub because * ngx_http_zstd_request_coding_weight() walks headers_in.headers instead. */ #if (nginx_version >= 1023000) #define NGX_HTTP_ZSTD_AE_NEXT(ae) ((const ngx_table_elt_t *) (ae)->next) #else #define NGX_HTTP_ZSTD_AE_NEXT(ae) ((const ngx_table_elt_t *) NULL) #endif /* * src/core/ngx_string.c: ngx_strncasecmp() — faithful upstream copy. * Compares up to n bytes case-insensitively; stops early on NUL in s1. */ static inline ngx_int_t ngx_strncasecmp(u_char *s1, u_char *s2, size_t n) { ngx_uint_t c1, c2; while (n) { c1 = (ngx_uint_t) *s1++; c2 = (ngx_uint_t) *s2++; c1 = (c1 >= 'A' && c1 <= 'Z') ? (c1 | 0x20) : c1; c2 = (c2 >= 'A' && c2 <= 'Z') ? (c2 | 0x20) : c2; if (c1 == c2) { if (c1) { n--; continue; } return 0; } return c1 - c2; } return 0; } /* * src/core/ngx_string.c: ngx_strcasestrn(). * * Case-insensitive search for a NUL-terminated needle inside a * NUL-terminated haystack. `n` is strlen(needle) - 1 (the function does * the +1 internally), exactly as the upstream contract — and exactly how * ngx_http_zstd_accept_encoding() calls it with sizeof("zstd") - 2. * * This is a faithful copy of the upstream implementation. It relies on * s1 (the haystack, == ae->data) being NUL-terminated, which is the * property the fuzz target deliberately stresses. */ static inline u_char * ngx_strcasestrn(u_char *s1, char *s2, size_t n) { ngx_uint_t c1, c2; c2 = (ngx_uint_t) *s2++; c2 = (c2 >= 'A' && c2 <= 'Z') ? (c2 | 0x20) : c2; do { do { c1 = (ngx_uint_t) *s1++; if (c1 == 0) { return NULL; } c1 = (c1 >= 'A' && c1 <= 'Z') ? (c1 | 0x20) : c1; } while (c1 != c2); } while (ngx_strncasecmp(s1, (u_char *) s2, n) != 0); return --s1; } /* * src/core/ngx_string.c: ngx_decode_base64() and its shared decode core * ngx_decode_base64_internal() (standard alphabet variant only, i.e. the * "+/" table -- this module never calls the base64url or basic64 * NGX_ESCAPE variants). Faithful upstream copy: rejects the input up * front unless its length is a multiple of 4 (dst->len is set from that * check, exactly mirroring ngx_base64_decoded_length()'s ((len + 3) / 4) * * 3 for a 4-aligned len), decodes 4 source bytes to 3 destination bytes * per group, and treats '=' padding and any byte outside the alphabet as * a hard decode failure (NGX_ERROR) rather than skipping it -- unlike * some non-nginx base64 decoders, this is NOT lenient about embedded * whitespace or padding placement. ngx_http_zstd_dcz_decode_digest() * relies on exactly this strictness: a malformed byte sequence must * decode-fail, not silently produce wrong bytes. */ static ngx_int_t ngx_decode_base64_internal(ngx_str_t *dst, ngx_str_t *src, const u_char *basis) { size_t len; u_char *d, *s; for (len = 0; len < src->len; len++) { if (src->data[len] == '=') { break; } if (basis[src->data[len]] == 77) { return NGX_ERROR; } } if (len % 4 == 1) { return NGX_ERROR; } s = src->data; d = dst->data; while (len > 3) { *d++ = (u_char) (basis[s[0]] << 2 | basis[s[1]] >> 4); *d++ = (u_char) (basis[s[1]] << 4 | basis[s[2]] >> 2); *d++ = (u_char) (basis[s[2]] << 6 | basis[s[3]]); s += 4; len -= 4; } if (len > 1) { *d++ = (u_char) (basis[s[0]] << 2 | basis[s[1]] >> 4); } if (len > 2) { *d++ = (u_char) (basis[s[1]] << 4 | basis[s[2]] >> 2); } dst->len = (size_t) (d - dst->data); return NGX_OK; } static const u_char ngx_base64_decode_table[] = { 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 62, 77, 77, 77, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 77, 77, 77, 77, 77, 77, 77, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 77, 77, 77, 77, 77, 77, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77 }; static inline ngx_int_t ngx_decode_base64(ngx_str_t *dst, ngx_str_t *src) { /* * src/core/ngx_string.c: ngx_decode_base64() rejects input whose * length is not a multiple of 4 -- the "=" padding is REQUIRED, not * optional (unlike the "url" variant). This is exactly the strict * gate the dcz Available-Dictionary path relies on: the fuzz seeds * cover both the padded-32-byte-digest case (which is a multiple of * 4) and unpadded/misaligned lengths (rejected here). */ if (src->len % 4 != 0) { return NGX_ERROR; } return ngx_decode_base64_internal(dst, src, ngx_base64_decode_table); } #endif /* NGX_ZSTD_FUZZ_SHIM_H */ myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/regressions/000077500000000000000000000000001525232545500237725ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/regressions/README.md000066400000000000000000000031321525232545500252500ustar00rootroot00000000000000# Regression corpus One file per PAST crash, named by its libFuzzer artifact hash (`crash-`), copied here after triage instead of being discarded. Replayed by `fuzzing.yml`'s `fuzz-regression` job **before** the time-boxed fresh run — a bug that was fixed once must be caught again in seconds if it regresses, not only if the fresh budget happens to rediscover the same input. Add a case with: ```bash cp ci/fuzz/crash- ci/fuzz/regressions/crash-- ``` then `ci/fuzz/fuzz_accept_encoding ci/fuzz/regressions/*` must abort before the fix and pass after it — verify both, the same as any other regression test's negative control. ## Seeded case (PR2 step 24) `crash-wildcard-precedence-flip` (`*, zstd;q=0`) is not a crash this target found on its own — libFuzzer's short bounded runs against the real parser stay clean (verified: 693576 execs / 10s, 0 crashes). It exists to prove the replay mechanism itself works end to end, and to give the mutation pass evidence: it trips the semantic differential oracle in `fuzz_accept_encoding.c` (CI5) when the explicit/wildcard precedence in `ngx_http_zstd_coding_weight()` is flipped, and passes clean (0ms) against the real parser. See `ci/adoption-findings.md` for the exact commands and both outcomes. The workflow step's empty-glob guard (`shopt -s nullglob`) still matters for whenever this directory is briefly empty again after a cleanup, or before the first case existed — an empty directory is a real, valid state, not a bug, and the step reports its case count explicitly either way rather than silently no-op'ing forever. myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/regressions/crash-wildcard-precedence-flip000066400000000000000000000000131525232545500316210ustar00rootroot00000000000000*, zstd;q=0myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/regressions_dcz/000077500000000000000000000000001525232545500246325ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/regressions_dcz/README.md000066400000000000000000000025341525232545500261150ustar00rootroot00000000000000# dcz regression corpus One file per PAST `fuzz_dcz` crash, named `crash-dcz-`, copied here after triage instead of being discarded. Replayed by `fuzzing.yml`'s `fuzz-regression` job **before** the time-boxed fresh run, so a bug that was fixed once is caught again in seconds if it regresses. Separate from `../regressions/` on purpose: that directory feeds `fuzz_accept_encoding`, whose input language is an `Accept-Encoding` header value. This target takes an RFC 8941 byte sequence. Replaying one corpus through the other target proves nothing. Add a case with: ```bash cp ci/fuzz/crash-dcz- ci/fuzz/regressions_dcz/crash-dcz- ``` then verify both halves, the same as any other regression test's negative control: `ci/fuzz/fuzz_dcz ci/fuzz/regressions_dcz/*` must abort before the fix and pass after it. ## Recorded cases `crash-dcz-framing-ok-payload-bad` (`::`) — the two-byte input whose framing is well-formed but whose payload is empty. It crashed CI on 2026-08-25: PR #172 split `ngx_http_zstd_dcz_decode_digest()`'s failure return into `NGX_DECLINED` (framing) and `NGX_ERROR` (payload), but this harness still asserted the old two-value contract and trapped on the new `NGX_ERROR`. The production code was correct; the oracle was stale. Verified red/green: exit 77 against the pre-fix harness, exit 0 after. myguard-labs-nginx-zstd-module-3486b5b/ci/fuzz/regressions_dcz/crash-dcz-framing-ok-payload-bad000066400000000000000000000000021525232545500326260ustar00rootroot00000000000000::myguard-labs-nginx-zstd-module-3486b5b/ci/linter/000077500000000000000000000000001525232545500217265ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/README.md000066400000000000000000000563441525232545500232210ustar00rootroot00000000000000# ci/linter — local lint gate Mirrors the cheap half of remote CI so a push does not burn a round-trip on a finding shellcheck could have named in two seconds. Every script is standalone; `run-all.sh` runs them all and `.githooks/pre-commit` runs `run-all.sh --staged`. ## Layout | Script | Covers | Gate | |---|---|---| | `lint-c.sh` | `src/*.[ch]` | flawfinder ≥4, cppcheck (warning/performance/portability), semgrep ≥WARNING (`p/c`, `p/security-audit`) | | `lint-nginx.sh` | `src/*.[ch]` | nginx conventions: libc alloc/str/num/io instead of `ngx_*`, hard tabs, >80 columns, trailing whitespace, `ngx_config.h` include order | | `lint-sh.sh` | `*.sh`, `*.bash`, `.githooks/*` | shellcheck `-S warning` | | `lint-python.sh` | `*.py` | `ruff check` + `ruff format --check` | | `lint-perl.sh` | `ci/t/*.t`, `*.pl`, `*.pm` | `perl -c` + perlcritic severity ≥4 | | `lint-yaml.sh` | `*.yml`, `*.yaml` | yamllint (errors block, warnings visible), actionlint + zizmor (`--persona=pedantic`) on `.github/workflows/` | | `lint-ci-runners.sh` | `.github/workflows/` | fork PRs never select the self-hosted pool; `pull_request_target` forbidden; every `runs-on` names labels that exist, including on workflows no pull request can reach | | `lint-ci-ports.sh` | `.github/workflows/` | every port-binding job declares a distinct `TEST_BASE_PORT` band, binds it, and verifies it above the FIRST binding step | | `lint-ci-cadence.sh` | `.github/workflows/` | a `workflow_call` member carries no `push:`/`pull_request:` of its own, so it runs once per change rather than twice on two uncancellable concurrency keys (`schedule:` allowed) | | `lint-ci-secrets.sh` | `.github/workflows/` | a `workflow_call` member declares the secrets it needs with `required: true`; callers wire them by name and never use `secrets: inherit` | | `lint-docs-drift.sh` | `.github/workflows/`, `README.md` | every workflow documented, every documented workflow exists | | `lint-spelling.sh` | all tracked files | codespell over prose, comments and log strings; vendored trees excluded via `lib.sh` | | `run-all.sh` | all of the above | runs every check, reports once | | `install-linters.sh` | — | apt-get → pipx → cpan → upstream binary | | `lib.sh` | — | sourced helpers (file selection, missing-tool failure) | | `workflow_policy.py` | — | the repo-policy checks the `ci-*`/`docs-drift` wrappers call (`runners`, `ports`, `docs`, `cadence`, `secrets`) | | `selftest.sh` | — | negative controls for the gate itself; run before the linters in `lint.yml`. Includes the set-equality control: every checker is named in `lint.yml`'s `LINT_ONLY` | | `fixtures/policy/` | — | trees the policy checks must go RED on — the known bypasses, the runner-label and step-ordering cases, and the four ways a `secrets:` declaration goes wrong; `clean/` and the `-ok` trees must stay GREEN | Rule config lives at the repo root so editors and these scripts agree: `.yamllint` (workflow-shaped YAML), `.perlcriticrc` (Test::Nginx-shaped Perl). Both carry the reason for every relaxation; read them before adding another. Thresholds deliberately match `.github/workflows/security-scanners.yml`. Move one there and move it here **in the same commit**, or local-green stops predicting remote-green — the only reason this directory exists. The `ci-*`, `docs-drift` and `sync-stamp` rows are **repo-policy** checks, not general linters, and that is why they are here rather than left to actionlint or zizmor. Those two read a workflow against general knowledge: syntax, and a catalogue of known attack shapes. These encode facts only this repo knows: which self-hosted labels exist, that the runners are persistent and shared with the Debian package builds, which port band each job owns, that `ci.yml` is the single orchestrator, that a member's secret surface is declared at the member, which files an adopter copies, and which file documents the pipeline. Each one goes red when a NEW workflow is added without a property every existing workflow happens to have — the case where copying an existing file is the only thing between the repo and a regression, and nothing enforces the copy. `clang-tidy` is **CI-only**: it needs `ngx_auto_config.h`, which exists only in a configured nginx tree. A hook cannot assume one, and a check that skips itself when the tree is missing is a vacuous gate. ## 1. Install the linters ```sh ci/linter/install-linters.sh # install what is missing ci/linter/install-linters.sh --check # report only ``` Preference order, and why each tool lands where it does: ### apt-get (preferred — distro-managed, no PEP 668 fight) ```sh sudo apt-get update sudo apt-get install -y --no-install-recommends \ shellcheck cppcheck flawfinder yamllint clang-tidy \ libperl-critic-perl perl pipx cpanminus ``` ### pip / pipx (Python tools Debian does not carry at the needed version) Use `pipx`, not `pip3`: Debian 12+ marks the system interpreter externally-managed, so a bare `pip3 install` fails and `--break-system-packages` is a worse answer than a venv per tool. ```sh pipx install 'ruff==0.16.1' # pinned to the CI version on purpose pipx install 'semgrep==1.173.0' # pinned to the CI version on purpose ``` `ruff` and `semgrep` are pinned because an unpinned upgrade changes findings under you and local stops matching CI. Bump each here and in its CI consumer together -- `ruff` in `install-linters.sh`, `semgrep` in `security-scanners.yml` too. ### cpan (Perl modules apt does not carry on every target release) ```sh sudo cpanm --notest Test::Nginx::Socket # also what makes `perl -c` work on ci/t/*.t sudo cpanm --notest Perl::Critic # only if libperl-critic-perl was unavailable ``` `--notest`: Test::Nginx's own suite wants a live nginx and a free port, which an install step has no business demanding. ```sh pipx install zizmor # GitHub Actions security audit ``` `zizmor` is deliberately **not** pinned, unlike ruff and semgrep: its rule set is the whole point, and a frozen security scanner stops finding what it was added for. A new rule going red is a finding to triage, not drift to suppress. ### upstream binary (no apt/pip/cpan source) ```sh ver=1.7.7 sha=023070a287cd8cccd71515fedc843f1985bf96c436b7effaecce67290e7e0757 curl -fsSL -o actionlint.tgz \ "https://github.com/rhysd/actionlint/releases/download/v${ver}/actionlint_${ver}_linux_amd64.tar.gz" echo "$sha actionlint.tgz" | sha256sum -c - # must pass before the next line tar -xzf actionlint.tgz actionlint && sudo install -m0755 actionlint /usr/local/bin/ ``` Do not pipe the tarball straight into `tar`: that installs whatever the network returned. The digest is from the release's `actionlint_${ver}_checksums.txt` and is pinned beside the version in `install-linters.sh` too — bump both together. Make sure `~/.local/bin` is on `PATH` for the pipx-installed tools. ## 2. Enable the pre-commit hook The hook is tracked at `.githooks/pre-commit` so a change to the gate arrives as a reviewable diff. Git does not use it until you point `core.hooksPath` at the directory — once per clone: ```sh git config core.hooksPath .githooks ``` Verify it is live: ```sh git config --get core.hooksPath # -> .githooks ``` Bypass in an emergency with `git commit --no-verify`. **This replaces the `pre-commit` framework hook.** `core.hooksPath` makes git ignore `.git/hooks/` entirely, including the hook `pre-commit install` writes there. The repo's `.pre-commit-config.yaml` still exists and covers overlapping ground (whitespace fixers, private-key detection, gitleaks, flawfinder, semgrep, cppcheck, ruff, shellcheck, actionlint). Pick one: - `git config core.hooksPath .githooks` — this directory: also covers Perl, yamllint and the nginx conventions, no Python framework needed. - `pipx install pre-commit && pre-commit install` — the framework: also runs the whitespace/EOF fixers, `detect-private-key` and `gitleaks` over the staged patch, but not `lint-nginx.sh`, `lint-perl.sh` or `lint-yaml.sh`'s yamllint/zizmor passes. Then leave `core.hooksPath` unset. Running both means running flawfinder, semgrep, cppcheck and ruff twice per commit. ## 3. Use it ```sh ci/linter/run-all.sh # every tracked file ci/linter/run-all.sh --staged # what the hook runs ci/linter/run-all.sh src/foo.c # named files LINT_ONLY="c nginx" ci/linter/run-all.sh LINT_SKIP_SEMGREP=1 ci/linter/run-all.sh # loud opt-out of the slowest pass LINT_JOBS=1 ci/linter/run-all.sh # serial, for bisecting a hang ci/linter/run-all.sh --list ``` Exit codes: `0` clean, `1` findings, `2` a linter is missing. ### Speed, and why it is shaped this way Measured 2026-07-31 on the self-hosted build host (i9-14900HX, 32 threads) with **no CI job running** — see the caveat below before comparing against your own numbers: | | before | after | |---|---|---| | full tree | 3.8s | **1.45s** | | one C file | 2.9s | **1.31s** | | full tree, `LINT_JOBS=1` | 3.2s | 2.57s | Re-measure on an idle box or not at all. This host also runs six self-hosted CI runner slots, and at load average ~50 the same full-tree run took 2.2s to 12.4s across six back-to-back attempts — the run-to-run spread is wider than the entire improvement, so a busy-box A/B measures the neighbours, not the change. Check `/proc/loadavg` first. Two changes, only one of which is really about speed: - **`semgrep --metrics=off`.** The end-of-scan POST to semgrep.dev was 2.76s of a 2.76s scan; without it the same scan is 1.27s. More than half the hook's wall clock was telemetry. - **`semgrep --jobs=1`.** A *correctness* fix. semgrep-core defaults to one OCaml domain per core and each domain opens its own io_uring ring against this host's 8 MB `RLIMIT_MEMLOCK`, which is shared with the self-hosted CI runners. When the runners are busy it exhausts and semgrep-core aborts with `Unix_error: Cannot allocate memory io_uring_queue_init`, exit 2 — a red commit gate caused by neighbouring load, not by the diff. Reproduced 3/3 on a busy box, 0/3 on an idle one. `src/` is three files, so nothing was gained by the parallelism in the first place. `security-scanners.yml` carries the same two flags; they run on the same host and must stay in sync. - **`run-all.sh` fans the checkers out** (`LINT_JOBS`, default one slot per checker). Each checker's output is buffered and replayed whole in glob order, never streamed: findings carry a `file:line` but not a checker name, so interleaved output is unattributable. Fixed order also keeps two runs of the same dirty tree byte-comparable. The floor is now semgrep's own startup. If this creeps back over ~2s, scope semgrep — do not drop a checker. Suppress one justified `lint-nginx.sh` finding with a trailing `/* NOLINT-nginx */` on that line. Whole-rule suppression is deliberately not supported: the exception belongs next to the code that needs it, where review can see the reason. ## Verify before trusting A green gate proves nothing until it has been seen red. Every probe below was run against this tree and observed failing; re-run them after changing a threshold. ```sh # shell: SC2164 + SC2115 -> exit 1 printf '#!/bin/bash\ncd /tmp/x\nrm -rf "${B}/"*\n' > _p.sh LINT_ONLY=sh ci/linter/run-all.sh _p.sh ; rm _p.sh # C + nginx conventions: malloc/strcpy -> exit 1 from both lint-c and lint-nginx printf '#include \nvoid f(void){char*p=malloc(4);strcpy(p,"ab");}\n' > src/_probe.c LINT_ONLY="c nginx" ci/linter/run-all.sh src/_probe.c ; rm src/_probe.c # python: unused import -> exit 1 printf 'import os\nx=1\n' > _p.py LINT_ONLY=python ci/linter/run-all.sh _p.py ; rm _p.py # perl: string eval + interpolated system() -> exit 1 printf 'my $x = 1;\nsystem("ls $x");\neval "1";\n' > _p.pl LINT_ONLY=perl ci/linter/run-all.sh _p.pl ; rm _p.pl # yaml: unterminated flow sequence -> exit 1 printf 'a: [1,\n' > _p.yml LINT_ONLY=yaml ci/linter/run-all.sh _p.yml ; rm _p.yml # workflow security: zizmor -> exit 1 on template-injection, artipacked, # unpinned-uses and excessive-permissions, all from these seven lines cat > .github/workflows/_probe.yml <<'EOF' name: probe on: [pull_request] jobs: p: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - run: echo "${{ github.event.pull_request.title }}" EOF LINT_ONLY=yaml ci/linter/run-all.sh .github/workflows/_probe.yml rm .github/workflows/_probe.yml # CI runner policy: a self-hosted runner on a PR-reachable workflow -> exit 1 cat > .github/workflows/_probe.yml <<'EOF' name: probe on: pull_request: jobs: p: runs-on: [self-hosted, linux] steps: - run: python3 ci/tools/test_runtime.py --nginx x --module y EOF LINT_ONLY=ci-runners ci/linter/run-all.sh # -> runs-on is not a trust split LINT_ONLY=ci-ports ci/linter/run-all.sh # -> starts the driver, no band LINT_ONLY=docs-drift ci/linter/run-all.sh # -> workflow not in README.md rm .github/workflows/_probe.yml # Runner LABELS, on a workflow no pull request can reach. Nothing else reads # these: actionlint validates labels for a literal `runs-on` only, and every # self-hosted selector here is a fromJSON(...) ternary it stays silent on. cat > .github/workflows/_probe.yml <<'EOF' name: probe on: schedule: - cron: "0 4 * * 1" jobs: p: runs-on: ${{ github.event.pull_request.head.repo.fork && 'ubuntu-latest' || fromJSON('["self-hosted","buidler02","lxc"]') }} steps: - run: echo probe EOF LINT_ONLY=ci-runners ci/linter/run-all.sh # -> not an approved selector (label typo) rm .github/workflows/_probe.yml # a secret wired at the caller that the member never declared -- both halves # read as correct on their own; the secret is dropped at the call boundary LINT_ONLY=ci-secrets ci/linter/run-all.sh # (see fixtures/policy/secrets-*) # an unstamped file in the shared set. A NEW probe rather than an edit to a # tracked workflow: the cleanup for that is `git checkout `, which also # discards any uncommitted work you had in it. cat > .github/workflows/_probe.yml <<'EOF' name: probe on: workflow_dispatch: jobs: p: runs-on: ubuntu-latest steps: - run: echo probe EOF LINT_ONLY=sync-stamp ci/linter/run-all.sh # -> MISSING stamp rm .github/workflows/_probe.yml # docs drift, the other direction: a README reference to a workflow that is gone printf '\nSee .github/workflows/nosuch.yml\n' >> README.md LINT_ONLY=docs-drift ci/linter/run-all.sh ; git checkout README.md # a citation naming a step ci/PROMPT.md does not define -- what a renumber or a # deleted step leaves behind, and what nothing else in the tree can see printf '\nSee step 99 for details.\n' >> README.md LINT_ONLY=prompt-steps ci/linter/run-all.sh ; git checkout README.md # the other direction, a hole in the sequence itself. selftest.sh covers this # against a generated tree, so no edit to the real 2100-line spec is needed: # ci/linter/selftest.sh 2>&1 | grep prompt-steps # missing tool -> exit 2, never a silent skip printf 'x = 1\n' > _p.py PATH="$(echo "$PATH" | tr : '\n' | grep -v "$HOME/.local/bin" | paste -sd:)" \ LINT_ONLY=python ci/linter/run-all.sh _p.py ; rm _p.py # the hook itself blocks the commit printf '#!/bin/bash\ncd /tmp/x\n' > _bad.sh && git add _bad.sh git commit -m probe # -> blocked git reset -q HEAD _bad.sh && rm _bad.sh ``` Note SC2086 is INFO and correctly does **not** trip `lint-sh.sh`; probe with a warning-severity finding or you will misread the gate. ### Workflow security (zizmor) `actionlint` reads a workflow as syntax; `zizmor` reads it as an attack surface — template injection into `run:`, `pull_request_target`, credentials persisted by `actions/checkout`, actions pinned to a mutable tag, over-broad `permissions:`. On a repo with **self-hosted runners** that class of mistake is arbitrary code execution on the build host, which is why it is a gate and not advice. Run at `--persona=pedantic`: the default persona already passes on this tree, so gating on it could never go red. Pedantic is what caught the `matrix.*` interpolations in `ci-deep.yml` (now passed through `env:`) and the undocumented CodeQL permissions. `--offline`, so a commit hook never needs a token. The online audits only add repo-settings context; that belongs in a periodic review. Inapplicable finding → `# zizmor: ignore[rule]` on the line, with the reason. Never a blanket disable in a `zizmor.yml`. ## In CI `.github/workflows/lint.yml` runs `install-linters.sh` then `LINT_ONLY="nginx sh python perl yaml spelling ci-runners ci-ports ci-cadence ci-secrets sync-stamp docs-drift" run-all.sh` — the same entry point as the hook, so a clone that never enabled `core.hooksPath` still cannot land a regression. That allowlist is narrower than the glob `run-all.sh` uses, so a checker added to `ci/linter/` runs locally and in the hook while being **absent from every PR** — a gate missing from the one path where it is load-bearing. `selftest.sh` now cross-checks the two ("every checker is named in lint.yml LINT_ONLY") and goes red on the gap; `c` is the one deliberate exclusion, for the reason below. Before that control existed, `ci-cadence` shipped and had never run remotely. A derived module's list will legitimately differ from this one. `lint.yml` is wired into the `ci.yml` orchestrator and runs on `ubuntu-latest`, taking no self-hosted slot. The `c` checker is left out there because `security-scanners.yml` already runs flawfinder/clang-tidy/semgrep over `src/` at the same thresholds. That is also why `lint-c.sh` must be edited in the same commit as that workflow. ## Extending - New repo-policy check: a subcommand in `workflow_policy.py` plus a thin `lint-.sh` wrapper. Keep the wrapper's `lint_files` pattern as the RELEVANCE test only — every one of these checks compares whole SETS, so a narrowed file list would let a deletion through whenever the counterpart file was the only one staged. **Then add a fixture under `fixtures/policy/` and a `policy_` line in `selftest.sh`.** A policy check whose red path is never exercised is indistinguishable from one that cannot go red — all three of these shipped bypassable by valid YAML (a `.yaml` extension, an inline `on: [pull_request]`, a comment after a job key) and every one of them reported clean while doing it. Point a check at a fixture tree with `WORKFLOW_POLICY_ROOT=`. - **Parse workflows with PyYAML, never with a regex over the file text.** The three checks here were originally regex-based, and the file argued in a docstring that a YAML parse would add a dependency that might be missing. Both halves were wrong: yamllint already makes PyYAML a hard dependency of the same gate, and every regex was walked past by valid YAML that GitHub reads exactly as the shape the regex expected. `workflow_policy.py` now exits 2 when PyYAML is absent rather than degrading. - New file type: drop a `ci/linter/lint-.sh` in place — `run-all.sh` picks it up by glob. Keep "no files of this kind" exiting 0, and fail with exit 2 (via `need`) when the tool is absent. - New nginx convention: one more `rule ` call in `lint-nginx.sh`. - New dependency: add it to `install-linters.sh` **and** to the apt/pip/cpan lists above, so a fresh clone is one command from armed. ## Coverage table — file type → linter → rule profile Derived at skeleton-adoption step 37 (2026-08-22). Every tracked file type is listed, with the checker that reaches it and the profile it is gated at. A type with no checker carries the reason it needs none — an unexplained blank is a gap, not a decision. | Tracked type | Count | Checker(s) | Rule profile | Gated where | |---|---|---|---|---| | `src/*.c`, `src/*.h` | 4 | `lint-c`, `lint-nginx`, ast-grep | flawfinder ≥4 · cppcheck warning/perf/portability · semgrep ≥WARNING (`p/c`,`p/security-audit`) · nginx conventions · 11 own + vendored structural rules | `security-scanners.yml` (CI) + pre-commit; **not** `lint.yml` — see § the `c` exclusion | | `ci/**/*.[ch]` | 9 | flawfinder, cppcheck, semgrep, ast-grep | same C profiles as `src/`, minus the nginx-convention pass (pool discipline does not apply to harness code) | pre-commit + `lint.yml` (ast-grep) | | `*.sh`, `.githooks/*` | 32 | `lint-sh` | shellcheck `-S warning` | `lint.yml` + pre-commit | | `*.py` | 17 | `lint-python` | `ruff check` + `ruff format --check` | `lint.yml` + pre-commit | | `*.t`, `*.pl`, `*.pm` | 4 | `lint-perl` | `perl -c` + perlcritic ≥4 | `lint.yml` + pre-commit | | `.github/workflows/*.yml` | 11 | `lint-yaml`, `lint-ci-{runners,ports,cadence,secrets}`, `lint-docs-drift` | yamllint · actionlint · zizmor `--persona=pedantic` · 5 repo policy checks | `lint.yml` + pre-commit | | other `*.yml`, `*.yaml` | 65 | `lint-yaml` | yamllint only (actionlint/zizmor are workflow-scoped) | `lint.yml` + pre-commit | | `*.md` | 30 | `lint-spelling`; `README.md` also `lint-docs-drift` | codespell · workflow/README set-equality | `lint.yml` + pre-commit | | all tracked prose/comments | — | `lint-spelling` | codespell, vendored trees excluded in `lib.sh` | `lint.yml` + pre-commit | | `*.key`, `*.zst`, `*.patch`, `*.dict`, `*.suppress` | 11 | none | binary/fixture/generated data with no syntax a linter can assert; `*.key` are test-only material | — | ### Justified exclusions - **The `c` checker is absent from `lint.yml`'s `LINT_ONLY`** — flawfinder and semgrep already run over `src/` in `security-scanners.yml` at the *same* thresholds `lint-c.sh` mirrors. Running them twice per PR buys queue time, not coverage. `lint-c.sh` remains the local half of that mirror. - **clang-tidy is CI-only.** It needs `ngx_auto_config.h`, i.e. a configured nginx tree, which a local hook cannot assume. A checker that skips itself when the tree is missing is a vacuous gate, so it lives only in `security-scanners.yml`, where the tree is built first. ### Gaps found at step 37 — both closed 1. **Three C files were reached by no checker at all.** `ci/fuzz/fuzz_accept_encoding.c`, `ci/fuzz/ngx_shim.h` and `ci/tests/unit/test_accept_encoding.c` fell outside both the linter selectors (`^src/.*\.[ch]$`) and the pre-commit hook globs, which stopped at `ci/tools/`. The fuzz harness is C that parses attacker-shaped input, so this was the gap worth closing first. The four C hooks now select `^(src|ci)/.*\.[ch]$`; all four pass on the newly covered files. 2. **ast-grep and gitleaks were hook-only.** Both run from `.pre-commit-config.yaml`, and no workflow runs pre-commit, so a clone that never set `core.hooksPath` bypassed both — exactly the failure mode `lint.yml`'s header says the remote run exists to prevent. `lint.yml` now runs each directly, as its own step, invoked the way the hook invokes it. `run-all.sh` still does not reach them: they are not `ci/linter/lint-*.sh`, and the selftest's set-equality control only covers checkers that are. If either step is removed from `lint.yml`, nothing will notice. ### Verifying these two gates Both were confirmed able to fail, not just to run: - **ast-grep** — a planted `strcpy()` in `ci/tests/unit/` produced `error[c-unbounded-string-copy]` and the pipeline exited 123. - **gitleaks** — use a NON-EXAMPLE secret. AWS's own `AKIAIOSFODNN7EXAMPLE` is allowlisted by the default ruleset and reports "no leaks found", which reads exactly like a broken gate. The probe recipe above the hook in `.pre-commit-config.yaml` produces `leaks found: 1` and exit 1. Note that piping the run into `tail` masks that exit status — check it unpiped. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/ci_topology.py000077500000000000000000000237341525232545500246430ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright (C) 2026 Thijs Eilander # SPDX-License-Identifier: BSD-2-Clause """Protect the measured four-lane PR/deep topology and hosted workflow calls.""" from __future__ import annotations import copy import pathlib import sys import yaml ROOT = pathlib.Path(__file__).resolve().parents[2] def load(name: str) -> dict: with (ROOT / ".github" / "workflows" / name).open(encoding="utf-8") as src: return yaml.safe_load(src) def needs(job: dict) -> set[str]: value = job.get("needs", []) return {value} if isinstance(value, str) else set(value) def edge_errors(filename: str, jobs: dict, edges: dict[str, set[str]]) -> list[str]: return [ f"{filename}:{job} breaks a four-lane dependency chain" for job, expected in edges.items() if job not in jobs or needs(jobs[job]) != expected ] def check_ci(ci: dict) -> list[str]: errors: list[str] = [] expected = { "lint": "./.github/workflows/lint.yml", "build-test": "./.github/workflows/build-test.yml", "arch-package": "./.github/workflows/arch-package.yml", "security-scanners": "./.github/workflows/security-scanners.yml", "harness-fault-arms": "./.github/workflows/harness-fault-arms.yml", "windows-build": "./.github/workflows/windows-build.yml", } jobs = ci.get("jobs", {}) if set(jobs) != set(expected): errors.append("ci.yml must contain only the measured PR workflow set") for job, target in expected.items(): if job not in jobs or jobs[job].get("uses") != target or needs(jobs[job]): errors.append(f"ci.yml:{job} must remain an independent call to {target}") trigger = ci.get("on", ci.get(True, {})) trigger_names = set(trigger) if isinstance(trigger, dict) else set(trigger or []) if trigger_names != {"pull_request", "push", "workflow_dispatch"}: errors.append("ci.yml must remain PR/manual plus the focused master signal") elif not isinstance(trigger["push"], dict) or trigger["push"].get("branches") != [ "master" ]: errors.append("ci.yml push must remain limited to master") push_guard = "github.event_name != 'push'" for job in ( "lint", "build-test", "arch-package", "security-scanners", "windows-build", ): if jobs.get(job, {}).get("if") != push_guard: errors.append(f"ci.yml:{job} must skip the focused master signal") if jobs.get("harness-fault-arms", {}).get("if") is not None: errors.append("ci.yml:harness-fault-arms must run on the master signal") return errors def check_build(build: dict) -> list[str]: errors: list[str] = [] jobs = build.get("jobs", {}) allowed_build = { "resolve", "validation", "release-build-fanout", "build", "build-arm64", "build-old-libzstd", "linkage", "cvary-interop", "build-asan", "tests", "tests-asan", } if set(jobs) != allowed_build: errors.append("build-test.yml contains a job outside the four-lane design") build_edges = { "resolve": set(), "validation": set(), "release-build-fanout": {"resolve"}, "build": {"resolve"}, "build-arm64": {"resolve"}, "tests": {"resolve", "build"}, "build-asan": {"resolve", "release-build-fanout"}, "tests-asan": {"build-asan"}, "build-old-libzstd": {"resolve", "release-build-fanout"}, "cvary-interop": {"resolve", "release-build-fanout"}, "linkage": {"resolve", "cvary-interop"}, } errors.extend(edge_errors("build-test.yml", jobs, build_edges)) fanout = jobs.get("release-build-fanout", {}) if fanout.get("runs-on") != "ubuntu-24.04": errors.append("build-test.yml fan-out barrier must stay off the four-lane pool") steps = fanout.get("steps", []) if len(steps) != 1 or steps[0].get("run") != "sleep 10": errors.append( "build-test.yml fan-out barrier must preserve the ten-second " "best-effort eligibility lead for the delayed fan-out chains" ) if "coverage" in jobs: errors.append("coverage is a deep report, not a PR job") return errors def check_deep(deep: dict) -> list[str]: errors: list[str] = [] trigger = deep.get("on", deep.get(True, {})) trigger_names = set(trigger) if isinstance(trigger, dict) else set(trigger or []) if trigger_names != {"schedule", "workflow_dispatch"}: errors.append("ci-deep.yml must remain weekly/manual and outside PR CI") jobs = deep.get("jobs", {}) allowed_deep = { "build-flavors", "fuzz", "memcheck", "harness-memcheck", "helgrind", "scanners", "coverage", "codeql", "asan-soak", } if set(jobs) != allowed_deep: errors.append("ci-deep.yml contains a job outside the four-lane design") deep_edges = { "memcheck": set(), "coverage": {"memcheck"}, "codeql": {"coverage"}, "build-flavors": {"codeql"}, "scanners": {"build-flavors"}, "asan-soak": {"scanners"}, "harness-memcheck": set(), "helgrind": {"harness-memcheck"}, "fuzz": set(), } errors.extend(edge_errors("ci-deep.yml", jobs, deep_edges)) if ( jobs.get("fuzz", {}).get("strategy", {}).get("max-parallel") != 2 or jobs.get("build-flavors", {}).get("strategy", {}).get("max-parallel") != 1 ): errors.append( "CI Deep matrices must reserve two fuzz lanes and serialize flavors" ) fuzz_include = ( jobs.get("fuzz", {}).get("strategy", {}).get("matrix", {}).get("include", []) ) fuzz_targets = { entry.get("binary") for entry in fuzz_include if isinstance(entry, dict) } if len(fuzz_include) != 2 or fuzz_targets != { "fuzz_accept_encoding", "fuzz_dcz", }: errors.append("CI Deep must run both parser fuzz targets in its two lanes") return errors def findings(ci: dict, build: dict, deep: dict) -> list[str]: return check_ci(ci) + check_build(build) + check_deep(deep) def selftest( # pylint: disable=too-many-statements ci: dict, build: dict, deep: dict ) -> int: baseline = findings(ci, build, deep) if baseline: for error in baseline: print(f"FAIL topology baseline is invalid: {error}", file=sys.stderr) return 1 cases = [] changed = copy.deepcopy(ci) changed["jobs"]["fuzzing"] = {"uses": "./.github/workflows/fuzzing.yml"} cases.append(("long PR job", changed, build, deep)) changed = copy.deepcopy(ci) changed["jobs"]["lint"]["needs"] = "build-test" cases.append(("serialized PR workflow call", changed, build, deep)) changed = copy.deepcopy(ci) changed["jobs"]["security-scanners"]["uses"] = "./.github/workflows/valgrind.yml" cases.append(("rewired PR workflow call", changed, build, deep)) changed = copy.deepcopy(ci) changed["jobs"]["arch-package"]["uses"] = "./.github/workflows/valgrind.yml" cases.append(("rewired hosted package workflow", changed, build, deep)) changed = copy.deepcopy(ci) del changed["jobs"]["build-test"]["if"] cases.append(("duplicate master build", changed, build, deep)) changed = copy.deepcopy(ci) del changed["jobs"]["arch-package"]["if"] cases.append(("duplicate master package build", changed, build, deep)) changed = copy.deepcopy(ci) changed[True]["push"] = None cases.append(("unbounded master signal", changed, build, deep)) changed = copy.deepcopy(build) changed["jobs"]["build"]["needs"] = ["resolve", "validation"] cases.append(("lost PR lane overlap", ci, changed, deep)) changed = copy.deepcopy(build) changed["jobs"]["validation"]["needs"] = "build" cases.append(("serialized validation lane", ci, changed, deep)) changed = copy.deepcopy(build) changed["jobs"]["build-arm64"]["needs"] = "build-arm64" cases.append(("invalid hosted arm64 dependency", ci, changed, deep)) changed = copy.deepcopy(build) changed["jobs"]["release-build-fanout"]["steps"][0]["run"] = "true" cases.append(("lost best-effort fan-out eligibility lead", ci, changed, deep)) changed = copy.deepcopy(deep) changed["jobs"]["fuzz"]["strategy"]["max-parallel"] = 3 cases.append(("five-lane deep fan-out", ci, build, changed)) changed = copy.deepcopy(deep) changed["jobs"]["fuzz"]["strategy"]["matrix"]["include"].pop() cases.append(("missing deep fuzz target", ci, build, changed)) changed = copy.deepcopy(deep) changed[True]["pull_request"] = {} cases.append(("deep PR trigger", ci, build, changed)) changed = copy.deepcopy(build) changed["jobs"]["unlisted"] = {} cases.append(("unlisted PR child job", ci, changed, deep)) changed = copy.deepcopy(deep) changed["jobs"]["unlisted"] = {} cases.append(("unlisted deep child job", ci, build, changed)) failed = False for label, ci_doc, build_doc, deep_doc in cases: if findings(ci_doc, build_doc, deep_doc): print(f"ok topology rejects {label}") else: print(f"FAIL topology accepted {label}", file=sys.stderr) failed = True return int(failed) def main() -> int: try: ci, build, deep = load("ci.yml"), load("build-test.yml"), load("ci-deep.yml") except (OSError, yaml.YAMLError) as exc: print(f"lint-ci-topology: could not parse workflows: {exc}", file=sys.stderr) return 2 if len(sys.argv) == 2 and sys.argv[1] == "--selftest": return selftest(ci, build, deep) if len(sys.argv) != 1: print(f"usage: {sys.argv[0]} [--selftest]", file=sys.stderr) return 2 errors = findings(ci, build, deep) for error in errors: print(f"lint-ci-topology: {error}", file=sys.stderr) if errors: return 1 print( "lint-ci-topology: PR/deep workflows preserve the measured four-lane topology" ) return 0 if __name__ == "__main__": raise SystemExit(main()) myguard-labs-nginx-zstd-module-3486b5b/ci/linter/codespell-ignore.txt000066400000000000000000000032611525232545500257240ustar00rootroot00000000000000# codespell ignore list -- read by ci/linter/lint-spelling.sh when present. # # EMPTY ON PURPOSE, and shipped anyway so the seam is documented rather than # discovered. codespell's dictionary flags plenty of real identifiers, and an # adopter meets that on the first run: nginx-cache-turbo-module measured 151 # findings, 149 of them correct spellings in that tree -- its own domain # acronym, an abbreviation for a well-known HTTP header, a URL path segment, # another header name, and bytes inside a fuzz corpus input. # # Without a file to put those in, the options are a red gate on arrival or a # blanket suppression, and the second one retires one of the cheapest real # checks in the set. # # Add one word per line, lowercase, with a comment saying why it is correct # HERE. An entry earns its place by being WRONG for this tree, never by being # inconvenient -- a word that is genuinely misspelled gets fixed instead. # # Note the file is itself linted, so spell out an example rather than quoting # the flagged token: writing the word here is what puts it in the tree. # # Corpus files are byte-exact fuzz inputs and must never be "corrected" -- # editing one silently changes what the fuzzer replays. lib.sh already excludes # ci/fuzz/corpus/, so those need no entry here. # Token-end index in the Accept-Encoding scanner, paired with the start index # two letters earlier in the alphabet. Both are local size_t cursors in # ci/fuzz/fuzz_accept_encoding.c, not an abbreviation of "the". te # nginx's own field name on ngx_http_request_t -- the filename extension of the # request URI. Spelled without the final "sion" by nginx itself since 0.1, so # correcting it would stop the code compiling. exten myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixture-selector.py000077500000000000000000000055121525232545500256120ustar00rootroot00000000000000#!/usr/bin/env python3 """Print (or write) the approved runner selector the -ok policy fixture must use. WHY THIS EXISTS ci/linter/fixtures/policy/schedule-only-runner-labels-ok/ is the POSITIVE control for the runner-label check: same shape as the drift fixture, but a selector `workflow_policy.py` must accept. Hardcoding a selector in it makes that control wrong for every adopter, because step 13 of ci/PROMPT.md tells a hosted-only adopter to turn SELF_HOSTED_ALLOWED off. The fixture then asserts a selector the adopter deliberately rejects, the control fails, and the failure looks like a bug in the checker rather than a stale fixture. Reported by the nginx-cache-turbo-module adoption, 2026-08-10. Deriving the selector from workflow_policy.py makes the fixture correct by construction: whatever a repo approves is what its positive control asserts. USAGE ci/linter/fixture-selector.py print the selector ci/linter/fixture-selector.py --write F rewrite F's `runs-on:` line in place Called by ci/linter/selftest.sh before the -ok fixture is checked. EXIT 0 selector printed/written, 1 SELF_HOSTED_ALLOWED is False (a hosted-only repo has no approved self-hosted selector, so the fixture pair does not apply). """ from __future__ import annotations import importlib.util import pathlib import re import sys HERE = pathlib.Path(__file__).resolve().parent def approved_selector() -> str | None: """APPROVED_SELECTOR, or None if this repo allows no self-hosted runners.""" spec = importlib.util.spec_from_file_location( "workflow_policy", HERE / "workflow_policy.py" ) if spec is None or spec.loader is None: return None mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) if not getattr(mod, "SELF_HOSTED_ALLOWED", False): return None return mod.APPROVED_SELECTOR def main(argv: list[str]) -> int: sel = approved_selector() if sel is None: # Hosted-only repo: no approved self-hosted selector exists, so there is # nothing for the positive control to assert. Silent exit 1 lets the # caller skip the rewrite without failing the selftest. return 1 if len(argv) >= 2 and argv[0] == "--write": path = pathlib.Path(argv[1]) text = path.read_text(encoding="utf-8") # Only the first job-level `runs-on:`; the fixture has exactly one. new, n = re.subn( r"(?m)^(\s*runs-on: ).*$", lambda m: m.group(1) + sel, text, count=1 ) if n != 1: print(f"{path}: no runs-on: line to rewrite", file=sys.stderr) return 1 if new != text: path.write_text(new, encoding="utf-8") return 0 print(sel) return 0 if __name__ == "__main__": raise SystemExit(main(sys.argv[1:])) myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/000077500000000000000000000000001525232545500235775ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/README.md000066400000000000000000000042131525232545500250560ustar00rootroot00000000000000# ci/linter/fixtures/ — trees the linters must go RED on **Nothing here is a workflow this repo runs. Do not "fix" these files.** Every file under `policy/` is deliberately wrong in one specific way. They are the negative controls for `ci/linter/workflow_policy.py`, driven by `ci/linter/selftest.sh`, which points a check at one of these trees with `WORKFLOW_POLICY_ROOT=` and asserts the exit status. `policy/clean/` is the positive control and must stay clean. Every sibling encodes a bypass that valid YAML used to walk straight through: | Fixture | Encodes | |---|---| | `clean/` | a plain, correct workflow — all three checks green | | `bypass-yaml-extension/` | `.yaml`, invisible while `workflows()` globbed `*.yml` | | `bypass-inline-events/` | `on: [pull_request]`, `on: pull_request`, inline `pull_request_target` | | `bypass-commented-job-key/` | `runtime: # comment`, which emptied the job list | ## Why a scanner will flag these, and why that is correct These workflows select self-hosted runners for fork pull requests, use `pull_request_target`, and declare no `permissions:`. That is the point — a fixture that passed a security scanner would not be testing anything. The repo's own gate does not see them: `ci/linter/lint-yaml.sh` scopes actionlint and zizmor to paths matching `^\.github/workflows/`, which these are not (they live under `ci/linter/fixtures/.../.github/workflows/`). yamllint does read them, and they are valid YAML, so it passes. If a future scanner is added that walks the whole tree, **exclude this directory rather than editing the fixtures.** Making these files pass a security scanner disarms every control they provide, and the resulting green is exactly the vacuous gate the whole `selftest.sh` layer exists to detect. ## Adding one Create `policy//` with its own `.github/workflows/` and a `README.md` saying which bypass it encodes and which subcommand must go red. Add a `policy_` line to `ci/linter/selftest.sh`. Then verify it in BOTH directions: red on the current parser, and green once you revert the fix it guards — a control that has never been observed failing is a control you have not tested. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/000077500000000000000000000000001525232545500250765ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/bypass-commented-job-key/000077500000000000000000000000001525232545500317065ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/bypass-commented-job-key/.github/000077500000000000000000000000001525232545500332465ustar00rootroot00000000000000workflows/000077500000000000000000000000001525232545500352245ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/bypass-commented-job-key/.githubruntime.yml000066400000000000000000000002431525232545500374310ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/bypass-commented-job-key/.github/workflows--- name: runtime on: workflow_call: jobs: runtime: # owns the 189xx band runs-on: ubuntu-latest steps: - run: python3 ci/tools/test_runtime.py myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/bypass-commented-job-key/README.md000066400000000000000000000006541525232545500331720ustar00rootroot00000000000000# fixture: a trailing comment on a job key `_jobs()` split on `^ (\w[\w-]*):\s*$`, so `runtime: # note` matched nothing and the whole file yielded ZERO jobs. `ports` then reported "no runtime-bearing jobs" -- clean, exit 0 -- for a workflow that starts the runtime driver with no port band. That is the vacuous-gate shape these linters exist to prevent, occurring inside a linter. Workflow: `.github/workflows/runtime.yml`. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/bypass-inline-events/000077500000000000000000000000001525232545500311555ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/bypass-inline-events/.github/000077500000000000000000000000001525232545500325155ustar00rootroot00000000000000workflows/000077500000000000000000000000001525232545500344735ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/bypass-inline-events/.githubscalar.yml000066400000000000000000000001671525232545500364670ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/bypass-inline-events/.github/workflows--- name: scalar on: pull_request jobs: evil: runs-on: [self-hosted, builder02] steps: - run: echo pwn seq.yml000066400000000000000000000001661525232545500360110ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/bypass-inline-events/.github/workflows--- name: seq on: [pull_request] jobs: evil: runs-on: [self-hosted, builder02] steps: - run: echo pwn target.yml000066400000000000000000000002321525232545500365010ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/bypass-inline-events/.github/workflows--- name: target on: [pull_request_target] jobs: evil: runs-on: ubuntu-latest steps: - run: echo base-repo secrets against head-repo code myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/bypass-inline-events/README.md000066400000000000000000000006471525232545500324430ustar00rootroot00000000000000# fixture: inline event syntax `on: [pull_request]` and `on: pull_request` mean exactly what the indented mapping form means. The reachability test used to be a regex for an INDENTED MAPPING KEY, so both inline spellings walked past the runner-trust check with a bare self-hosted `runs-on:`. `.github/workflows/seq.yml`, `.github/workflows/scalar.yml` and `.github/workflows/target.yml`: `runners` must report all three. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/bypass-yaml-extension/000077500000000000000000000000001525232545500313515ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/bypass-yaml-extension/.github/000077500000000000000000000000001525232545500327115ustar00rootroot00000000000000workflows/000077500000000000000000000000001525232545500346675ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/bypass-yaml-extension/.githubprobe.yaml000066400000000000000000000002411525232545500366570ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/bypass-yaml-extension/.github/workflows--- name: probe on: pull_request: jobs: evil: runs-on: [self-hosted, builder02] steps: - run: echo executes PR-authored code on the build host myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/bypass-yaml-extension/README.md000066400000000000000000000005631525232545500326340ustar00rootroot00000000000000# fixture: a workflow with the .yaml extension GitHub reads `*.yml` and `*.yaml` alike; `workflows()` used to glob `*.yml` only, so this self-hosted, PR-triggered, undocumented workflow was invisible to every policy check. `runners` and `docs` must both go red here. The documented workflow list deliberately mentions no file: `docs` must report the undocumented gate. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/clean/000077500000000000000000000000001525232545500261605ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/clean/.github/000077500000000000000000000000001525232545500275205ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/clean/.github/workflows/000077500000000000000000000000001525232545500315555ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/clean/.github/workflows/ci.yml000066400000000000000000000022251525232545500326740ustar00rootroot00000000000000--- name: ci on: pull_request: jobs: build: runs-on: ubuntu-latest steps: - run: echo build # Two shapes the ordering check must NOT flag, kept in the positive control so # a false positive shows up as a broken control rather than as a mystery red. same-step: runs-on: ubuntu-latest env: TEST_BASE_PORT: '19600' # Test::Nginx reads this, not TEST_BASE_PORT, so a prove job that declares # a band without wiring it here binds 1984 regardless -- see the # prove-band-not-passed-through fixture for that red. TEST_NGINX_PORT: '19600' steps: # One run block that verifies and then binds. Step indices cannot separate # these, so the check falls back to position within the text. - run: | bash ci/tools/max-port.sh "$TEST_BASE_PORT" prove -v ci/t/ word-boundary: runs-on: ubuntu-latest env: TEST_BASE_PORT: '19700' steps: # "improve" is not "prove". A substring match would read this as a binder # sitting above the verifier and report a finding. - run: echo improve the fixture - run: bash ci/tools/max-port.sh "$TEST_BASE_PORT" myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/clean/README.md000066400000000000000000000004301525232545500274340ustar00rootroot00000000000000# fixture: clean Positive control for ci/linter/selftest.sh. Every policy check must report clean against this tree, so that a RED result on a sibling fixture is attributable to the bypass that fixture encodes and not to the fixture shape. Workflow: `.github/workflows/ci.yml`. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/member-with-push-ok/000077500000000000000000000000001525232545500307025ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/member-with-push-ok/.github/000077500000000000000000000000001525232545500322425ustar00rootroot00000000000000workflows/000077500000000000000000000000001525232545500342205ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/member-with-push-ok/.githubbuild-test.yml000066400000000000000000000002241525232545500370150ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/member-with-push-ok/.github/workflowsname: Build&Test on: workflow_call: {} schedule: - cron: '0 3 * * 1' jobs: build: runs-on: ubuntu-latest steps: - run: make myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/member-with-push-ok/README.md000066400000000000000000000007431525232545500321650ustar00rootroot00000000000000# fixture: member-with-push-ok The green half of the `member-with-push` pair: a `workflow_call` member whose second trigger is `schedule:`, which is the INTENDED shape -- codeql.yml and ci-deep.yml are reached both from ci.yml and on their own cadence, and that is not a duplicate run of the same tree. This exists so the red next door means "a member carries a PR-duplicating trigger" and not merely "a member carries two triggers". Workflow: `.github/workflows/build-test.yml`. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/member-with-push/000077500000000000000000000000001525232545500302735ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/member-with-push/.github/000077500000000000000000000000001525232545500316335ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/member-with-push/.github/workflows/000077500000000000000000000000001525232545500336705ustar00rootroot00000000000000build-test.yml000066400000000000000000000002151525232545500364060ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/member-with-push/.github/workflowsname: Build&Test on: workflow_call: {} push: branches: [main] jobs: build: runs-on: ubuntu-latest steps: - run: make myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/member-with-push/README.md000066400000000000000000000017131525232545500315540ustar00rootroot00000000000000# fixture: member-with-push A `workflow_call` member that also carries its own `push: branches: [main]`. `workflow_call` does not suppress a member's own triggers, so this runs twice per change -- once on the PR, once on the merge commit, against a tree identical to the head that already passed. The two runs get different concurrency keys, so `cancel-in-progress` cannot collapse them. The reason this needs a fixture at all: BOTH runs are green. Nothing fails, nothing is slower, and the duplicate is visible only in the runner bill and in a README that has quietly stopped describing what runs when. Downstream this shape reached six of seven members before review caught it. `cadence` must go red here. Its pair, `member-with-push-ok`, is the same file with `schedule:` instead -- without that twin, this red would be equally consistent with "any member carrying a second trigger is flagged", which is not the rule. Workflow: `.github/workflows/build-test.yml`. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/prove-band-not-passed-through/000077500000000000000000000000001525232545500326645ustar00rootroot00000000000000.github/000077500000000000000000000000001525232545500341455ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/prove-band-not-passed-throughworkflows/000077500000000000000000000000001525232545500362025ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/prove-band-not-passed-through/.githubruntime.yml000066400000000000000000000002351525232545500404100ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/prove-band-not-passed-through/.github/workflowsname: runtime on: workflow_call: jobs: runtime: runs-on: ubuntu-latest env: TEST_BASE_PORT: "18976" steps: - run: prove -v ci/t/ README.md000066400000000000000000000020371525232545500340660ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/prove-band-not-passed-through# fixture: prove-band-not-passed-through A `prove` job that declares a unique `TEST_BASE_PORT` and never passes it on. Every check above this one is satisfied: the band is declared, it is distinct from every sibling, its width does not overrun a neighbour. The job still binds 1984, because Test::Nginx reads `TEST_NGINX_PORT` and has never heard of `TEST_BASE_PORT`. A declaration nothing consumes is decoration. `check_ports()` had the equivalent guard for the runtime driver (`--port "$TEST_BASE_PORT"`) but keyed it on `starts_runtime`, so the `prove` path had no pass-through check at all: the change that made `prove` a first-class binder for the declaration-required branch extended what must declare a band without extending what must pass it through. Ported from `labs/nginx-error-abuse-module` (raised by CodeRabbit on that repo's PR #48) as the REF49 carry-back recorded in that module's TODO.md. The skeleton had the declaration half already and this half not at all. `ports` must go red here. Workflow: `.github/workflows/runtime.yml`. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/prove-only-binder-exempt/000077500000000000000000000000001525232545500317515ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/prove-only-binder-exempt/.github/000077500000000000000000000000001525232545500333115ustar00rootroot00000000000000workflows/000077500000000000000000000000001525232545500352675ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/prove-only-binder-exempt/.githubruntime.yml000066400000000000000000000001661525232545500375000ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/prove-only-binder-exempt/.github/workflowsname: runtime on: workflow_call: jobs: runtime: runs-on: ubuntu-latest steps: - run: prove -v ci/t/ myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/prove-only-binder-exempt/README.md000066400000000000000000000020461525232545500332320ustar00rootroot00000000000000# fixture: prove-only-binder-exempt `check_ports()`'s declaration-required branch — the one its own comment calls "THE CHECK THAT MATTERS MOST" — keyed on `starts_runtime` (`RUNTIME_DRIVER in body`, i.e. `ci/tools/test_runtime.py`) rather than on `BINDERS`/`BINDER_RE`. A job whose only binder is `prove` was therefore exempt from the "declare `TEST_BASE_PORT`" requirement, even though `prove` is already a `BINDERS` member and `_order_finding()` already treats it as one thing that binds the band. The exemption is invisible from every other angle. Such a job declares no band, so the uniqueness check has nothing to collide; it silently takes Test::Nginx's hardcoded default and collides with any other runtime job pinned to the same runner. Found downstream in `nginx-api-abuse-module` as a failed negative control: deleting a prove-only job's band left `ports` GREEN, reporting "all with distinct port bands". `ports` must go red here. Workflow: `.github/workflows/runtime.yml` — a job that runs `prove -v ci/t/` and declares no `TEST_BASE_PORT`. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-apt-chained-unzip/000077500000000000000000000000001525232545500325545ustar00rootroot00000000000000.github/000077500000000000000000000000001525232545500340355ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-apt-chained-unzipworkflows/000077500000000000000000000000001525232545500360725ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-apt-chained-unzip/.githubci.yml000066400000000000000000000003251525232545500372100ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-apt-chained-unzip/.github/workflows--- name: ci on: pull_request: jobs: build: runs-on: ubuntu-latest steps: - run: wget -q -O payload https://example.invalid/payload - run: apt-get install -y unzip && unzip payload -d tool README.md000066400000000000000000000003341525232545500337540ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-apt-chained-unzip# fixture: provenance-apt-chained-unzip An `apt-get install unzip` package token must not hide a separate unverified archive extraction chained after it. `provenance` must go red. Workflow: `.github/workflows/ci.yml`. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-bare-wget-unverified/000077500000000000000000000000001525232545500332475ustar00rootroot00000000000000.github/000077500000000000000000000000001525232545500345305ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-bare-wget-unverifiedworkflows/000077500000000000000000000000001525232545500365655ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-bare-wget-unverified/.githubci.yml000066400000000000000000000003701525232545500377030ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-bare-wget-unverified/.github/workflows--- name: ci on: pull_request: jobs: build: runs-on: ubuntu-latest steps: - run: echo build - run: | wget -q "https://example.invalid/tool.tgz" - run: | tar -xzf tool.tgz ./tool/configure README.md000066400000000000000000000013741525232545500344540ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-bare-wget-unverified# fixture: provenance-bare-wget-unverified Same defect as `provenance-unverified-extract`, written with the download spelling four of this repo's own jobs actually use: `wget -q ""` with **no** `-O`. The first version of the scoping regex required `-O`/`-o`, so a job written this way matched no download token at all and was skipped WHOLESALE -- not passed, never examined. That is a gate failing open: the seven jobs it silently excluded were verified only by luck, and a future unverified copy of the idiom would have been invisible. `curl -O` and PowerShell `Invoke-WebRequest -OutFile` were blind the same way; `ci-deep.yml:scanners` is a real site the narrow regex never scanned. `provenance` must go red here. Workflow: `.github/workflows/ci.yml`. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-cache-hit-skips-verify/000077500000000000000000000000001525232545500335125ustar00rootroot00000000000000.github/000077500000000000000000000000001525232545500347735ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-cache-hit-skips-verifyworkflows/000077500000000000000000000000001525232545500370305ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-cache-hit-skips-verify/.githubci.yml000066400000000000000000000010151525232545500401430ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-cache-hit-skips-verify/.github/workflows--- name: ci on: pull_request: jobs: build: runs-on: ubuntu-latest steps: - run: echo build - name: Cache tool tarball id: tool-tarball uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 with: path: tool.tgz key: tool-src - name: Download tool if: steps.tool-tarball.outputs.cache-hit != 'true' run: wget -q -O tool.tgz "https://example.invalid/tool.tgz" - run: | tar -xzf tool.tgz ./tool/configure README.md000066400000000000000000000012751525232545500347170ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-cache-hit-skips-verify# fixture: provenance-cache-hit-skips-verify The trap this check exists to close, not just a duplicate of `provenance-unverified-extract`: the download step is gated `if: steps.tool-tarball.outputs.cache-hit != 'true'`, so on a cache HIT no download step runs at all -- and still nothing in the job asserts a trust-anchor before `tar -xzf` unpacks whatever `actions/cache` restored. A checker that only looked at the download step's presence/absence would read this job as having "no download to gate" on a cache hit and pass it; this one must still go red because the extract/execute step itself has no anchor before it, regardless of which path fed the file. Workflow: `.github/workflows/ci.yml`. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-chained-unzip/000077500000000000000000000000001525232545500317725ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-chained-unzip/.github/000077500000000000000000000000001525232545500333325ustar00rootroot00000000000000workflows/000077500000000000000000000000001525232545500353105ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-chained-unzip/.githubci.yml000066400000000000000000000002621525232545500364260ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-chained-unzip/.github/workflows--- name: ci on: pull_request: jobs: build: runs-on: ubuntu-latest steps: - run: curl -fsSL -o payload https://example.invalid/payload && unzip payload -d tool myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-chained-unzip/README.md000066400000000000000000000003061525232545500332500ustar00rootroot00000000000000# fixture: provenance-chained-unzip An unverified ZIP extraction chained after a download with `&&` remains an extraction boundary. `provenance` must go red. Workflow: `.github/workflows/ci.yml`. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-curl-O-unverified/000077500000000000000000000000001525232545500325335ustar00rootroot00000000000000.github/000077500000000000000000000000001525232545500340145ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-curl-O-unverifiedworkflows/000077500000000000000000000000001525232545500360515ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-curl-O-unverified/.githubci.yml000066400000000000000000000004211525232545500371640ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-curl-O-unverified/.github/workflows--- name: ci on: pull_request: jobs: build: runs-on: ubuntu-latest steps: - run: echo build - run: | curl -fsSLO --connect-timeout 30 "https://example.invalid/tool.tgz" - run: | tar -xzf tool.tgz ./tool/configure README.md000066400000000000000000000011021525232545500337250ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-curl-O-unverified# fixture: provenance-curl-O-unverified Sibling of `provenance-bare-wget-unverified` for the `curl -O` spelling (`windows-build.yml:mingw` uses `curl -fsSLO`). The pre-fix scoping regex required a separate `-O ` argument pair and did not match the bundled `-fsSLO` form, so a job written this way was skipped wholesale rather than checked. Exists because the comment on `DOWNLOAD_RE` claims three spellings are covered; without a probe per spelling, a regression in two of them passes the selftest. `provenance` must go red here. Workflow: `.github/workflows/ci.yml`. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-exec-downloaded-binary/000077500000000000000000000000001525232545500335605ustar00rootroot00000000000000.github/000077500000000000000000000000001525232545500350415ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-exec-downloaded-binaryworkflows/000077500000000000000000000000001525232545500370765ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-exec-downloaded-binary/.githubci.yml000066400000000000000000000003701525232545500402140ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-exec-downloaded-binary/.github/workflows--- name: ci on: pull_request: jobs: build: runs-on: ubuntu-latest steps: - run: echo build - run: | wget -q -O tool "https://example.invalid/tool" - run: | chmod +x tool ./tool --version README.md000066400000000000000000000006551525232545500347660ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-exec-downloaded-binary# fixture: provenance-exec-downloaded-binary No archive is involved: a STANDALONE executable is downloaded, marked executable, and run directly. The tar/unzip patterns miss this completely, so the job passed while running unverified attacker-controllable code on the runner -- the same privilege boundary the archive cases exist to guard, one step shorter. `provenance` must go red here. Workflow: `.github/workflows/ci.yml`. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-iwr-outfile-unverified/000077500000000000000000000000001525232545500336405ustar00rootroot00000000000000.github/000077500000000000000000000000001525232545500351215ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-iwr-outfile-unverifiedworkflows/000077500000000000000000000000001525232545500371565ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-iwr-outfile-unverified/.githubci.yml000066400000000000000000000005021525232545500402710ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-iwr-outfile-unverified/.github/workflows--- name: ci on: pull_request: jobs: build: runs-on: windows-latest steps: - run: echo build - shell: pwsh run: | Invoke-WebRequest -Uri "https://example.invalid/tool.tgz" -OutFile tool.tgz - shell: pwsh run: | tar -xzf tool.tgz ./tool/configure README.md000066400000000000000000000007051525232545500350420ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-iwr-outfile-unverified# fixture: provenance-iwr-outfile-unverified Sibling of `provenance-bare-wget-unverified` for PowerShell's `Invoke-WebRequest -OutFile` (`windows-build.yml:msvc` uses it). A Windows runner unpacking an unverified tarball is the same privilege boundary as a Linux one; the pre-fix scoping regex knew only `wget`/`curl`, so every PowerShell download job was invisible to this check. `provenance` must go red here. Workflow: `.github/workflows/ci.yml`. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-mixed-codeql-unzip/000077500000000000000000000000001525232545500327525ustar00rootroot00000000000000.github/000077500000000000000000000000001525232545500342335ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-mixed-codeql-unzipworkflows/000077500000000000000000000000001525232545500362705ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-mixed-codeql-unzip/.githubci.yml000066400000000000000000000003641525232545500374110ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-mixed-codeql-unzip/.github/workflows--- name: ci on: pull_request: jobs: build: runs-on: ubuntu-latest steps: - run: curl -fsSL -o payload https://example.invalid/payload - run: | unzip payload -d tool unzip -qq "$db/src.zip" -d "$src" README.md000066400000000000000000000003431525232545500341520ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-mixed-codeql-unzip# fixture: provenance-mixed-codeql-unzip The local CodeQL database exception applies only to its own extraction. It must not exempt an unverified ZIP extraction earlier in the same step. Workflow: `.github/workflows/ci.yml`. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-noncomparing-sha256/000077500000000000000000000000001525232545500327345ustar00rootroot00000000000000.github/000077500000000000000000000000001525232545500342155ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-noncomparing-sha256workflows/000077500000000000000000000000001525232545500362525ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-noncomparing-sha256/.githubci.yml000066400000000000000000000004601525232545500373700ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-noncomparing-sha256/.github/workflows--- name: ci on: pull_request: jobs: build: runs-on: ubuntu-latest steps: - run: echo build - run: | wget -q -O tool.tgz "https://example.invalid/tool.tgz" - run: | sha256sum tool.tgz - run: | tar -xzf tool.tgz ./tool/configure README.md000066400000000000000000000012451525232545500341360ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-noncomparing-sha256# fixture: provenance-noncomparing-sha256 The anchor half failing open, mirroring the scoping half. A step runs `sha256sum tool.tgz` -- which PRINTS a digest to the log and compares it to nothing -- and the extract step follows. The first `TRUST_ANCHOR_RE` matched the bare word `sha256sum`, so this job read as anchored while validating no bytes at all. A lone `FOO_SHA256:` env declaration nothing tests had the same effect. An anchor must show a COMPARISON: `sha256sum -c`, or a captured digest tested against a pinned `*_SHA256`. `provenance-verified-ok` is the green control for exactly that shape. `provenance` must go red here. Workflow: `.github/workflows/ci.yml`. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-pipe-unzip/000077500000000000000000000000001525232545500313345ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-pipe-unzip/.github/000077500000000000000000000000001525232545500326745ustar00rootroot00000000000000workflows/000077500000000000000000000000001525232545500346525ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-pipe-unzip/.githubci.yml000066400000000000000000000002561525232545500357730ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-pipe-unzip/.github/workflows--- name: ci on: pull_request: jobs: build: runs-on: ubuntu-latest steps: - run: wget -q -O payload https://example.invalid/payload | unzip payload -d tool myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-pipe-unzip/README.md000066400000000000000000000002421525232545500326110ustar00rootroot00000000000000# fixture: provenance-pipe-unzip A pipe does not remove the unverified ZIP extraction boundary. `provenance` must go red. Workflow: `.github/workflows/ci.yml`. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-subshell-unzip/000077500000000000000000000000001525232545500322205ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-subshell-unzip/.github/000077500000000000000000000000001525232545500335605ustar00rootroot00000000000000workflows/000077500000000000000000000000001525232545500355365ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-subshell-unzip/.githubci.yml000066400000000000000000000002571525232545500366600ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-subshell-unzip/.github/workflows--- name: ci on: pull_request: jobs: build: runs-on: ubuntu-latest steps: - run: wget -q -O payload https://example.invalid/payload; (unzip payload -d tool) myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-subshell-unzip/README.md000066400000000000000000000002761525232545500335040ustar00rootroot00000000000000# fixture: provenance-subshell-unzip Running an unverified ZIP extraction inside a subshell remains an extraction boundary. `provenance` must go red. Workflow: `.github/workflows/ci.yml`. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-unverified-extract/000077500000000000000000000000001525232545500330445ustar00rootroot00000000000000.github/000077500000000000000000000000001525232545500343255ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-unverified-extractworkflows/000077500000000000000000000000001525232545500363625ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-unverified-extract/.githubci.yml000066400000000000000000000004041525232545500374760ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-unverified-extract/.github/workflows--- name: ci on: pull_request: jobs: build: runs-on: ubuntu-latest steps: - run: echo build - run: | wget -q -O tool.tgz "https://example.invalid/tool.tgz" - run: | tar -xzf tool.tgz ./tool/configure README.md000066400000000000000000000006711525232545500342500ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-unverified-extract# fixture: provenance-unverified-extract The bypass class this repo hit: a step downloads a tarball with `wget -O`, and a later step in the SAME job extracts it with `tar -xzf` and immediately runs the unpacked binary's `configure` -- with no `gpg --verify`, no `sha256sum` comparison, and no delegation to `ci-build.sh` / `fetch-verified-nginx.sh` anywhere in between. `provenance` must go red here. Workflow: `.github/workflows/ci.yml`. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-unverified-unzip/000077500000000000000000000000001525232545500325375ustar00rootroot00000000000000.github/000077500000000000000000000000001525232545500340205ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-unverified-unzipworkflows/000077500000000000000000000000001525232545500360555ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-unverified-unzip/.githubci.yml000066400000000000000000000002741525232545500371760ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-unverified-unzip/.github/workflows--- name: ci on: pull_request: jobs: build: runs-on: ubuntu-latest steps: - run: curl -fsSL -o payload https://example.invalid/payload - run: unzip payload -d tool README.md000066400000000000000000000004171525232545500337410ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-unverified-unzip# fixture: provenance-unverified-unzip An archive's extension is not its format: a downloaded ZIP can be named `payload` and `unzip payload` still extracts it. With no trust anchor before that extraction, `provenance` must go red. Workflow: `.github/workflows/ci.yml`. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-verified-ok/000077500000000000000000000000001525232545500314405ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-verified-ok/.github/000077500000000000000000000000001525232545500330005ustar00rootroot00000000000000workflows/000077500000000000000000000000001525232545500347565ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-verified-ok/.githubci.yml000066400000000000000000000005761525232545500361040ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-verified-ok/.github/workflows--- name: ci on: pull_request: jobs: build: runs-on: ubuntu-latest steps: - run: echo build - run: | wget -q -O tool.tgz "https://example.invalid/tool.tgz" - run: | actual="$(sha256sum tool.tgz | awk '{print $1}')" [ "$actual" = "deadbeef" ] || exit 1 - run: | tar -xzf tool.tgz ./tool/configure myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-verified-ok/README.md000066400000000000000000000006551525232545500327250ustar00rootroot00000000000000# fixture: provenance-verified-ok The green sibling of `provenance-unverified-extract`: the SAME job, with a `sha256sum` comparison step inserted between the download and the extract/execute step. Run as a PAIR -- without this fixture, the red on `provenance-unverified-extract` would be equally consistent with "any download in a job is flagged", not specifically "an unverified one is". Workflow: `.github/workflows/ci.yml`. provenance-verify-helper-after-extract/000077500000000000000000000000001525232545500345055ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy.github/000077500000000000000000000000001525232545500360455ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-verify-helper-after-extractworkflows/000077500000000000000000000000001525232545500401025ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-verify-helper-after-extract/.githubci.yml000066400000000000000000000004521525232545500412210ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-verify-helper-after-extract/.github/workflows--- name: ci on: pull_request: permissions: contents: read jobs: build: runs-on: ubuntu-latest steps: - run: wget -q https://nginx.org/download/nginx-1.2.3.tar.gz - run: | tar -xzf nginx-1.2.3.tar.gz ci/tools/verify-nginx-tarball.sh nginx-1.2.3.tar.gz README.md000066400000000000000000000005121525232545500357620ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/provenance-verify-helper-after-extract# fixture: provenance-verify-helper-after-extract The same-step ordering bypass: the job downloads and extracts an nginx archive, then invokes the shared verification helper after extraction. Merely naming the reviewed helper must not anchor commands that already consumed untrusted bytes. Workflow: `.github/workflows/ci.yml`. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/schedule-only-runner-labels-ok/000077500000000000000000000000001525232545500330275ustar00rootroot00000000000000.github/000077500000000000000000000000001525232545500343105ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/schedule-only-runner-labels-okworkflows/000077500000000000000000000000001525232545500363455ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/schedule-only-runner-labels-ok/.githubnightly.yml000066400000000000000000000006611525232545500405510ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/schedule-only-runner-labels-ok/.github/workflows--- # Byte-for-byte the sibling fixture's file with the pool label spelled right. # Schedule-only and therefore outside the fork-trust question, but the label # half of the check still runs over it -- and must stay silent here. name: nightly on: schedule: - cron: "0 4 * * 1" workflow_dispatch: {} permissions: {} jobs: soak: runs-on: ${{ fromJSON(vars.POOL || '["ubuntu-latest"]') }} steps: - run: echo soak README.md000066400000000000000000000021041525232545500342240ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/schedule-only-runner-labels-ok# fixture: schedule-only-runner-labels-ok The positive control for `schedule-only-runner-labels`, and the reason that one means anything. Same shape — schedule-only, not reachable from any pull request — but the selector is the approved one: the pool read from `vars.POOL`, with a GitHub-hosted fallback. Expected: `runners` exit 0. `ci/linter/selftest.sh` regenerates this `runs-on:` line through a temporary copy before checking it, so a hosted-only adopter that turns `SELF_HOSTED_ALLOWED` off does not see this control fail against a selector it deliberately rejects. Without this, a red on the drift fixture is equally consistent with "the label check caught the typo" and "non-PR-reachable workflows are now flagged unconditionally". Those are different checks and only one of them is wanted; the pair separates them. It is a separate fixture rather than a second workflow inside the red one because `policy_` asserts an exit STATUS, not a finding count. A clean workflow sitting beside a dirty one in the same tree changes nothing about the exit code and would prove nothing. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/schedule-only-runner-labels/000077500000000000000000000000001525232545500324205ustar00rootroot00000000000000.github/000077500000000000000000000000001525232545500337015ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/schedule-only-runner-labelsworkflows/000077500000000000000000000000001525232545500357365ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/schedule-only-runner-labels/.githubnightly.yml000066400000000000000000000010051525232545500401330ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/schedule-only-runner-labels/.github/workflows--- # Schedule-only, so nothing a fork can reach -- and until 2026-08-02 that meant # the runner selector was not read at all. The pool label below is misspelled # (`buidler02`); the shape is otherwise the approved trust split byte for byte, # so only the label check can tell this apart from a correct file. name: nightly on: schedule: - cron: "0 4 * * 1" workflow_dispatch: {} permissions: {} jobs: soak: runs-on: ${{ fromJSON('["self-hosted","builder02","lxc"]') }} steps: - run: echo soak README.md000066400000000000000000000046311525232545500336240ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/schedule-only-runner-labels# fixture: schedule-only-runner-labels Encodes the bypass: **a drifted runner selector in a workflow that no pull request can reach was checked by nothing at all.** `check_runners` used to `continue` past any workflow whose triggers did not include `pull_request` or `workflow_call`, because the fork-trust question only applies to a job a fork can cause to run. That skipped the exact-string membership test with it, and that membership test is the only thing in this toolchain that reads the pool labels. actionlint does not cover the gap. It validates runner labels for a LITERAL `runs-on` only; every selector in this repo is a `fromJSON(...)` expression, which it walks past without a word. A quiet linter and a clean linter print the same thing. Measured in the real tree, 2026-08-02: `builder02` -> `buidler02` in `build-test.yml` (`workflow_call`, PR-reachable) was reported; the same edit in `bump.yml` and `ci-deep.yml` (`schedule` + `workflow_dispatch`) was silent, on both this check and actionlint. Six selectors had no label checking anywhere. The failure mode is why it is worth a gate: a selector naming a pool this repo does not own is not a lint error and not a dispatch error. It is a queued job that never picks up a runner, on a weekly schedule nobody is watching. `nightly.yml` here is that shape — schedule-only, with the pool inlined into `runs-on` instead of read from `vars.POOL`. Expected: `runners` exit 1. Why an inlined pool rather than the mistyped `buidler02` this fixture used to carry: since 2026-08-13 the pool label lives in the `POOL` repo variable and appears nowhere in the tree, so a label typo is no longer expressible *here*. It is still expressible in the GitHub UI, and it fails the same way it always did: `["self-hosted","buidler02","lxc"]` is valid JSON and non-empty, so it parses, skips the fallback, and queues against a label nobody answers. No linter in this repo can see the variable's value. What remains spellable, and what this now encodes, is a selector that has drifted from the approved form: an inlined label set, a resurrected fork ternary, a hand-edited fallback. That drift fails the same way the typo did. The GREEN half of this control is the sibling fixture `schedule-only-runner-labels-ok` — the same file using the approved selector, which must stay silent. Without it, a red here could be "non-PR-reachable workflows are now always flagged" rather than "the drift is caught". myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-inherit-external/000077500000000000000000000000001525232545500320265ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-inherit-external/.github/000077500000000000000000000000001525232545500333665ustar00rootroot00000000000000workflows/000077500000000000000000000000001525232545500353445ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-inherit-external/.githubci.yml000066400000000000000000000002141525232545500364570ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-inherit-external/.github/workflowsname: CI on: pull_request: jobs: release: uses: myguard-labs/shared-workflows/.github/workflows/release.yml@v1 secrets: inherit myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-inherit-external/README.md000066400000000000000000000017201525232545500333050ustar00rootroot00000000000000# fixture: secrets-inherit-external `secrets: inherit` on a call to a reusable workflow in ANOTHER REPOSITORY -- `owner/repo/.github/workflows/x.yml@ref` rather than `./.github/workflows/x.yml`. The reason this needs its own fixture, next to `secrets-inherit`: the caller loop used to filter to local members BEFORE it judged `inherit`, so this call was skipped entirely. That inverted the severity. A local `inherit` overshares within one repository; this one hands the caller's entire secret set to code maintained somewhere else, at a ref that can move under you. No member workflow is checked in here, and that is the point -- the member is external and unreadable from this tree, so the check cannot pair the call against a declaration. `inherit` is rejected on the call site alone. `secrets` must go red here. Its local twin is `secrets-inherit`; its green twin is `secrets-typed-ok`, which names the secret at the call site. Workflows: `.github/workflows/ci.yml`. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-inherit/000077500000000000000000000000001525232545500302065ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-inherit/.github/000077500000000000000000000000001525232545500315465ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-inherit/.github/workflows/000077500000000000000000000000001525232545500336035ustar00rootroot00000000000000build-test.yml000066400000000000000000000002531525232545500363230ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-inherit/.github/workflowsname: Build&Test on: workflow_call: secrets: GH_SUBMODULE_TOKEN: required: true jobs: build: runs-on: ubuntu-latest steps: - run: make ci.yml000066400000000000000000000001631525232545500346420ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-inherit/.github/workflowsname: CI on: pull_request: jobs: build-test: uses: ./.github/workflows/build-test.yml secrets: inherit myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-inherit/README.md000066400000000000000000000013661525232545500314730ustar00rootroot00000000000000# fixture: secrets-inherit A caller reaching a member with `secrets: inherit` instead of naming what the member needs. The member here is correctly typed, so the only defect is at the call site -- `inherit` hands that member the caller's ENTIRE secret set, including every secret it has no business reading. The reason this needs a fixture: `inherit` works. The pipeline is green, the member gets its token, and the only symptom is a blast radius that silently grows every time an unrelated secret is added to the repo. Nothing in actionlint or zizmor objects to it. `secrets` must go red here. Its green twin is `secrets-typed-ok`, the same topology with the secret named explicitly at the call site. Workflows: `.github/workflows/{ci,build-test}.yml`. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-malformed-external/000077500000000000000000000000001525232545500323325ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-malformed-external/.github/000077500000000000000000000000001525232545500336725ustar00rootroot00000000000000workflows/000077500000000000000000000000001525232545500356505ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-malformed-external/.githubci.yml000066400000000000000000000002121525232545500367610ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-malformed-external/.github/workflowsname: CI on: pull_request: jobs: release: uses: myguard-labs/shared-workflows/.github/workflows/release.yml@v1 secrets: false README.md000066400000000000000000000026721525232545500335410ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-malformed-external# fixture: secrets-malformed-external `secrets:` holding neither a mapping nor `inherit` -- here `false` -- on a call to a reusable workflow in ANOTHER REPOSITORY. The reason this needs its own fixture, next to `secrets-inherit-external`: the caller loop rejected `inherit` above the local-member filter but still validated the SHAPE of `secrets:` below it. An external call carrying `secrets: false` was dropped by that filter before the shape was ever judged, and `secrets` reported CLEAN over a caller GitHub will refuse to start. `secrets: []` took the same path. The filter exists to gate `declared` lookups, which mean nothing for a member this tree cannot read. It was never meant to gate whether the call is valid at all, and that is the distinction this fixture pins. No member workflow is checked in here, deliberately -- as in `secrets-inherit-external`, the member is external and unreadable, so the shape of the call site is the only thing there is to judge. `secrets` must exit **2** here, not 1. A `secrets:` value this checker cannot interpret means the check did not RUN over that call, which is a different claim from "ran and found a problem" -- see `PolicyError` in `workflow_policy.py`. That is also why `policy_msg_` cannot assert this fixture: it hardcodes exit 1. Its local twin is `secrets-undeclared` (a mapping, judged against a declaration); its `inherit` twin is `secrets-inherit-external`. Workflows: `.github/workflows/ci.yml`. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-no-required-key/000077500000000000000000000000001525232545500315645ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-no-required-key/.github/000077500000000000000000000000001525232545500331245ustar00rootroot00000000000000workflows/000077500000000000000000000000001525232545500351025ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-no-required-key/.githubbuild-test.yml000066400000000000000000000003221525232545500376760ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-no-required-key/.github/workflowsname: Build&Test on: workflow_call: secrets: GH_SUBMODULE_TOKEN: description: token for the private submodule checkout jobs: build: runs-on: ubuntu-latest steps: - run: make myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-no-required-key/README.md000066400000000000000000000014141525232545500330430ustar00rootroot00000000000000# fixture: secrets-no-required-key A member declaring `GH_SUBMODULE_TOKEN` as a real mapping -- with a `description:` -- but no `required:` key. This is the shape a careful author produces when documenting a secret without typing it. The reason this needs its OWN fixture, distinct from both `secrets-untyped` and `secrets-optional`: it is the second mutant that survived. `secrets-untyped` spells the declaration `GH_SUBMODULE_TOKEN:` with nothing under it, which YAML parses to null -- and null `is not True`, so the VALUE branch catches it even with the missing-key branch deleted. Only a spec that is a genuine mapping lacking the key can distinguish the two. `secrets` must go red here. Its green twin is `secrets-typed-ok`. Workflow: `.github/workflows/build-test.yml`. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-optional/000077500000000000000000000000001525232545500303715ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-optional/.github/000077500000000000000000000000001525232545500317315ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-optional/.github/workflows/000077500000000000000000000000001525232545500337665ustar00rootroot00000000000000build-test.yml000066400000000000000000000002541525232545500365070ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-optional/.github/workflowsname: Build&Test on: workflow_call: secrets: GH_SUBMODULE_TOKEN: required: false jobs: build: runs-on: ubuntu-latest steps: - run: make myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-optional/README.md000066400000000000000000000015061525232545500316520ustar00rootroot00000000000000# fixture: secrets-optional A member declaring `GH_SUBMODULE_TOKEN` with `required: false`. Distinct from `secrets-untyped`: the `required:` key IS present, so a check that only asks "was it typed at all" passes this. The reason this needs its OWN fixture: it is the mutant that survived. Disarming the `required is not True` branch left `secrets-untyped` red anyway, because that fixture trips the missing-key branch instead -- so the value test had no control of its own and could have been deleted silently. `required: false` is not a weaker version of the rule, it is the absence of it: GitHub starts the call with the secret unset, the member reads an empty string, and the failure surfaces downstream of the cause. `secrets` must go red here. Its green twin is `secrets-typed-ok`. Workflow: `.github/workflows/build-test.yml`. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-required-not-wired/000077500000000000000000000000001525232545500322725ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-required-not-wired/.github/000077500000000000000000000000001525232545500336325ustar00rootroot00000000000000workflows/000077500000000000000000000000001525232545500356105ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-required-not-wired/.githubbuild-test.yml000066400000000000000000000002531525232545500404070ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-required-not-wired/.github/workflowsname: Build&Test on: workflow_call: secrets: GH_SUBMODULE_TOKEN: required: true jobs: build: runs-on: ubuntu-latest steps: - run: make ci.yml000066400000000000000000000001361525232545500367260ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-required-not-wired/.github/workflowsname: CI on: pull_request: jobs: build-test: uses: ./.github/workflows/build-test.yml README.md000066400000000000000000000016151525232545500334750ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-required-not-wired# fixture: secrets-required-not-wired The mirror image of `secrets-undeclared`: here the MEMBER declares `GH_SUBMODULE_TOKEN` with `required: true` and the CALLER passes nothing. The reason this needs a fixture: it is the direction the check missed on first writing. Both halves look right in isolation again -- the member's declaration is exactly the endorsed shape, and the caller is a plain `uses:` with no secrets block, which is correct for every OTHER member in a repo like this one. GitHub does refuse to start the call, so this fails loudly rather than silently. That is the whole point of `required: true`. But it fails on the first run after merge, and the caller/member pairing is checkable at review time instead. `secrets` must go red here. Its green twin is `secrets-typed-ok`, the same member with the caller actually wiring the secret. Workflows: `.github/workflows/{ci,build-test}.yml`. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-typed-ok/000077500000000000000000000000001525232545500303005ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-typed-ok/.github/000077500000000000000000000000001525232545500316405ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-typed-ok/.github/workflows/000077500000000000000000000000001525232545500336755ustar00rootroot00000000000000build-test.yml000066400000000000000000000002531525232545500364150ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-typed-ok/.github/workflowsname: Build&Test on: workflow_call: secrets: GH_SUBMODULE_TOKEN: required: true jobs: build: runs-on: ubuntu-latest steps: - run: make ci.yml000066400000000000000000000002471525232545500347370ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-typed-ok/.github/workflowsname: CI on: pull_request: jobs: build-test: uses: ./.github/workflows/build-test.yml secrets: GH_SUBMODULE_TOKEN: ${{ secrets.GH_SUBMODULE_TOKEN }} myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-typed-ok/README.md000066400000000000000000000011341525232545500315560ustar00rootroot00000000000000# fixture: secrets-typed-ok The endorsed shape, and the green half of three pairs: a member declaring `GH_SUBMODULE_TOKEN` with `required: true`, and a caller wiring that one secret by name. Without this fixture, the reds in `secrets-inherit`, `secrets-untyped` and `secrets-undeclared` would be equally consistent with "any workflow mentioning a secret is flagged" -- which is the opposite of the rule. The rule is that secrets must be declared at the member and wired explicitly, so the correctly declared and wired case has to be provably green. Workflows: `.github/workflows/{ci,build-test}.yml`. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-undeclared/000077500000000000000000000000001525232545500306525ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-undeclared/.github/000077500000000000000000000000001525232545500322125ustar00rootroot00000000000000workflows/000077500000000000000000000000001525232545500341705ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-undeclared/.githubbuild-test.yml000066400000000000000000000001601525232545500367640ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-undeclared/.github/workflowsname: Build&Test on: workflow_call: {} jobs: build: runs-on: ubuntu-latest steps: - run: make ci.yml000066400000000000000000000002471525232545500353110ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-undeclared/.github/workflowsname: CI on: pull_request: jobs: build-test: uses: ./.github/workflows/build-test.yml secrets: GH_SUBMODULE_TOKEN: ${{ secrets.GH_SUBMODULE_TOKEN }} myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-undeclared/README.md000066400000000000000000000013461525232545500321350ustar00rootroot00000000000000# fixture: secrets-undeclared A caller passing `GH_SUBMODULE_TOKEN` to a member whose `workflow_call: {}` declares no secrets at all. This is the shape every member in the reference repo has today, which is correct only while no caller passes anything. The reason this needs a fixture: the secret is dropped at the call boundary in silence. The caller looks right -- the secret is named, wired from `${{ secrets.* }}`, visible in review -- and the member simply never receives it. Both halves read as correct in isolation; only the pairing is wrong. `secrets` must go red here. Its green twin is `secrets-typed-ok`, the same call against a member that declares what it is being passed. Workflows: `.github/workflows/{ci,build-test}.yml`. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-untyped/000077500000000000000000000000001525232545500302345ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-untyped/.github/000077500000000000000000000000001525232545500315745ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-untyped/.github/workflows/000077500000000000000000000000001525232545500336315ustar00rootroot00000000000000build-test.yml000066400000000000000000000002241525232545500363470ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-untyped/.github/workflowsname: Build&Test on: workflow_call: secrets: GH_SUBMODULE_TOKEN: jobs: build: runs-on: ubuntu-latest steps: - run: make myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/secrets-untyped/README.md000066400000000000000000000013571525232545500315210ustar00rootroot00000000000000# fixture: secrets-untyped A `workflow_call` member declaring `GH_SUBMODULE_TOKEN:` with no `required:` key. The secret IS declared, so the caller-side check has nothing to say and the name shows up where a reader expects it. The reason this needs a fixture: an untyped (or `required: false`) secret restores exactly the failure the declaration was supposed to remove. GitHub will start the call with the secret absent, the member sees an empty string, and it dies downstream of the cause -- a checkout that 404s, a curl that 401s. `required: true` is what makes it refuse to start instead. `secrets` must go red here. Its green twin is `secrets-typed-ok`, the same declaration with `required: true`. Workflow: `.github/workflows/build-test.yml`. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/verify-after-bind/000077500000000000000000000000001525232545500304135ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/verify-after-bind/.github/000077500000000000000000000000001525232545500317535ustar00rootroot00000000000000workflows/000077500000000000000000000000001525232545500337315ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/verify-after-bind/.githubci.yml000066400000000000000000000006761525232545500350600ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/verify-after-bind/.github/workflows--- name: ci on: pull_request: jobs: test-nginx: runs-on: ubuntu-latest env: TEST_BASE_PORT: '19200' steps: - run: echo build # Binds the band. Everything the verifier below could have caught for # this step has already happened by the time it runs. - run: prove -v ci/t/ - run: bash ci/tools/max-port.sh "$TEST_BASE_PORT" - run: python3 ci/tools/test_runtime.py --port "$TEST_BASE_PORT" myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/policy/verify-after-bind/README.md000066400000000000000000000011401525232545500316660ustar00rootroot00000000000000# fixture: verify-after-bind Encodes the defect this repo shipped until 2026-08-02: `build-test.yml` ran `ci/tools/max-port.sh` *between* `prove` and the runtime suite. Every existing check reads that job as correct — it declares a distinct band, it passes the band to the driver, and the verifier is present — so nothing went red while the first binder ran unguarded and the one failure the verifier exists to name arrived inside `prove` as an unattributed bind error or timeout. The verifier is a property of a POSITION, not of a job. `ports` must go red here. Workflow: `.github/workflows/ci.yml`. myguard-labs-nginx-zstd-module-3486b5b/ci/linter/fixtures/semgrep-blocking.c.fixture000066400000000000000000000001651525232545500306620ustar00rootroot00000000000000#include void copy_untrusted(char *destination, const char *source) { strcpy(destination, source); } myguard-labs-nginx-zstd-module-3486b5b/ci/linter/install-linters.sh000077500000000000000000000327711525232545500254230ustar00rootroot00000000000000#!/usr/bin/env bash # ci/linter/install-linters.sh -- install every linter ci/linter/ needs. # # Order of preference, per tool: apt-get (distro-managed, no PEP 668 fight) # -> pipx/pip (Python tools apt does not carry) -> cpan (Perl modules apt does # not carry) -> upstream binary (actionlint). Nothing here is a silent # skip: a tool that fails to install prints why and the script exits non-zero, # so a half-installed toolchain cannot masquerade as a working gate. # # Usage: # ci/linter/install-linters.sh install what is missing # ci/linter/install-linters.sh --check report only, install nothing # ci/linter/install-linters.sh --force reinstall even if present # # Needs sudo for the apt-get and cpan steps; the pipx step installs into # ~/.local/bin (make sure it is on PATH). # # Side effects: apt-get install, pipx install, cpanm install, and one curl of # the actionlint release tarball into /usr/local/bin. # # Extend: add a line to the APT/PIPX/CPAN lists. Anything needing a bespoke # install gets its own function at the bottom, next to install_actionlint. set -uo pipefail CHECK=0 FORCE=0 case "${1:-}" in --check) CHECK=1 ;; --force) FORCE=1 ;; -h|--help) sed -n '2,23p' "$0"; exit 0 ;; "") ;; *) echo "unknown argument: $1" >&2; exit 2 ;; esac SUDO="" [ "$(id -u)" -eq 0 ] || SUDO="sudo" # toolapt package -- checked with command -v APT_TOOLS=( "shellcheck:shellcheck" # sh/bash "cppcheck:cppcheck" # C "flawfinder:flawfinder" # C, risky-API scan "yamllint:yamllint" # YAML "clang-tidy:clang-tidy" # C, CI-only (needs a configured nginx tree) "bear:bear" # records the build's real compiler argv into # compile_commands.json, so clang-tidy parses # the same TU that ships instead of a # hand-kept -I list that drifts from configure "perlcritic:libperl-critic-perl" # Perl test suite "perl:perl" ) # Python IMPORTS, not executables, so `command -v` cannot see them and they # need their own list -- checked with `python3 -c "import "`. # # PyYAML is a hard requirement of ci/linter/workflow_policy.py, which refuses # to run without it rather than falling back to a regex scan. Every policy # check it makes was bypassable by valid YAML while it parsed workflows by # regex, so "degrade quietly to the old behaviour" is the one option that is # worse than failing the job. PY_MODULES=( "yaml:python3-yaml" # ci/linter/workflow_policy.py ) # Not in Debian/Ubuntu at the versions this repo targets -> Python packaging. # pipx, not pip: PEP 668 marks the system interpreter externally-managed, so a # bare `pip3 install` fails on Debian 12+ and `--break-system-packages` is a # worse answer than an isolated venv per tool. PIPX_TOOLS=( "ruff:ruff==0.16.3" # Python lint + format check, pinned: # an unpinned ruff changes findings under # you and local stops matching CI, same # reasoning as the semgrep pin below. "zizmor:zizmor" # GitHub Actions security audit. Not # pinned: its rule set is the point, and a # frozen security scanner stops finding # what it was added for. A new rule going # red is a finding to triage, not drift. "semgrep:semgrep==1.173.0" # C, pinned to the CI version on purpose: # an unpinned semgrep changes findings # under you and local stops matching CI. "codespell:codespell" # ci/linter/lint-spelling.sh. Unpinned: the # dictionary is the point, and a new entry # going red is one typo to fix, not drift. "pre-commit:pre-commit" # .githooks/pre-commit runs the # .pre-commit-config.yaml hooks too, and # treats its absence as exit 2 rather than # skipping the secret and C SAST gates. ) # apt has no libtest-nginx-perl on every target release; cpan always does. # # "Test::Nginx" IS the openresty test suite -- there is no separately-named # openresty distribution to look for. The CPAN dist is AGENT/Test-Nginx-*.tar.gz # (AGENT = agentzh / Yichun Zhang, openresty's author); confirm with # `cpanm --info Test::Nginx`. The giveaway in an installed tree is the # Test::Nginx::Socket::Lua* modules, which only that dist ships. CPAN_MODULES=( "Test::Nginx::Socket" # the ci/t/*.t suite; also makes `perl -c` work "Perl::Critic" # fallback if libperl-critic-perl was missing ) have() { command -v "$1" >/dev/null 2>&1; } step() { printf '\n== %s\n' "$*"; } rc=0 # A MISSING line must also move the exit status. Printing "MISSING" and still # exiting 0 makes --check advisory: lint.yml gates on the exit code, so a # half-armed toolchain reads as a pass and the missing checker is simply never # run. The printf is the human half; rc=1 is the machine half. report() { local path if path="$(command -v "$1" 2>/dev/null)"; then printf '%-14s %s\n' "$1" "$path" else printf '%-14s %s\n' "$1" MISSING rc=1 fi } have_mod() { python3 -c "import $1" >/dev/null 2>&1; } if [ "$CHECK" -eq 1 ]; then step "linter status" for e in "${APT_TOOLS[@]}" "${PIPX_TOOLS[@]}"; do report "${e%%:*}"; done report actionlint # gitleaks and ast-grep are this module's own gates (.pre-commit-config.yaml # and ci/ast-grep/), not the skeleton's. A tool the checker set needs but # --check never names is invisible to lint.yml, so the gate can be absent on # a fresh clone while --check reports everything fine. report gitleaks report ast-grep for e in "${PY_MODULES[@]}"; do mod="${e%%:*}" if have_mod "$mod"; then printf '%-14s ok (python module)\n' "$mod" else printf '%-14s MISSING (apt: %s)\n' "$mod" "${e##*:}" rc=1 fi done if perl -MTest::Nginx::Socket -e1 2>/dev/null; then printf '%-14s ok\n' 'Test::Nginx' else printf '%-14s MISSING\n' 'Test::Nginx' rc=1 fi exit "$rc" fi step "apt-get" NEED=() for e in "${APT_TOOLS[@]}"; do tool="${e%%:*}"; pkg="${e##*:}" if [ "$FORCE" -eq 1 ] || ! have "$tool"; then NEED+=("$pkg"); fi done for e in "${PY_MODULES[@]}"; do mod="${e%%:*}"; pkg="${e##*:}" if [ "$FORCE" -eq 1 ] || ! have_mod "$mod"; then NEED+=("$pkg"); fi done if [ "${#NEED[@]}" -gt 0 ]; then echo "installing: ${NEED[*]}" $SUDO apt-get update -qq || rc=1 $SUDO apt-get install -y --no-install-recommends "${NEED[@]}" || rc=1 else echo "nothing to do" fi step "pipx" have pipx || $SUDO apt-get install -y --no-install-recommends pipx || rc=1 # A pinned spec (tool==version) must end up at THAT version, so presence of the # binary is not enough to skip the install: `have semgrep` is true for any # version, including the one the pin was just bumped away from. Compare the # installed version against the pin and reinstall when it differs. # # --force is required on the reinstall path. A bare `pipx install` against an # existing venv prints "already seems to be installed", keeps the old version # and exits 0 -- so the previous `pipx install || pipx install --force` fallback # could never fire, and the pin silently never moved. for e in "${PIPX_TOOLS[@]}"; do tool="${e%%:*}"; spec="${e#*:}" want="" ; case "$spec" in *==*) want="${spec##*==}" ;; esac need=0 if [ "$FORCE" -eq 1 ] || ! have "$tool"; then need=1 elif [ -n "$want" ]; then # Unpinned tools (zizmor) have no target version to compare against and # are intentionally left at whatever is installed. got="$("$tool" --version 2>/dev/null | head -1)" # --version output varies (bare "1.2.3" vs "tool 1.2.3"); match the # version as a whole word rather than assuming a format. case " $got " in *" $want "*) ;; *) need=1 ;; esac fi if [ "$need" -eq 1 ]; then echo "installing: $spec" pipx install --force "$spec" || rc=1 fi done step "cpan" have cpanm || $SUDO apt-get install -y --no-install-recommends cpanminus || rc=1 for m in "${CPAN_MODULES[@]}"; do if [ "$FORCE" -eq 1 ] || ! perl -M"$m" -e1 2>/dev/null; then echo "installing: $m" # --notest: Test::Nginx's own suite wants a live nginx binary and a # free port, which is not something an install step should demand. $SUDO cpanm --notest "$m" || rc=1 fi done install_actionlint() { # No apt package on the target releases and no pip/cpan equivalent: a Go # binary from the upstream release, pinned by version AND sha256. # # The digest is COMPARED, not printed. Printing `sha256sum` and moving on # reads as a verification step but installs whatever arrived -- worse than # no check, because it stops anyone from adding a real one. Version and # digest sit on adjacent lines for the same reason .github/versions.env # does it: a version must not be able to move while its digest stays behind. # From the upstream release's actionlint__checksums.txt. local ver="1.7.7" local sha="023070a287cd8cccd71515fedc843f1985bf96c436b7effaecce67290e7e0757" local tmp tmp="$(mktemp -d)" curl -fsSL -o "$tmp/al.tgz" \ "https://github.com/rhysd/actionlint/releases/download/v${ver}/actionlint_${ver}_linux_amd64.tar.gz" || return 1 if ! printf '%s %s\n' "$sha" "$tmp/al.tgz" | sha256sum -c - >/dev/null 2>&1; then echo "actionlint ${ver}: sha256 mismatch" >&2 echo " expected: $sha" >&2 echo " got: $(sha256sum < "$tmp/al.tgz" | cut -d' ' -f1)" >&2 rm -rf "$tmp" return 1 fi tar -xzf "$tmp/al.tgz" -C "$tmp" actionlint || return 1 $SUDO install -m0755 "$tmp/actionlint" /usr/local/bin/actionlint || return 1 rm -rf "$tmp" } step "actionlint" if [ "$FORCE" -eq 1 ] || ! have actionlint; then install_actionlint || { echo "actionlint install failed" >&2; rc=1; } else echo "present: $(command -v actionlint)" fi # --------------------------------------------------------------------------- # Two tools this module gates on that the skeleton does not use. Both are # entries in .pre-commit-config.yaml, so .githooks/pre-commit exits non-zero # without them -- but a checker that is merely absent from --check is invisible # to CI, which is the failure mode worth more than the install itself. # # gitleaks staged-secret scan (.pre-commit-config.yaml) # ast-grep the vendored ci/ast-grep/ structural ruleset (PR #127) # # Same shape as install_actionlint: upstream release, version AND sha256 on # adjacent lines, digest compared rather than printed. # --------------------------------------------------------------------------- install_gitleaks() { local ver="8.30.1" local sha="551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb" local tmp tmp="$(mktemp -d)" curl -fsSL -o "$tmp/gl.tgz" \ "https://github.com/gitleaks/gitleaks/releases/download/v${ver}/gitleaks_${ver}_linux_x64.tar.gz" || return 1 if ! printf '%s %s\n' "$sha" "$tmp/gl.tgz" | sha256sum -c - >/dev/null 2>&1; then echo "gitleaks ${ver}: sha256 mismatch" >&2 echo " expected: $sha" >&2 echo " got: $(sha256sum < "$tmp/gl.tgz" | cut -d' ' -f1)" >&2 rm -rf "$tmp" return 1 fi tar -xzf "$tmp/gl.tgz" -C "$tmp" gitleaks || return 1 $SUDO install -m0755 "$tmp/gitleaks" /usr/local/bin/gitleaks || return 1 rm -rf "$tmp" } step "gitleaks" if [ "$FORCE" -eq 1 ] || ! have gitleaks; then install_gitleaks || { echo "gitleaks install failed" >&2; rc=1; } else echo "present: $(command -v gitleaks)" fi install_astgrep() { # npm/cargo would also work; the release binary keeps this script free of a # Node or Rust toolchain requirement, same reasoning as actionlint. local ver="0.44.1" local sha="611f9e5e76f2611ecea1a35dd3468ceedf600641a11224b80341d79c6ee7b9dd" local tmp tmp="$(mktemp -d)" curl -fsSL -o "$tmp/sg.zip" \ "https://github.com/ast-grep/ast-grep/releases/download/${ver}/app-x86_64-unknown-linux-gnu.zip" || return 1 if ! printf '%s %s\n' "$sha" "$tmp/sg.zip" | sha256sum -c - >/dev/null 2>&1; then echo "ast-grep ${ver}: sha256 mismatch" >&2 echo " expected: $sha" >&2 echo " got: $(sha256sum < "$tmp/sg.zip" | cut -d' ' -f1)" >&2 rm -rf "$tmp" return 1 fi unzip -q -o "$tmp/sg.zip" -d "$tmp" || return 1 # The zip ships `ast-grep` and the `sg` alias; Debian already owns /usr/bin/sg # (shadow's setgid tool), so install ONLY the long name and let the config # be invoked as `ast-grep`. Installing `sg` here would shadow a system # binary on PATH order alone. $SUDO install -m0755 "$tmp/ast-grep" /usr/local/bin/ast-grep || return 1 rm -rf "$tmp" } step "ast-grep" if [ "$FORCE" -eq 1 ] || ! have ast-grep; then install_astgrep || { echo "ast-grep install failed" >&2; rc=1; } else echo "present: $(command -v ast-grep)" fi step "result" if [ "$rc" -ne 0 ]; then echo "one or more installs FAILED -- ci/linter/ is not fully armed" >&2 exit 1 fi echo "all linters installed; verify with: ci/linter/install-linters.sh --check" myguard-labs-nginx-zstd-module-3486b5b/ci/linter/lib.sh000077500000000000000000000057141525232545500230420ustar00rootroot00000000000000#!/usr/bin/env bash # ci/linter/lib.sh -- shared helpers for the ci/linter/lint-*.sh scripts. # # Sourced, never executed. Provides: # repo_root absolute path of the checkout # lint_files emit the file list to lint, one per line, NUL-safe # callers use `mapfile -t`. Honours LINT_MODE: # staged (default in the git hook) -- staged files only # all -- every tracked file # and an explicit file list passed in "$@" by run-all.sh. # mapfile_checked [args...] # populate an array only after its producer succeeds # need hard-fail with an install hint when a linter is absent # say / warn / die consistent output # # Missing tools are a HARD FAILURE, never a silent skip: a gate that quietly # disappears when its tool is uninstalled reports green while checking nothing. # Run ci/linter/install-linters.sh to get them all. set -euo pipefail repo_root() { git rev-parse --show-toplevel; } say() { printf ' %s\n' "$*"; } warn() { printf ' WARN: %s\n' "$*" >&2; } die() { printf 'ERROR: %s\n' "$*" >&2; exit 2; } need() { command -v "$1" >/dev/null 2>&1 && return 0 die "$1 not found. Install it: $2 (or run ci/linter/install-linters.sh)" } mapfile_checked() { local array_name="$1" tmp rc; shift tmp="$(mktemp)" || die "mktemp failed" if [ "${MAPFILE_CHECKED_FAULT:-}" = "partial-error" ]; then printf '%s\n' plausible-prefix >"$tmp" rm -f "$tmp" return 23 fi if "$@" >"$tmp"; then mapfile -t "$array_name" <"$tmp" rm -f "$tmp" return 0 else rc=$? rm -f "$tmp" return "$rc" fi } # Paths never linted: byte-exact fuzz inputs, vendored trees, build output. # # `(^|/)vendor/` and not `^ci/vendor/`: the pattern matches a vendored tree at # any depth, so a second one added later is excluded without editing this regex. # Linting vendored code produces findings nobody may fix -- they are upstream's, # and a hand-edit is reverted by the next refresh. Same regex shape as the # top-level exclude in .pre-commit-config.yaml, so the hook and these scripts # select the same files. LINT_EXCLUDE_RE='^ci/fuzz/corpus/|^ci/fuzz/regressions/|(^|/)vendor/|(^|/)objs/|(^|/)\.build/|(^|/)node_modules/|(^|/)\.venv/' # lint_files [explicit files...] lint_files() { local match_re="$1"; shift local list if [ "$#" -gt 0 ]; then list=$(printf '%s\n' "$@") elif [ "${LINT_MODE:-all}" = "staged" ]; then list=$(git diff --cached --name-only --diff-filter=ACMR) || die "git diff --cached failed" else list=$(git ls-files) || die "git ls-files failed" fi printf '%s\n' "$list" \ | grep -Ev "$LINT_EXCLUDE_RE" \ | grep -E "$match_re" \ | while read -r f; do [ -f "$f" ] && printf '%s\n' "$f"; done \ || true } myguard-labs-nginx-zstd-module-3486b5b/ci/linter/lint-c.sh000077500000000000000000000077241525232545500234650ustar00rootroot00000000000000#!/usr/bin/env bash # ci/linter/lint-c.sh -- C static analysis for src/ and ci/ *.[ch]. # # Mirrors .github/workflows/security-scanners.yml exactly: # flawfinder gate at >=4 (below 4 is risky-API grep noise; gating on it # trains everyone to --no-verify) # semgrep gate at >=WARNING with p/c + p/security-audit # plus cppcheck, which CI does not run and which is cheap enough locally. # # If a threshold moves in that workflow, move it here in the SAME commit -- # otherwise local-green stops predicting remote-green, which is the only # reason this script exists. # # NOT here, deliberately: # clang-tidy -- needs ngx_auto_config.h, i.e. a configured nginx tree. A # local hook cannot assume one, and a check that skips itself # when the tree is missing is a vacuous gate. CI-only. # # Usage: ci/linter/lint-c.sh [files...] (no args => LINT_MODE, default all) # Env: LINT_MODE=staged|all # LINT_SKIP_SEMGREP=1 explicit, loud opt-out for slow machines # # Extend: add a scanner as one more block below; keep the CI mirror comment # accurate or the header above becomes a lie. # shellcheck source=ci/linter/lib.sh . "$(git rev-parse --show-toplevel)/ci/linter/lib.sh" mapfile_checked FILES lint_files '^(src|ci)/.*\.[ch]$' "$@" [ "${#FILES[@]}" -gt 0 ] || { echo "lint-c: no C files to check"; exit 0; } echo "lint-c: ${#FILES[@]} file(s)" rc=0 # fuzz.dict's coding-name tokens are generated from the real # ngx_http_zstd_coding_weight(..., "TOKEN", ...) call sites in src/*.[ch] # (ci/fuzz/gen_dict.sh). A merely incomplete dictionary still produces a # green crash-only fuzz run, so this is the only thing that would ever # catch a new coding name added without updating the dictionary. say "fuzz.dict coding-token drift (ci/fuzz/gen_dict.sh --check)" bash "$(git rev-parse --show-toplevel)/ci/fuzz/gen_dict.sh" --check || rc=1 need flawfinder "apt-get install flawfinder" say "flawfinder (gate >=4)" flawfinder --minlevel=4 --error-level=4 --quiet "${FILES[@]}" || rc=1 need cppcheck "apt-get install cppcheck" say "cppcheck (error,warning)" # --error-exitcode=1 makes any reported id fail. missingInclude/unusedFunction # are suppressed: this is a module compiled INTO nginx, so its headers and its # ngx_http_* callbacks are never resolvable or called from this tree alone. cppcheck --quiet --error-exitcode=1 \ --enable=warning,performance,portability \ --inline-suppr \ --suppress=missingInclude --suppress=missingIncludeSystem \ --suppress=unusedFunction --suppress=unknownMacro \ --suppress=normalCheckLevelMaxBranches \ "${FILES[@]}" || rc=1 if [ -n "${LINT_SKIP_SEMGREP:-}" ]; then warn "semgrep SKIPPED via LINT_SKIP_SEMGREP -- CI still gates on it" else need semgrep "pipx install semgrep==1.173.0" say "semgrep (gate >=WARNING)" # --quiet is deliberately absent: it hides semgrep-core's own crash text # (io_uring/RLIMIT_MEMLOCK) and turns a diagnosable failure into a bare # exit 2 that reads like a real finding. # # --jobs=1 is a correctness flag, not a speed flag. semgrep-core defaults to # one OCaml domain per core (32 here), each of which opens its own io_uring # ring against the 8 MB RLIMIT_MEMLOCK this host shares with the self-hosted # CI runners. Under runner load it exhausts and semgrep-core dies with # `Unix_error: Cannot allocate memory io_uring_queue_init` -> exit 2, which # this script reports as a finding. Observed 3/3 crashed on a 3-file scan # while runners were busy, 0/3 on an idle box: a load-dependent false RED. # src/ is three files, so the parallelism was buying nothing to begin with. # # --metrics=off: no scan-summary POST to semgrep.dev. Measured 2.76s -> 1.27s # on this tree, i.e. more than half the wall clock was that upload. semgrep scan --config p/c --config p/security-audit \ --severity=WARNING --severity=ERROR --error \ --jobs=1 --metrics=off "${FILES[@]}" || rc=1 fi exit "$rc" myguard-labs-nginx-zstd-module-3486b5b/ci/linter/lint-ci-cadence.sh000077500000000000000000000035171525232545500252120ustar00rootroot00000000000000#!/usr/bin/env bash # Copyright (C) 2026 Thijs Eilander # SPDX-License-Identifier: BSD-2-Clause # # ci/linter/lint-ci-cadence.sh -- a workflow_call member has ci.yml as its only # PR entry point, and does not carry a second `push:`/`pull_request:` of its own. # # The rule and the reasoning live in ci/linter/workflow_policy.py (subcommand # `cadence`); this wrapper exists so run-all.sh picks the check up by glob and # LINT_ONLY=ci-cadence selects it. # # The failure it prevents is invisible in the one place people look. Both runs # are GREEN: the member runs once on the PR and once on the merge commit, over a # tree identical to the head that just passed. Different concurrency keys, so # cancel-in-progress cannot collapse them. Nothing goes red, nothing is slower, # and the only evidence is the runner bill plus a README whose description of # what runs when has quietly stopped being true. # # `schedule:` is allowed on a member -- codeql.yml and ci-deep.yml are reached # from ci.yml AND on their own cadence, which is the intended shape. # # Usage: ci/linter/lint-ci-cadence.sh [files...] Env: LINT_MODE=staged|all # Extend: the single-orchestrator topology this assumes is ci.yml; a module that # deliberately keeps a member on push: for its own branch wants the rule relaxed # in workflow_policy.py, not a bypass here. # shellcheck source=ci/linter/lib.sh . "$(git rev-parse --show-toplevel)/ci/linter/lib.sh" mapfile_checked FILES lint_files '^\.github/workflows/.*\.ya?ml$' "$@" [ "${#FILES[@]}" -gt 0 ] || { echo "lint-ci-cadence: no workflow files to check"; exit 0; } need python3 "apt-get install python3" # Whole-tree by nature: the question is which workflows are members of the # ci.yml tree, so the file list only decides whether this checker is relevant. exec python3 "$(git rev-parse --show-toplevel)/ci/linter/workflow_policy.py" cadence myguard-labs-nginx-zstd-module-3486b5b/ci/linter/lint-ci-ports.sh000077500000000000000000000033271525232545500247760ustar00rootroot00000000000000#!/usr/bin/env bash # Copyright (C) 2026 Thijs Eilander # SPDX-License-Identifier: BSD-2-Clause # # ci/linter/lint-ci-ports.sh -- every runtime-bearing CI job owns a distinct # test port band, actually binds the band it declared, and proves the band free # BEFORE the first step that binds it. # # The rule and the reasoning live in ci/linter/workflow_policy.py (subcommand # `ports`); this wrapper exists so run-all.sh picks the check up by glob and # LINT_ONLY=ci-ports selects it. # # The failure it prevents does not look like a port bug. build-test.yml and # asan.yml pin the SAME self-hosted runner and have DISJOINT concurrency groups, # so nothing serialises them: two jobs run ci/tools/test_runtime.py at once, on # the same default port, and the second one's listener collides with the first # one's server. What CI reports is an "Address already in use" in one job and a # missing-response assertion in the other -- two unrelated-looking reds, one # cause, and a rerun usually hides it. # # Usage: ci/linter/lint-ci-ports.sh [files...] Env: LINT_MODE=staged|all # Extend: the band width and the driver's default live in # ci/tools/test_runtime.py and ci/tools/max-port.sh; this only checks the wiring. # shellcheck source=ci/linter/lib.sh . "$(git rev-parse --show-toplevel)/ci/linter/lib.sh" mapfile_checked FILES lint_files '^\.github/workflows/.*\.ya?ml$' "$@" [ "${#FILES[@]}" -gt 0 ] || { echo "lint-ci-ports: no workflow files to check"; exit 0; } need python3 "apt-get install python3" # Whole-tree by nature: band UNIQUENESS is a property of the set of workflows, # so the file list only decides whether this checker is relevant to the change. exec python3 "$(git rev-parse --show-toplevel)/ci/linter/workflow_policy.py" ports myguard-labs-nginx-zstd-module-3486b5b/ci/linter/lint-ci-provenance.sh000077500000000000000000000031161525232545500257630ustar00rootroot00000000000000#!/usr/bin/env bash # Copyright (C) 2026 Thijs Eilander # SPDX-License-Identifier: BSD-2-Clause # # ci/linter/lint-ci-provenance.sh -- every step that extracts an archive or # executes a downloaded artifact must be preceded, in the same job, by a # gpg/sha256 trust-anchor assertion. A cache-hit path is not exempt: a cached # tarball is exactly as untrusted as a fresh download until re-checked. # # The rule and the detector live in ci/linter/workflow_policy.py (subcommand # `provenance`); this wrapper exists so run-all.sh picks the check up by glob # and LINT_ONLY=ci-provenance selects it. # # The failure it prevents: a release/origin/cache compromise becomes code # execution on the self-hosted runner the moment CI unpacks or runs what it # just fetched with no verification -- audit sha e289021 F3. # # Usage: ci/linter/lint-ci-provenance.sh [files...] Env: LINT_MODE=staged|all # Extend: a new trust-anchor shape (a detached signature against a different # tool, a different pinned-hash naming convention) goes in TRUST_ANCHOR_RE in # workflow_policy.py, not here. # shellcheck source=ci/linter/lib.sh . "$(git rev-parse --show-toplevel)/ci/linter/lib.sh" mapfile_checked FILES lint_files '^\.github/workflows/.*\.ya?ml$' "$@" [ "${#FILES[@]}" -gt 0 ] || { echo "lint-ci-provenance: no workflow files to check"; exit 0; } need python3 "apt-get install python3" # Whole-tree by nature: a download in one step and its extract/exec several # steps later are the same JOB, so the file list only decides relevance. exec python3 "$(git rev-parse --show-toplevel)/ci/linter/workflow_policy.py" provenance myguard-labs-nginx-zstd-module-3486b5b/ci/linter/lint-ci-runners.sh000077500000000000000000000040241525232545500253160ustar00rootroot00000000000000#!/usr/bin/env bash # Copyright (C) 2026 Thijs Eilander # SPDX-License-Identifier: BSD-2-Clause # # ci/linter/lint-ci-runners.sh -- fork PRs must never select the self-hosted # runners, and every runner selector must name labels that exist. # # The rule and the reasoning live in ci/linter/workflow_policy.py (subcommand # `runners`); this wrapper exists so run-all.sh picks the check up by glob and # LINT_ONLY=ci-runners selects it, like every other checker here. # # Why it is a GATE and not advice: the self-hosted runners are persistent and shared # with the Debian package builds. A pull_request-triggered job executes scripts # from the PR head, so one workflow missing the fork fallback hands an arbitrary # contributor code execution on the build host. This is a public template repo; # taking outside contributions is the point. # # The second half is a LABEL check, and it covers workflows the trust half does # not: a schedule-only workflow has no fork to keep off the build host, but a # typo in its pool labels is still a job that never starts. actionlint cannot # cover that gap -- it validates labels only for a literal `runs-on`, and every # self-hosted selector here is a `fromJSON(...)` ternary, on which it stays # silent. Silence and a pass are the same output. # # Usage: ci/linter/lint-ci-runners.sh [files...] Env: LINT_MODE=staged|all # Extend: the approved runner selector is APPROVED_SELECTOR in workflow_policy.py. # shellcheck source=ci/linter/lib.sh . "$(git rev-parse --show-toplevel)/ci/linter/lib.sh" mapfile_checked FILES lint_files '^\.github/workflows/.*\.ya?ml$' "$@" [ "${#FILES[@]}" -gt 0 ] || { echo "lint-ci-runners: no workflow files to check"; exit 0; } need python3 "apt-get install python3" # The check is whole-tree by nature -- a runner policy is a property of the SET # of workflows, not of one file -- so the file list above is only used to decide # whether this checker is RELEVANT to the change, never to narrow what it reads. exec python3 "$(git rev-parse --show-toplevel)/ci/linter/workflow_policy.py" runners myguard-labs-nginx-zstd-module-3486b5b/ci/linter/lint-ci-secrets.sh000077500000000000000000000031551525232545500252760ustar00rootroot00000000000000#!/usr/bin/env bash # Copyright (C) 2026 Thijs Eilander # SPDX-License-Identifier: BSD-2-Clause # # ci/linter/lint-ci-secrets.sh -- a workflow_call member declares the secrets it # needs by name and typed `required: true`; callers never use `secrets: inherit`. # # The rule and the reasoning live in ci/linter/workflow_policy.py (subcommand # `secrets`); this wrapper exists so run-all.sh picks the check up by glob and # LINT_ONLY=ci-secrets selects it. # # The failure it prevents is a silent one at the call boundary. An undeclared # secret is not passed at all, so the member sees an empty string and dies # somewhere downstream of the cause -- a checkout that 404s, a curl that 401s -- # with nothing pointing back at the missing declaration. # # This repo declares no secrets and is vacuously green on the member half by # design; see the subcommand docstring. The caller half is live today. # # Usage: ci/linter/lint-ci-secrets.sh [files...] Env: LINT_MODE=staged|all # Extend: the stance (typed-per-member, never inherit) is a repo policy decision # recorded in workflow_policy.py, not something to bypass here. # shellcheck source=ci/linter/lib.sh . "$(git rev-parse --show-toplevel)/ci/linter/lib.sh" mapfile_checked FILES lint_files '^\.github/workflows/.*\.ya?ml$' "$@" [ "${#FILES[@]}" -gt 0 ] || { echo "lint-ci-secrets: no workflow files to check"; exit 0; } need python3 "apt-get install python3" # Whole-tree by nature: the check pairs callers with the members they call, so # the file list only decides whether this checker is relevant. exec python3 "$(git rev-parse --show-toplevel)/ci/linter/workflow_policy.py" secrets myguard-labs-nginx-zstd-module-3486b5b/ci/linter/lint-ci-topology.sh000077500000000000000000000002001525232545500254660ustar00rootroot00000000000000#!/usr/bin/env bash # Protect the measured four-lane workflow topology. set -euo pipefail exec python3 ci/linter/ci_topology.py myguard-labs-nginx-zstd-module-3486b5b/ci/linter/lint-docs-drift.sh000077500000000000000000000034001525232545500252640ustar00rootroot00000000000000#!/usr/bin/env bash # Copyright (C) 2026 Thijs Eilander # SPDX-License-Identifier: BSD-2-Clause # # ci/linter/lint-docs-drift.sh -- README.md and .github/workflows/ describe the # same pipeline. # # The rule and the reasoning live in ci/linter/workflow_policy.py (subcommand # `docs`); this wrapper exists so run-all.sh picks the check up by glob and # LINT_ONLY=docs-drift selects it. # # Both drift directions are silent and neither fails anything else: # * a workflow added with no README row is a gate nobody knows exists, so # nobody notices when a later refactor deletes it; # * a README row for a workflow that no longer exists is a badge that 404s and # a documented guarantee the repo does not actually have. # This is a TEMPLATE repo: its README is the specification nine sibling modules # are rolled out against, so a stale claim there propagates. # # Structural facts only -- no exact job counts, no durations, no subtest tallies. # Those are the brittle claims that make a drift check annoying enough to delete, # and the ones a lane-timing comment already owns. # # Usage: ci/linter/lint-docs-drift.sh [files...] Env: LINT_MODE=staged|all # Extend: add a structural fact to check_docs(); resist adding a count. # shellcheck source=ci/linter/lib.sh . "$(git rev-parse --show-toplevel)/ci/linter/lib.sh" mapfile_checked FILES lint_files '^(\.github/workflows/.*\.ya?ml|README\.md)$' "$@" [ "${#FILES[@]}" -gt 0 ] || { echo "lint-docs-drift: no workflow or README changes"; exit 0; } need python3 "apt-get install python3" # Whole-tree by nature: the check compares two SETS, so a narrowed file list # would let a workflow deletion pass whenever README.md was the only file staged. exec python3 "$(git rev-parse --show-toplevel)/ci/linter/workflow_policy.py" docs myguard-labs-nginx-zstd-module-3486b5b/ci/linter/lint-nginx.sh000077500000000000000000000111571525232545500243610ustar00rootroot00000000000000#!/usr/bin/env bash # ci/linter/lint-nginx.sh -- nginx-specific source conventions for src/*.[ch]. # # What no generic C linter knows: an nginx module must use the nginx allocator, # the nginx string/number helpers and the nginx source style, because it is # compiled into a worker with a pool-based lifetime and a shared coding # standard. flawfinder/cppcheck/semgrep all pass code that leaks a malloc() # into a request pool or calls atoi() on attacker input. # # Checks (each one line per finding, "file:line: rule: text"): # libc-alloc malloc/calloc/realloc/free -> ngx_palloc/ngx_pcalloc/ngx_pfree # libc-str strcpy/strcat/sprintf/strncpy -> ngx_cpymem/ngx_snprintf # libc-num atoi/atol/strtol on request data -> ngx_atoi/ngx_atoof # libc-io bare printf/fprintf(stderr) -> ngx_log_error # tabs hard tab in source -> nginx style is 4 spaces # width line >80 columns -> nginx style limit # trailing trailing whitespace # include .c must include ngx_config.h before ngx_core.h # # Suppress a single justified line with a trailing /* NOLINT-nginx */ . # Suppressing a whole rule is not supported on purpose: the exception belongs # next to the code that needs it, where review can see the reason. # # Usage: ci/linter/lint-nginx.sh [files...] Env: LINT_MODE=staged|all # Extend: add a rule as one more `rule ` call. # shellcheck source=ci/linter/lib.sh . "$(git rev-parse --show-toplevel)/ci/linter/lib.sh" mapfile_checked FILES lint_files '^src/.*\.[ch]$' "$@" [ "${#FILES[@]}" -gt 0 ] || { echo "lint-nginx: no C files to check"; exit 0; } echo "lint-nginx: ${#FILES[@]} file(s)" rc=0 # rule -- report every matching line not marked NOLINT. rule() { local name="$1" re="$2" msg="$3" hits hits=$(grep -nE "$re" "${FILES[@]}" 2>/dev/null | grep -v 'NOLINT-nginx' || true) [ -n "$hits" ] || return 0 printf '%s\n' "$hits" | sed "s/^/ /; s/$/ [$name: $msg]/" rc=1 } rule libc-alloc '(^|[^_[:alnum:]])(malloc|calloc|realloc|free)[[:space:]]*\(' \ 'use ngx_palloc/ngx_pcalloc/ngx_pfree' rule libc-str '(^|[^_[:alnum:]])(strcpy|strcat|sprintf|strncpy|strncat)[[:space:]]*\(' \ 'use ngx_cpymem/ngx_snprintf' rule libc-num '(^|[^_[:alnum:]])(atoi|atol|strtol|strtoul)[[:space:]]*\(' \ 'use ngx_atoi/ngx_atoof' rule libc-io '(^|[^_[:alnum:]])(printf|fprintf|perror)[[:space:]]*\(' \ 'use ngx_log_error/ngx_conf_log_error' rule tabs $'\t' 'nginx style is 4 spaces, no hard tabs' rule trailing '[[:space:]]+$' 'trailing whitespace' rule width '^.{81,}$' 'nginx style limit is 80 columns' for f in "${FILES[@]}"; do case "$f" in *.c) # ngx_config.h defines the feature macros every later nginx header # reads; including ngx_core.h first silently changes the build. Look at # the FIRST ngx_ include, not a fixed head -N window: these files open # with a long licence/design comment that would push the includes out # of any window and make the check vacuous. # Angle brackets only: a local "ngx_http__*.h" is this module's # own header and carries its own ngx_config.h include -- matching it # here reported every well-formed file. # Either spelling, not just <...>. A module whose .c files open with # #include "ngx_http__module.h" satisfies the ordering contract # when THAT header leads with -- looking only at angle # includes reports the first one after it as a violation. Two false # positives on arrival in nginx-cache-turbo-module (2026-08-10), whose # every .c has that shape; this repo's own .c files include # directly, so the gap never showed here. first_ngx=$(grep -nE '^[[:space:]]*#[[:space:]]*include[[:space:]]*[<"]ngx_' "$f" | head -1 || true) case "$first_ngx" in *'"'*) # Leading local ngx_ header: the contract moves to that header. local_h="src/$(printf '%s' "$first_ngx" | sed -E 's/.*"([^"]+)".*/\1/')" if [ -f "$local_h" ] \ && grep -nE '^[[:space:]]*#[[:space:]]*include[[:space:]]*/dev/null || rc=1 done need perlcritic "apt-get install libperl-critic-perl (or: cpan Perl::Critic)" say "perlcritic (severity >=4)" perlcritic --severity 4 --quiet "${FILES[@]}" || rc=1 exit "$rc" myguard-labs-nginx-zstd-module-3486b5b/ci/linter/lint-python.sh000077500000000000000000000031631525232545500245550ustar00rootroot00000000000000#!/usr/bin/env bash # ci/linter/lint-python.sh -- ruff lint + format check over tracked *.py. # # ruff replaces flake8/pyflakes/isort/black here: one binary, no venv, and it # is what the superrepo's tools/ scripts are already checked with. It covers # ci/linter/workflow_policy.py and ci/tools/test_runtime.py -- both gate logic, # so an unlinted break here is a check that stops checking rather than a script # that stops running. # # Keep this dispatched even when the count is low: an adopted module that adds # its first ci/tools/ helper gets it checked from the first commit. Run an # extension census (`git ls-files | sed -n 's/.*\.//p' | sort | uniq -c`) when # deriving a linter set for a new module -- a tracked-but-unlinted language is # a silent pass in exactly the way a selector matching zero files is. # # `ruff format --check` reports formatting WITHOUT rewriting: a linter that # edits files behind a commit hook changes what you are about to commit. # # Usage: ci/linter/lint-python.sh [files...] Env: LINT_MODE=staged|all # Extend: per-rule config goes in a [tool.ruff] block in pyproject.toml at the # repo root, not in flags here, so editors and CI see the same rules. # shellcheck source=ci/linter/lib.sh . "$(git rev-parse --show-toplevel)/ci/linter/lib.sh" mapfile_checked FILES lint_files '\.py$' "$@" [ "${#FILES[@]}" -gt 0 ] || { echo "lint-python: no Python files to check"; exit 0; } echo "lint-python: ${#FILES[@]} file(s)" need ruff "apt-get install ruff (or: pipx install ruff)" rc=0 say "ruff check" ruff check "${FILES[@]}" || rc=1 say "ruff format --check" ruff format --check "${FILES[@]}" || rc=1 exit "$rc" myguard-labs-nginx-zstd-module-3486b5b/ci/linter/lint-sh.sh000077500000000000000000000021561525232545500236470ustar00rootroot00000000000000#!/usr/bin/env bash # ci/linter/lint-sh.sh -- shellcheck over every *.sh / *.bash in the tree. # # -S warning, not info: the info tier (SC2015, SC2086 in safe positions) fires # on existing build scripts and would land this gate red on arrival, which only # teaches everyone to --no-verify. Same floor as the shellcheck the actionlint # hook runs inside `run:` blocks (SHELLCHECK_OPTS=-Swarning), so an SC2164 is # not ignored in a .sh file while blocking in a workflow. # # Usage: ci/linter/lint-sh.sh [files...] Env: LINT_MODE=staged|all # Extend: raise to -S info only together with a pass that fixes the backlog. # shellcheck source=ci/linter/lib.sh . "$(git rev-parse --show-toplevel)/ci/linter/lib.sh" # .githooks/* has no extension but is bash; an unchecked commit hook is the # one script whose bug silently disables every other check here. mapfile_checked FILES lint_files '\.(sh|bash)$|^\.githooks/' "$@" [ "${#FILES[@]}" -gt 0 ] || { echo "lint-sh: no shell files to check"; exit 0; } echo "lint-sh: ${#FILES[@]} file(s)" need shellcheck "apt-get install shellcheck" shellcheck -S warning -x "${FILES[@]}" say "clean" myguard-labs-nginx-zstd-module-3486b5b/ci/linter/lint-spelling.sh000077500000000000000000000057431525232545500250570ustar00rootroot00000000000000#!/usr/bin/env bash # ci/linter/lint-spelling.sh -- codespell over prose: docs, comments, log # strings, error messages. # # The findings this catches are cosmetic one at a time and expensive in # aggregate: a typo in an error message is the string a future maintainer greps # for and does not find, and a typo in a README is the sentence a reader stops # to re-parse. Neither shows up in any other gate here -- every checker in this # directory reads code as code, and prose is where this repo keeps most of its # reasoning. # # SCOPE: every tracked file. The vendored upstream test suite is already outside # lint_files' reach via LINT_EXCLUDE_RE, which is load-bearing here rather than # incidental -- 95 of the 98 raw findings on this tree came from # ci/vendor/nginx-tests/, and none of them are misspellings. The directives # below are how you keep a deliberate example out of the findings: same line as # the word, naming the word -- never a file-wide skip. # `isnt` is a Test::More predicate. # codespell:ignore isnt # `clen` is that suite's content-length variable. # codespell:ignore clen # Pointing codespell at vendored code produces a permanently red gate made # entirely of false positives, which is the shape that teaches --no-verify. # # No --write-changes, ever. codespell picks one candidate from an ambiguous # list -- the vendored variable above maps to "clean, clan" -- and an auto-fixer # would silently pick wrong inside a string literal or an identifier. # # Extend, cheapest first: a one-off deliberate example gets an inline # `codespell:ignore ` on its own line. A genuine domain word that recurs # goes in ci/linter/codespell-ignore.txt, one per line. Neither is a reason to # widen LINT_EXCLUDE_RE, which blinds every checker here to whole files. # # Usage: ci/linter/lint-spelling.sh [files...] Env: LINT_MODE=staged|all # # VERIFY BEFORE TRUSTING (a green checker proves nothing until seen red): # printf '# teh gate must catch this\n' > _p.sh # codespell:ignore teh # ci/linter/lint-spelling.sh _p.sh # expect a finding, exit 65 # rm _p.sh # shellcheck source=ci/linter/lib.sh . "$(git rev-parse --show-toplevel)/ci/linter/lib.sh" mapfile_checked FILES lint_files '.' "$@" [ "${#FILES[@]}" -gt 0 ] || { echo "lint-spelling: no files to check"; exit 0; } echo "lint-spelling: ${#FILES[@]} file(s)" need codespell "pipx install codespell" say "codespell" ROOT="$(repo_root)" IGNORE="$ROOT/ci/linter/codespell-ignore.txt" args=() [ -f "$IGNORE" ] && args+=(--ignore-words "$IGNORE") # Normalise codespell's exit status into this suite's 0/1/2 contract. It exits # 65 on findings (its own convention); run-all.sh happens to bucket that as # "findings", but the hook, lint.yml and a human reading the raw status all see # an undocumented value -- and a future codespell using 2 would be reported as # "tool missing", masking a real red. Reported by the # nginx-cache-turbo-module adoption 2026-08-10. rc=0 codespell "${args[@]}" -- "${FILES[@]}" || rc=1 exit "$rc" myguard-labs-nginx-zstd-module-3486b5b/ci/linter/lint-yaml.sh000077500000000000000000000061601525232545500241760ustar00rootroot00000000000000#!/usr/bin/env bash # ci/linter/lint-yaml.sh -- yamllint over every *.yml/*.yaml, plus actionlint # over .github/workflows/*. # # Workflows are code that fails silently: a bad `if:` expression, an unknown # context or a shell error inside a `run:` block does not stop the run, it # makes the job pass while checking nothing. actionlint is the only checker # here that reads them as GitHub Actions rather than as YAML. # # -ignore 'label ".+" is unknown' # actionlint 1.7.7 carries a FIXED list of runner images and flags newer # ones (ubuntu-26.04, ubuntu-26.04-arm) as unknown. Every such report is # about the linter's age, not the workflow. Drop the ignore once # actionlint is new enough to know the labels this repo targets. # SHELLCHECK_OPTS=-Swarning # actionlint's embedded shellcheck otherwise reports at info while # lint-sh.sh gates at warning -- same finding, two verdicts. # # No --strict on yamllint, on purpose: warnings (long inline `run:` lines) stay # visible without failing the gate, errors still block. The rule set lives in # .yamllint at the repo root so editors and this script agree. # # actionlint and zizmor are not alternatives: actionlint reads a workflow as # SYNTAX (bad if:, unknown context, shell error in run:), zizmor reads it as an # ATTACK SURFACE (template injection, dangerous triggers, leaked credentials, # unpinned actions, over-broad permissions). This repo targets self-hosted # runners, where a workflow-level mistake is arbitrary code execution on the # build host -- so the security pass is not optional here. # # --persona=pedantic, not the default: the default already passes on this tree, # so gating on it would never go red. Pedantic is what caught the matrix # interpolations in ci-deep.yml and the undocumented CodeQL permissions. Findings # that are genuinely inapplicable get a `# zizmor: ignore[rule]` at the line, # with the reason -- never a blanket rule disable in zizmor.yml. # # --offline: no API calls from a commit hook. The online audits need a token and # only add repo-settings context, which belongs in a periodic review, not here. # # Usage: ci/linter/lint-yaml.sh [files...] Env: LINT_MODE=staged|all # Extend: yamllint rules live in .yamllint at the repo root. # shellcheck source=ci/linter/lib.sh . "$(git rev-parse --show-toplevel)/ci/linter/lib.sh" mapfile_checked FILES lint_files '\.ya?ml$' "$@" [ "${#FILES[@]}" -gt 0 ] || { echo "lint-yaml: no YAML files to check"; exit 0; } echo "lint-yaml: ${#FILES[@]} file(s)" rc=0 need yamllint "apt-get install yamllint" say "yamllint" yamllint "${FILES[@]}" || rc=1 WF=() for f in "${FILES[@]}"; do [[ "$f" =~ ^\.github/workflows/ ]] && WF+=("$f") done if [ "${#WF[@]}" -gt 0 ]; then need actionlint "go install github.com/rhysd/actionlint/cmd/actionlint@latest (see install-linters.sh)" say "actionlint (${#WF[@]} workflow(s))" SHELLCHECK_OPTS=-Swarning \ actionlint -ignore 'label ".+" is unknown' "${WF[@]}" || rc=1 need zizmor "pipx install zizmor" say "zizmor (workflow security, pedantic)" zizmor --offline --persona=pedantic --no-progress "${WF[@]}" || rc=1 fi exit "$rc" myguard-labs-nginx-zstd-module-3486b5b/ci/linter/run-all.sh000077500000000000000000000110341525232545500236360ustar00rootroot00000000000000#!/usr/bin/env bash # ci/linter/run-all.sh -- run every ci/linter/lint-*.sh and report once. # # This is what .githooks/pre-commit calls and what a human should call before # pushing. It does NOT stop at the first failure: seeing all four failures in # one run beats four commit attempts. # # Usage: # ci/linter/run-all.sh lint every tracked file # ci/linter/run-all.sh --staged lint only staged files (hook mode) # ci/linter/run-all.sh --list list the checks and exit # ci/linter/run-all.sh ... lint exactly these files # # Env: # LINT_MODE=staged|all same as --staged / default # LINT_ONLY="c sh" run only these checkers (names below) # LINT_SKIP_SEMGREP=1 loud opt-out of the slowest C pass # LINT_JOBS=N parallel checkers (default 0 = one per checker, # 1 = serial; use 1 when bisecting a hang) # # Exit: 0 all clean, 1 findings, 2 a linter is missing (install-linters.sh). # # The checkers are independent, so they run concurrently and each one's output # is buffered to its own file and replayed whole, in the fixed glob order. Never # stream them interleaved: findings carry a file:line but not a checker name, so # interleaving makes them unattributable, which is the whole reason to buffer. # Order stays glob order and not completion order so two runs of a dirty tree # produce the same transcript. # # Extend: drop a new ci/linter/lint-.sh in place -- it is picked up by # glob, no edit here. Keep the "no files of this kind" case exiting 0. set -uo pipefail ROOT="$(git rev-parse --show-toplevel)" # || exit 2, not a bare cd: every checker resolves its paths relative to the # repo root, so failing to get there would lint the wrong tree -- or nothing at # all -- and report clean. Same "could not run" status as a missing linter. cd "$ROOT" || exit 2 MODE="${LINT_MODE:-all}" FILES=() for a in "$@"; do case "$a" in --staged) MODE=staged ;; --all) MODE=all ;; --list) ls ci/linter/lint-*.sh; exit 0 ;; -h|--help) sed -n '2,30p' "$0"; exit 0 ;; *) FILES+=("$a") ;; esac done export LINT_MODE="$MODE" echo "== lint (mode: $MODE) ==" # Selected checkers, in glob order. Built first so the replay loop below can # walk the same list it launched. ALL=() SEL=() for s in ci/linter/lint-*.sh; do name="$(basename "$s" .sh)"; name="${name#lint-}" ALL+=("$name") if [ -n "${LINT_ONLY:-}" ] && [[ " $LINT_ONLY " != *" $name "* ]]; then continue fi SEL+=("$name") done # An empty selection is "could not run", never "clean". A typo in LINT_ONLY # used to leave both loops below iterating zero times, printing "all linters # clean" and exiting 0 -- a gate that silently lints nothing is worse than no # gate. Same exit 2 as a missing linter. if [ "${#SEL[@]}" -eq 0 ]; then if [ "${#ALL[@]}" -eq 0 ]; then echo "no ci/linter/lint-*.sh found -- wrong tree?" >&2 else echo "LINT_ONLY=\"${LINT_ONLY:-}\" matched no checker; known: ${ALL[*]}" >&2 fi exit 2 fi JOBS="${LINT_JOBS:-0}" [ "$JOBS" -eq 0 ] 2>/dev/null && JOBS="${#SEL[@]}" [ "$JOBS" -ge 1 ] 2>/dev/null || JOBS=1 # mktemp -d, not a fixed path: two checkouts (or a hook racing a manual run) # would otherwise share buffers. Trap covers the Ctrl-C path too. BUF="$(mktemp -d "${TMPDIR:-/tmp}/lint-buf.XXXXXX")" trap 'rm -rf "$BUF"' EXIT INT TERM for name in "${SEL[@]}"; do # Bounded fan-out: block until a slot frees. `wait -n` needs bash 4.3+, # which install-linters.sh already assumes. while [ "$(jobs -rp | wc -l)" -ge "$JOBS" ]; do wait -n; done { "ci/linter/lint-$name.sh" "${FILES[@]+"${FILES[@]}"}" \ > "$BUF/$name.out" 2>&1 # Exit status travels in a file, not in $?: the reaping `wait` below is # collective, so per-child statuses are not otherwise recoverable. echo "$?" > "$BUF/$name.rc" } & done wait rc=0 missing=0 failed=() for name in "${SEL[@]}"; do cat "$BUF/$name.out" # A missing .rc means the subshell died before writing one (OOM-killer, # SIGKILL). Treat it as a failure, never as a pass. crc="$(cat "$BUF/$name.rc" 2>/dev/null || echo 1)" case "$crc" in 0) ;; 2) missing=1; failed+=("$name(tool missing)") ;; *) rc=1; failed+=("$name") ;; esac done if [ "$missing" -eq 1 ]; then echo "== FAIL: ${failed[*]} -- run ci/linter/install-linters.sh ==" >&2 exit 2 fi if [ "$rc" -ne 0 ]; then echo "== FAIL: ${failed[*]} ==" >&2 exit 1 fi echo "== all linters clean ==" myguard-labs-nginx-zstd-module-3486b5b/ci/linter/selftest.sh000077500000000000000000000550461525232545500241300ustar00rootroot00000000000000#!/usr/bin/env bash # ci/linter/selftest.sh -- negative controls for the lint gate itself. # # The gate is the thing that decides whether everything else is allowed to # land, so "the gate ran and said clean" has to be distinguishable from "the # gate ran nothing and said clean". That distinction is not observable from a # green lint job: a selector typo used to make run-all.sh print # "== all linters clean ==" and exit 0 having executed zero checkers. # # Every case here asserts the FAILING direction -- a check that only ever # asserts the passing direction cannot detect its own disarming. # # Usage: ci/linter/selftest.sh # Exit: 0 all controls held, 1 one or more did not. # # Runs in about a second: no case invokes a real checker (LINT_ONLY values are # either bogus or rejected before dispatch), so no linter needs to be installed. # # Extend: add a case() line. Keep each case asserting a specific exit status, # and prefer a case where the OLD, broken behaviour would have passed. set -uo pipefail ROOT="$(git rev-parse --show-toplevel)" cd "$ROOT" || exit 2 rc=0 # case -- command... case_() { local want="$1" desc="$2"; shift 2 local out got out="$("$@" 2>&1)"; got=$? if [ "$got" -eq "$want" ]; then echo "ok $desc (exit $got)" else echo "FAIL $desc: expected exit $want, got $got" >&2 printf ' | %s\n' "${out//$'\n'/$'\n | '}" >&2 rc=1 fi } # The regression itself: an unmatched selector must be "could not run" (2), # not "clean" (0). case_ 2 "unknown LINT_ONLY exits 2" \ env LINT_ONLY=nosuchchecker ci/linter/run-all.sh # ...including when only SOME of the listed names are bogus in a way that # leaves nothing selected. case_ 2 "LINT_ONLY of only-bogus names exits 2" \ env LINT_ONLY="c-lang shellscript" ci/linter/run-all.sh # The error names the offending value and the known checkers, or nobody can # act on it. # Captured first, not piped into grep: `set -o pipefail` would otherwise hand # the pipeline run-all.sh's exit 2 and the assertion would read as failed no # matter what the message said. msg="$(env LINT_ONLY=nosuchchecker ci/linter/run-all.sh 2>&1)" if printf '%s\n' "$msg" | grep -q 'matched no checker; known: .*sh'; then echo "ok unmatched-selector message names value and known checkers" else echo "FAIL unmatched-selector message is not actionable" >&2 rc=1 fi # Positive control: the selector still selects. --list is used rather than a # real run so this stays independent of which linters are installed. case_ 0 "--list works" ci/linter/run-all.sh --list case_ 0 "CI topology negative controls hold" \ python3 ci/linter/ci_topology.py --selftest # The CodeQL SARIF filter decides which findings reach the Security tab. Its # dangerous direction is dropping a result it should have kept -- silently, # since a dropped finding leaves no trace. Its own controls assert the KEEP # side (module results, and results with no resolvable location). case_ 0 "CodeQL SARIF filter scoping controls hold" \ python3 ci/tools/filter-codeql-sarif.py --selftest # Work-list consumers used process substitution, whose exit status is not # visible to mapfile. Exercise each real consumer with a producer that emits a # plausible prefix and then fails; all three must propagate its exact status. case_ 23 "lint work list rejects partial output followed by failure" \ env MAPFILE_CHECKED_FAULT=partial-error ci/linter/lint-sh.sh ci/linter/lib.sh case_ 23 "dictionary work list rejects partial output followed by failure" \ env MAPFILE_CHECKED_FAULT=partial-error ci/fuzz/gen_dict.sh --check # shellcheck disable=SC2317 # Invoked indirectly through case_ below. dict_crlf_rejected_() ( local backup out got mode backup="$(mktemp "${TMPDIR:-/tmp}/fuzz-dict.XXXXXX")" || exit 2 cp ci/fuzz/fuzz.dict "$backup" || exit 2 trap 'cp "$backup" ci/fuzz/fuzz.dict; rm -f "$backup"' EXIT INT TERM while IFS= read -r line || [ -n "$line" ]; do printf '%s\r\n' "$line" done <"$backup" >ci/fuzz/fuzz.dict for mode in check regenerate; do if [ "$mode" = check ]; then out="$(ci/fuzz/gen_dict.sh --check 2>&1)"; got=$? else out="$(ci/fuzz/gen_dict.sh 2>&1)"; got=$? fi if [ "$got" -ne 1 ] || ! printf '%s\n' "$out" | grep -qF 'contains CRLF line endings'; then printf '%s: expected exit 1 with CRLF diagnosis, got %d\n%s\n' \ "$mode" "$got" "$out" >&2 exit 1 fi done ) case_ 0 "dictionary CRLF rejection covers check and regeneration" \ dict_crlf_rejected_ case_ 23 "coverage work list rejects partial output followed by failure" \ env MAPFILE_CHECKED_FAULT=partial-error ci/tools/coverage.sh \ --selftest-work-list "$ROOT" case_ 23 "docs directive list rejects partial output followed by failure" \ env MAPFILE_CHECKED_FAULT=partial-error ci/tools/test_docs_ci_drift.sh \ --selftest-directive-list case_ 23 "docs workflow list rejects partial output followed by failure" \ env MAPFILE_CHECKED_FAULT=partial-error ci/tools/test_docs_ci_drift.sh \ --selftest-workflow-list # ---------------------------------------------------------------------------- # workflow_policy.py -- the red path of each policy check. # # Every fixture below encodes a bypass that VALID YAML used to walk straight # through while the checks parsed workflows by regex. Each was verified in both # directions when it was written: red on the current parser, green on the # regex one. They are committed rather than planted in .github/workflows/ at # runtime so no cleanup failure can leave a probe workflow in the live tree. # # Extend: add a fixture directory with its own .github/workflows/ and a README # stating which bypass it encodes, then a policy_ line here. policy_() { # policy_ local want="$1" fixture="$2" cmd="$3" case_ "$want" "policy $cmd: $fixture" \ env "WORKFLOW_POLICY_ROOT=ci/linter/fixtures/policy/$fixture" \ python3 ci/linter/workflow_policy.py "$cmd" } # policy_msg_ -- red, and red for the STATED # reason. # # Exit status alone cannot tell a finding from a crash: an uncaught # AttributeError also exits 1, so a fixture asserting only `1` stays green when # the branch it covers is replaced by a traceback. That is not hypothetical -- # deleting the `isinstance(spec, dict)` guard in check_secrets() made # secrets-untyped die on `None.get()` and every exit-1 control still passed. # Use this wherever the guard under test is what stops a crash. # # Captured first, not piped: pipefail would hand the pipeline the checker's # exit 1 and the assertion would read as failed whatever the message said. policy_msg_() { local fixture="$1" cmd="$2" want_re="$3" out got out="$(env "WORKFLOW_POLICY_ROOT=ci/linter/fixtures/policy/$fixture" \ python3 ci/linter/workflow_policy.py "$cmd" 2>&1)"; got=$? if [ "$got" -eq 1 ] && printf '%s\n' "$out" | grep -qE "$want_re"; then echo "ok policy $cmd: $fixture (finding, not a crash)" else echo "FAIL policy $cmd: $fixture: expected exit 1 matching /$want_re/, got $got" >&2 printf '%s\n' "$out" | sed 's/^/ | /' >&2 rc=1 fi } # THE control that makes the rest mean anything: a fixture tree that is simply # a valid workflow must be GREEN on all three. Without it, a red on any bypass # fixture could be the fixture shape rather than the bypass. policy_ 0 clean runners policy_ 0 clean ports policy_ 0 clean docs policy_ 0 clean provenance # Deliberately NO `policy_ 0 clean cadence`: the clean fixture has no # workflow_call member, so that line would assert green over an empty set -- # vacuous, and indistinguishable from the check being broken. The green control # for cadence is member-with-push-ok below, which has a member to clear. # A `.yaml` workflow was invisible to every check: unchecked runner, unchecked # ports, undocumented gate. policy_ 1 bypass-yaml-extension runners policy_ 1 bypass-yaml-extension docs # `on: [pull_request]` / `on: pull_request` are the same trigger as the mapping # form; both used to skip the runner-trust check, as did an inline # pull_request_target. policy_ 1 bypass-inline-events runners # `runtime: # comment` used to yield zero jobs, so a runtime-bearing job with # no port band reported "no runtime-bearing jobs" and exit 0. policy_ 1 bypass-commented-job-key ports # A band verifier placed BELOW the first binder. Declaration, pass-through and # uniqueness all hold, so the presence checks stay green -- this repo's own # build-test.yml sat in exactly this shape until 2026-08-02. policy_ 1 verify-after-bind ports # A job whose only binder is `prove` (not ci/tools/test_runtime.py) used to be # exempt from the "declare TEST_BASE_PORT" requirement, even though `prove` is # already a BINDERS member and the ordering check already treats it as one. # Found downstream as a failed negative control: deleting a prove-only job's # band left this check green. policy_ 1 prove-only-binder-exempt ports # The other half of the same defect: a prove job may declare a unique band, # satisfy every check above, and still bind 1984, because Test::Nginx reads # TEST_NGINX_PORT and has never heard of TEST_BASE_PORT. The driver's # pass-through guard is keyed on starts_runtime, so it never covered `prove`. policy_ 1 prove-band-not-passed-through ports # A workflow_call member carrying its own `push:` runs twice per change and BOTH # runs are green, so nothing else in the toolchain notices. Every member here is # correct today, but only a per-file comment says so, and a comment does not # survive the next workflow copied in. These run as a PAIR: the -ok fixture is # the same file with `schedule:` (the intended shape), and without it the red # above is equally consistent with "any second trigger is flagged". policy_ 1 member-with-push cadence policy_ 0 member-with-push-ok cadence # The three ways a secret goes missing between a caller and a member, none of # which fails anything at the time it is introduced. `inherit` is green and # merely over-broad; an untyped declaration starts the call with an empty # string; an undeclared secret is dropped at the boundary while both halves # read as correct in isolation. These run as a GROUP with secrets-typed-ok: # this repo declares no secrets, so without a green fixture that DOES, the # three reds would be equally consistent with "any mention of a secret is # flagged" -- and the live check would be asserting green over an empty set. policy_ 1 secrets-inherit secrets # The same defect one repository over. The caller loop filtered to local # members BEFORE judging `inherit`, so a call to # `owner/repo/.github/workflows/x.yml@ref` was skipped -- the case where # `inherit` is WORST, since the secret set crosses a repository boundary to a # moving ref. Distinct from secrets-inherit: reordering that filter back below # the inherit branch leaves the local fixture red and only this one green. policy_ 1 secrets-inherit-external secrets # The third case that filter used to swallow: an external call whose `secrets:` # is neither a mapping nor `inherit`. `inherit` was hoisted above the filter, # but the SHAPE check stayed below it, so `secrets: false` on an external member # reported clean over a caller GitHub refuses to start. Exit 2, not 1 -- an # uninterpretable `secrets:` means the check did not run over that call, so # policy_msg_ (which hardcodes exit 1) cannot assert this one. Moving the shape # check back below `if not local` turns this fixture green while every other # secrets fixture stays exactly as it is. policy_ 2 secrets-malformed-external secrets # Message-asserted: a null spec is exactly the input that crashes `.get()` if # the isinstance guard is removed, and a traceback also exits 1. policy_msg_ secrets-untyped secrets 'declares secret .* untyped' # secrets-optional is NOT redundant with secrets-untyped. Disarming the # `required is not True` test left secrets-untyped red anyway -- it trips the # missing-key branch -- so the value test had no control of its own. A mutant # that survives is a branch nothing is gating. policy_ 1 secrets-optional secrets # ...and secrets-no-required-key is not redundant with EITHER. `NAME:` with # nothing under it parses to null, and null `is not True`, so the value branch # covers secrets-untyped even when the missing-key branch is deleted. Only a # spec that is a real mapping without the key separates the two. policy_ 1 secrets-no-required-key secrets policy_ 1 secrets-undeclared secrets # The mirror of secrets-undeclared, and the direction the check missed when # first written: member requires it, caller wires nothing. policy_ 1 secrets-required-not-wired secrets policy_ 0 secrets-typed-ok secrets # A step downloads a tarball; a later step in the same job extracts it and # runs the unpacked configure with no gpg/sha256 trust-anchor assertion # anywhere in between -- the class this repo hit in build-test.yml/ci-deep.yml # after the ci-build.sh fix (a direct download/extract/execute path added # outside that already-verified helper). These run as a PAIR: the -ok fixture # is the same job with a sha256 comparison step inserted, so the red above is # not equally consistent with "any download in a job is flagged". policy_ 1 provenance-unverified-extract provenance policy_ 0 provenance-verified-ok provenance # A reviewed verification helper named later in the SAME run block cannot # retroactively protect extraction that occurred earlier in that block. policy_ 1 provenance-verify-helper-after-extract provenance # The trap this check exists to close: the download step is gated # `cache-hit != 'true'`, so a cache HIT skips it entirely and a checker that # only looks for a download step present would read the job as having # nothing to verify. This must still go red -- a cached tarball is exactly as # untrusted as a fresh one until re-checked. policy_ 1 provenance-cache-hit-skips-verify provenance # The scoping regex must know every spelling that writes a file from the # network, not just `-O`/`-o`. A bare `wget ""` is the idiom four of this # repo's own jobs use; while the regex required -O those jobs matched no # download token and were skipped wholesale rather than checked -- a gate # failing open. Same class for `curl -O` and `Invoke-WebRequest -OutFile`. policy_ 1 provenance-bare-wget-unverified provenance # One probe per spelling the DOWNLOAD_RE comment claims to cover. Without # these two, a regression in the curl -O or Invoke-WebRequest branch passes # the selftest while the comment still advertises them as handled. policy_ 1 provenance-curl-O-unverified provenance policy_ 1 provenance-iwr-outfile-unverified provenance # The ANCHOR half failing open, not the scoping half: a bare `sha256sum f` # prints a digest and compares nothing, yet used to satisfy TRUST_ANCHOR_RE. # provenance-verified-ok is the green control for a real comparison. policy_ 1 provenance-noncomparing-sha256 provenance # No archive at all: download a standalone binary, chmod +x, run it. The # tar/unzip patterns miss this entirely, yet it is the same privilege # boundary one step shorter. policy_ 1 provenance-exec-downloaded-binary provenance # ZIP files need not retain a .zip extension. The detector must still reject # an unverified archive passed to unzip under an extensionless local name. policy_ 1 provenance-unverified-unzip provenance policy_ 1 provenance-chained-unzip provenance policy_ 1 provenance-mixed-codeql-unzip provenance policy_ 1 provenance-pipe-unzip provenance policy_ 1 provenance-subshell-unzip provenance policy_ 1 provenance-apt-chained-unzip provenance # A mistyped pool label in a schedule-only workflow. The trust half of the # runners check does not apply to a workflow no fork can reach, and skipping it # used to skip the LABEL membership test with it -- while actionlint, the only # other thing that reads runner labels, stays silent on the `fromJSON(...)` # selectors this repo uses everywhere. Six selectors in bump.yml and ci-deep.yml # had no label checking at all. These two run as a PAIR: the -ok fixture is the # same file spelled correctly, and without it the red below is equally # consistent with "every non-PR-reachable workflow is now flagged". policy_ 1 schedule-only-runner-labels runners # The -ok fixture is checked through a TEMPORARY COPY whose selector is # regenerated from workflow_policy.py, rather than against the tracked file's # hardcoded selector. A hosted-only adopter, which step 13 tells to turn # SELF_HOSTED_ALLOWED off, otherwise sees this # POSITIVE control fail against a selector the fixture still spells the # skeleton's way: correct behaviour, confusing symptom, and it cost the # nginx-cache-turbo-module adoption a debugging detour on 2026-08-10. # # A COPY, not the tracked file: a selftest that rewrites a tracked fixture # leaves the tree dirty on every run, and a test which mutates the repo it is # testing is a worse problem than the stale fixture it fixes. ok_src="ci/linter/fixtures/policy/schedule-only-runner-labels-ok" if approved="$(python3 ci/linter/fixture-selector.py 2>/dev/null)" \ && [ -n "$approved" ] && [ -d "$ok_src" ]; then ok_tmp="$(mktemp -d "${TMPDIR:-/tmp}/lint-okfix.XXXXXX")" cp -r "$ok_src/." "$ok_tmp/" python3 ci/linter/fixture-selector.py --write \ "$ok_tmp/.github/workflows/nightly.yml" case_ 0 "policy runners: schedule-only-runner-labels-ok (selector from workflow_policy)" \ env "WORKFLOW_POLICY_ROOT=$ok_tmp" \ python3 ci/linter/workflow_policy.py runners rm -rf "$ok_tmp" else # Hosted-only repo: SELF_HOSTED_ALLOWED is off, so there is no approved # self-hosted selector for a positive control to assert. Say so rather # than silently skipping -- a control that vanishes is the failure mode # this whole file exists to catch. echo "skip policy runners: schedule-only-runner-labels-ok (SELF_HOSTED_ALLOWED off: hosted-only)" fi # WIRING CONTROLS. These assert that a checker is reachable at all, which is a # weaker claim than "it goes red on a defect" -- the red-direction probes need # real tools and live in each checker's header instead, so this file keeps its # no-linter-required property. They exist because both failures below were # silent: a checker nobody dispatches and a hook nobody runs look exactly like a # clean tree. # # core.hooksPath REPLACES .git/hooks/, so `pre-commit install` writes a file git # never reads. Measured 2026-08-02: trailing whitespace and a missing final # newline committed clean past the hooks whose only job is those two things. # .githooks/pre-commit therefore has to invoke pre-commit itself. case_ 0 "the commit hook invokes the pre-commit-config hooks" \ grep -q '^ *pre-commit run' .githooks/pre-commit # run-all.sh dispatches by glob, so a checker that is not executable, or is # named outside the lint-*.sh pattern, is silently not run. # Every glob-discovered checker is named in lint.yml's LINT_ONLY. # # run-all.sh picks checkers up by glob, but CI narrows the run to an explicit # allowlist, and nothing connected the two: a checker added to ci/linter/ ran # locally and in the pre-commit hook while being silently absent from every PR. # That is not a hypothetical -- ci-cadence shipped in 2026-08-06 and had never # run remotely when this control was written. A gate that runs everywhere except # the merge path is the one place it is load-bearing. missing="" only="$(sed -n 's/^ *LINT_ONLY: *//p' .github/workflows/lint.yml)" for s in ci/linter/lint-*.sh; do n="${s#ci/linter/lint-}"; n="${n%.sh}" # "c" is deliberately excluded in CI (see lint.yml's header): the src/ # scanners run in security-scanners.yml instead. [ "$n" = "c" ] && continue printf '%s\n' "$only" | tr ' ' '\n' | grep -qx "$n" || missing="$missing $n" done if [ -z "$missing" ]; then echo "ok every checker is named in lint.yml LINT_ONLY" else echo "FAIL checkers absent from lint.yml LINT_ONLY:$missing" >&2 echo " | they run locally and in the hook, but never on a PR" >&2 rc=1 fi case_ 0 "lint-spelling is dispatched by run-all.sh" \ bash -c 'ci/linter/run-all.sh --list | grep -q lint-spelling.sh' # The control above closes the loop for checkers that ARE a ci/linter/lint-*.sh # script. Two gates in lint.yml are not: ast-grep and gitleaks live in # .pre-commit-config.yaml and are invoked as their own `run:` steps, because # run-all.sh discovers checkers by the lint-*.sh glob and can never reach them. # That left them structurally unguarded -- delete either step from lint.yml and # every control here still passed, which is how they came to be hook-only (and # so absent from every PR) until 2026-08-22. # # Keying on the step NAME rather than the tool binary is deliberate: the name is # what the job summary shows, and a step renamed out of this list is a step # whose disappearance from the UI nobody would notice either. for step in \ 'ast-grep (structural sinks, gate on promoted rules)' \ 'gitleaks (whole tracked tree)' do if grep -qF -- "- name: $step" .github/workflows/lint.yml; then echo "ok lint.yml still runs the non-script gate: $step" else echo "FAIL lint.yml no longer runs: $step" >&2 echo " | it is not a ci/linter/lint-*.sh script, so run-all.sh" >&2 echo " | cannot dispatch it and no other control covers it" >&2 rc=1 fi done # The prompt-steps temp root went with the checker (see below), leaving only # $badroot. Its trap is registered here because `trap ... EXIT` REPLACES any # earlier EXIT trap rather than adding to it. # # lint-prompt-steps.* and lint-sync-stamp.sh are reference-only: they check # ci/PROMPT.md step citations and the skeleton-shared .github/ content stamps, # neither of which this module has. Dropped with their selftest cases rather # than left asserting against a checker that is not here. badroot="" trap 'rm -rf ${badroot:+"$badroot"}' EXIT INT TERM # Unparsable YAML is "could not run" (2), never "clean" -- GitHub may still # read a file this parser rejects, so a verdict over the rest of the tree would # be unsupported. Fixture is generated: a committed broken-YAML file would trip # yamllint on the real tree. # Assigned into the variable the EXIT trap above already covers; a second # `trap ... EXIT` here would replace that one and leak this temp root. badroot="$(mktemp -d)" mkdir -p "$badroot/.github/workflows" printf 'on: [pull_request\njobs: {\n' > "$badroot/.github/workflows/broken.yml" case_ 2 "policy runners: unparsable YAML is exit 2, not clean" \ env "WORKFLOW_POLICY_ROOT=$badroot" python3 ci/linter/workflow_policy.py runners if [ "$rc" -eq 0 ]; then echo "== lint gate selftest: all controls held ==" else echo "== lint gate selftest: FAILED ==" >&2 fi exit "$rc" myguard-labs-nginx-zstd-module-3486b5b/ci/linter/workflow_policy.py000077500000000000000000001333331525232545500255420ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright (C) 2026 Thijs Eilander # SPDX-License-Identifier: BSD-2-Clause """Repo-policy checks over .github/workflows/ that no off-the-shelf linter makes. ci/linter/workflow_policy.py runners runner trust boundary + pool labels ci/linter/workflow_policy.py ports per-job test port bands ci/linter/workflow_policy.py docs README <-> workflows drift ci/linter/workflow_policy.py cadence one PR entry point per member ci/linter/workflow_policy.py secrets typed per-member secrets, no inherit ci/linter/workflow_policy.py provenance extract/exec steps trust-anchored Each subcommand is wrapped by a ci/linter/lint-*.sh so run-all.sh picks it up by glob and a human can select it with LINT_ONLY. Exit: 0 clean, 1 findings, 2 could not run. WHY THESE ARE NOT actionlint OR zizmor RULES. Both of those read a workflow against GENERAL knowledge -- syntax, and a catalogue of known attack shapes. The checks here encode facts about THIS repo that no general tool can know: which self-hosted labels exist, that the runners are persistent and shared with package builds, which port band each job owns, which files document the pipeline, that ci.yml is the single orchestrator, and that a member's secret surface is declared at the member. They are the checks that go red when a NEW workflow is added without the property every existing one happens to have -- the case where copying an existing file is the only thing standing between the repo and a regression, and nothing enforces the copy. """ from __future__ import annotations import os import pathlib import re import sys try: import yaml except ModuleNotFoundError: # pragma: no cover - exercised by the selftest doc print( "workflow_policy: PyYAML not installed (apt-get install python3-yaml).\n" " Refusing to run rather than degrading to a regex scan: every check\n" " here was bypassable by VALID YAML while it parsed workflows by\n" " regex, which is the vacuous-gate shape this file exists to prevent.", file=sys.stderr, ) raise SystemExit(2) from None # WORKFLOW_POLICY_ROOT points the checks at a fixture tree instead of the repo. # It exists so ci/linter/selftest.sh can assert the FAILING direction of every # check against a committed, readable fixture -- rather than by planting a file # in the live .github/workflows/ and hoping the cleanup runs. A check whose red # path is never exercised is indistinguishable from one that cannot go red. ROOT = pathlib.Path( os.environ.get("WORKFLOW_POLICY_ROOT") or pathlib.Path(__file__).resolve().parents[2] ) WORKFLOWS = ROOT / ".github" / "workflows" class PolicyError(Exception): """Could not run the check (exit 2) -- never confused with "clean".""" # The one approved runner selector: the pool is a repo VARIABLE, with a # GitHub-hosted fallback baked into the expression. # # runs-on: ${{ fromJSON(vars.POOL || '["ubuntu-latest"]') }} # # Set `POOL` on a repo and its jobs run on the self-hosted pool; leave it unset # and they fall back to `ubuntu-latest`. That is the access-granting mechanism: # per repo, from the GitHub UI, with no pool label anywhere in the tree. # # What replaced what, so this is not "simplified" back later: every selector # used to inline `["self-hosted","builder02","lxc"]` behind a # `github.event.pull_request.head.repo.fork` ternary, in 15 places. The fork arm # kept a fork PR's code off the persistent build host; the duplication meant a # mistyped label was a job that queued forever, which is why this check existed. # # Both concerns moved, deliberately (2026-08-13): # - The pool label is now ONE value in the GitHub UI, so there is nothing left # in the tree to spell wrong and 15 copies cannot disagree. The typo did not # become impossible, though, only single-sited: `POOL` set to # `["self-hosted","buidler02","lxc"]` is valid JSON and non-empty, so # `fromJSON` parses it and the `||` fallback never fires. That job queues # against a label nobody answers, exactly as before. Only a MALFORMED value # fails loudly (fromJSON parse error) and only an UNSET one falls back to # hosted. Nothing in this repo can check the variable's contents; that is # the residual risk of moving it out of the tree. # - Fork trust is now GitHub's `all_external_contributors` approval policy: # every fork PR needs an explicit approval before any job runs. Verified # set on this repo 2026-08-13. # # That second move is why `pull_request_target` stays forbidden below and why # that ban matters MORE than it used to. `pull_request_target` runs the base # repo's workflow, with full secrets, against the fork's code -- and it would # read `vars.POOL` like any other job, putting unreviewed fork code on the build # host. The approval policy does not gate it. The ban in check_runners() is the # only thing in this repo that does. APPROVED_SELECTOR = "${{ fromJSON(vars.POOL || '[\"ubuntu-latest\"]') }}" # Adopters that own no self-hosted pool set this to None: there is then no # approved self-hosted selector, every job must be GitHub-hosted, and the # positive-control fixture pair does not apply (ci/linter/fixture-selector.py # exits 1, selftest.sh skips that case). Kept as an explicit switch because a # hosted-only repo must not have to edit the check itself to be correct -- # reported by the nginx-cache-turbo-module adoption, 2026-08-10. SELF_HOSTED_ALLOWED = True # GitHub-hosted runner images. The skeleton only ever needed the Ubuntu arm; # this module also builds on Windows (windows-build.yml: MSVC and MinGW), which # is a genuinely different platform rather than a pool choice, so its images # belong here too. Kept as an explicit image list, NOT a permissive # ".*-latest": the whole point of this check is that an unrecognised label is a # finding, and a wildcard would wave through a self-hosted pool called # "builder-latest". # # macos-* is listed for completeness; no workflow here uses it today. HOSTED = re.compile(r"(?:ubuntu|windows|macos)-(?:latest|[0-9]+(?:\.[0-9]+)?)(?:-arm)?") # The runtime driver. A job that starts it is a "runtime-bearing" job and owes # the port-band declaration checked below. RUNTIME_DRIVER = "ci/tools/test_runtime.py" # The band verifier, and everything that BINDS the band. The ordering check # below needs both: a verify step is only a guard for the binders that come # after it, and `prove` is a binder even though it is not the runtime driver. BAND_VERIFIER = "ci/tools/max-port.sh" # Word-bounded on purpose: a bare "prove" substring also matches `approve`, # `improve` and `prover`, and a run block that merely says "improve the fixture" # is not a binder. A shell comment inside a run block still counts -- treating a # commented-out binder as absent is the safe direction here, since the finding is # about the step that DOES bind. BINDERS = ( re.escape(RUNTIME_DRIVER), r"(? list[pathlib.Path]: """Every workflow file, BOTH extensions. GitHub reads `*.yml` and `*.yaml` alike. Globbing one of them made all three checks below skip a `.yaml` workflow entirely -- an undocumented, unchecked, possibly self-hosted PR entry point that reported clean. """ return sorted( [*WORKFLOWS.glob("*.yml"), *WORKFLOWS.glob("*.yaml")], key=lambda p: p.name, ) def load(path: pathlib.Path) -> dict: """Parse a workflow. A file that will not parse is exit 2, never clean.""" try: doc = yaml.safe_load(path.read_text(encoding="utf-8")) except yaml.YAMLError as exc: raise PolicyError(f"{path.name}: unparsable YAML: {exc}") from exc if not isinstance(doc, dict): raise PolicyError(f"{path.name}: top level is not a mapping") return doc def events(doc: dict) -> set[str]: """The trigger names, from any of the three legal `on:` spellings. `on: [pull_request]`, `on: pull_request` and the indented mapping form mean exactly the same thing to GitHub. Only the mapping form was recognised before, so the two others walked past the runner-trust check. PyYAML resolves the bare key `on` to the boolean True (YAML 1.1 truthiness), so the True key is where the node actually lands unless it was quoted. """ node = doc.get("on", doc.get(True)) if isinstance(node, str): return {node} if isinstance(node, list): return {str(e) for e in node} if isinstance(node, dict): return {str(k) for k in node} return set() def jobs(doc: dict) -> list[tuple[str, dict]]: """(name, node) for every job. A non-mapping `jobs:` yields nothing.""" node = doc.get("jobs") if not isinstance(node, dict): return [] return [(str(k), v) for k, v in node.items() if isinstance(v, dict)] def selector(node) -> str: """Render a `runs-on:` node back to one comparable string. The list and mapping forms are rendered rather than flattened so that a `runs-on: [self-hosted, builder02]` fails the membership test as ONE finding naming the whole selector, instead of one confusing finding per label. """ if isinstance(node, str): return node.strip() if isinstance(node, list): return "[" + ", ".join(selector(v) for v in node) + "]" if isinstance(node, dict): return "{" + ", ".join(f"{k}: {selector(v)}" for k, v in node.items()) + "}" return str(node) def report(name: str, errors: list[str], ok_msg: str) -> int: if errors: print(f"{name}: {len(errors)} finding(s)", file=sys.stderr) for e in errors: print(f" {e}", file=sys.stderr) return 1 print(f"{name}: {ok_msg}") return 0 # -------------------------------------------------------------------------- # runners def check_runners() -> int: errors: list[str] = [] for path in workflows(): doc = load(path) trigger = events(doc) # pull_request_target runs with the BASE repo's secrets against the # HEAD's code. There is no configuration of it that is safe here, so it # is refused outright rather than conditioned. if "pull_request_target" in trigger: errors.append(f"{path.name}: pull_request_target is forbidden") # PR-REACHABLE, not just pull_request-triggered. Every heavy workflow in # this repo is `workflow_call`-only and reached from ci.yml, whose sole # trigger IS pull_request -- and a called workflow inherits the caller's # event payload, so `github.event.pull_request.head.repo.fork` is # populated inside it. A check that only looked for a literal # `pull_request:` trigger would therefore pass every single one of them # while checking nothing, which is precisely the vacuous-gate shape this # repo keeps re-learning. pr_reachable = bool(trigger & {"pull_request", "workflow_call"}) wf_jobs = jobs(doc) if not wf_jobs: if pr_reachable: errors.append(f"{path.name}: PR-reachable workflow declares no jobs") continue for job, node in wf_jobs: # A `uses:` job is a call into another workflow, which carries its # own runs-on and is checked as its own file. It owes no runner. if "uses" in node: continue if "runs-on" not in node: # Only a PR-reachable job OWES a runner declaration here: that is # the trust question. A schedule-only job without one is invalid # to GitHub itself and is not this check's business. if pr_reachable: errors.append( f"{path.name}:{job} is PR-reachable and declares no runs-on" ) continue runner = selector(node["runs-on"]) approved = SELF_HOSTED_ALLOWED and runner == APPROVED_SELECTOR if approved or HOSTED.fullmatch(runner): continue if pr_reachable: errors.append( f"{path.name}:{job} has runs-on: {runner} -- must be " "GitHub-hosted or the approved vars.POOL selector " "(see APPROVED_SELECTOR in ci/linter/workflow_policy.py)" ) continue # NOT PR-reachable, so there is no fork-trust question to answer -- # but the check still runs, because nothing else in the toolchain # looks at these selectors at all. # # actionlint validates runner labels only for a LITERAL `runs-on`. # Every selector here is a `fromJSON(...)` expression, so actionlint # goes QUIET on all of them -- and silence reads exactly like a pass. # Skipping schedule-only workflows would leave bump.yml and # ci-deep.yml (six selectors) unchecked by anything: measured # 2026-08-02 against the old inline labels, a typo in build-test.yml # was reported while the same edit in bump.yml and ci-deep.yml was # silent on both this check and actionlint. # # The pool label itself now lives in the `POOL` repo variable and is # no longer spellable here, so what this catches is a selector that # has drifted from the approved form -- an inlined label set, a # resurrected fork ternary, a hand-edited fallback. Those fail at # dispatch as a job that never picks up a runner, on a weekly # schedule nobody is watching. errors.append( f"{path.name}:{job} has runs-on: {runner} -- not the approved " "selector. This workflow is not PR-reachable, so the finding is " "about the SELECTOR SHAPE, not fork trust: nothing else " "validates it (actionlint skips non-literal runs-on) and a " "drifted selector is a job that never starts " "(see APPROVED_SELECTOR in ci/linter/workflow_policy.py)" ) return report( "lint-ci-runners", errors, "no pull_request_target; every runs-on is hosted or the vars.POOL selector", ) # -------------------------------------------------------------------------- # ports def _body(node: dict) -> str: """A job node re-serialised, for the substring questions asked below. The job SPLIT is structural (see jobs()); only the "does this job mention the driver / --port" questions are textual, and those are asked of this normalised dump rather than of the file. Two consequences, both wanted: a comment can no longer hide a job from the split -- `runtime: # note` used to yield ZERO jobs and a cheerful "no runtime-bearing jobs" -- and a port mentioned only in a comment no longer counts as a declaration. """ dumped = yaml.safe_dump(node, default_flow_style=False, sort_keys=False) # Shell comments inside a `run:` block survive the YAML round-trip, so prose # that merely NAMES the driver used to make a job runtime-bearing and demand # a band. Strip comments and non-binding invocations before the substring # questions below are asked. # # Split on ESCAPED newlines as well as real ones. safe_dump folds a # multi-line `run:` into one quoted scalar containing literal "\n", so a # naive per-dumped-line strip cuts at the first '#' and discards every real # command after it -- including the `--port $TEST_BASE_PORT` that proves a # job is wired. That direction fails OPEN, which is the worse one. kept = [] for line in re.split(r"\\n|\n", dumped): code = line.split("#", 1)[0] if NON_BINDING_RE.search(code): continue kept.append(code) return "\n".join(kept) def _steps(node: dict) -> list[str]: """Each step's `run:` text, in declaration order. Non-run steps keep their slot as an empty string so an index comparison stays an ORDER comparison.""" out: list[str] = [] for step in node.get("steps") or []: if not isinstance(step, dict): out.append("") continue run = step.get("run") out.append(run if isinstance(run, str) else "") return out def _order_finding(where: str, node: dict) -> str | None: """A band verifier that runs after something has already bound the band. The verify step is not a property of the job, it is a property of a POSITION: it can only speak for the steps below it. `build-test.yml` shipped it between `prove` and the runtime suite, which reads as guarded and left the first binder unguarded -- the failure max-port.sh exists to name still arrived inside `prove` as a bind error or a timeout with no cause attached. Only jobs that already carry the verifier are checked here; whether a binding job must carry one at all is the declaration check above. """ runs = _steps(node) verify = next((i for i, r in enumerate(runs) if BAND_VERIFIER in r), None) if verify is None: return None first_bind = next((i for i, r in enumerate(runs) if BINDER_RE.search(r)), None) if first_bind is None or first_bind > verify: return None # Same step: a multi-line `run:` block may legitimately verify and then bind. # Index order cannot separate those, so fall back to position within the text. if first_bind == verify: run = runs[verify] # Bound to a local so the None case is handled rather than assumed. # first_bind == verify means BINDER_RE already matched this step, so # `bind` cannot be None in practice -- but calling .start() straight off # the search result makes that an invariant mypy cannot see, and a # future edit to BINDER_RE would turn it into an AttributeError inside a # linter: exit 2 "could not run" for what is really a clean step. bind = BINDER_RE.search(run) if bind is None or run.index(BAND_VERIFIER) < bind.start(): return None return ( f"{where} binds the band earlier in the same step than it runs " f"{BAND_VERIFIER} (step {verify + 1}) -- verify first, then bind" ) return ( f"{where} runs {BAND_VERIFIER} at step {verify + 1}, after step " f"{first_bind + 1} has already bound the band -- the verifier only " "guards the steps below it, so move it above the FIRST binder" ) def check_ports() -> int: errors: list[str] = [] bands: dict[str, str] = {} # port value -> "file:job" that claimed it for path in workflows(): doc = load(path) for job, node in jobs(doc): body = _body(node) declared = re.search(r"(?m)^\s*TEST_BASE_PORT:\s*[\"']?(\d+)", body) starts_runtime = RUNTIME_DRIVER in body binds_band = BINDER_RE.search(body) is not None where = f"{path.name}:{job}" order = _order_finding(where, node) if order: errors.append(order) # THE CHECK THAT MATTERS MOST. A new runtime-bearing job added later # with no band is invisible to the uniqueness check below (it # declares nothing to collide), silently takes the driver's default # --port, and reintroduces exactly the cross-job collision the bands # exist to prevent: two jobs pinned to the same runner, disjoint # concurrency groups, nothing serialising them, both binding 18880. # Any BINDER (not just the runtime driver) owes this declaration -- # `prove` and coverage.sh are binders too (see BINDERS above), and a # job whose only binder is `prove` was exempt here while still being # treated as a binder by the ordering check. That gap is a live # negative-control failure downstream: deleting a prove-only job's # band left this check GREEN. if binds_band and not declared: errors.append( f"{where} binds a port (via " f"{RUNTIME_DRIVER if starts_runtime else 'prove/coverage.sh'}) " "without declaring TEST_BASE_PORT -- it would take the " "default port and collide with any other runtime job on " "the same runner" ) continue if not declared: continue port = declared.group(1) if port in bands: errors.append( f"{where} and {bands[port]} both claim TEST_BASE_PORT " f"{port} -- bands must be disjoint across ALL workflows" ) else: bands[port] = where # The same requirement for a `prove` binder. Test::Nginx reads # TEST_NGINX_PORT and knows nothing about TEST_BASE_PORT, so a job # can declare a unique band, satisfy every check above, and still # bind 1984. Distinct from the driver case only in which variable # carries the value: there it is an argument, here an env var. if PROVE_RE.search(body): nginx_port = re.search( r"(?m)^\s*TEST_NGINX_PORT:\s*[\"']?([^\"'\n]+)", body ) # Either spelling is correct: an expression referring to the # band, or the literal band value. What matters is that the # number prove binds is the one this job declared -- a # TEST_NGINX_PORT naming some OTHER port is the same defect as # not setting it at all. wired = nginx_port is not None and ( "TEST_BASE_PORT" in nginx_port.group(1) or nginx_port.group(1).strip() == port ) # TEST_NGINX_RANDOMIZE is the one case where a runtime-bearing # job legitimately does NOT bind its declared band. It is what # makes `prove -jN` safe: Test::Nginx's Util.pm gen_rand_port # picks a port PER PARALLEL JOB and binds only ports it has # proved free, so pinning TEST_NGINX_PORT would re-share the # resource the randomization just separated. The band is still # declared and still swept, which is what reserves the job's # territory on the shared runner. # # Reported by the nginx-cache-turbo-module adoption 2026-08-10; # that repo's prove job randomizes and this one does not, so the # gap was invisible here. if re.search(r"(?m)^\s*TEST_NGINX_RANDOMIZE:", body): wired = True if not wired: errors.append( f"{where} declares TEST_BASE_PORT but never passes it " "to prove as TEST_NGINX_PORT -- Test::Nginx does not " "read TEST_BASE_PORT, so it would bind its 1984 " "default anyway" ) # A declared band that is not passed through is decoration: the # driver still binds its default. if starts_runtime and "--port" not in body: errors.append( f"{where} declares TEST_BASE_PORT but never passes --port; " "the driver would bind its default anyway" ) if starts_runtime and "TEST_BASE_PORT" not in body.split("--port")[-1][:40]: errors.append( f"{where} passes --port with something other than " "$TEST_BASE_PORT -- the declaration and the bind must be " "the same value or they drift" ) return report( "lint-ci-ports", errors, f"{len(bands)} runtime job(s), all with distinct port bands" if bands else "no runtime-bearing jobs", ) # -------------------------------------------------------------------------- # docs def check_docs() -> int: """Every workflow is documented, and every documented workflow exists. The drift this catches is silent in both directions and neither direction fails anything else: a workflow added without a README row is a gate nobody knows exists (so nobody notices when it is later removed), and a README row for a deleted workflow is a badge that 404s and a claim of coverage the repo does not have. Structural facts only -- deliberately NOT exact job counts or durations, which are the brittle claims that get a drift check deleted. """ errors: list[str] = [] readme = ROOT / "README.md" if not readme.is_file(): print("lint-docs-drift: no README.md", file=sys.stderr) return 2 text = readme.read_text(encoding="utf-8") names = {p.name for p in workflows()} for name in sorted(names): if name not in text: errors.append( f"{name} exists under .github/workflows/ but is not mentioned " "in README.md -- an undocumented gate" ) # Only PATH-QUALIFIED references. A bare "ci.yml" in prose could mean any # file; ".github/workflows/ci.yml" is unambiguously a claim that this repo # has that workflow, which is the claim worth checking. for ref in sorted(set(re.findall(r"\.github/workflows/([\w.-]+\.ya?ml)", text))): if ref not in names: errors.append( f"README.md references .github/workflows/{ref}, which does not " "exist -- a dead link or a stale badge" ) return report( "lint-docs-drift", errors, f"{len(names)} workflow(s), all documented in README.md", ) # -------------------------------------------------------------------------- # cadence def check_cadence() -> int: """A `workflow_call` member carries no second entry point of its own. `workflow_call` does not suppress a member's own triggers. A member reached from ci.yml that ALSO carries `push:` runs twice per change: once on the PR, once on the merge commit, against a tree identical to the PR head that already passed. The two runs get different concurrency keys, so `cancel-in-progress` does not collapse them, and BOTH are green -- the only symptoms are the bill and a README that no longer describes what runs when. Nothing else catches this. Every member here is correct today, but that correctness is asserted only by a comment in each file ("No push/pull_request trigger here on purpose"), and a comment does not survive the next workflow copied in from a repo with a different topology. Downstream this cost five stray post-merge runs across six of seven members before review caught it. `schedule:` is explicitly allowed: codeql.yml and ci-deep.yml are reachable both from ci.yml and on their own cadence, which is the intended shape and not a duplicate run of the same tree. """ errors: list[str] = [] for path in workflows(): trigger = events(load(path)) if "workflow_call" not in trigger: continue for dupe in sorted(trigger & {"push", "pull_request"}): errors.append( f"{path.name} is a workflow_call member and also carries " f"`{dupe}:` -- it would run twice per change, on two " "concurrency keys that cannot cancel each other. Reach it " "from ci.yml only (schedule: is allowed)" ) return report( "lint-ci-cadence", errors, "every workflow_call member has ci.yml as its only PR entry point", ) # -------------------------------------------------------------------------- # provenance # # A step that extracts an archive or executes a downloaded artifact with no # nearby trust-anchor assertion is code execution on the self-hosted runner # the moment the origin, a release, or a poisoned actions/cache entry is # compromised -- audit sha e289021 F3, re-opened by direct-download steps # added after the ci-build.sh fix. A cache HIT is not exempt: a cached # tarball is exactly as untrusted as a fresh download until re-checked, so # this check does not care whether the step is gated by a # `cache-hit != 'true'` condition -- it looks at whether the JOB proves # trust somewhere before the extract/execute point, which a cache-skipped # download step cannot do on its own. # What counts as "this step unpacks or runs something fetched from outside # the repo". Deliberately textual (same _body()/_steps() shape as the ports # check): a `tar -xzf` or a direct `./configure`/binary invocation is the # actual privilege boundary, not the download line above it. EXTRACT_OR_EXEC_RE = re.compile( r"(? bool: """Return whether an extract-token spelling is not a shell invocation.""" line_start = run.rfind("\n", 0, token_start) + 1 while line_start: previous_end = line_start - 1 previous_start = run.rfind("\n", 0, previous_end) + 1 if not run[previous_start:previous_end].rstrip().endswith("\\"): break line_start = previous_start prefix = run[line_start:token_start] if prefix.lstrip().startswith("#"): return True logical_prefix = prefix.replace("\\\n", " ") return bool(re.search(r"\bapt(?:-get)?\s+install\b[^;&|\n]*$", logical_prefix)) # What counts as a trust-anchor assertion having been made ON THIS PATH # before the extract/exec point. Any one of: # - a detached-signature check (gpg --verify) # - a checksum COMPARISON -- `sha256sum -c`, or a digest captured into a # variable that is then tested against a pinned `*_SHA256` # - delegating to a helper this repo already verified for the SAME # property (ci-build.sh, fetch-verified-nginx.sh, # verify-nginx-tarball.sh) # # A bare `sha256sum tool.tgz` PRINTS a digest and compares nothing, and a # lone `FOO_SHA256:` env declaration is a pinned value nothing tests. Both # used to satisfy this regex, so a job could look anchored while validating # no bytes at all -- the anchor half failing open the same way the scoping # half did. Matching a comparison means an `if`/`test`/`[`/`||`/`&&` or a # `-c` check must appear with the digest, not merely the digest command. TRUST_ANCHOR_RE = re.compile( # detached signature r"gpg\b(?:\s+\S+)*\s+--\S*verify" # sha256sum -c / --check against a manifest r"|sha256sum\b[^\n]*\s-(?:c\b|-check\b)" # digest captured, then compared -- either order, same step or later r"|sha256sum\b[^\n]*\n?[^\n]*(?:\bif\b|\btest\b|\[|!=|==|\|\||&&)" # PowerShell: Get-FileHash captured, then compared (windows-build.yml:msvc # does `$actual = (Get-FileHash ...).Hash` then `if ($actual -ne $s.sha)`). # -ne/-eq are the comparison operators, not != / ==. r"|Get-FileHash\b[\s\S]*?(?:-ne\b|-eq\b|\bif\b)" r"|(?:\bif\b|\btest\b|\[|!=|==)[^\n]*_SHA256\b" r"|_SHA256\b[^\n]*(?:!=|==|\|\||&&)" # helpers whose own verification this repo already reviewed r"|ci-build\.sh|fetch-verified-nginx\.sh|verify-nginx-tarball\.sh" ) # A step that only DOWNLOADS and does not itself extract or execute is not a # finding on its own -- the extract/exec step downstream is what this check # anchors on. Restricting the scan to run: text that mentions a # download-shaped command keeps the check from firing on, say, a `tar -xzf` # of a repo-local artifact this job built itself (build-test.yml's ccache / # coverage tarballs never leave the runner and have no external origin to # forge). # # Every spelling that WRITES A FILE from the network must appear here, not # just the `-O`/`-o` ones: `wget ` with no -O (the idiom four of this # repo's own jobs use), `curl -O`/`-OJ`, and PowerShell's # `Invoke-WebRequest -OutFile` (windows-build.yml) all fetch to disk. An # omitted spelling does not make the check lenient -- it makes the whole JOB # unscanned, so a future unverified copy of that idiom is invisible rather # than flagged. That is a gate failing open, so the scoping regex is # deliberately broader than the extract/exec one. DOWNLOAD_RE = re.compile( r"\bwget\b|\bcurl\b(?:\s+\S+)*\s+-\S*[oO]\b|\bcurl\b[^\n]*\s-\S*O|" r"Invoke-WebRequest\b|actions/cache@" ) def check_provenance() -> int: """An extract/execute step with no trust-anchor assertion anywhere earlier in the same job is unverified code execution on the self-hosted runner. Scoped to jobs that show a download-shaped command (wget/curl writing a file, or an actions/cache restore feeding one) SOMEWHERE in the job -- a job with no external fetch at all has nothing for this check to gate. Within such a job, every extract/exec step must be preceded (same step or an earlier one, in source order) by a trust-anchor assertion: this check does not attempt to prove the anchor covers the SAME artifact byte for byte, only that the job asserts one at all before it unpacks or runs something -- the gap this exists to close is "no check ran", not "the check that ran was subtly wrong". Both halves have been tightened once already after review found them failing open: the scoping regex missed bare `wget ` / `curl -O` / `Invoke-WebRequest`, which skipped whole JOBS rather than passing them, and the anchor regex accepted a non-comparing `sha256sum` or an untested `*_SHA256` declaration. Each spelling has its own red fixture; add one with any future widening, or the claim in these comments outruns what the selftest actually proves. """ errors: list[str] = [] for path in workflows(): doc = load(path) for job, node in jobs(doc): runs = _steps(node) if not any(DOWNLOAD_RE.search(r) for r in runs): continue anchored = False for i, run in enumerate(runs): # Same-step trust anchors count only before extraction, in # the same order the shell consumes the commands. for extract in EXTRACT_OR_EXEC_RE.finditer(run): if is_package_or_comment_token(run, extract.start()): continue local = LOCAL_CODEQL_DATABASE_UNZIP_RE.search(run, extract.start()) if local is not None and local.start() == extract.start(): continue if not anchored and not TRUST_ANCHOR_RE.search( run[: extract.start()] ): errors.append( f"{path.name}:{job} step {i + 1} extracts or executes " "a downloaded artifact with no gpg/sha256 trust-anchor " "assertion earlier in the job -- verify before " "extraction/execution, cache-hit path included" ) # A same-step anchor only protects later commands. Recording # it before the extraction check lets `tar ...; verify ...` # satisfy the gate after untrusted bytes already executed. if TRUST_ANCHOR_RE.search(run): anchored = True return report( "lint-ci-provenance", errors, "every download-then-extract/execute step is trust-anchored first", ) # -------------------------------------------------------------------------- # secrets def check_secrets() -> int: """A member declares the secrets it needs, by name and typed; no `inherit`. `workflow_call: {}` with no `secrets:` stanza is not "this member needs no secrets" -- it is "this member can never receive one". An undeclared secret is simply not passed, so a member that starts needing a token gets an empty string and fails somewhere downstream of the cause: a checkout that 404s, a curl that 401s. Nothing says the secret was dropped at the call boundary. The stance this enforces is (a) of the two available: every member names the secrets it needs with `required: true`, and callers wire them explicitly. The rejected alternative is `secrets: inherit` at the caller, which is less boilerplate and widens every member's blast radius to the caller's FULL secret set -- including the ones that member has no business reading. A member's secret surface should be visible at the member. `required: true` over `required: false` is the load-bearing half. A false (or omitted) `required` restores exactly the silent-empty-string failure this check exists to prevent; true makes GitHub refuse to start the call. NOTE FOR AN ADOPTER: this repo declares no secrets, and that is the correct state for it -- every job is `contents: read` over a public tree. The check is therefore VACUOUSLY GREEN here by design, and its value is entirely in the moment an adopter adds the first one. It is not a no-op: the caller-side half below goes red on `secrets: inherit` and on a secret passed to a member that never declared it, both of which are reachable in this repo today. """ errors: list[str] = [] declared: dict[str, set[str]] = {} for path in workflows(): data = load(path) # Same `on:`-spelling quirk events() documents: PyYAML resolves the bare # key to True. Only the mapping form can carry a secrets: block. node = data.get("on", data.get(True)) # `workflow_call:` (null) and `workflow_call: {}` are the same workflow # to GitHub. Both must register as members, or a caller passing a secret # to a null-spelled member reads as "unknown member" and goes unchecked. if not isinstance(node, dict) or "workflow_call" not in node: continue call = node["workflow_call"] if call is not None and not isinstance(call, dict): raise PolicyError(f"{path.name}: workflow_call is not a mapping") secrets = (call or {}).get("secrets") or {} if not isinstance(secrets, dict): raise PolicyError(f"{path.name}: workflow_call.secrets is not a mapping") declared[path.name] = set(secrets) for name, spec in sorted(secrets.items()): # These two branches must not overlap. An earlier split tested # `"required" not in spec` here and the VALUE below, but every spec # missing the key yields None from .get(), and `None is not True`, # so the value branch decided both cases and deleting this one # changed no verdict -- a mutant that survived every fixture. This # branch now owns only the not-a-mapping shape, which .get() cannot # be asked about at all; the value branch owns every mapping. if not isinstance(spec, dict): errors.append( f"{path.name} declares secret `{name}` untyped -- give it " "`required: true`, or a member that stops being passed it " "fails on an empty string instead of refusing to start" ) elif spec.get("required") is not True: errors.append( f"{path.name} declares secret `{name}` with " # str(): a YAML-facing message must not print Python's # `False` for what the author wrote as `false`. f"`required: {str(spec.get('required')).lower()}` -- use " "`required: true`; " "an optional secret reintroduces the silent empty string" ) # Caller side. `uses: ./.github/workflows/X.yml` with a `secrets:` block. for path in workflows(): for job, spec in (load(path).get("jobs") or {}).items(): if not isinstance(spec, dict): continue uses = spec.get("uses") if not isinstance(uses, str): continue passed = spec.get("secrets") local = uses.startswith("./.github/workflows/") # `inherit` is judged BEFORE the local-member filter, because the # blast radius it hands over is worst for the calls this filter used # to drop: `owner/repo/.github/workflows/x.yml@ref` sends the # caller's entire secret set to ANOTHER REPOSITORY. Everything below # this point compares against `declared`, which only knows local # members, so the rest of the loop stays local-only. if passed == "inherit": errors.append( f"{path.name} job `{job}` uses `secrets: inherit` -- name " "the secrets the member needs instead; inherit hands it the " "caller's entire secret set, including ones it never reads" ) continue if passed is not None and not isinstance(passed, dict): # Shape is judged BEFORE the local-member filter, for the same # reason `inherit` is: `secrets:` that is neither a mapping nor # `inherit` is a caller GitHub will refuse, and dropping it here # because the member happens to live in another repository # reports CLEAN over a call that cannot work. The filter below # exists only to gate `declared` lookups, which are meaningless # for an external member -- it was never meant to gate validity. # # Without this, a bare string reaching `set(passed)` below is # iterated CHARACTER by character, reporting a finding per # letter -- red for the wrong reason. raise PolicyError( f"{path.name} job `{job}`: `secrets:` is neither a mapping " f"nor `inherit` ({type(passed).__name__})" ) if not local: continue member = uses.rsplit("/", 1)[-1] # The other direction: a member that REQUIRES a secret nobody wires. # GitHub refuses to start that call, which is the loud failure # `required: true` was chosen for -- but it fails on the first run # after merge, and the pairing is checkable here at review time. # `inherit` and every non-mapping shape have already been rejected # above, so `passed` here is a mapping or None -- None meaning the # caller wires nothing, which is exactly what this loop reports. supplied = set(passed) if passed is not None else set() for name in sorted(declared.get(member, set()) - supplied): errors.append( f"{path.name} job `{job}` calls {member}, which requires " f"secret `{name}`, but does not pass it -- GitHub refuses " "to start the call" ) if passed is None: continue if member not in declared: # Not our business here: `docs`/`cadence` cover missing members. continue for name in sorted(set(passed) - declared[member]): errors.append( f"{path.name} job `{job}` passes secret `{name}` to " f"{member}, which does not declare it -- it is silently " "dropped at the call boundary" ) total = sum(len(v) for v in declared.values()) return report( "lint-ci-secrets", errors, f"{len(declared)} workflow_call member(s), {total} typed secret(s), no inherit", ) COMMANDS = { "runners": check_runners, "ports": check_ports, "docs": check_docs, "cadence": check_cadence, "secrets": check_secrets, "provenance": check_provenance, } def main(argv: list[str]) -> int: if len(argv) != 2 or argv[1] not in COMMANDS: print(f"usage: {argv[0]} {{{'|'.join(COMMANDS)}}}", file=sys.stderr) return 2 if not WORKFLOWS.is_dir(): print(f"no {WORKFLOWS} -- wrong tree?", file=sys.stderr) return 2 try: return COMMANDS[argv[1]]() except PolicyError as exc: # 2, not 1: the check did not RUN. A workflow this file cannot parse is # also a workflow GitHub may read differently, so reporting "clean" or # even "findings" over the rest of the tree would be a claim the run # cannot support. print(f"workflow_policy: {exc}", file=sys.stderr) return 2 if __name__ == "__main__": raise SystemExit(main(sys.argv)) myguard-labs-nginx-zstd-module-3486b5b/ci/skeleton-findings.md000066400000000000000000000053721525232545500244050ustar00rootroot00000000000000# Skeleton adoption findings Findings from the `ci/PROMPT.md` skeleton-adoption rollout (phase 5, PR2) that do not fit the mechanical step acceptance criteria and need a record for step 33's reconciliation pass. ## Step 18 — windows-build.yml has no reference equivalent `windows-build.yml` (MSVC x64 static + MinGW-w64 x64 dynamic build+smoke) has no counterpart in `nginx-skeleton-module`. **Decision: keep as-is.** Reason: it gates something no other workflow in this module or the reference covers at all — the `NGX_WIN32` build paths, which no Linux job compiles, and the MSVC-vs-MinGW module-loading difference (static-only vs dynamic .so). Folding it into an existing member (e.g. `build-test.yml`) would mix a Windows-only toolchain into a Linux self-hosted-runner workflow for no benefit; it is not a duplicate of any reference gate under another name. It already carries a badge and a `## CI` table row (added before this phase); step 20 only had to reposition it to the end of both lists, after CI Deep. ## Step 17 — versions.env / sha256 pinning NOT ported wholesale The reference's `.github/versions.env` + `.github/scripts/{load-versions, compute-versions,fetch-verify}.sh` pin nginx/Angie by version string AND a committed sha256, fetched from the network. This module's OWN existing convention (`ci/tools/ci-build.sh`'s header, `build-test.yml`'s `resolve` job, `codeql.yml`/`valgrind.yml`/`fuzzing.yml`/ `security-scanners.yml`/`ci-deep.yml`) resolves nginx mainline dynamically at CI run time and verifies the tarball via PGP signature against vendored keys in `ci/tools/keys/` — deliberately chosen over a fetched sha256 pin. The `ci-build.sh` header explicitly documents this as a considered rejection of "the sibling nginx-skeleton-module repo['s]" sha256 approach, citing audit finding e289021-F3: fetching a verification digest from the same origin that serves the artifact gives an origin compromise the ability to substitute all three. Porting `versions.env` as specified ("also port, adapting paths") would install a second, competing version-pinning mechanism across 8 workflow files that already have a coherent, documented one — this is a judgment call beyond a mechanical port and was escalated to the supervisor rather than decided unilaterally by the phase-5 worker. See the worker's escalation packet for step 17/PR #129 (or its successor) for the decision once made. `.github/actions/build-cache/` and the three `.github/scripts/*.sh` were likewise left unported pending that decision, since they are wired to the `versions.env` convention specifically (`asan.yml`'s reference version loads `.github/scripts/load-versions.sh`; this module's ported `asan.yml` instead reuses the existing dynamic-resolve + PGP-verify pattern, so it does not depend on them). myguard-labs-nginx-zstd-module-3486b5b/ci/suppressions/000077500000000000000000000000001525232545500232065ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/suppressions/lsan.suppress000066400000000000000000000022261525232545500257530ustar00rootroot00000000000000# LeakSanitizer suppressions for the ASAN+UBSAN smoke tests. # # Scope: nginx CORE allocations that are still reachable when a worker is # terminated by SIGTERM. nginx does not free every allocation at exit by # design -- the process is about to die and the pool teardown is skipped -- # so these are exit-time noise, not leaks. # # The three below are the complete set observed on builder02 (56320 bytes in # 3 allocations, every run): the connection/event/read-write event arrays # allocated in ngx_event_process_init (src/event/ngx_event.c:755, :762, :774). # Every frame is pure nginx core; none of them touch this module. # # Deliberately NARROW: suppression is by allocating function, so a leak from # ngx_http_zstd_* -- or from any other core path -- is still reported and # still fails the step. Do NOT widen this to a blanket `leak:ngx_*`; that # would suppress the module's own allocations, which reach LSan through # ngx_palloc and friends, and turn the whole leak gate into a no-op. # # If a new core-exit allocation shows up, add it here with its file:line and # a note on why it is exit-time-reachable -- do not relax the pattern. leak:ngx_event_process_init myguard-labs-nginx-zstd-module-3486b5b/ci/suppressions/valgrind.suppress000066400000000000000000000044731525232545500266320ustar00rootroot00000000000000# Valgrind suppressions for nginx-zstd-module. # # DELIBERATELY EMPTY. Read this before adding a block. # # What was here until 2026-08-22: the 28-block generic nginx suppression file # that circulates online. Every block still carried the unedited # `` placeholder, and the set named # `ngx_http_lua_*` and `drizzle_state_connect` -- symbols this module does not # link. It had been adopted wholesale, never derived. # # Most of it was merely inert. One block was not: # # { # Memcheck:Cond # obj:* } # # `Memcheck:Cond` with `obj:*` matches a use-of-uninitialised-value conditional # in ANY object at ANY depth, including ngx_http_zstd_filter_module.so. It muted # that entire error class -- the class most likely to catch a real bug in this # module's header parsing -- across both the memcheck and helgrind lanes, since # ci/tools/soak.sh feeds this file to both. # # Measured, not inferred. A four-line C program reading one uninitialised int: # without suppressions valgrind exits 99, 1 uninitialised error # with that block alone valgrind exits 0, 0 errors # So `Memcheck lite (60s soak)` was reporting green by construction for the Cond # class, and would have kept doing so however long the soak ran. # # Empty is the correct default: it suppresses nothing and therefore hides # nothing. A soak that goes red on genuine nginx-core noise is a real result to # triage, not a reason to restore a file nobody derived. # # ADDING A BLOCK -- the rules, so this does not silently regrow: # # 1. Derive it, never copy it. ci/tools/soak.sh already passes # --gen-suppressions=all, so a real soak prints the exact block to paste. # 2. Name it. A placeholder name means nobody decided what is being # suppressed; that is the state this file was just rescued from. # 3. Anchor it to a named frame that is NOT ours -- an `ngx_*` core symbol or # a libc entry point reached from one. A block whose stack could match a # frame in src/ suppresses our own bug. # 4. Never use `obj:*`, and never let `...` be the only frame constraint. # Both match arbitrary code, which is a mute rather than a suppression. # 5. Say WHY in a comment above the block: which run produced it, and why the # finding is not this module's to fix. myguard-labs-nginx-zstd-module-3486b5b/ci/t/000077500000000000000000000000001525232545500206745ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/t/00-filter.t000066400000000000000000002722461525232545500226000ustar00rootroot00000000000000use Test::Nginx::Socket; use File::Basename; use lib 'lib'; my $dirname = dirname(__FILE__); # local: this process is the test run, but perlcritic is right that a bare # assignment to %ENV leaks into anything that runs after it. local $ENV{'TEST_NGINX_PERL_PATH'} = "$ENV{'PWD'}/$dirname"; my @dynamic_modules; if (defined $ENV{'TEST_NGINX_BINARY'}) { my $nginx_dir = dirname($ENV{'TEST_NGINX_BINARY'}); for my $module_name (qw(ngx_http_zstd_filter_module.so ngx_http_zstd_static_module.so)) { my $module_path = "$nginx_dir/$module_name"; push @dynamic_modules, $module_path if -f $module_path; } } add_block_preprocessor(sub { my $block = shift; return if !@dynamic_modules; my $main_config = join "\n", map { "load_module $_;" } @dynamic_modules; $block->set_value("main_config", $main_config); }); no_long_string(); log_level 'debug'; repeat_each(3); plan 'no_plan'; run_tests(); __DATA__ === TEST 1: zstd off --- config location /filter { zstd off; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- response_headers Content-Length: 59738 !Content-Encoding --- no_error_log [error] === TEST 2: zstd off (with accept-encoding header) --- config location /filter { zstd off; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter Accept-Encoding: gzip,zstd --- response_headers Content-Length: 59738 !Content-Encoding --- no_error_log [error] === TEST 3: zstd on --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: gzip, zstd --- response_headers !Content-Length Transfer-Encoding: chunked Content-Encoding: zstd Content-type: text/plain --- no_error_log [error] === TEST 4: zstd on (without accept-encoding header) --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- response_headers Content-Length: 59738 !Content-Encoding --- no_error_log [error] === TEST 5: zstd on (without zstd component in accept-encoding header) --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: gzip, br --- response_headers Content-Length: 59738 !Content-Encoding --- no_error_log [error] === TEST 5a: repeated Accept-Encoding accepts zstd on a later field line --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: gzip Accept-Encoding: zstd --- response_headers !Content-Length Transfer-Encoding: chunked Content-Encoding: zstd Content-type: text/plain --- no_error_log [error] === TEST 5b: repeated Accept-Encoding keeps explicit zstd;q=0 over wildcard --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: * Accept-Encoding: zstd;q=0 --- response_headers Content-Length: 59738 !Content-Encoding --- no_error_log [error] === TEST 6: zstd zstd_min_length (greater than min_length) --- config location /filter { zstd on; zstd_min_length 1024; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd, br --- response_headers !Content-Length Transfer-Encoding: chunked Content-Encoding: zstd Content-type: text/plain --- no_error_log [error] === TEST 7: zstd zstd_min_length (less than length) --- config location /filter { zstd on; zstd_types text/plain; zstd_min_length 60k; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd, br --- response_headers Content-Length: 59738 !Content-Encoding --- no_error_log [error] === TEST 8 zstd & gzip --- config location /filter { zstd on; zstd_min_length 1024; zstd_types text/plain; gzip on; gzip_min_length 1024; gzip_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd, gzip, br --- response_headers !Content-Length Transfer-Encoding: chunked Content-Encoding: zstd Content-type: text/plain --- no_error_log [error] === TEST 9 zstd & gzip (Accept-Encoding start with gzip) --- config location /filter { zstd on; zstd_min_length 1024; zstd_types text/plain; gzip on; gzip_min_length 1024; gzip_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: gzip, zstd, br --- response_headers !Content-Length Transfer-Encoding: chunked Content-Encoding: zstd Content-type: text/plain --- no_error_log [error] === TEST 10 zstd & gzip (hit gzip) --- config location /filter { zstd on; zstd_min_length 60k; zstd_types text/plain; gzip on; gzip_min_length 1024; gzip_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd, gzip, br --- response_headers !Content-Length Transfer-Encoding: chunked Content-Encoding: gzip Content-type: text/plain --- no_error_log [error] === TEST 11 zstd on (file does not exist) --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test2; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: gzip, br --- error_code: 404 === TEST 12 zstd off (file does not exist) --- config location /filter { zstd off; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test2; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: gzip, br --- error_code: 404 === TEST 13: RFC 7231 quality value - q=0 (explicitly reject) --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd;q=0, gzip;q=1 --- response_headers Content-Length: 59738 !Content-Encoding --- no_error_log [error] === TEST 14: RFC 7231 quality value - q=0.0 (explicitly reject) --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd;q=0.0, gzip;q=1 --- response_headers Content-Length: 59738 !Content-Encoding --- no_error_log [error] === TEST 15: RFC 7231 quality value - q=0.5 (accept with lower priority) --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd;q=0.5 --- response_headers !Content-Length Transfer-Encoding: chunked Content-Encoding: zstd Content-type: text/plain --- no_error_log [error] === TEST 16: RFC 7231 quality value - q=1.0 (highest priority) --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd;q=1.0 --- response_headers !Content-Length Transfer-Encoding: chunked Content-Encoding: zstd Content-type: text/plain --- no_error_log [error] === TEST 17: zstd with max_length (exceeds limit) --- config location /filter { zstd on; zstd_types text/plain; zstd_max_length 10k; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers Content-Length: 59738 !Content-Encoding --- no_error_log [error] === TEST 18: zstd with max_length (within limit) --- config location /filter { zstd on; zstd_types text/plain; zstd_max_length 100k; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers !Content-Length Transfer-Encoding: chunked Content-Encoding: zstd Content-type: text/plain --- no_error_log [error] === TEST 19: zstd compression level 3 --- config location /filter { zstd on; zstd_comp_level 3; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers !Content-Length Transfer-Encoding: chunked Content-Encoding: zstd Content-type: text/plain --- no_error_log [error] === TEST 20: zstd compression level 10 (high) --- config location /filter { zstd on; zstd_comp_level 10; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers !Content-Length Transfer-Encoding: chunked Content-Encoding: zstd Content-type: text/plain --- no_error_log [error] === TEST 21: zstd with multiple content types --- config location /filter { zstd on; zstd_types text/plain text/html application/json; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers !Content-Length Transfer-Encoding: chunked Content-Encoding: zstd Content-type: text/plain --- no_error_log [error] === TEST 22: zstd - mixed quality values (prefer highest) --- config location /filter { zstd on; zstd_types text/plain; gzip on; gzip_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd;q=0.9, gzip;q=0.8 --- response_headers !Content-Length Transfer-Encoding: chunked Content-Encoding: zstd Content-type: text/plain --- no_error_log [error] === TEST 23: zstd - gzip preferred via quality --- config location /filter { zstd on; zstd_types text/plain; gzip on; gzip_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd;q=0.5, gzip;q=0.9 --- response_headers !Content-Length Transfer-Encoding: chunked Content-Encoding: zstd Content-type: text/plain --- no_error_log [error] === TEST 24: zstd filter preserves HEAD pass-through behaviour --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request HEAD /filter --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd !Content-Length --- no_error_log [error] === TEST 25: zstd filter skips 204 responses --- config location /filter { zstd on; zstd_types text/plain; return 204; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- error_code: 204 --- response_headers !Content-Encoding --- no_error_log [error] === TEST 26: zstd filter skips 205 responses --- config location /filter { zstd on; zstd_types text/plain; return 205; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- error_code: 205 --- response_headers !Content-Encoding --- no_error_log [error] === TEST 27: zstd filter skips 304 responses --- config location /filter { zstd on; zstd_types text/plain; return 304; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- error_code: 304 --- response_headers !Content-Encoding --- no_error_log [error] === TEST 28: zstd filter compresses 403 responses above min_length --- config location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; default_type text/plain; return 403 "forbidden body\n"; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- error_code: 403 --- response_headers !Content-Length Transfer-Encoding: chunked Content-Encoding: zstd Content-Type: text/plain --- no_error_log [error] === TEST 29: zstd filter compresses 404 responses above min_length --- config location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; default_type text/plain; return 404 "not found body\n"; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- error_code: 404 --- response_headers !Content-Length Transfer-Encoding: chunked Content-Encoding: zstd Content-Type: text/plain --- no_error_log [error] === TEST 29a: zstd filter compresses 410 responses above min_length # 410 Gone joins 403/404 as an error status with a compressible body, # matching what nginx/nginx#1466 (approved for 1.31.6) proposes for core # gzip. --- config location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; default_type text/plain; return 410 "gone body\n"; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- error_code: 410 --- response_headers !Content-Length Transfer-Encoding: chunked Content-Encoding: zstd Content-Type: text/plain --- no_error_log [error] === TEST 29b: a 409 response stays uncompressed (the 4xx exceptions are a list, not a range) --- config location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; default_type text/plain; return 409 "conflict body\n"; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- error_code: 409 --- response_headers !Content-Encoding Content-Type: text/plain --- no_error_log [error] === TEST 30: no infinite loop / CPU spin on a zero-length proxied body # Regression for the recurring "100% CPU infinite loop" class: # 7f86e5b, 2af5889, 924c9bf, PR #23/#49. # An empty upstream body with Content-Encoding still set must terminate # (emit a valid empty zstd frame) and not spin. Test::Nginx enforces a # request timeout, so a hang fails the test instead of running forever. --- config location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/empty; } location /empty { default_type text/plain; return 200 ""; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- error_code: 200 --- timeout: 5 --- no_error_log [error] === TEST 31: no infinite loop on a single-byte body below the stream-in size # Same loop class — a tiny body must flush a terminal frame and stop. --- config location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/one; } location /one { default_type text/plain; return 200 "x"; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- error_code: 200 --- timeout: 5 --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 32: $zstd_ratio computation path on a large body (overflow guard) # Regression for 064895c "integer overflow in compression ratio calc". # $zstd_ratio is a log-phase variable; its value is asserted to be a # finite N.NNN string by tools/test_encoding.py (which can read it). Here # we exercise the computation path itself — a ~58 KB body makes # bytes_in*1000 large, the exact arithmetic that overflowed pre-064895c. # A clean compressed response with no error proves the math did not trap. --- config location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; set $unused $zstd_ratio; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 33: zstd composes correctly with sub_filter (filter ordering) # Regression for the recurring filter-order class: f4ba115, 2d2e641, # cae80f9, 3f73e15, 8a6e370, 18c778d. zstd must run AFTER sub_filter so # the substitution is present in the (decompressed) output, not skipped. --- config location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; sub_filter 'ORIGINAL' 'REWRITTEN'; sub_filter_once off; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/src; } location /src { default_type text/plain; return 200 "ORIGINAL ORIGINAL ORIGINAL\n"; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 34: negative compression level produces a valid zstd stream # Regression for cc9f6ec / b58c7cd: negative levels are accepted by # zstd_comp_level but were never exercised by a test. --- config location /filter { zstd on; zstd_comp_level -5; zstd_min_length 1; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 35: explicit non-default zstd_types only compresses listed types # Regression for 46f95bf "passed default mime/types to zstd_types parser": # a type NOT in the list must not be compressed. --- config location /json { zstd on; zstd_min_length 1; zstd_types application/json; default_type text/plain; return 200 "plain text not in zstd_types\n"; } --- request GET /json --- more_headers Accept-Encoding: zstd --- response_headers !Content-Encoding --- no_error_log [error] === TEST 36: zstd_types match DOES compress the listed type # Positive half of TEST 35 — application/json is listed, so it compresses. --- config location /json { zstd on; zstd_min_length 1; zstd_types application/json; default_type application/json; return 200 "{\"compress\":\"this is a json body long enough\"}\n"; } --- request GET /json --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 37: max_length enforced when Content-Length is known (proxy) # Pins the documented contract from d94b220 / f065cb6: when the response # length IS known (the common proxied case), a body larger than # zstd_max_length must NOT be compressed. The complementary "length # unknown / chunked -> cannot be enforced" half is a documented behaviour # (see README) not cleanly unit-testable via `return` (which always sets # Content-Length); it is covered by the docs, not by a brittle test. --- config location /filter { zstd on; zstd_min_length 1; zstd_max_length 4; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/big; } location /big { default_type text/plain; return 200 "this body is far larger than the 4 byte max_length\n"; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers !Content-Encoding --- no_error_log [error] === TEST 38: zstd_window_log caps the window and still produces valid output # Regression for the zstd_window_log memory-bounding directive. With a # 15-bit (32 KB) window and a body well over 32 KB, zstd must still emit # a well-formed stream: the directive bounds per-request memory, it must # not corrupt the response. Served from the on-disk test fixture (~58 KB) # so the capped window is genuinely exercised. --- config location /filter { zstd on; zstd_min_length 1; zstd_window_log 15; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 39: $zstd_bytes_in / $zstd_bytes_out are emitted for a compressed response # Asserts the byte-count log variables. $zstd_bytes_in and # $zstd_bytes_out are log-phase variables backed by ctx->bytes_in/out # (the same counters $zstd_ratio derives from). Referencing them via # `set` exercises the get_handler; a clean compressed response with no # error proves the handler resolves both fields without faulting. # Exact-value correctness (and consistency with $zstd_ratio) is verified # end-to-end in tools/test_encoding.py, which can read the access log. --- config location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; set $unused_in $zstd_bytes_in; set $unused_out $zstd_bytes_out; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 40: zstd_bypass skips compression when the predicate is truthy # A request header maps to a bypass variable; when it is set to a # non-empty value other than "0", the response must be served identity # even though the client supports zstd and the type/size qualify. --- http_config map $http_x_no_zstd $zstd_off { default 0; "1" 1; } --- config location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; zstd_bypass $zstd_off; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd X-No-Zstd: 1 --- response_headers Content-Length: 59738 !Content-Encoding --- no_error_log [error] === TEST 41: zstd_bypass value "0" does NOT bypass (still compresses) # Same config, but the bypass variable resolves to "0", which per # ngx_http_test_predicates is falsy — compression must still happen. # Pins the "0"/empty == not-bypassed contract. --- http_config map $http_x_no_zstd $zstd_off { default 0; "1" 1; } --- config location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; zstd_bypass $zstd_off; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers !Content-Length Content-Encoding: zstd --- no_error_log [error] === TEST 42: zstd_max_length is enforced on a chunked upstream (no Content-Length) # Regression for the length-independent input cap. TEST 37 covers the # known-Content-Length case (rejected in the header filter). This covers # the genuine DoS vector: an upstream that streams chunked with NO # Content-Length, so the header-filter check is skipped. A mock TCP # backend returns a chunked body far larger than zstd_max_length; # compression has already started, so the only safe action is to abort # the request — the worker must not be fed unbounded input. We assert # the dedicated abort message is logged. --- config location /filter { zstd on; zstd_min_length 1; zstd_max_length 100; zstd_types text/plain; proxy_http_version 1.1; proxy_buffering off; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/up; } location /up { proxy_http_version 1.1; proxy_pass http://127.0.0.1:$TEST_NGINX_RAND_PORT_1/; } --- tcp_listen: $TEST_NGINX_RAND_PORT_1 --- tcp_no_close --- tcp_reply eval "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n" . sprintf("%x\r\n", 5000) . ("A" x 5000) . "\r\n0\r\n\r\n" --- request GET /filter --- more_headers Accept-Encoding: zstd --- ignore_response --- error_log input exceeded zstd_max_length (100) after === TEST 43: known Content-Length response round-trips (pledged-src-size) # Regression for the ZSTD_CCtx_setPledgedSrcSize optimisation. A # proxied response with an exact Content-Length takes the pledged-size # path in init_cctx; an off-by-anything pledge would make # ZSTD_compressStream2 error or corrupt the stream. Assert the response # is zstd-encoded, chunked (filter strips the length), and crucially # decompresses back to the exact original bytes. --- config location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/src; } location /src { default_type text/plain; return 200 "pledged-src-size round-trip body, long enough to compress and exercise the known-Content-Length path end to end\n"; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers !Content-Length Content-Encoding: zstd --- response_body_filters eval sub { my $zstd = $_[0]; require File::Temp; my ($tfh, $tmp) = File::Temp::tempfile("zstd_t43_XXXXXX", TMPDIR => 1, UNLINK => 1); binmode($tfh); print $tfh $zstd; close($tfh); # List-form pipe open: no shell, so the temp path is never re-parsed. open(my $r, "-|", "zstd", "-dqc", $tmp) or do { unlink $tmp; return "ERR" }; local $/; my $d = <$r>; close($r); unlink $tmp; return $d; } --- response_body pledged-src-size round-trip body, long enough to compress and exercise the known-Content-Length path end to end === TEST 44: chunked no-Content-Length body > one ZSTD_CStreamOutSize buffer round-trips # Regression for the multi-output-buffer use-after-free / NULL-deref: # get_buf()'s early return ("buffer_out not full -> keep current out_buf") # did not check that ctx->out_buf was still non-NULL. On a chunked / # no-Content-Length response large enough to need more than one # ZSTD_CStreamOutSize output buffer, the body filter's recycle guard # (ctx->out_buf = NULL after ngx_chain_update_chains) left out_buf NULL # while buffer_out still looked non-full; the next compress() dereferenced # it ("ctx->out_buf->last += ...") -> worker SIGSEGV, the response # truncated at exactly 131072 decoded bytes with no zstd end-of-frame. # Single-buffer responses never recycle so never crashed, which is why # the homepage worked but wp-admin (large chunked CSS) broke in prod. # # A mock TCP backend streams a chunked body with NO Content-Length, far # larger than one ~128 KB output buffer and highly compressible. The # response must come back zstd-encoded, decompress cleanly (no premature # end), and equal the original byte-for-byte (asserted via a len:md5 # checksum so the body need not be inlined). A pre-fix module crashes # the worker here / yields a short body; the fixed module round-trips. --- config location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; proxy_http_version 1.1; proxy_buffering on; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/up; } location /up { proxy_http_version 1.1; proxy_pass http://127.0.0.1:$TEST_NGINX_RAND_PORT_1/; } --- tcp_listen: $TEST_NGINX_RAND_PORT_1 --- tcp_no_close --- tcp_reply eval my $unit = "ABCDEFGHIJ0123456789zstd-multibuffer-regression-payload-"; my $body = $unit x 5000; # ~290 KB, >2x ZSTD_CStreamOutSize my $hdr = "HTTP/1.1 200 OK\r\n" . "Content-Type: text/plain\r\n" . "Transfer-Encoding: chunked\r\n" . "Connection: close\r\n\r\n"; my $out = $hdr; # emit in 8 KB chunks so it is genuinely chunked, no Content-Length for (my $i = 0; $i < length($body); $i += 8192) { my $c = substr($body, $i, 8192); $out .= sprintf("%x\r\n", length($c)) . $c . "\r\n"; } $out .= "0\r\n\r\n"; $out --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers !Content-Length Content-Encoding: zstd --- response_body_filters eval sub { my $zstd = $_[0]; require File::Temp; my ($tfh, $tmp) = File::Temp::tempfile("zstd_t44_XXXXXX", TMPDIR => 1, UNLINK => 1); binmode($tfh); print $tfh $zstd; close($tfh); # List-form pipe open: no shell, so the temp path is never re-parsed. open(my $r, "-|", "zstd", "-dqc", $tmp) or do { unlink $tmp; return "ERR-OPEN" }; local $/; my $d = <$r>; close($r); my $rc = $?; unlink $tmp; return "ERR-DECODE rc=$rc" if $rc != 0; # premature end -> non-zero require Digest::MD5; return length($d) . ":" . Digest::MD5::md5_hex($d); } --- response_body eval my $unit = "ABCDEFGHIJ0123456789zstd-multibuffer-regression-payload-"; my $body = $unit x 5000; require Digest::MD5; length($body) . ":" . Digest::MD5::md5_hex($body) === TEST 45: zstd_max_cctx_memory rejects parameters that exceed the budget # Per-request CCtx memory hardening: a budget of 1 KB with level 19 is # wildly insufficient (level 19 needs ~80–90 MB), so nginx must refuse # to start. The same test also exercises the no-STATIC_LINKING_ONLY # build path: that build cannot honour the directive and rejects it # unconditionally with a different but equally clear message. Either # way, must_die. --- config location /filter { zstd on; zstd_min_length 1; zstd_comp_level 19; zstd_max_cctx_memory 1k; zstd_types text/plain; return 200 "x"; } --- request GET /filter --- must_die === TEST 46: Accept-Encoding "notzstd, zstd" still negotiates zstd # Regression for the multi-occurrence parser fix. The first "zstd" # substring lives inside the unrelated token "notzstd", which the # delimiter check correctly rejects; the parser must then walk on to # the next list element, find the standalone "zstd" token, and accept # the encoding. Pre-fix this returned identity because only the first # occurrence was examined. --- config location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; return 200 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; } --- request GET /filter --- more_headers Accept-Encoding: notzstd, zstd --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 47: Vary: Accept-Encoding is emitted when gzip_vary is on # The filter sets r->gzip_vary = 1 whenever a request enters a # zstd-enabled location, but only the core nginx code actually emits # the "Vary: Accept-Encoding" header — and only when gzip_vary is on. # Without this header shared caches (CDNs, reverse proxies) cannot # distinguish a zstd-encoded variant from the identity one and will # serve the wrong body to clients that do not accept zstd. --- config gzip_vary on; location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; return 200 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd Vary: Accept-Encoding --- no_error_log [error] === TEST 48: Vary: Accept-Encoding is emitted even when the client does not accept zstd # Same location as TEST 47, but the client only accepts gzip. The # response is identity, yet the Vary header must still appear so that # downstream caches keep zstd and identity variants apart. This locks # the "set gzip_vary before negotiating the encoding" behaviour. --- config gzip_vary on; location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; return 200 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; } --- request GET /filter --- more_headers Accept-Encoding: gzip --- response_headers !Content-Encoding Vary: Accept-Encoding --- no_error_log [error] === TEST 48a: Vary: Accept-Encoding is emitted with gzip_vary OFF (accepting client) # G5, the cell that used to be broken. No "gzip_vary" anywhere — its # compiled-in default is off, which is what most deployments actually # run. Before G5 this response was zstd-encoded with NO Vary header at # all: a shared cache stored it and served that zstd body to clients # that cannot decode zstd. The module now emits the field itself, so # correctness no longer depends on an operator directive it does not # own. Paired with TEST 47 (gzip_vary on) this is the dynamic half of # the gzip_vary x Accept-Encoding matrix. --- config location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; return 200 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd Vary: Accept-Encoding --- no_error_log [error] === TEST 48b: Vary: Accept-Encoding is emitted with gzip_vary OFF (non-accepting client) # The identity arm of TEST 48a, and the more dangerous fill order: a # cache that stores THIS response without Vary pins every later client # to identity, silently losing compression for everyone. Emitting the # field before negotiating the encoding is what keeps the two variants # apart, with gzip_vary off. --- config location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; return 200 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; } --- request GET /filter --- more_headers Accept-Encoding: gzip --- response_headers !Content-Encoding Vary: Accept-Encoding --- no_error_log [error] === TEST 49: zstd_buffers with a small custom value still produces a valid stream # The zstd_buffers directive was previously not exercised by any test. # A very small buffer count forces the body filter through the # multi-buffer output path on a single response, so this also guards # against regressions in chunk accounting under buffer pressure. --- config location /filter { zstd on; zstd_min_length 1; zstd_buffers 4 4k; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers !Content-Length Transfer-Encoding: chunked Content-Encoding: zstd --- no_error_log [error] === TEST 50: zstd_target_cblock_size produces a valid zstd stream # Locks the target_cblock_size advanced parameter path. The size is # small enough to force the encoder to honour the cap on a sizeable # input. On libzstd < 1.5.6 this directive only logs a warning at # config time and is otherwise ignored, but the response must still # be a well-formed zstd stream either way. --- config location /filter { zstd on; zstd_min_length 1; zstd_target_cblock_size 4k; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 51: zstd_long on compresses cleanly # Long-range mode enables the zstd long-distance matcher. The flag # was previously configured but never exercised in tests, so a silent # regression in the parameter wiring would have gone unnoticed. A # response that is large enough to be worth compressing is enough to # prove the parameter path stays well-formed. --- config location /filter { zstd on; zstd_min_length 1; zstd_long on; zstd_window_log 17; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 52: unknown-length (chunked) body is eligible regardless of size # TEST 6/7 cover min_length on the known-Content-Length path. A response with # no Content-Length takes the other branch: there is no length to test, so the # min_length gate is skipped and the body is ALWAYS eligible — even when it is # smaller than zstd_min_length (documented behaviour). The previous version of # this test used "return 200" upstream, which sets a Content-Length and so # never exercised this path at all (it was rejected by the known-length gate). # Use a raw chunked HTTP/1.1 backend so the response reaching the filter # genuinely has no Content-Length. --- config location /filter { zstd on; zstd_min_length 4096; zstd_types text/plain; proxy_http_version 1.1; proxy_set_header Connection ""; proxy_pass http://127.0.0.1:$TEST_NGINX_RAND_PORT_1/; } --- raw_request eval "GET /filter HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\nAccept-Encoding: zstd\r\n\r\n" --- tcp_listen: $TEST_NGINX_RAND_PORT_1 --- tcp_reply eval "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n4\r\ntiny\r\n0\r\n\r\n" --- response_headers !Content-Length Content-Encoding: zstd --- no_error_log [error] === TEST 53: zstd_comp_level 1 (minimum) produces a valid stream # Boundary coverage for the comp_level slot. Existing tests use 3, # 10, -5, and 19. Level 1 is the documented minimum positive level # and the fastest setting; an off-by-one in the bounds post-handler # would have caught here. --- config location /filter { zstd on; zstd_min_length 1; zstd_comp_level 1; zstd_types text/plain; return 200 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 54: zstd_comp_level 22 (maximum) produces a valid stream # Boundary coverage for the comp_level slot at libzstd's documented # maximum. The body is intentionally small so the test stays cheap; # the assertion is that the encoder accepts the level and emits a # valid stream, not that it produces any specific ratio. --- config location /filter { zstd on; zstd_min_length 1; zstd_comp_level 22; zstd_types text/plain; return 200 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 55: Accept-Encoding parser tolerates tab OWS around the q-value # RFC 7230 OWS is "*( SP / HTAB )". Most clients only ever send # spaces, but the parser is required to accept tabs too. The # rewritten parser walks OWS explicitly; this locks that contract. --- config location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; return 200 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; } --- request GET /filter --- more_headers Accept-Encoding: zstd ; q=0.5 --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 56: Accept-Encoding parser is case-insensitive on the coding name # The HTTP coding names are token-class, which RFC 7231 treats as # case-insensitive. A client that sends "ZSTD" must still negotiate # zstd. ngx_strncasecmp inside the parser is what makes this work; # this locks that we keep using the case-insensitive comparator. --- config location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; return 200 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; } --- request GET /filter --- more_headers Accept-Encoding: ZSTD --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 57: Accept-Encoding parser ignores stray empty list elements # RFC 7230 allows empty list members (",,zstd,,"). The parser must # skip them and still match the standalone zstd token. This guards # the OWS-and-comma skip loop at the top of the per-element walk. --- config location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; return 200 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; } --- request GET /filter --- more_headers Accept-Encoding: ,, zstd ,, --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 58: subrequests are never zstd-encoded # The filter explicitly returns NGX_DECLINED for r != r->main: only # the top-level response gets compressed, never the body of an # auth-request, addition_module, or SSI subrequest. This drives an # auth_request subrequest whose own location has zstd on; the # subrequest's response body must NOT be returned with # Content-Encoding: zstd (and the outer response, which returns 204 # anyway, must not gain one either). --- config location = /auth { internal; zstd on; zstd_min_length 1; zstd_types text/plain; return 204; } location /filter { auth_request /auth; return 204; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers !Content-Encoding --- error_code: 204 --- no_error_log [error] === TEST 59: zstd directive inside an if-block compiles and applies # The "zstd" command is registered with NGX_HTTP_LIF_CONF, so it must # be usable inside an "if (...) { ... }" block. This proves the # directive parses in that context and that the resulting location's # merged config takes effect (the if-branch turns zstd off while the # enclosing location had it on). --- config location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; if ($arg_nozstd) { zstd off; } return 200 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; } --- request GET /filter?nozstd=1 --- more_headers Accept-Encoding: zstd --- response_headers !Content-Encoding --- no_error_log [error] === TEST 60: filter bails out when the upstream has already set Content-Encoding # If an upstream response already carries Content-Encoding (here: # identity-with-a-pre-set-header, simulated by add_header on a # proxied response), the zstd filter must not re-encode it. The # response should keep the upstream's encoding marker and the body # should NOT be wrapped in a zstd frame. --- config location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/upstream; } location /upstream { add_header Content-Encoding "identity"; return 200 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: identity --- no_error_log [error] === TEST 61: RFC 9110 wildcard "*" makes zstd acceptable # Per RFC 9110 §12.5.3 "*" matches any coding not explicitly listed, so a # client sending only "*" accepts zstd. (The pre-RFC2 parser ignored "*".) --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: * --- response_headers !Content-Length Transfer-Encoding: chunked Content-Encoding: zstd --- no_error_log [error] === TEST 62: explicit zstd;q=0 overrides a permissive wildcard # An explicit "zstd" token decides the result even against "*;q=1". --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd;q=0, *;q=1 --- response_headers Content-Length: 59738 !Content-Encoding --- no_error_log [error] === TEST 63: malformed qvalue with trailing junk (q=1x) is not acceptable --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd;q=1x --- response_headers Content-Length: 59738 !Content-Encoding --- no_error_log [error] === TEST 64: qvalue with a fourth decimal digit (q=0.0001) is malformed --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd;q=0.0001 --- response_headers Content-Length: 59738 !Content-Encoding --- no_error_log [error] === TEST 65: 206 Partial Content is not compressed (Content-Range preserved) # RFC4: an upstream 206 carries a Content-Range computed against its selected # representation; the filter must not apply a new content coding to it. --- config location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; proxy_http_version 1.1; proxy_set_header Connection ""; proxy_pass http://127.0.0.1:$TEST_NGINX_RAND_PORT_1/; } --- raw_request eval "GET /filter HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\nAccept-Encoding: zstd\r\n\r\n" --- tcp_listen: $TEST_NGINX_RAND_PORT_1 --- tcp_reply eval "HTTP/1.1 206 Partial Content\r\nContent-Type: text/plain\r\nContent-Range: bytes 0-9/100\r\nContent-Length: 10\r\nConnection: close\r\n\r\nAAAAAAAAAA" --- response_headers !Content-Encoding Content-Range: bytes 0-9/100 --- error_code: 206 --- no_error_log [error] === TEST 66: zstd_bypass_vary appends the named field to Vary # S1: header-driven bypass must be advertised to shared caches via Vary. # G5: this location has no "gzip_vary on", and the module now emits # "Vary: Accept-Encoding" itself rather than depending on that directive, # so the compressed response carries both fields. Before G5 it carried # only X-No-Compression — a zstd body that never told a shared cache it # was negotiated on Accept-Encoding. --- config location /filter { zstd on; zstd_types text/plain; zstd_bypass $http_x_no_compression; zstd_bypass_vary X-No-Compression; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd Vary: X-No-Compression, Accept-Encoding --- no_error_log [error] === TEST 67: zstd_target_cblock_size is accepted and still produces a valid stream # C1: on libzstd >= 1.5.6 the directive applies; on older it is a warned no-op. # Either way the config must load and the response must be a valid zstd stream. --- config location /filter { zstd on; zstd_types text/plain; zstd_target_cblock_size 4096; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 68: zstd_bypass identity arm still advertises Vary # S1: when the bypass predicate fires (X-No-Compression present), the response # must be served identity (no Content-Encoding: zstd) yet STILL carry # Vary: X-No-Compression so a shared cache keys the bypassed variant separately # from the compressed one. This is the cache-poisoning arm TEST 66 omitted. # G5: the bypassed identity response must ALSO carry Accept-Encoding. The # same URI serves zstd to a request that does not trip the predicate, so # an identity body cached without Accept-Encoding in the key is served to # clients that should have been compressed (and vice versa on the reverse # fill order). This location has no "gzip_vary on"; the module emits the # field itself. --- config location /filter { zstd on; zstd_types text/plain; zstd_bypass $http_x_no_compression; zstd_bypass_vary X-No-Compression; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd X-No-Compression: 1 --- response_headers Content-Encoding: Vary: X-No-Compression, Accept-Encoding --- no_error_log [error] === TEST 68b: below zstd_min_length never reaches the bypass decision, no bypass Vary # A31b-F4: zstd_bypass_vary must key only the responses for which this # module actually reaches the zstd_bypass fork (the compressed and # bypassed-identity variants). A response declined earlier by an # eligibility gate -- here, body smaller than zstd_min_length -- would be # identity regardless of the bypass predicate's value, so no bypass-driven # cache variance exists and no "Vary: X-No-Compression" line is added. # Accept-Encoding is still absent: this location has no gzip_vary and the # module's own Vary: Accept-Encoding push also sits below this gate. --- config location /filter { zstd on; zstd_types text/plain; zstd_min_length 60k; zstd_bypass $http_x_no_compression; zstd_bypass_vary X-No-Compression; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd X-No-Compression: 1 --- response_headers Content-Encoding: !Vary --- no_error_log [error] === TEST 68c: content type outside zstd_types never reaches the bypass decision, no bypass Vary # A31b-F4 negative control for the content-type eligibility gate: same # claim as TEST 68b, different gate. The fixture is served as text/plain # by default; restricting zstd_types to a type it does not match makes the # content-type gate the one that declines, above the bypass_vary push. --- config location /filter { zstd on; zstd_types application/json; zstd_bypass $http_x_no_compression; zstd_bypass_vary X-No-Compression; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd X-No-Compression: 1 --- response_headers Content-Encoding: !Vary --- no_error_log [error] === TEST 69: quoted-string in coding-name position does not fabricate zstd # A quoted-string can never be a valid coding. With the name scan not # stopping at '"', the comma inside `"a,zstd "` split the element and the # trailing `zstd ` was mis-read as a real coding (phantom-token accept). The # whole quoted blob is a single non-coding element and MUST decline. This is # the coding-NAME-position arm of the quoted-comma class; TEST in the fuzz # corpus (30_quoted_name_phantom) gates it under the differential oracle too. --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: "a,zstd ";q=1 --- response_headers Content-Length: 59738 !Content-Encoding --- no_error_log [error] === TEST 70: a real zstd element after a quoted-string element still negotiates # Guard against over-fixing TEST 69: `"a",zstd` is a quoted-string element # (declined) followed by a separate, valid `zstd` coding element — that zstd # token must still be honoured. Confirms the name-scan '"' stop ends the # quoted element rather than swallowing the following comma-separated token. --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: "a",zstd --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 71: omitted directives use the web MIME defaults at the 1024-byte boundary # application/json is in the default type list. The response is exactly 1024 # bytes, proving the default zstd_min_length boundary is inclusive. --- config location /json { zstd on; default_type application/json; return 200 $arg_body; } --- request eval "GET /json?body=" . ("x" x 1024) --- more_headers Accept-Encoding: zstd --- response_headers !Content-Length Transfer-Encoding: chunked Content-Encoding: zstd Content-Type: application/json --- no_error_log [error] === TEST 72: omitted zstd_min_length skips a 1023-byte HTML response # text/html was already the module's default type, so this isolates the # changed 1024-byte minimum length gate from the expanded MIME-type list. --- config location /html { zstd on; default_type text/html; return 200 $arg_body; } --- request eval "GET /html?body=" . ("x" x 1023) --- more_headers Accept-Encoding: zstd --- response_headers Content-Length: 1023 !Content-Encoding Content-Type: text/html --- no_error_log [error] === TEST 73: omitted zstd_types includes text/csv --- config location /csv { zstd on; default_type text/csv; return 200 $arg_body; } --- request eval "GET /csv?body=" . ("x" x 1024) --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd Content-Type: text/csv --- no_error_log [error] === TEST 74: omitted zstd_types includes application/x-ndjson --- config location /ndjson { zstd on; default_type application/x-ndjson; return 200 $arg_body; } --- request eval "GET /ndjson?body=" . ("x" x 1024) --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd Content-Type: application/x-ndjson --- no_error_log [error] === TEST 75: omitted zstd_types includes application/json-seq --- config location /json-seq { zstd on; default_type application/json-seq; return 200 $arg_body; } --- request eval "GET /json-seq?body=" . ("x" x 1024) --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd Content-Type: application/json-seq --- no_error_log [error] === TEST 76: omitted zstd_types includes application/wasm --- config location /wasm { zstd on; default_type application/wasm; return 200 $arg_body; } --- request eval "GET /wasm?body=" . ("x" x 1024) --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd Content-Type: application/wasm --- no_error_log [error] === TEST 77: omitted zstd_types includes text/wgsl --- config location /wgsl { zstd on; default_type text/wgsl; return 200 $arg_body; } --- request eval "GET /wgsl?body=" . ("x" x 1024) --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd Content-Type: text/wgsl --- no_error_log [error] === TEST 78: qvalue followed by whitespace then trailing junk is malformed --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd;q=1 garbage --- response_headers Content-Length: 59738 !Content-Encoding --- no_error_log [error] === TEST 79: qvalue followed by tab then trailing junk is malformed --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd;q=0.5 junk --- response_headers Content-Length: 59738 !Content-Encoding --- no_error_log [error] === TEST 80: repeated "q" parameter is malformed (RFC 9110 permits at most one) --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd;q=0.5;q=0.9 --- response_headers Content-Length: 59738 !Content-Encoding --- no_error_log [error] === TEST 81: "q" parameter present without "=value" is malformed --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd;q --- response_headers Content-Length: 59738 !Content-Encoding --- no_error_log [error] === TEST 82: qvalue leading digit not 0 or 1 is malformed (q=2) --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd;q=2 --- response_headers Content-Length: 59738 !Content-Encoding --- no_error_log [error] === TEST 83: non-"q" parameter with an escaped quoted-pair does not confuse the delimiter scan --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd;foo="a\"b,c";q=0.8, gzip;q=0.1 --- response_headers_like Content-Encoding: zstd --- no_error_log [error] === TEST 84: unterminated quoted-string in a non-"q" parameter runs to end without OOB read, defaults q=1 --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd;foo="unterminated --- response_headers_like Content-Encoding: zstd --- no_error_log [error] === TEST 85: Content-Encoding is emitted exactly once on the wire # nginx >= 1.23 links same-name response headers via ngx_table_elt_t.next # and expects every producer to NULL-terminate the chain it starts, as # core's own gzip filter and gzip_static do. The stale pointer is not # reachable through plain HTTP with current core, and there is no # in-harness readback either: add_header ($sent_http_*) evaluates before # this module's header filter by deliberate module order (filter/config # places ngx_http_headers_filter_module after this module, mirroring # core's gzip placement), so headers_out holds no Content-Encoding at # evaluation time. This test therefore pins the wire contract only; see # the sibling static-module TEST 29, where the content-phase handler # runs early enough for the $sent_http_* readback to observe the entry. --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 86: OWS around the "=" of a q parameter is accepted (RFC 9110 BWS) # Covers the two optional-whitespace skips that bracket the "=" in # ngx_http_zstd_eval_qvalue: whitespace after the parameter name and # whitespace before its value. Both loops were unexecuted — every prior test # wrote "q=..." with no spaces, so a tolerated-by-grammar header shape was # entirely untested. q=1 keeps zstd acceptable, so the response compresses. --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd; q = 1 --- response_headers_like Content-Encoding: zstd --- no_error_log [error] === TEST 87: "q=" with the value missing at end-of-field is rejected # The `p >= end` guard right after the "=" is consumed: the field ends before # any qvalue digit. Distinct from TEST 81 ("q" with no "=" at all) — this one # reaches the is_q branch and runs out of input inside it. Malformed # parameter => the element is dropped => zstd is not acceptable => identity. --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd;q= --- response_headers ! Content-Encoding --- no_error_log [error] === TEST 88: an unquoted non-q parameter value is skipped to the delimiter # The `else { p++; }` arm of the non-q value scanner. TEST 83 covered the # quoted-string arm; an ordinary unquoted token value never exercised the # byte-at-a-time path. The parameter is ignored and the following "q=0" # still parses, so zstd is NOT acceptable here — proving the scanner stopped # at the ';' rather than swallowing the rest of the field. --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd;foo=bar;q=0 --- response_headers ! Content-Encoding --- no_error_log [error] === TEST 89: a quoted-string starting mid-value is skipped, parsing continues # A non-q parameter whose value mixes unquoted bytes and a quoted-string # ("a\"b\""): the scanner steps byte-at-a-time until the '"', then hands off to # ngx_http_zstd_skip_quoted for the quoted run, then resumes. Both arms of that # loop therefore run for a single parameter. The trailing q=1 must still be # found and parsed, proving the quoted region was skipped as one unit and the # cursor landed on the ';' — not swallowed past it (which would drop the q and # silently change the negotiated weight). # # NOTE: the early-return guard in skip_quoted (p >= end || *p != '"') is NOT # reachable from here, or from any HTTP input: the only call site tests # *p == '"' first. It is defensive-only, left uncovered deliberately. --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd;foo=a"b";q=1 --- response_headers_like Content-Encoding: zstd --- no_error_log [error] === TEST 90: a HEAD advertises the same Content-Encoding its GET would produce # RFC 9110 §9.3.2: a HEAD response carries the header fields the equivalent GET # would have sent, with no body. So a compressible HEAD MUST still advertise # "Content-Encoding: zstd" (a client uses it to size/negotiate the later GET), # while sending zero body bytes. # # This pins the direction deliberately, because the opposite assertion looks # just as plausible: "no body, so no Content-Encoding". Verified against a live # server before writing it — HEAD and GET emit identical Content-Encoding here. # # Note on the module's own r->header_only short-circuit in the header filter: # it is NOT what handles this request. r->header_only is set by nginx's # ngx_http_header_filter_module, which runs AFTER the zstd filter in the chain, # so on a plain HEAD the flag is still 0 when zstd inspects the response. That # branch only fires for internally generated header-only responses, and is left # uncovered rather than faked with a test that would assert the wrong contract. --- config location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request HEAD /filter --- more_headers Accept-Encoding: zstd --- error_code: 200 --- response_headers Content-Encoding: zstd --- response_body --- no_error_log [error] === TEST 90b: a HEAD advertises the same zstd_bypass_vary Vary field as GET # This PR moved the r->header_only short-circuit ABOVE the # zstd_bypass_vary Vary-append block in ngx_http_zstd_header_filter() # (it used to run after the content-type check, now runs right after the # encoding/status/length gates and before it). TEST 90 already proves # header_only is not what handles a real HEAD here (see its comment); this # arm proves the reorder did not change what a real HEAD emits for the # OTHER header the filter appends unconditionally on this code path -- # zstd_bypass_vary's Vary field -- by checking it is present and identical # on both HEAD and the equivalent GET. --- config location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; zstd_bypass $http_x_no_compression; zstd_bypass_vary X-No-Compression; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request HEAD /filter --- more_headers Accept-Encoding: zstd --- error_code: 200 --- response_headers Content-Encoding: zstd Vary: X-No-Compression, Accept-Encoding --- response_body --- no_error_log [error] === TEST 91: an SSI subrequest is not separately zstd-compressed # The header filter checks r != r->main early and declines for subrequests, # so only the main request negotiates a content coding. Without that guard # the subrequest would be compressed on its own and its zstd frame would be # spliced into the parent's body as opaque bytes. # # The assertion is the error-log line the filter emits when declining the # subrequest, counted over the whole request: the parent here is text/html # with zstd off, so a correctly guarded run reaches "zstd: compressing # response" zero times. Asserting only on the assembled body would not work — # with the guard removed the page still reads correctly, because the parent's # own filter chain re-processes the spliced output, so the body is identical # in both states and would guard nothing. --- log_level: debug --- config location /page.html { ssi on; default_type text/html; root $TEST_NGINX_SERVER_ROOT/html; } location /inc { zstd on; zstd_min_length 1; zstd_types text/plain; default_type text/plain; return 200 "INCLUDED-PAYLOAD-INCLUDED-PAYLOAD-INCLUDED-PAYLOAD"; } --- user_files >>> page.html BEGINEND --- request GET /page.html --- more_headers Accept-Encoding: zstd --- response_body BEGININCLUDED-PAYLOAD-INCLUDED-PAYLOAD-INCLUDED-PAYLOADEND --- error_log zstd: skip, subrequest --- no_error_log zstd: compressing response [error] === TEST 92: sub_filter partial-match sync bufs must not reset the CCtx mid-stream # Regression for the silent-truncation bug: the body filter inferred # "first body data" from ctx->buffer_in.src == NULL, while add_data # reloaded buffer_in.src from every incoming buffer. The sub filter # emits a data-less sync carrier (pos == NULL, ngx_http_sub_filter # line ~516) whenever an in-memory input buffer is entirely absorbed # into a cross-buffer match candidate ("looked") and produces no # output. Loading that carrier re-armed the first-call check, the next # invocation re-ran ZSTD_CCtx_reset(), and everything libzstd had # buffered but not yet flushed was silently discarded -- the response # still ended as ONE well-formed frame (200 + valid zstd), just with # the pre-reset content missing. Seen in production as truncated HTML # on sub_filter-rewritten pages fed by a slow chunked upstream. # # The mock backend streams three chunks with pauses; chunk 2 lies # strictly inside the sub_filter FROM pattern, so it is fully absorbed # into the match state and triggers the sync carrier. A pre-fix module # loses chunk 1 ("EARLY-CONTENT ..."); the fixed module round-trips # the whole rewritten body. --- config location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; sub_filter 'http://upstream.example' 'https://very-long-replacement-host.example'; sub_filter_once off; sub_filter_types text/plain; proxy_http_version 1.1; proxy_buffering off; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/up; } location /up { proxy_http_version 1.1; proxy_pass http://127.0.0.1:$TEST_NGINX_RAND_PORT_1/; } --- tcp_listen: $TEST_NGINX_RAND_PORT_1 --- tcp_no_close --- tcp_reply_delay: 100ms --- tcp_reply eval my $hdr = "HTTP/1.1 200 OK\r\n" . "Content-Type: text/plain\r\n" . "Transfer-Encoding: chunked\r\n" . "Connection: close\r\n\r\n"; my $seg1 = "EARLY-CONTENT that must survive the match holdback http://up"; my $seg2 = "stream.exa"; # entirely inside the FROM pattern: absorbed, no output my $seg3 = "mple/path LATE-CONTENT after the match completes\n"; my $chunk = sub { sprintf("%x\r\n%s\r\n", length($_[0]), $_[0]) }; [ $hdr . $chunk->($seg1), $chunk->($seg2), $chunk->($seg3) . "0\r\n\r\n", ] --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- response_body_filters eval sub { my $zstd = $_[0]; require File::Temp; my ($tfh, $tmp) = File::Temp::tempfile("zstd_t92_XXXXXX", TMPDIR => 1, UNLINK => 1); binmode($tfh); print $tfh $zstd; close($tfh); # List-form pipe open: no shell, so the temp path is never re-parsed. open(my $r, "-|", "zstd", "-dqc", $tmp) or do { unlink $tmp; return "ERR" }; local $/; my $d = <$r>; close($r); my $rc = $?; unlink $tmp; return "ERR-DECODE rc=$rc" if $rc != 0; return $d; } --- response_body EARLY-CONTENT that must survive the match holdback https://very-long-replacement-host.example/path LATE-CONTENT after the match completes --- no_error_log [error] === TEST 93: a coding name running straight into a DQUOTE does not negotiate # RFC 9110 12.5.3 `codings` is a token and '"' is not a tchar, so `zstd"x` # advertises no coding. The name scan stops on '"' for the phantom-token # guard, but the stopping byte went unexamined, so the truncated name still # compared equal and we compressed for a client that never offered zstd -- # a parser differential against a strict intermediary, and a divergence from # nginx's own ngx_http_gzip_accept_encoding(), which requires ',', ';', ' ' # or end after the name. --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd"x --- response_headers Content-Length: 59738 !Content-Encoding --- no_error_log [error] === TEST 94: a DQUOTE-suffixed name followed by a quoted blob does not negotiate # The other half of TEST 93: `zstd"a,b"` hides a comma inside the quoted run. # The element must decline AND the quote-aware skip must swallow the whole # blob, so the bytes after the in-quote comma cannot start a phantom element. --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd"a,b" --- response_headers Content-Length: 59738 !Content-Encoding --- no_error_log [error] === TEST 95: zstd_comp_level -1 is applied, not silently replaced # NGX_CONF_UNSET is -1, and -1 is a valid documented level (ZSTD_minCLevel() # ..-1). The old merge could not tell "operator asked for -1" from "nothing # configured", so it overwrote the request with the inherited server value. # This location inherits 9 from http_config and asks for -1. The COMPRESSED # SIZE is the oracle -- the response is chunked, so there is no # Content-Length to read; the body filter reports the byte count instead. # Level -1 is far cheaper than 9 and must produce a visibly larger body. # TEST 96 pins what an inherited 9 produces on the same fixture: pre-fix the # two were byte-for-byte identical, which is exactly what makes this pair a # negative control rather than a smoke test. --- http_config zstd_comp_level 9; --- config location /filter { zstd on; zstd_types text/plain; zstd_comp_level -1; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- response_body_filters eval sub { "zstd_bytes=" . length($_[0]) } --- response_body chomp zstd_bytes=4796 --- no_error_log [error] === TEST 96: an inherited zstd_comp_level 9 is the control for TEST 95 # Same fixture, same server-level 9, no location override -- so this is the # size TEST 95 must NOT produce. Keep the two in step: if the fixture or the # linked libzstd changes, both numbers move, and they must stay different. --- http_config zstd_comp_level 9; --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- response_body_filters eval sub { "zstd_bytes=" . length($_[0]) } --- response_body chomp zstd_bytes=3560 --- no_error_log [error] === TEST 97: junk after OWS following a coding name does not negotiate # The DQUOTE guard added for TEST 93 sat BEFORE the OWS skip, so it caught # `zstd"x` and missed everything hiding behind a space: `zstd "x` and plain # `zstd x` both still negotiated. RFC 9110 12.5.3 allows only ';', ',' or # end of field after a coding name, which is the same rule # ngx_http_gzip_accept_encoding() applies. Found by CodeRabbit on PR #142. --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd "x --- response_headers Content-Length: 59738 !Content-Encoding --- no_error_log [error] === TEST 98: a bare token after OWS following a coding name does not negotiate # The no-quotes arm of TEST 97. `zstd x` is two tokens with no separator, # not a coding with a parameter, so the element offers nothing. --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd x --- response_headers Content-Length: 59738 !Content-Encoding --- no_error_log [error] === TEST 99: OWS before ';' and ',' still negotiates # Guard against over-fixing TEST 97/98: OWS between the name and a real # separator is legal (RFC 9110 OWS), so these must still compress. --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: gzip , zstd ;q=1 --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 100: multi-chunk streamed body across several body-filter callbacks # Regression for the O(1)-tail append bug: ctx->last_in is retracked to # &ctx->in whenever add_data drains the last retained link (ctx->in becomes # NULL), because ngx_free_chain() overwrites that consumed link's ->next # with the pool's free-chain head immediately afterward. Without the # retrack, the NEXT body-filter callback's append splices its copied chain # into ngx_pool_t.chain instead of onto ctx->in -- every callback after the # first is silently dropped, the compressor never receives the rest of the # body, and the response hangs waiting for a last_buf that never arrives. # Multiple real (non-special, proxy_buffering off) chunks with pauses in # between exercise several separate ngx_http_zstd_body_filter() callbacks # against the same request, each one draining ctx->in completely before the # next chunk arrives -- the exact interleaving the bug needs. --- config location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; proxy_http_version 1.1; proxy_buffering off; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/up; } location /up { proxy_http_version 1.1; proxy_pass http://127.0.0.1:$TEST_NGINX_RAND_PORT_1/; } --- tcp_listen: $TEST_NGINX_RAND_PORT_1 --- tcp_no_close --- tcp_reply_delay: 100ms --- tcp_reply eval my $hdr = "HTTP/1.1 200 OK\r\n" . "Content-Type: text/plain\r\n" . "Transfer-Encoding: chunked\r\n" . "Connection: close\r\n\r\n"; my $chunk = sub { sprintf("%x\r\n%s\r\n", length($_[0]), $_[0]) }; [ $hdr . $chunk->("first-chunk-"), $chunk->("second-chunk-"), $chunk->("third-chunk-"), $chunk->("fourth-chunk-tail\n") . "0\r\n\r\n", ] --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- response_body_filters eval sub { my $zstd = $_[0]; require File::Temp; my ($tfh, $tmp) = File::Temp::tempfile("zstd_t100_XXXXXX", TMPDIR => 1, UNLINK => 1); binmode($tfh); print $tfh $zstd; close($tfh); open(my $r, "-|", "zstd", "-dqc", $tmp) or do { unlink $tmp; return "ERR" }; local $/; my $d = <$r>; close($r); my $rc = $?; unlink $tmp; return "ERR-DECODE rc=$rc" if $rc != 0; return $d; } --- response_body first-chunk-second-chunk-third-chunk-fourth-chunk-tail --- no_error_log [error] === TEST 101: pledged size just below the configured buffer size still round-trips # Boundary regression for skipping ZSTD_compressBound() when the pledged # Content-Length is already >= the configured zstd_buffers size (that # skip cannot shrink buf_size below what the call would have produced -- # see the get_buf() comment). zstd_buffers is pinned to a small, exact # size (1 buffer of 512 bytes) so the boundary sits at an exact byte # count instead of the 128 KB default. This case pledges 511 bytes -- # ONE BYTE UNDER buf_size -- which must still take the ORIGINAL # ZSTD_compressBound() path (pledged_size < buf_size), unaffected by the # new skip branch. Correctness oracle: byte-exact round-trip. --- config eval " location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; zstd_buffers 1 512; proxy_pass http://127.0.0.1:\$TEST_NGINX_SERVER_PORT/src; } location /src { default_type text/plain; return 200 \"" . ("A" x 511) . "\"; }" --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers !Content-Length Content-Encoding: zstd --- response_body_filters eval sub { my $zstd = $_[0]; require File::Temp; my ($tfh, $tmp) = File::Temp::tempfile("zstd_t101_XXXXXX", TMPDIR => 1, UNLINK => 1); binmode($tfh); print $tfh $zstd; close($tfh); open(my $r, "-|", "zstd", "-dqc", $tmp) or do { unlink $tmp; return "ERR" }; local $/; my $d = <$r>; close($r); my $rc = $?; unlink $tmp; return "ERR-DECODE rc=$rc" if $rc != 0; return $d; } --- response_body eval "A" x 511 --- no_error_log [error] === TEST 102: pledged size EXACTLY EQUAL to the configured buffer size still round-trips # The exact boundary the skip's ">=" comparison turns on: pledged_size == # buf_size (512 == 512) takes the NEW skip path (get_buf(): # "(size_t) ctx->pledged_size >= buf_size"). ZSTD_compressBound(512) + 64 # is provably >= 512, so the clamp this call feeds would never have fired # here either -- this case is the one boundary value where that claim # must hold exactly, not just in the general case covered by TEST 103. --- config eval " location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; zstd_buffers 1 512; proxy_pass http://127.0.0.1:\$TEST_NGINX_SERVER_PORT/src; } location /src { default_type text/plain; return 200 \"" . ("B" x 512) . "\"; }" --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers !Content-Length Content-Encoding: zstd --- response_body_filters eval sub { my $zstd = $_[0]; require File::Temp; my ($tfh, $tmp) = File::Temp::tempfile("zstd_t102_XXXXXX", TMPDIR => 1, UNLINK => 1); binmode($tfh); print $tfh $zstd; close($tfh); open(my $r, "-|", "zstd", "-dqc", $tmp) or do { unlink $tmp; return "ERR" }; local $/; my $d = <$r>; close($r); my $rc = $?; unlink $tmp; return "ERR-DECODE rc=$rc" if $rc != 0; return $d; } --- response_body eval "B" x 512 --- no_error_log [error] === TEST 103: pledged size well ABOVE the configured buffer size still round-trips # Deep into the skip path: a body several times larger than buf_size, # well past the boundary, still round-trips byte-exact with the # ZSTD_compressBound() call skipped on the first buffer. (4000 bytes, # not larger: nginx's config-file tokenizer has its own line-length # limit on a quoted "return" literal, well under a size that would # still usefully exercise this boundary -- 4000 is comfortably above # both buf_size=512 and that unrelated limit.) --- config eval " location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; zstd_buffers 4 512; proxy_pass http://127.0.0.1:\$TEST_NGINX_SERVER_PORT/src; } location /src { default_type text/plain; return 200 \"" . ("C" x 4000) . "\"; }" --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers !Content-Length Content-Encoding: zstd --- response_body_filters eval sub { my $zstd = $_[0]; require File::Temp; my ($tfh, $tmp) = File::Temp::tempfile("zstd_t103_XXXXXX", TMPDIR => 1, UNLINK => 1); binmode($tfh); print $tfh $zstd; close($tfh); open(my $r, "-|", "zstd", "-dqc", $tmp) or do { unlink $tmp; return "ERR" }; local $/; my $d = <$r>; close($r); my $rc = $?; unlink $tmp; return "ERR-DECODE rc=$rc" if $rc != 0; return $d; } --- response_body eval "C" x 4000 --- no_error_log [error] === TEST 104: chained Accept-Encoding — "gzip" then "zstd" negotiates zstd # RFC 9110 section 5.3: repeated field lines are the single comma-joined # field, so these two lines ARE "gzip, zstd" and accept zstd. Parsing only # the first line saw "gzip" and served the body uncompressed. --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: gzip Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 105: chained Accept-Encoding — "zstd" then "gzip" negotiates zstd --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd Accept-Encoding: gzip --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 106: chained Accept-Encoding — "zstd;q=0" on the FIRST line declines # The duplicate-coding rule: an explicit q=0 anywhere in the field is # final and cannot be upgraded by a later line. Serving zstd here would # hand a body to a client that explicitly refused it. --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd;q=0 Accept-Encoding: gzip --- response_headers Content-Length: 59738 !Content-Encoding --- no_error_log [error] === TEST 107: chained Accept-Encoding — "zstd;q=0" on a LATER line declines --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: gzip Accept-Encoding: zstd;q=0 --- response_headers Content-Length: 59738 !Content-Encoding --- no_error_log [error] === TEST 108: chained Accept-Encoding — q=0 then q=1 accepts # Duplicate field lines are comma-joined in received order, so this follows # the parser's existing last-explicit-token-wins rule. --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd;q=0 Accept-Encoding: zstd;q=1 --- response_headers !Content-Length Transfer-Encoding: chunked Content-Encoding: zstd --- no_error_log [error] === TEST 108b: chained Accept-Encoding — q=1 then q=0 declines # Negative control for TEST 108: reversing the same fields must reverse the # decision, proving that field order reaches the live negotiation path. --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: zstd;q=1 Accept-Encoding: zstd;q=0 --- response_headers Content-Length: 59738 !Content-Encoding --- no_error_log [error] === TEST 109: chained Accept-Encoding — wildcard "*" on a later line accepts --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: gzip Accept-Encoding: * --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 110: chained Accept-Encoding — three field lines, zstd on the third --- config location /filter { zstd on; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/test; } location /test { root $TEST_NGINX_PERL_PATH/suite/; } --- request GET /filter --- more_headers Accept-Encoding: gzip Accept-Encoding: br Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 111: streaming zstd_max_length cap — body exactly AT the cap still compresses # Boundary coverage for the ctx->bytes_in (uint64_t) vs zlcf->max_length # (ssize_t) cap comparison, using the same mock chunked/no-Content-Length # upstream fixture as TEST 42. bytes_in accumulates per received chunk and # the check only fires once bytes_in EXCEEDS the cap, so a body whose # size equals the cap exactly must still pass through and compress # normally. The 32-bit-off_t cast defect this cap's comparison guards # against is not reachable on this (64-bit off_t) harness -- see # ci/tools/test_max_length_cap_unit.sh for the fixture that drives it # under a genuine 32-bit off_t build. --- config location /filter { zstd on; zstd_min_length 1; zstd_max_length 5000; zstd_types text/plain; proxy_http_version 1.1; proxy_buffering off; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/up; } location /up { proxy_http_version 1.1; proxy_pass http://127.0.0.1:$TEST_NGINX_RAND_PORT_1/; } --- tcp_listen: $TEST_NGINX_RAND_PORT_1 --- tcp_no_close --- tcp_reply eval "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n" . sprintf("%x\r\n", 5000) . ("A" x 5000) . "\r\n0\r\n\r\n" --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 112b: $zstd_ratio and $zstd_bytes_* carry their final values at the log phase # TESTs 32 and 39 reference the variables via `set`, which evaluates them # during rewrite -- before the filter has run -- so they only ever exercise # the not_found guard. The formatted values exist for one consumer, the # access log, which evaluates at the log phase after the response # completed. Two requests on one instance: the first compresses a body # under a log_format naming all three variables, the second serves the # log file back. The file is cleared when the server root is set up and # then accumulates one line per repetition of the block (repeat_each is # 3), so the whole body is anchored: every line must be a well-formed # record and there are at most three -- a later repetition cannot hide # behind an earlier line. --- post_setup_server_root eval 'unlink "$ENV{TEST_NGINX_SERVER_ROOT}/logs/zstdvars.log";' --- http_config log_format zstdvars 'ratio=$zstd_ratio in=$zstd_bytes_in out=$zstd_bytes_out'; --- config location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; default_type text/plain; access_log $TEST_NGINX_SERVER_ROOT/logs/zstdvars.log zstdvars; return 200 "log-phase variables: this body is long enough to compress and to compress again and again\n"; } location = /vars.log { alias $TEST_NGINX_SERVER_ROOT/logs/zstdvars.log; default_type text/plain; access_log off; } --- request eval ["GET /filter", "GET /vars.log"] --- more_headers eval ["Accept-Encoding: zstd", ""] --- response_headers eval ["Content-Encoding: zstd", "!Content-Encoding"] --- response_body_like eval [qr/./s, qr/\A(?:ratio=\d+\.\d{3} in=[1-9]\d+ out=[1-9]\d+\n){1,3}\z/] --- no_error_log [error] === TEST 112: streaming zstd_max_length cap — body one byte OVER the cap aborts # Companion to TEST 111, same mock upstream fixture as TEST 42 (which # covers a body grossly over the cap; this is the boundary case). One # byte past the cap must still trip the abort path -- confirms the > # (not >=) comparison at the boundary. As with TEST 42, the aborted # connection has no clean chunked terminator, so this only asserts the # error_log line under --- ignore_response; Test::Nginx::Socket's # response-body checks are skipped together with the parse they depend # on, so no additional "body never completed" assertion is available # through this harness for a deliberately truncated response. --- config location /filter { zstd on; zstd_min_length 1; zstd_max_length 4999; zstd_types text/plain; proxy_http_version 1.1; proxy_buffering off; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/up; } location /up { proxy_http_version 1.1; proxy_pass http://127.0.0.1:$TEST_NGINX_RAND_PORT_1/; } --- tcp_listen: $TEST_NGINX_RAND_PORT_1 --- tcp_no_close --- tcp_reply eval "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n" . sprintf("%x\r\n", 5000) . ("A" x 5000) . "\r\n0\r\n\r\n" --- request GET /filter --- more_headers Accept-Encoding: zstd --- ignore_response --- error_log input exceeded zstd_max_length (4999) after myguard-labs-nginx-zstd-module-3486b5b/ci/t/01-static.t000066400000000000000000001507411525232545500225760ustar00rootroot00000000000000use Test::Nginx::Socket; use File::Basename; use File::Spec; use lib 'lib'; my @dynamic_modules; if (defined $ENV{'TEST_NGINX_BINARY'}) { my $nginx_dir = dirname($ENV{'TEST_NGINX_BINARY'}); for my $module_name (qw(ngx_http_zstd_filter_module.so ngx_http_zstd_static_module.so)) { my $module_path = "$nginx_dir/$module_name"; push @dynamic_modules, $module_path if -f $module_path; } } add_block_preprocessor(sub { my $block = shift; return if !@dynamic_modules; my $main_config = join "\n", map { "load_module $_;" } @dynamic_modules; $block->set_value("main_config", $main_config); }); no_long_string(); log_level 'debug'; repeat_each(3); check_accum_error_log(); plan 'no_plan'; run_tests(); # COVERAGE NOTES on gaps found by the git-history CI audit # (kept above __DATA__ so they are not parsed into any test block): # # Gap 2 — f7e2ef3 ("clear Accept-Ranges in static module"), since # REVERSED: the handler now opts IN via r->allow_ranges = 1, covered # fail-first by TESTS 36–37. The history, for the record: f7e2ef3's # clear was a no-op (the handler never set r->allow_ranges, and # ngx_http_range_filter_module.c bails without it — empirically # verified at the time: identical responses on pre-fix and fixed # builds), and its rationale misread the RFC it cited. RFC 9110 §14.2 # has ranges address the SELECTED REPRESENTATION — under # Content-Encoding: zstd, the .zst bytes — which is not an # "undecipherable fragment" hazard but exactly the coherent, # resumable-download semantics gzip_static has always provided by # setting r->allow_ranges = 1 (strong validator, byte-identical # representation on disk). The filter module's clear remains correct # for the opposite reason: a stream generated on the fly has nothing # stable to seek into. # # Gap 3 — HTTP/2 transport axis (8281baa bug-B class): # the build enables --with-http_v2_module but nothing tests the h2 # path. HTTP/2 in nginx requires TLS + ALPN/Upgrade negotiation; h2c # (cleartext) does not work without Upgrade in nginx config. A Python # test without TLS cannot easily drive the h2 path. The bug-B defect # (empty-buffer, flush-state-machine, c->buffered accounting) is # already well-covered by test_proxy_unbuffered_truncation.py # (HTTP/1.1) and the matrix under ASAN; the h2-specific framing path # would be redundant effort without adding coverage for a new code # path. Left for future work when/if CI adds TLS test infrastructure. __DATA__ === TEST 1: zstd_static off --- config location /test { zstd_static off; root ../suite; } --- request GET /test --- response_headers Content-Length: 59738 ETag: "5be17d33-e95a" !Content-Encoding --- no_error_log [error] === TEST 2: zstd_static off (with accept-encoding header) --- config location /test { zstd_static off; root ../suite; } --- request GET /test Accept-Encoding: gzip,zstd --- response_headers Content-Length: 59738 ETag: "5be17d33-e95a" !Content-Encoding --- no_error_log [error] === TEST 3: zstd_static on --- config location /test { zstd_static on; root ../suite; } --- request GET /test --- more_headers Accept-Encoding: gzip, zstd --- response_headers Content-Length: 3717 ETag: "5be17d33-e85" Content-Encoding: zstd --- no_error_log [error] === TEST 4: zstd_static on (without accept-encoding header) --- config location /test { zstd_static on; root ../suite; } --- request GET /test --- response_headers Content-Length: 59738 ETag: "5be17d33-e95a" !Content-Encoding --- no_error_log [error] === TEST 5: zstd_static on (without zstd component in accept-encoding header) --- config location /test { zstd_static on; root ../suite; } --- request GET /test --- more_headers Accept-Encoding: gzip, br --- response_headers Content-Length: 59738 ETag: "5be17d33-e95a" !Content-Encoding --- no_error_log [error] === TEST 6: zstd_static always --- config location /test { zstd_static always; root ../suite; } --- request GET /test --- more_headers Accept-Encoding: gzip, br --- response_headers Content-Length: 3717 ETag: "5be17d33-e85" Content-Encoding: zstd --- no_error_log [error] === TEST 7: zstd_static always (without accept-encoding header) --- config location /test { zstd_static always; root ../suite; } --- request GET /test --- response_headers Content-Length: 3717 ETag: "5be17d33-e85" Content-Encoding: zstd --- no_error_log [error] === TEST 8: zstd_static always (without zstd component in accept-encoding header) --- config location /test { zstd_static always; root ../suite; } --- request GET /test --- more_headers Accept-Encoding: gzip, br --- response_headers Content-Length: 3717 ETag: "5be17d33-e85" Content-Encoding: zstd --- no_error_log [error] === TEST 9: zstd_static always (file does not exist) --- config location /test2 { zstd_static always; root ../suite; } --- request GET /test2 --- more_headers Accept-Encoding: gzip, br --- error_code: 404 === TEST 10: zstd_static on (file does not exist) --- config location /test2 { zstd_static on; root ../suite; } --- request GET /test2 --- more_headers Accept-Encoding: gzip, br --- error_code: 404 === TEST 11: zstd_static off (file does not exist) --- config location /test2 { zstd_static off; root ../suite; } --- request GET /test2 --- more_headers Accept-Encoding: gzip, br --- error_code: 404 === TEST 12: zstd_static on with quality value q=0 (reject) --- config location /test { zstd_static on; root ../suite; } --- request GET /test --- more_headers Accept-Encoding: zstd;q=0, gzip;q=1 --- response_headers Content-Length: 59738 ETag: "5be17d33-e95a" !Content-Encoding --- no_error_log [error] === TEST 13: zstd_static on with quality value q=0.5 (accept lower) --- config location /test { zstd_static on; root ../suite; } --- request GET /test --- more_headers Accept-Encoding: zstd;q=0.5 --- response_headers Content-Length: 3717 ETag: "5be17d33-e85" Content-Encoding: zstd --- no_error_log [error] === TEST 14: zstd_static always with q=0 (still serve zst) --- config location /test { zstd_static always; root ../suite; } --- request GET /test --- more_headers Accept-Encoding: zstd;q=0 --- response_headers Content-Length: 3717 ETag: "5be17d33-e85" Content-Encoding: zstd --- no_error_log [error] === TEST 15: zstd_static on with gzip_vary and gzip support --- config location /test { zstd_static on; gzip_vary on; root ../suite; } --- request GET /test --- more_headers Accept-Encoding: gzip, zstd --- response_headers Content-Length: 3717 ETag: "5be17d33-e85" Content-Encoding: zstd --- no_error_log [error] === TEST 16: zstd_static on with gzip_vary but no zstd support --- config location /test { zstd_static on; gzip_vary on; root ../suite; } --- request GET /test --- more_headers Accept-Encoding: gzip --- response_headers Content-Length: 59738 ETag: "5be17d33-e95a" !Content-Encoding --- no_error_log [error] === TEST 17: zstd_static on - HEAD request --- config location /test { zstd_static on; root ../suite; } --- request HEAD /test --- more_headers Accept-Encoding: zstd --- response_headers Content-Length: 3717 ETag: "5be17d33-e85" Content-Encoding: zstd --- no_error_log [error] === TEST 18: zstd_static on - POST request (not GET/HEAD) --- config location /test { zstd_static on; root ../suite; } --- request POST /test --- more_headers Accept-Encoding: zstd --- error_code: 405 --- response_headers !Content-Encoding === TEST 19: zstd_dict_file loads and serves correctly (filter path) # Regression for the untested zstd_dict_file feature, which had three # distinct historical bug fixes with NO test: 0fb40d9 (CDict leak on # cleanup), 50f27a8 (version-specific init error handling), f735a5d # (cleanup handler size). This is the first test that exercises a # dictionary at all: nginx must start with zstd_dict_file set and still # produce a valid compressed response. # zstd_dict_file is an http{}-context directive (NGX_HTTP_MAIN_CONF), so # it goes in --- http_config, not --- main_config (global, before http{}). # Relative paths resolve against the *configuration* prefix (conf/), while # --- user_files land in /html, so use the absolute servroot # token Test::Nginx exports for this exact purpose. --- http_config zstd_dict_file_unsafe on; zstd_dict_file $TEST_NGINX_SERVER_ROOT/html/zstd.dict; --- config location /filter { zstd on; zstd_min_length 1; zstd_types text/plain; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/src; } location /src { default_type text/plain; return 200 "dictionary compressed body, long enough to compress\n"; } --- user_files >>> zstd.dict the quick brown fox jumps over the lazy dog 0123456789 dictionary sample payload for zstd training corpus padding padding padding --- request GET /filter --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 20: zstd_static handles a long URI without buffer overflow # Regression for 9789448 "buffer overflow when appending .zst extension". # A long request path stresses the .zst path-build reservation made via # ngx_http_map_uri_to_path(sizeof(".zst")). Missing file -> clean 404, # no crash, no ASAN abort (the ASAN test job runs this binary too). --- config location /s/ { zstd_static on; root ../suite; } --- request eval "GET /s/" . ("a" x 2000) . "/nonexistent-resource-name" --- error_code: 404 --- response_headers !Content-Encoding --- no_error_log [alert] === TEST 21: zstd_static rejects a file whose contents are not a zstd frame # Defence-in-depth: a .zst whose first 4 bytes are not the zstd magic # (truncated download, mistakenly renamed text, `cp foo.txt foo.zst`) # must NOT be served with Content-Encoding: zstd — the client would # receive an undecodable body. The handler pread()s the leading 4 # bytes, checks them against ZSTD_MAGICNUMBER / ZSTD_MAGIC_SKIPPABLE_*, # and declines on mismatch. The fixture below contains plain ASCII # ("HELO ...") with no zstd magic; no uncompressed fallback file is # placed alongside it, so the request falls through to a clean 404. --- config location /bogus { zstd_static on; root html; } --- user_files >>> bogus.zst HELO this is not a zstd frame --- request GET /bogus --- more_headers Accept-Encoding: zstd --- error_code: 404 --- response_headers !Content-Encoding !Vary --- error_log is not a zstd frame === TEST 22: zstd_static always does NOT set Vary even with gzip_vary on # Locks intentional behaviour: in "always" mode the handler unconditionally # serves the precompressed .zst and never sets r->gzip_vary. Vary: # Accept-Encoding would mis-key shared caches for a response that does # not actually vary on Accept-Encoding (the same .zst comes back no # matter what the client sends), so the absence of Vary here is the # correct contract. TEST 6-8 cover "always" without gzip_vary; this # locks that adding gzip_vary on at the location does not flip the # behaviour by accident. --- config gzip_vary on; location /test { zstd_static always; root ../suite; } --- request GET /test --- more_headers Accept-Encoding: gzip, br --- response_headers Content-Length: 3717 ETag: "5be17d33-e85" Content-Encoding: zstd !Vary --- no_error_log [error] === TEST 23: zstd_static rejects an empty .zst file # A zero-byte .zst cannot satisfy the 4-byte pread() magic check; # the handler must decline rather than serve an empty body with # Content-Encoding: zstd. TEST 21 covers the wrong-magic case; # this locks the truncated-to-zero edge specifically. Like TEST 21, # no uncompressed fallback is placed alongside empty.zst, so a # benign ENOENT on the fallback path is expected and not asserted # against. --- config location /empty { zstd_static on; root html; } --- user_files >>> empty.zst --- request GET /empty --- more_headers Accept-Encoding: zstd --- error_code: 404 --- response_headers !Content-Encoding !Vary === TEST 24: zstd_static declines a directory-style request # A request whose URI ends in "/" maps to a path with a trailing # slash; appending ".zst" would produce ".../.zst". The handler # short-circuits at the URI-suffix check (uri.data[uri.len - 1] # == '/') and declines without touching the filesystem, so the # request falls through to the normal directory-index machinery # rather than being answered with Content-Encoding: zstd. The # fallback then 403s the directory and logs the missing index file # — that log line is from the regular static handler, not from # zstd_static, and is expected here. The contract being locked is # only !Content-Encoding (i.e. zstd_static did not falsely claim # the response was zstd-encoded). --- config location /dir/ { zstd_static on; root ../suite; } --- request GET /dir/ --- more_headers Accept-Encoding: zstd --- error_code: 404 --- response_headers !Content-Encoding === TEST 25: zstd_static on sets Vary even when declining for a non-accepting client # Subtle behaviour at static.c:204 — when zstd_static is "on" and # the .zst exists, the handler sets r->gzip_vary = 1 *before* # declining for a client that does not accept zstd. That keeps the # response cacheable by intermediaries that key on Vary, so a later # request from a zstd-capable client through the same shared cache # gets the encoded variant rather than the identity one. Without # this, a CDN that saw the identity response first would pin all # subsequent clients to it. --- config gzip_vary on; location /test { zstd_static on; root ../suite; } --- request GET /test --- more_headers Accept-Encoding: gzip --- response_headers !Content-Encoding Vary: Accept-Encoding --- no_error_log [error] === TEST 25a: zstd_static on sets Vary with gzip_vary OFF (non-accepting client) # G5: the gzip_vary-off cell of TEST 25, and the one that was broken. # With no "gzip_vary on", the handler used to return early WITHOUT even # probing for the .zst, so this identity response carried no Vary at # all — a shared cache then pinned every later client, including # zstd-capable ones, to the identity variant. The handler now emits the # field itself before declining. --- config location /test { zstd_static on; root ../suite; } --- request GET /test --- more_headers Accept-Encoding: gzip --- response_headers !Content-Encoding Vary: Accept-Encoding --- no_error_log [error] === TEST 25b: zstd_static on sets Vary with gzip_vary OFF (accepting client) # The served-.zst arm of the same cell: the compressed representation # must announce that it was negotiated on Accept-Encoding, or a cache # hands it to a client that cannot decode it. Together with TEST 15, # TEST 16 and TEST 25a this completes the static handler's # gzip_vary x Accept-Encoding matrix. --- config location /test { zstd_static on; root ../suite; } --- request GET /test --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd Vary: Accept-Encoding --- no_error_log [error] === TEST 26: zstd_static serves a .zst under directio without a failed pread # Regression for the O_DIRECT magic-probe bug. With "directio" active the # open_file_cache opens the .zst with O_DIRECT; the 4-byte, unaligned # magic pread() then fails EINVAL, wrongly declining every .zst above the # threshold and spamming NGX_LOG_CRIT. The fix skips the probe when # of.is_directio. Assert the precompressed frame is still served (200 + # Content-Encoding: zstd) and no CRIT/[error] appears. # # NOTE: O_DIRECT support is filesystem-dependent (tmpfs may not honour it), # so this deterministically catches the "declined + CRIT" regression only # where O_DIRECT actually engages; everywhere else it still asserts correct # serving. directio 1 = every file >= 1 byte is eligible (test.zst is 3717B). --- config location /test { zstd_static on; directio 1; root ../suite; } --- request GET /test --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- error_code: 200 --- no_error_log [error] === TEST 27: zstd_static "on" declining does not suppress the gzip fallback # Regression for the gzip-fallback latch. ngx_http_zstd_ok() latches # r->gzip_tested=1 / r->gzip_ok=0 as a side effect; the static handler # used to call it (before the .zst existence check), so when the .zst was # absent it declined but left gzip permanently marked "not ok" for the # request. A later gzip filter/handler then short-circuited on the cached # decision and served identity instead of gzip. The fix routes the static # decision through the side-effect-free ngx_http_zstd_accepts(). # # Reproduces with the always-present gzip *filter* (no gzip_static needed): # request a plain file that has NO sibling .zst. zstd_static declines; the # core static handler serves it; the gzip filter must still compress it. # Pre-fix: Content-Encoding is absent (gzip suppressed). Post-fix: gzip. --- config location /gz/ { zstd_static on; gzip on; gzip_min_length 1; gzip_types text/plain; root html; } --- user_files >>> gz/plain.txt gzip fallback body long enough to exceed gzip_min_length and actually compress padding padding padding padding --- request GET /gz/plain.txt --- more_headers Accept-Encoding: gzip, zstd --- response_headers Content-Encoding: gzip --- error_code: 200 --- no_error_log [error] === TEST 28: zstd_static coexists with gzip_static; .gz still served # Interop guard, sibling of TEST 27 but for the gzip_static *module*. # NOTE: unlike the gzip *filter* (TEST 27), gzip_static is a CONTENT_PHASE # handler. It is a built-in module, so its handler is pushed onto the # content-phase array before the dynamically-loaded zstd_static handler and # runs FIRST -- it serves the .gz before zstd_static ever runs, so the # ngx_http_zstd_ok() latch could not suppress it even on the pre-fix code. # (Verified: this test passes on both pre-fix and the fixed tree; the real # latch regression is covered by TEST 27 via the post-content gzip filter.) # Kept as a coexistence contract: with both directives on and a .gz but no # .zst present, zstd_static declines and gzip_static serves the .gz -- the # two static handlers must not fight over the request. # # The request has NO sibling .zst (so zstd_static declines) but DOES have a # real gzip-compressed plain.txt.gz (so gzip_static must serve it). --- config location /gzs/ { zstd_static on; gzip_static on; root html; } --- user_files eval my $body = "gzip_static fallback body long enough to matter " x 4; my $gz; require IO::Compress::Gzip; IO::Compress::Gzip::gzip(\$body => \$gz) or die "gzip failed: $IO::Compress::Gzip::GzipError"; ">>> gzs/plain.txt\n$body>>> gzs/plain.txt.gz\n$gz"; --- request GET /gzs/plain.txt --- more_headers Accept-Encoding: gzip, zstd --- response_headers Content-Encoding: gzip --- error_code: 200 --- no_error_log [error] === TEST 29: Content-Encoding entry leaves the headers_out chain terminated # Sibling of filter TEST 85 for the static module: the .zst sidecar path # pushes its own Content-Encoding entry and must terminate the header # chain the same way (see core ngx_http_gzip_static_module.c). --- config location /test { zstd_static on; add_header X-Sent-Content-Encoding $sent_http_content_encoding; root ../suite; } --- request GET /test --- more_headers Accept-Encoding: gzip, zstd --- response_headers Content-Encoding: zstd X-Sent-Content-Encoding: zstd --- error_code: 200 --- no_error_log [error] === TEST 30: a .zst that is a DIRECTORY is declined, origin served instead # of.is_dir branch in the static handler. ngx_open_cached_file() succeeds on a # directory, so without the is_dir check the handler would proceed to serve a # directory fd as a response body. Creating "dir.txt.zst/" as a real directory # (via a file nested inside it) makes the sibling lookup hit a directory; the # handler must decline and let the plain origin be served. --- config location /isdir/ { zstd_static on; root html; } --- user_files >>> isdir/dir.txt plain origin body served because the .zst sibling is a directory >>> isdir/dir.txt.zst/keep.txt this file only exists to make dir.txt.zst a directory --- request GET /isdir/dir.txt --- more_headers Accept-Encoding: zstd --- response_headers ! Content-Encoding ! Vary --- response_body plain origin body served because the .zst sibling is a directory --- error_code: 200 --- no_error_log [error] === TEST 31: a .zst declaring a >8MB window is declined (streaming frame) # The frame header (built byte-by-byte from RFC 8878 §3.1.1.1) declares # a 128 MB decompression window: magic, descriptor 0x00 (no # Single_Segment), Window_Descriptor 0x88 = exponent 17 -> 1 << 27. # That is what a Node streaming encoder stamps at high levels when not # told the input size — the file decodes fine with the zstd CLI but # every browser rejects it before decoding, so the handler must decline # from the header alone (trailing zeros stand in for the never-read # block data) and let the identity origin be served. --- config location /bw/ { zstd_static on; root html; } --- user_files eval ">>> bw/big.js\nbig-window stream body\n>>> bw/big.js.zst\n" . pack("C*", 0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x88, 0x00, 0x00, 0x00) --- request GET /bw/big.js --- more_headers Accept-Encoding: gzip, zstd --- response_headers ! Content-Encoding --- response_body big-window stream body --- error_code: 200 --- error_log big.js.zst declares a 134217728-byte decompression window above the 8 MB limit browsers enforce for Content-Encoding: zstd recompress with a window log <= 23 === TEST 31b: a .zst with reserved frame descriptor bit 0x08 is declined # RFC 8878 reserves Frame_Header_Descriptor bit 3. The static probe is the # browser-facing guard before we emit Content-Encoding: zstd, so it must not # serve a frame that libzstd itself rejects. The eval block keeps that # decoder-backed oracle attached to the handcrafted fixture when zstd is # available locally; hosts without the CLI still run the live fallback check. --- config location /reserved/ { zstd_static on; root html; } --- user_files eval my $bad = pack("C*", 0x28, 0xB5, 0x2F, 0xFD, 0x08, 0x00, 0x00); if (system("zstd", "--version") == 0) { require File::Temp; my ($fh, $path) = File::Temp::tempfile(); binmode $fh; print $fh $bad; close $fh; open my $old_stderr, ">&", \*STDERR or die "dup STDERR: $!"; open STDERR, ">", File::Spec->devnull() or die "redirect STDERR: $!"; my $rc = system("zstd", "-q", "-t", $path); open STDERR, ">&", $old_stderr or die "restore STDERR: $!"; unlink $path; die "decoder unexpectedly accepted reserved descriptor bit 0x08" if $rc == 0; } ">>> reserved/bit.js\nreserved-bit origin body\n>>> reserved/bit.js.zst\n" . $bad --- request GET /reserved/bit.js --- more_headers Accept-Encoding: gzip, zstd --- response_headers ! Content-Encoding --- response_body reserved-bit origin body --- error_code: 200 --- error_log frame header sets reserved Frame_Header_Descriptor bit 0x08 declining static variant === TEST 32: a single-segment .zst with >8MB content size is declined # Single-segment frames carry no Window_Descriptor — the window IS the # frame content size, read from behind the optional dictionary id. # Descriptor 0xA0 = Single_Segment with a 4-byte content size; the # little-endian field declares 20 MB, over the browser cap, so the # handler must decline this layout too. --- config location /bw/ { zstd_static on; root html; } --- user_files eval ">>> bw/single.js\nbig-window single-segment body\n>>> bw/single.js.zst\n" . pack("C*", 0x28, 0xB5, 0x2F, 0xFD, 0xA0, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x00) --- request GET /bw/single.js --- more_headers Accept-Encoding: gzip, zstd --- response_headers ! Content-Encoding --- response_body big-window single-segment body --- error_code: 200 --- error_log single.js.zst declares a 20971520-byte decompression window above the 8 MB limit browsers enforce for Content-Encoding: zstd recompress with a window log <= 23 === TEST 33: the window check runs under directio too # The probe historically skipped O_DIRECT files (unaligned preads fail # EINVAL — see #75); it now uses an aligned read so validation still # runs. The .zst is padded past the "directio 512" threshold so the # open really is O_DIRECT, and the oversized declared window must be # declined the same as TEST 31. Verified fail-first: on the # directio-skip build this file is served as Content-Encoding: zstd and # every assertion here fails. The "aligned probe" debug line is the # positive witness that is_directio was really set for this request — # without it the block would also pass vacuously through the stack-read # path on filesystems where O_DIRECT does not take. --- config location /bw/ { zstd_static on; directio 512; root html; } --- user_files eval ">>> bw/dio.js\nbig-window directio body\n>>> bw/dio.js.zst\n" . pack("C*", 0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x88) . ("\0" x 1018) --- request GET /bw/dio.js --- more_headers Accept-Encoding: gzip, zstd --- response_headers ! Content-Encoding --- response_body big-window directio body --- error_code: 200 --- error_log dio.js.zst declares a 134217728-byte decompression window aligned probe on directio file === TEST 34: the directio probe follows directio_alignment up to the cap # Review: the probe geometry follows the operator's declared alignment # between the 4 KB floor and the NGX_HTTP_ZSTD_STATIC_DIO_PROBE_MAX # (64 KB) ceiling, so storage declaring a larger block size than the # floor still gets an aligned read, and a failed validation read # DECLINES rather than serving unvalidated. 16 KB is inside that band # and so is honoured verbatim: the witness line must show a 16384-byte # probe and the oversized window must still be declined. TEST 34b pins # the other end of the band. --- config location /bw/ { zstd_static on; directio 512; directio_alignment 16k; root html; } --- user_files eval ">>> bw/dioal.js\nbig-window directio alignment body\n>>> bw/dioal.js.zst\n" . pack("C*", 0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x88) . ("\0" x 1018) --- request GET /bw/dioal.js --- more_headers Accept-Encoding: gzip, zstd --- response_headers ! Content-Encoding --- response_body big-window directio alignment body --- error_code: 200 --- error_log dioal.js.zst declares a 134217728-byte decompression window 16384-byte aligned probe on directio file === TEST 34b: the probe alignment is capped, and the cap applies to a client that does NOT accept zstd # s1 regression, two properties in one block. # # (1) CAP. "directio_alignment" is ngx_conf_set_off_slot on an off_t # with no upper bound in core (ngx_http_core_module.c), and core spends # it on the copy filter's BODY buffer, where a large value is a # throughput choice. O_DIRECT legality is a property of the device's # logical block size, not of that directive, so an 18-byte frame-HEADER # probe has nothing to gain from scaling with it. Uncapped, the probe # did ngx_pmemalign(align * 2) plus a 2*align O_DIRECT read per # request; at "directio_alignment 1m" that is a 2 MB allocation and a # 2 MB read to inspect 18 bytes. The probe alignment is now clamped to # NGX_HTTP_ZSTD_STATIC_DIO_PROBE_MAX, so the witness must say # 65536-byte, NOT 1048576-byte. # # (2) NON-ACCEPTING CLIENT. TESTs 26/33/34/46 all send an accepting # client, so nothing covered the directio probe on the path a client # that does not accept zstd takes — which is precisely the path the # amplification was reachable on, because #202 moved the # "return NGX_DECLINED" for such a client to AFTER the probe (the probe # result is required to decide whether Vary is truthful). This block # sends "Accept-Encoding: gzip" and still asserts the probe ran, so the # cap is pinned on the request shape that motivated it. # # Falsifiability, both directions: # - remove the clamp -> the witness line reads "1048576-byte aligned # probe", the 65536 pattern does not match, and this block goes red. # - restore the pre-#202 early return for a non-accepting client -> # no probe runs at all, so neither the witness line NOR the Vary # header appears, and this block goes red on both. # The 1048576 negative assertion makes the first direction fail loudly # rather than merely stop matching. # # Vary is asserted because it is the reason the probe is allowed to run # for this client at all: the .zst is a valid, usable variant here (a # small window, unlike TEST 34), so the identity response really is # Accept-Encoding-dependent and must say so. The .zst is padded past # the "directio 512" threshold so the open really is O_DIRECT. --- config location /cap/ { zstd_static on; directio 512; directio_alignment 1m; root html; } --- user_files eval ">>> cap/noaccept.js\ncapped probe identity body\n>>> cap/noaccept.js.zst\n" . pack("C*", 0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x00, 0x19, 0x00, 0x00) . "hi\n" . ("\0" x 2048) --- request GET /cap/noaccept.js --- more_headers Accept-Encoding: gzip --- response_headers ! Content-Encoding Vary: Accept-Encoding --- response_body capped probe identity body --- error_code: 200 --- error_log 65536-byte aligned probe on directio file --- no_error_log 1048576-byte aligned probe on directio file [error] === TEST 35: concatenated frames are validated on the leading frame only # Contract pin for the documented scope (review): a valid small first # frame followed by an oversized second frame is SERVED — a regular # frame's header does not declare its compressed length, so the probe # cannot walk the sequence without decoding block chains. The README # scopes the guarantee to the leading frame and points at # `zstd -t --memory=8MB` as the complete pre-deploy check; this block # exists so any future change to that scope is a deliberate one. # First frame: 1 KB window, one raw last-block containing "hi\n". # Second frame: the 128 MB-window header from TEST 31. --- config location /bw/ { zstd_static on; root html; } --- user_files eval ">>> bw/concat.js\nconcat origin body\n>>> bw/concat.js.zst\n" . pack("C*", 0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x00, 0x19, 0x00, 0x00) . "hi\n" . pack("C*", 0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x88, 0x00, 0x00, 0x00) --- request GET /bw/concat.js --- more_headers Accept-Encoding: gzip, zstd --- response_headers Content-Encoding: zstd --- error_code: 200 --- no_error_log decompression window === TEST 36: a served sidecar advertises Accept-Ranges: bytes # gzip_static parity (see gzip_static's r->allow_ranges = 1): the # representation is the .zst bytes and the validator is strong, so # static-side ranges are coherent — resumable downloads included. # Fails on builds without the opt-in: no allow_ranges, no header. --- config location /test { zstd_static on; root ../suite; } --- request GET /test --- more_headers Accept-Encoding: gzip, zstd --- response_headers Content-Encoding: zstd Accept-Ranges: bytes --- no_error_log [error] === TEST 37: byte ranges slice the sidecar's bytes (206 + Content-Range) # RFC 9110 §14.2: ranges address the SELECTED REPRESENTATION — under # Content-Encoding: zstd that is the .zst bytes themselves, which a # client can fetch, resume and concatenate before decompressing. The # fixture is TEST 35's crafted 12-byte frame (magic + 1 KB window + # raw "hi\n" block), so the slice is byte-pinned: the first four bytes # are the frame magic. Unfixed code ignores Range and answers 200. --- config location /rg/ { zstd_static on; root html; } --- user_files eval ">>> rg/tiny.js\ntiny origin body\n>>> rg/tiny.js.zst\n" . pack("C*", 0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x00, 0x19, 0x00, 0x00) . "hi\n" --- request GET /rg/tiny.js --- more_headers Accept-Encoding: zstd Range: bytes=0-3 --- error_code: 206 --- response_headers Content-Encoding: zstd Content-Range: bytes 0-3/12 --- response_body eval pack("C*", 0x28, 0xB5, 0x2F, 0xFD) --- no_error_log [error] === TEST 38: a 4-byte skippable-magic-only file is declined (truncated skip header) # sec/g5-static-skippable-frame: ngx_http_zstd_static_probe_frame() must # see the 4-byte Frame_Size field before it can trust a skippable frame # at all. A file that is only the magic cannot supply it, so this must # fail closed exactly like any other truncated header — decline, fall # back to serving the uncompressed original. --- config location /skip/ { zstd_static on; root html; } --- user_files eval ">>> skip/magiconly.js\nmagic only origin\n>>> skip/magiconly.js.zst\n" . pack("C*", 0x50, 0x2A, 0x4D, 0x18) --- request GET /skip/magiconly.js --- more_headers Accept-Encoding: zstd --- response_headers ! Content-Encoding --- response_body magic only origin --- error_code: 200 --- error_log frame header truncated === TEST 39: a 7-byte skippable header (Frame_Size one byte short) is declined --- config location /skip/ { zstd_static on; root html; } --- user_files eval ">>> skip/seven.js\nseven byte origin\n>>> skip/seven.js.zst\n" . pack("C*", 0x50, 0x2A, 0x4D, 0x18, 0x00, 0x00, 0x00) --- request GET /skip/seven.js --- more_headers Accept-Encoding: zstd --- response_headers ! Content-Encoding --- response_body seven byte origin --- error_code: 200 --- error_log frame header truncated === TEST 40: an exact 8-byte skippable header with nothing after it is declined # A zero-length-payload skippable header parses fine (Frame_Size = 0) # and passes the bounds check (8 header bytes exactly fill of.size), so # the walk advances to offset 8 and probes again — there is nothing # left to read there, so the follow-up pread(2) returns 0 and the # handler declines exactly as it does for any other pread short-read, # rather than reading past EOF looking for a frame that isn't there. --- config location /skip/ { zstd_static on; root html; } --- user_files eval ">>> skip/eightonly.js\neight byte origin\n>>> skip/eightonly.js.zst\n" . pack("C*", 0x50, 0x2A, 0x4D, 0x18, 0x00, 0x00, 0x00, 0x00) --- request GET /skip/eightonly.js --- more_headers Accept-Encoding: zstd --- response_headers ! Content-Encoding --- response_body eight byte origin --- error_code: 200 --- error_log pread --- no_error_log reusing === TEST 41: a skippable Frame_Size that overflows 32-bit arithmetic is declined # Frame_Size 0xFFFFFFFF (4294967295) must not be added to the 8-byte # header offset with plain 32-bit/size_t arithmetic on a narrow # platform — the handler does the bounds check in 64-bit so this fails # closed instead of wrapping into a small, in-bounds offset. --- config location /skip/ { zstd_static on; root html; } --- user_files eval ">>> skip/overflow.js\noverflow origin\n>>> skip/overflow.js.zst\n" . pack("C*", 0x50, 0x2A, 0x4D, 0x18, 0xFF, 0xFF, 0xFF, 0xFF) --- request GET /skip/overflow.js --- more_headers Accept-Encoding: zstd --- response_headers ! Content-Encoding --- response_body overflow origin --- error_code: 200 --- error_log skippable frame declares a 4294967295-byte skip past end of file === TEST 42: a skippable Frame_Size declaring a skip past EOF is declined # Frame_Size 1000 in an 8-byte file: 8 + 1000 is nowhere near # overflowing, but it is far past of.size. This is the "declared skip # length past EOF" case, distinct from TEST 41's overflow case. --- config location /skip/ { zstd_static on; root html; } --- user_files eval ">>> skip/pasteof.js\npast eof origin\n>>> skip/pasteof.js.zst\n" . pack("C*", 0x50, 0x2A, 0x4D, 0x18, 0xE8, 0x03, 0x00, 0x00) --- request GET /skip/pasteof.js --- more_headers Accept-Encoding: zstd --- response_headers ! Content-Encoding --- response_body past eof origin --- error_code: 200 --- error_log skippable frame declares a 1000-byte skip past end of file === TEST 43: a valid dcz-style skippable prefix followed by a good regular frame IS served # The bypass fix must not break the legitimate shape it exists # alongside: RFC 9842 dcz frames are exactly a skippable frame ahead of # the real payload frame (see README "Standards-based dictionary # compression"). One skippable frame (Frame_Size 4, four bytes of # opaque payload) followed by a small-window regular frame must still # be served normally. --- config location /skip/ { zstd_static on; root html; } --- user_files eval ">>> skip/prefix.js\nprefix origin body\n>>> skip/prefix.js.zst\n" . pack("C*", 0x50, 0x2A, 0x4D, 0x18, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x00, 0x19, 0x00, 0x00) . "hi\n" --- request GET /skip/prefix.js --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- error_code: 200 --- no_error_log [error] === TEST 44: exactly four leading skippable frames followed by a regular frame are served # NGX_HTTP_ZSTD_STATIC_MAX_SKIP_FRAMES permits four leading skippable # frames. The handler must still probe the following regular frame, # rather than declining before it reads that frame. --- config location /skip/ { zstd_static on; root html; } --- user_files eval ">>> skip/four.js\nfour skips origin\n>>> skip/four.js.zst\n" . (pack("C*", 0x50, 0x2A, 0x4D, 0x18, 0x00, 0x00, 0x00, 0x00) x 4) . pack("C*", 0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x00, 0x19, 0x00, 0x00) . "hi\n" --- request GET /skip/four.js --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- error_code: 200 --- no_error_log [error] === TEST 44b: a fifth leading skippable frame is declined # The cap is enforced only after the fifth skippable frame is observed. # This keeps the walk bounded while allowing the regular frame after four. --- config location /skip/ { zstd_static on; root html; } --- user_files eval ">>> skip/toolong.js\ntoo long origin\n>>> skip/toolong.js.zst\n" . (pack("C*", 0x50, 0x2A, 0x4D, 0x18, 0x00, 0x00, 0x00, 0x00) x 5) . pack("C*", 0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x00, 0x19, 0x00, 0x00) . "hi\n" --- request GET /skip/toolong.js --- more_headers Accept-Encoding: zstd --- response_headers ! Content-Encoding --- response_body too long origin --- error_code: 200 --- error_log has at least 5 leading skippable frames === TEST 45: a skippable prefix hiding an oversized-window regular frame is REJECTED # THE BYPASS THIS ITEM FIXES: before this change, ANY skippable magic # made the probe return OK immediately, so prepending a trivial # skippable frame to an oversized-window regular frame skipped the 8 MB # window guard entirely — the whole point of the probe. The handler # must now resolve the skip and check the frame that actually follows. --- config location /skip/ { zstd_static on; root html; } --- user_files eval ">>> skip/bypass.js\nbypass origin body\n>>> skip/bypass.js.zst\n" . pack("C*", 0x50, 0x2A, 0x4D, 0x18, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x88, 0x00, 0x00, 0x00) . "payloadbytes" --- request GET /skip/bypass.js --- more_headers Accept-Encoding: zstd --- response_headers ! Content-Encoding --- response_body bypass origin body --- error_code: 200 --- error_log declares a 134217728-byte decompression window above the 8 MB limit browsers enforce for Content-Encoding: zstd === TEST 46: a dcz-style skippable prefix on a directio file IS served # M1 regression. Under directio the probe buffer is aligned and the # first read at offset 0 succeeds, but the skippable walk then moves # `pos` to 40 (the canonical dcz SHA-256 prefix: 8-byte skippable # header + 32-byte payload) and the NEXT read used that offset raw. An # O_DIRECT descriptor rejects an unaligned file offset with EINVAL, so # the read returned -1, the probe took the "aligned probe ... returned # -1" branch and DECLINED — every request for a dcz-shaped .zst on a # directio location silently lost its precompressed variant (a 404 with # "zstd_static always" and no identity file). The fix rounds the read # offset down to the alignment and parses the frame at its offset # inside the block. # # Fail-first control (observed): reverting the alignment (reading at # `pos` instead of `base`) turns this into "! Content-Encoding" plus # the "returned -1" error line, so both the 200-with-zstd assertion and # the --- no_error_log [error] assertion go red. # # The .zst is padded past the "directio 512" threshold so the open # really is O_DIRECT; "aligned probe on directio file" is the positive # witness that is_directio was set for this request, so the block # cannot pass vacuously through the stack-read path on a filesystem # where O_DIRECT does not take. --- config location /skip/ { zstd_static on; directio 512; root html; } --- user_files eval ">>> skip/dioprefix.js\ndirectio prefix origin\n>>> skip/dioprefix.js.zst\n" . pack("C*", 0x50, 0x2A, 0x4D, 0x18, 0x20, 0x00, 0x00, 0x00) . ("\0" x 32) . pack("C*", 0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x00, 0x19, 0x00, 0x00) . "hi\n" . ("\0" x 1024) --- request GET /skip/dioprefix.js --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- error_code: 200 --- no_error_log [error] === TEST 47: dictionary bypass defaults off and the sidecar still wins # This is the default-value negative control. A dictionary-carrying client # explicitly accepts dcz, but without zstd_static_dict_bypass the static # handler keeps its historical first claim on the response. --- config location /test { zstd_static on; root ../suite; } --- request GET /test --- more_headers Accept-Encoding: zstd, dcz Available-Dictionary: :AA==: --- response_headers Content-Length: 3717 ETag: "5be17d33-e85" Content-Encoding: zstd --- no_error_log [error] === TEST 48: dictionary bypass requires Available-Dictionary # Explicit dcz alone must not forfeit a usable sidecar. --- config location /test { zstd_static on; zstd_static_dict_bypass on; root ../suite; } --- request GET /test --- more_headers Accept-Encoding: zstd, dcz --- response_headers Content-Length: 3717 ETag: "5be17d33-e85" Content-Encoding: zstd --- no_error_log [error] === TEST 49: dictionary bypass does not treat wildcard as dcz # RFC 9842 requires an explicit dcz token; '*' cannot prove the client can # decode a dictionary response. --- config location /test { zstd_static on; zstd_static_dict_bypass on; root ../suite; } --- request GET /test --- more_headers Accept-Encoding: zstd, * Available-Dictionary: :AA==: --- response_headers Content-Length: 3717 ETag: "5be17d33-e85" Content-Encoding: zstd --- no_error_log [error] === TEST 50: dictionary bypass honors dcz q=0 across duplicate fields # Repeated Accept-Encoding lines are one list. An explicit refusal on either # line wins, so this stays on the sidecar path. --- config location /test { zstd_static on; zstd_static_dict_bypass on; root ../suite; } --- request GET /test --- more_headers Accept-Encoding: zstd, dcz Accept-Encoding: dcz;q=0 Available-Dictionary: :AA==: --- response_headers Content-Length: 3717 ETag: "5be17d33-e85" Content-Encoding: zstd --- no_error_log [error] === TEST 51: dictionary bypass also applies to zstd_static always # The bypass is a preference for the filter path, so it must precede the # always-mode shortcut. With no filter configured here, core serves identity; # the complete Vary key keeps that fallback separate from the sidecar. --- config location /test { zstd_static always; zstd_static_dict_bypass on; root ../suite; } --- request GET /test --- more_headers Accept-Encoding: zstd, dcz Available-Dictionary: :AA==: --- response_headers Content-Length: 59738 ETag: "5be17d33-e95a" ! Content-Encoding Vary: Accept-Encoding, Available-Dictionary, Sec-Fetch-Site --- no_error_log [error] === TEST 52: dictionary bypass inherits into a location --- config zstd_static on; zstd_static_dict_bypass on; location /test { root ../suite; } --- request GET /test --- more_headers Accept-Encoding: zstd, dcz Available-Dictionary: :AA==: --- response_headers Content-Length: 59738 ETag: "5be17d33-e95a" ! Content-Encoding Vary: Accept-Encoding, Available-Dictionary, Sec-Fetch-Site --- no_error_log [error] === TEST 53: location off overrides inherited dictionary bypass --- config zstd_static on; zstd_static_dict_bypass on; location /test { zstd_static_dict_bypass off; root ../suite; } --- request GET /test --- more_headers Accept-Encoding: zstd, dcz Available-Dictionary: :AA==: --- response_headers Content-Length: 3717 ETag: "5be17d33-e85" Content-Encoding: zstd --- no_error_log [error] === TEST 54: HEAD dictionary bypass keeps the cache partition # The response filter deliberately does not encode header-only responses, so # the static handler must emit the complete Vary key before declining. --- config location /test { zstd_static on; zstd_static_dict_bypass on; root ../suite; } --- request HEAD /test --- more_headers Accept-Encoding: zstd, dcz Available-Dictionary: :AA==: --- response_headers Content-Length: 59738 ETag: "5be17d33-e95a" ! Content-Encoding Vary: Accept-Encoding, Available-Dictionary, Sec-Fetch-Site --- no_error_log [error] === TEST 55: dictionary bypass preserves a subrequest sidecar # auth_request drives /only/test as a real subrequest. The response filter # refuses every subrequest, so the static handler must ignore dictionary # bypass there. Only the .zst sidecar exists: removing the r == r->main guard # makes the subrequest fall through to a 404 and the parent return 500. --- config location = /main { auth_request /only/test; root html; } location = /only/test { internal; zstd_static always; zstd_static_dict_bypass on; root html; } --- user_files eval open my $fixture, '<:raw', 'ci/t/suite/test.zst' or die "open zstd fixture: $!"; local $/; my $zst = <$fixture>; close $fixture or die "close zstd fixture: $!"; [ [ 'main' => "main response\n" ], [ 'only/test.zst' => $zst ], ] --- request GET /main --- more_headers Accept-Encoding: zstd, dcz Available-Dictionary: :AA==: --- response_body main response --- no_error_log [error] === TEST 56: the directio skip-frame walk reuses the block it already read # The canonical dcz shape (8-byte skippable header + 32-byte payload, # next frame at offset 40) with an alignment of 4096 or more puts BOTH # frames inside the very first aligned block. The walk still recomputed # base = 0 on iteration 2 and re-issued an identical 2*align O_DIRECT # pread for bytes already sitting in `hdr` — at "directio_alignment # 16k" that is a 32 KB re-read per skipped frame, to look at 18 bytes # it already had. # # TEST 46 pins that this shape is SERVED; it cannot see how many reads # it took, because both the cached and uncached paths serve identical # bytes. The "reusing ... block at offset 0" debug line is the witness # that the second read was actually elided, so this block asserts the # optimization engaged rather than merely that the response is right. # # Falsifiability: drop the cache guard (always pread) and the reuse # line never appears, so this block goes red while TEST 46 stays green # — which is precisely the regression TEST 46 cannot catch. # # The alignment witness is asserted too, so this cannot pass vacuously # on a filesystem where O_DIRECT does not engage: with no directio the # buffered path runs, neither line is logged, and both assertions fail. --- config location /skip/ { zstd_static on; directio 512; directio_alignment 16k; root html; } --- user_files eval ">>> skip/dioreuse.js\ndirectio reuse origin\n>>> skip/dioreuse.js.zst\n" . pack("C*", 0x50, 0x2A, 0x4D, 0x18, 0x20, 0x00, 0x00, 0x00) . ("\0" x 32) . pack("C*", 0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x00, 0x19, 0x00, 0x00) . "hi\n" . ("\0" x 1024) --- request GET /skip/dioreuse.js --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- error_code: 200 --- error_log 16384-byte aligned probe on directio file reusing 32768-byte block at offset 0 for next frame --- no_error_log [error] === TEST 57: the ordinary probe retains a canonical dcz prefix # The ordinary probe reads the canonical 40-byte skippable prefix and the # following maximum 18-byte frame header together. The reuse log proves the # second frame came from that request-local read-ahead instead of another # offset read; TEST 45 separately pins malformed and truncated skip handling. --- config location /skip/ { zstd_static on; root html; } --- user_files eval ">>> skip/read_ahead.js\nordinary read ahead origin\n>>> skip/read_ahead.js.zst\n" . pack("C*", 0x50, 0x2A, 0x4D, 0x18, 0x20, 0x00, 0x00, 0x00) . ("\0" x 32) . pack("C*", 0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x00, 0x19, 0x00, 0x00) . "hi\n" --- request GET /skip/read_ahead.js --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- error_code: 200 --- error_log reusing 58-byte block at offset 0 for next frame --- no_error_log [error] === TEST 58: malformed sidecar verdict is memoized per path and mtime # Three repeated requests hit the same malformed sidecar and fixed revision. # The first path is probed; the later requests use its worker-local verdict. # Matching only the stable message fragments makes # grep_error_log_out an exact count assertion independent of the temp path. # # Falsifiability: removing the cache lookup yields three "not a zstd frame" # matches and no cached-verdict matches. --- config error_log logs/error.log debug; location /memoized.txt { zstd_static on; root html; } --- user_files >>> memoized.txt 202601010000.00 identity fallback >>> memoized.txt.zst 202601010000.00 HELO malformed sidecar --- request GET /memoized.txt --- more_headers Accept-Encoding: zstd --- response_body identity fallback --- grep_error_log eval qr/(?:is not a zstd frame|cached malformed verdict)/ --- grep_error_log_out eval ["is not a zstd frame\n", "is not a zstd frame\ncached malformed verdict\n", "is not a zstd frame\ncached malformed verdict\n" . "cached malformed verdict\n"] --- no_error_log [alert] === TEST 59: dictionary bypass fails closed on duplicate Available-Dictionary # Regression for TEST 48's row: should_bypass() must count every # Available-Dictionary line, not stop at the first, and stand aside only # when exactly one is present. Two lines is the ambiguous case the dynamic # filter refuses outright (avail_dict_count > 1 in # ngx_http_zstd_filter_module.c); the static routing predicate must not # bypass to it here either, or the client forfeits a usable .zst sidecar # for a filter that then declines and falls back to identity. --- config location /test { zstd_static on; zstd_static_dict_bypass on; root ../suite; } --- request GET /test --- more_headers Accept-Encoding: zstd, dcz Available-Dictionary: :AA==: Available-Dictionary: :AA==: --- response_headers Content-Length: 3717 ETag: "5be17d33-e85" Content-Encoding: zstd --- no_error_log [error] === TEST 60: valid sidecar probe verdict is reused with open_file_cache # repeat_each(3) sends the same sidecar through one worker three times. The # first request probes; requests two and three must emit the debug witness. # Removing the GOOD-cache lookup keeps the response correct but makes this # exact-count oracle empty. --- config error_log logs/error.log debug; open_file_cache max=10 inactive=60s; open_file_cache_valid 60s; location /test { zstd_static on; root ../suite; } --- request GET /test --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- grep_error_log eval qr/cached good frame verdict/ --- grep_error_log_out eval ["", "cached good frame verdict\n", "cached good frame verdict\ncached good frame verdict\n"] --- no_error_log [alert] === TEST 61: valid verdict is not retained when open_file_cache is off # The standalone unit can pass good_valid=0 directly; this integration arm # proves the handler derives that value from nginx's real off sentinel. A # positive verdict surviving here would outlive the fresh descriptor/stat # contract that disabling open_file_cache requests. --- config error_log logs/error.log debug; open_file_cache off; location /test { zstd_static on; root ../suite; } --- request GET /test --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- grep_error_log eval qr/cached good frame verdict/ --- grep_error_log_out eval ["", "", ""] --- no_error_log [alert] myguard-labs-nginx-zstd-module-3486b5b/ci/t/02-conf-warn.t000066400000000000000000001136111525232545500231750ustar00rootroot00000000000000use Test::Nginx::Socket; use File::Basename; use File::Spec; use File::Temp qw(tempfile); use lib 'lib'; my $dirname = dirname(__FILE__); # Absolute path to the committed fixture dir (ci/t/suite, holding test + # test.zst). TEST 14 uses it so the fixture is independent of which safe, # runner-private servroot a local, PR, deep, or coverage invocation selects. our $suite_dir = File::Spec->rel2abs("$dirname/suite"); # local: this process is the test run, but perlcritic is right that a bare # assignment to %ENV leaks into anything that runs after it. local $ENV{'TEST_NGINX_PERL_PATH'} = "$ENV{'PWD'}/$dirname"; my @dynamic_modules; our $static_so; # the static .so alone, for the filterless block if (defined $ENV{'TEST_NGINX_BINARY'}) { my $nginx_dir = dirname($ENV{'TEST_NGINX_BINARY'}); for my $module_name (qw(ngx_http_zstd_filter_module.so ngx_http_zstd_static_module.so)) { my $module_path = "$nginx_dir/$module_name"; push @dynamic_modules, $module_path if -f $module_path; $static_so = $module_path if $module_name eq 'ngx_http_zstd_static_module.so' && -f $module_path; } } # The estimator-only directive (zstd_max_cctx_memory) needs a module built # with -DZSTD_STATIC_LINKING_ONLY. ci/tools/ci-build.sh leaves that out of # its release flavours on purpose (the deployable .so must not depend on # libzstd's static-only entry points), so CI Deep's Build & Test matrix runs # this file against a binary that refuses the directive. `nginx -V` echoes # the configure arguments, so the build shape is detectable here: blocks # that need the estimator skip themselves on a release-shape build, and # their complement (the refusal with its message) runs only there. our $static_linking = 0; if (defined $ENV{'TEST_NGINX_BINARY'}) { # Shell-free: the binary path is one argv word, however it is spelled, # and its stderr (where -V prints) is joined onto the pipe in the child. my $pid = open(my $vh, '-|'); die "fork for nginx -V: $!" if !defined $pid; if (!$pid) { open(STDERR, '>&', \*STDOUT) or exit 127; exec {$ENV{'TEST_NGINX_BINARY'}} $ENV{'TEST_NGINX_BINARY'}, '-V'; exit 127; } my $v = do { local $/; <$vh> }; close $vh; $static_linking = 1 if defined $v && $v =~ /-DZSTD_STATIC_LINKING_ONLY/; } # One byte over NGX_HTTP_ZSTD_MAX_DICT_SIZE (10 MB), generated rather than # committed, for the too-large refusal (TEST 25b). Exposed to config blocks # via $TEST_NGINX_ZSTD_HUGEDICT. File::Temp picks an unpredictable name # and opens it O_EXCL, so a pre-seeded symlink at a guessable path cannot # redirect the write on a shared runner, and the file goes at exit. The # loader refuses it at fstat() on size alone, so the file is extended with # a checked truncate rather than written: no 10 MiB scalar, no bytes read. my ($huge_fh, $huge_path) = tempfile("zstd-hugedict-XXXXXX", TMPDIR => 1, UNLINK => 1); truncate($huge_fh, 10 * 1024 * 1024 + 1) or die "hugedict: truncate: $!"; close $huge_fh or die "hugedict: close: $!"; local $ENV{'TEST_NGINX_ZSTD_HUGEDICT'} = $huge_path; add_block_preprocessor(sub { my $block = shift; return if !@dynamic_modules; # Blocks named "filterless" model the static-only deployment: load # ONLY the static .so, so ngx_http_zstd_filter_module is genuinely # absent from the cycle. Possible on dynamic builds alone — those # blocks skip_eval themselves away when the .so is not there. if (defined($block->name) && $block->name =~ /filterless/ && $static_so) { $block->set_value("main_config", "load_module $static_so;"); return; } my $main_config = join "\n", map { "load_module $_;" } @dynamic_modules; $block->set_value("main_config", $main_config); }); no_long_string(); log_level 'warn'; # Config-load warnings are emitted ONCE at startup; with repeat_each > 1 # the error.log is wiped between repeats and later iterations cannot # re-observe them (why these blocks live outside t/00-filter.t). repeat_each(1); plan 'no_plan'; run_tests(); __DATA__ === TEST 1: zstd_bypass_vary without zstd_bypass warns at config load # zstd_bypass_vary names the header the zstd_bypass decision varies on; # set alone it emits a Vary field no response varies on (silent cache # hit-rate degradation). merge_loc_conf warns — assert the warning is # actually emitted so the misconfig stays visible. --- config location /warn { zstd on; zstd_bypass_vary X-No-Compression; default_type text/plain; return 200 "hello world padding padding padding\n"; } --- request GET /warn --- more_headers Accept-Encoding: zstd --- error_log "zstd_bypass_vary" is set without a "zstd_bypass" predicate --- no_error_log [error] === TEST 2: zstd_bypass_vary WITH zstd_bypass does not warn # The complementary half: a correctly paired configuration must load # silently, so the warning cannot become noise on valid configs. --- config location /paired { zstd on; zstd_bypass $http_x_no_compression; zstd_bypass_vary X-No-Compression; default_type text/plain; return 200 "hello world padding padding padding\n"; } --- request GET /paired --- more_headers Accept-Encoding: zstd --- no_error_log eval [qr/zstd_bypass_vary.*without/, qr/\[error\]/] === TEST 2b: zstd_bypass on a direct $http_* predicate without vary warns # Inverse of TEST 1: a zstd_bypass predicate that reads a request header # DIRECTLY, with no zstd_bypass_vary alongside it, lets a shared cache # mix an identity response with a compressed one under the same key. --- config location /warn2b { zstd on; zstd_bypass $http_x_no_compression; default_type text/plain; return 200 "hello world padding padding padding\n"; } --- request GET /warn2b --- more_headers Accept-Encoding: zstd --- error_log without a "zstd_bypass_vary" --- no_error_log [error] === TEST 2c: zstd_bypass on a direct $cookie_* predicate without vary warns # Same hazard, cookie-driven bypass instead of a request header. --- config location /warn2c { zstd on; zstd_bypass $cookie_no_compression; default_type text/plain; return 200 "hello world padding padding padding\n"; } --- request GET /warn2c --- more_headers Accept-Encoding: zstd --- error_log without a "zstd_bypass_vary" --- no_error_log [error] === TEST 2d: direct predicate WITH zstd_bypass_vary stays silent # The correctly-paired configuration (TEST 2's config) must not trip the # new inverse warning either -- both directions of the coupling check # must agree on a valid config. --- config location /paired2d { zstd on; zstd_bypass $http_x_no_compression; zstd_bypass_vary X-No-Compression; default_type text/plain; return 200 "hello world padding padding padding\n"; } --- request GET /paired2d --- more_headers Accept-Encoding: zstd --- no_error_log eval [qr/without a "zstd_bypass_vary"/, qr/\[error\]/] === TEST 2e: a map-based zstd_bypass predicate without vary stays silent # Indirect variables (map results, etc.) are a documented operator # responsibility, not something this module can resolve; the raw # predicate text here is "$zstd_off", which contains neither "$http_" # nor "$cookie_", so the new check must not fire a false positive. --- http_config map $http_x_no_zstd $zstd_off { default 0; "1" 1; } --- config location /warn2e { zstd on; zstd_bypass $zstd_off; default_type text/plain; return 200 "hello world padding padding padding\n"; } --- request GET /warn2e --- more_headers Accept-Encoding: zstd --- no_error_log eval [qr/without a "zstd_bypass_vary"/, qr/\[error\]/] === TEST 2f: a response-only variable predicate without vary stays silent # A predicate that references neither a request header nor a cookie -- # here a response-side module variable -- must not trip the new check. --- config location /warn2f { zstd on; zstd_bypass $zstd_ratio; default_type text/plain; return 200 "hello world padding padding padding\n"; } --- request GET /warn2f --- more_headers Accept-Encoding: zstd --- no_error_log eval [qr/without a "zstd_bypass_vary"/, qr/\[error\]/] === TEST 3: zstd_static rejects an invalid enum value cleanly # Regression for the missing ngx_null_string sentinel in the # ngx_http_zstd_static[] ngx_conf_enum_t array. ngx_conf_set_enum_slot() # scans until name.len == 0; without the terminating { ngx_null_string, 0 } # an unmatched value walked off the array end (OOB read, wild-pointer # ngx_strcasecmp -> possible crash / ASAN abort). With the sentinel an # unknown value is a clean "invalid value" config error. The ASAN CI job # runs this binary, so a re-introduced OOB read aborts here. --- config location /bad { zstd_static maybe; root html; } --- must_die --- error_log invalid value "maybe" --- no_error_log [alert] === TEST 4: zstd_static accepts the last valid enum value ("always") # Positive counterpart: the sentinel must not shorten the valid range. # "always" is the final real entry, immediately before the sentinel — it # still parses and loads. --- config location /ok { zstd_static always; root html; } --- request GET /ok/nope --- error_code: 404 --- no_error_log invalid value === TEST 5: zstd_dict_file without zstd_dict_file_unsafe refuses to start # RFC1 gate (init_main_conf): a dictionary-compressed body is emitted as a # plain "Content-Encoding: zstd" that no generic client can decode and that # RFC 9842 (dcz) does not negotiate. Starting without the explicit # acknowledgement must be a hard config error, not a warning — otherwise the # non-standard mode ships silently. Guards the operator-acknowledgement gate. --- http_config zstd_dict_file $TEST_NGINX_SERVER_ROOT/html/zstd.dict; --- user_files >>> zstd.dict the quick brown fox jumps over the lazy dog --- config location /d { zstd on; default_type text/plain; return 200 "body"; } --- must_die --- error_log Set "zstd_dict_file_unsafe on;" to acknowledge you control both ends --- no_error_log [alert] === TEST 6: zstd_dict_file pointing at a missing file fails at config load # The open() failure path in merge_loc_conf. A dictionary that vanished (bad # path, un-deployed asset) must fail startup with the filename in the message, # rather than starting and silently compressing without the dictionary. --- http_config zstd_dict_file_unsafe on; zstd_dict_file $TEST_NGINX_SERVER_ROOT/html/does-not-exist.dict; --- config location /d { zstd on; default_type text/plain; return 200 "body"; } --- must_die --- error_log does-not-exist.dict" failed --- no_error_log [alert] === TEST 7: zstd_comp_level above the library maximum is rejected # ngx_http_zstd_comp_level bounds the level against ZSTD_minCLevel()/ # ZSTD_maxCLevel() at config load. Without the check libzstd would reject the # value per-request, turning one typo into a 500 on every response for the # location. Test above the upper bound rather than below the lower one: # ZSTD_minCLevel() is -131072, so a "clearly too negative" literal would have # to be enormous to stay invalid. # # The value is deliberately far above the bound rather than maxCLevel()+1: the # module reads the maximum from libzstd at runtime, so a literal 23 would stop # being invalid the day libzstd raises its ceiling, and this must_die block # would silently start passing for the wrong reason. --- config location /lvl { zstd on; zstd_comp_level 999999; default_type text/plain; return 200 "body"; } --- must_die --- error_log zstd compression level must be between --- no_error_log [alert] === TEST 8: zstd_window_log outside the library bounds is rejected # C3 regression: zstd_window_log is validated against # ZSTD_cParam_getBounds(ZSTD_c_windowLog) at config load, not hard-coded # constants. 99 is above upperBound on every supported libzstd; catching it # here turns a per-request 500 into a clear startup error. --- config location /wl { zstd on; zstd_window_log 99; default_type text/plain; return 200 "body"; } --- must_die --- error_log "zstd_window_log" must be 0 (default) or between --- no_error_log [alert] === TEST 9: zstd_window_log accepts 0 (explicit "library default") # Positive counterpart to TEST 8: the bounds check must special-case 0 rather # than comparing it against lowerBound (which is > 0), or "keep zstd's # level-derived default" would be unspellable. --- config location /wl0 { zstd on; zstd_window_log 0; zstd_min_length 1; zstd_types text/plain; default_type text/plain; return 200 "compress me compress me compress me compress me"; } --- request GET /wl0 --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 10: a non-numeric zstd_window_log is rejected as an invalid number # ngx_conf_zstd_set_num_slot_with_negatives' ngx_atoi failure path. The custom # slot parser exists to accept negative levels, so its error handling is the # module's own code rather than nginx's stock ngx_conf_set_num_slot. --- config location /wlx { zstd on; zstd_window_log abc; default_type text/plain; return 200 "body"; } --- must_die --- error_log invalid number --- no_error_log [alert] === TEST 11: a duplicate zstd_comp_level in one location is rejected # The "is duplicate" guard in ngx_conf_zstd_set_num_slot_with_negatives. The # custom slot must reject a repeated directive exactly like nginx's stock # num slot, or the second value would silently win. --- config location /dup { zstd on; zstd_comp_level 3; zstd_comp_level 5; default_type text/plain; return 200 "body"; } --- must_die --- error_log "zstd_comp_level" directive is duplicate --- no_error_log [alert] === TEST 12: zstd on without gzip_vary still emits Vary: Accept-Encoding # G5. Whether the response is zstd or identity depends on # Accept-Encoding; without Vary a shared cache serves the compressed # variant to a client that cannot decode it. This used to be a # config-load WARNING telling the operator to set "gzip_vary on" — # advice that was silently ignorable, and one missed warning poisoned a # cache. The module now emits the field itself, so the header is what # gets asserted, and the warning is gone: a warning about a directive # that no longer changes the outcome would be misleading. --- config location /gv { zstd on; zstd_min_length 1; default_type text/plain; return 200 "gzip_vary warning fixture body, long enough\n"; } --- request GET /gv --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd Vary: Accept-Encoding --- no_error_log eval [qr/gzip_vary/, qr/\[error\]/] === TEST 13: zstd on WITH gzip_vary emits exactly one Vary line # The duplicate-safety half. With "gzip_vary on" nginx emits the field # from r->gzip_vary, so the module must NOT push a second identical # line. Test::Nginx's response_headers compares the joined value of the # field, so a doubled emission reads as "Accept-Encoding, Accept-Encoding" # and fails here. --- config location /gv { zstd on; zstd_min_length 1; gzip_vary on; default_type text/plain; return 200 "gzip_vary warning fixture body, long enough\n"; } --- request GET /gv --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd Vary: Accept-Encoding --- no_error_log eval [qr/gzip_vary/, qr/\[error\]/] === TEST 14: zstd_static on without gzip_vary still emits Vary: Accept-Encoding # G5, static handler, on the DECLINE arm. ci/t/suite holds test + # test.zst, so this URI is Accept-Encoding-dependent; the client here # does not accept zstd, so the handler emits the Vary field and then # declines for the identity file to be served. That identity response # is exactly the one a shared cache must not pin every later client to, # and it must carry the field with no "gzip_vary on" configured — which # is what the old warning could only ask for. TEST 25 in 01-static.t is # the same arm WITH gzip_vary on. # # The fixture dir is addressed by ABSOLUTE path ($::suite_dir), not as # "root ../suite", so this assertion is independent of the suite's # runner-private servroot location. --- config eval " location /test { zstd_static on; root $::suite_dir; }" --- request GET /test --- more_headers Accept-Encoding: gzip --- response_headers !Content-Encoding Vary: Accept-Encoding --- no_error_log eval [qr/gzip_vary/, qr/\[error\]/] === TEST 15: zstd_static always does not vary and does not mention gzip_vary # "always" ignores Accept-Encoding: it is not a negotiated variant, so # it must NOT claim to vary on Accept-Encoding (that would fragment # every cache key for nothing). The G5 emission is deliberately scoped # to "on" only. See C5. --- config location /st/ { zstd_static always; root html; } --- user_files >>> st/plain.txt static warn fixture --- request GET /st/plain.txt --- response_headers !Vary --- no_error_log eval [qr/gzip_vary/, qr/\[error\]/] === TEST 16: an empty zstd_dict_file is rejected at config load # A 0-byte trained dictionary used to load as a silent no-op: the read is # complete (0 == size) and ZSTD_createCDict(buf, 0, level) returns a VALID # CDict, so nginx started and compressed with no dictionary at all -- after # the operator had explicitly set zstd_dict_file_unsafe on. The dcz loader # has always rejected an empty file; both loaders now agree. --- http_config zstd_dict_file_unsafe on; zstd_dict_file $TEST_NGINX_SERVER_ROOT/html/empty.dict; --- user_files >>> empty.dict --- config location /d { zstd on; default_type text/plain; return 200 "body"; } --- must_die --- error_log empty.dict" is empty --- no_error_log [alert] === TEST 17: a duplicate zstd_comp_level is rejected when the first value is -1 # Regression: NGX_CONF_UNSET is -1, and -1 is a valid documented level, so # the "is duplicate" guard tested the same value a real directive could # store. "zstd_comp_level -1; zstd_comp_level 5;" was therefore accepted # silently and 5 won. TEST 11 cannot catch this -- neither of its values is # the sentinel. The slot now tests NGX_HTTP_ZSTD_LEVEL_UNSET. --- config location /dup { zstd on; zstd_comp_level -1; zstd_comp_level 5; default_type text/plain; return 200 "body"; } --- must_die --- error_log "zstd_comp_level" directive is duplicate --- no_error_log [alert] === TEST 18: both modules in one location emit exactly ONE Vary: Accept-Encoding # The duplicate-emission cell the G5 matrix missed, and it was REAL: # with "zstd_static on" + "zstd on" + gzip_vary off, a non-accepting # client made the static handler emit Vary and DECLINE, after which the # filter emitted it again on the identity response it also declined to # encode -- two identical field lines, breaking the exactly-one contract # ngx_http_zstd_vary_accept_encoding() exists to keep. Reproduced with # curl before the guard (2 lines), 1 after. # # The response_headers check below asserts the VALUE, which Test::Nginx # joins with ", " when a field repeats -- so a regression reads as # "Accept-Encoding, Accept-Encoding" and fails here rather than passing # a mere presence check. That is the whole point of this block. --- config eval " location /test { zstd_static on; zstd on; zstd_types text/plain; root $::suite_dir; }" --- request GET /test --- more_headers Accept-Encoding: gzip --- response_headers !Content-Encoding Vary: Accept-Encoding --- no_error_log eval [qr/gzip_vary/, qr/\[error\]/] === TEST 19: both modules, accepting client, still exactly ONE Vary # The other half of TEST 18: the zstd client takes the static sidecar # path, which emits Vary and then SERVES rather than declining. Same # exactly-one contract on the arm that actually returns a body. --- config eval " location /test { zstd_static on; zstd on; zstd_types text/plain; root $::suite_dir; }" --- request GET /test --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd Vary: Accept-Encoding --- no_error_log eval [qr/gzip_vary/, qr/\[error\]/] === TEST 21: zstd_dict_strict_path refuses a ".." path component # The component walk explicitly rejects "." and ".." rather than # resolving them, because ".." would climb back above a component # already verified and make the walk's guarantee unstatable. --- http_config eval " zstd_dict_file_unsafe on; zstd_dict_strict_path on; zstd_dict_file \$TEST_NGINX_SERVER_ROOT/html/../html/zstd.dict;" --- user_files >>> zstd.dict the quick brown fox jumps over the lazy dog --- post_setup_server_root eval 'my $root = $ENV{TEST_NGINX_SERVER_ROOT} or die "TEST_NGINX_SERVER_ROOT unset"; chmod(0755, $root, "$root/html") == 2 or die "chmod strict-path fixture: $!";' --- config location /d { zstd on; default_type text/plain; return 200 "body"; } --- must_die --- error_log contains a "." or ".." component; refused by "zstd_dict_strict_path on" --- no_error_log [alert] === TEST 22: zstd_dict_strict_path refuses a symlinked intermediate component # The reason M3 exists: O_NOFOLLOW on the leaf alone guards only the last # component, so /srv/current/dict.bin with "current" a symlink is # followed silently by a plain open(). Walking one component at a time # with openat(O_NOFOLLOW) makes the symlinked "current" fail the walk. --- http_config eval " zstd_dict_file_unsafe on; zstd_dict_strict_path on; zstd_dict_file \$TEST_NGINX_SERVER_ROOT/html/current/zstd.dict;" --- post_setup_server_root eval 'my $root = $ENV{TEST_NGINX_SERVER_ROOT} or die "TEST_NGINX_SERVER_ROOT unset"; chmod(0755, $root, "$root/html") == 2 or die "chmod strict-path fixture: $!"; my $real = "$root/html/real-release"; mkdir $real or die "mkdir $real: $!"; open my $fh, ">", "$real/zstd.dict" or die "open zstd.dict: $!"; print $fh "the quick brown fox jumps over the lazy dog"; close $fh; symlink $real, "$root/html/current" or die "symlink current: $!";' --- config location /d { zstd on; default_type text/plain; return 200 "body"; } --- must_die --- error_log a symlink at any component is refused, not followed --- no_error_log [alert] === TEST 23: a tight zstd_max_cctx_memory without dcz dictionaries still loads # Regression pin for the dcz window-cap floor. ngx_http_zstd_dcz_window_cap() # walks down from NGX_HTTP_ZSTD_DCZ_MAX_WINDOW_LOG and, if nothing fit, pins # the cap to NGX_HTTP_ZSTD_DCZ_MIN_WINDOW_LOG and succeeds rather than # rejecting. That permissive fallthrough must stay permissive: a budget the # hard gate accepted has to keep loading, so the cap computation can never # turn a working "nginx -t" into a failure. Level 1 at the default window # estimates ~1.37 MB (libzstd 1.5.7), so 2m is tight but satisfiable — the # budget path is genuinely exercised rather than trivially slack, and the # location configures no zstd_dcz_dict_file, which is the scoping that makes # the dcz clamp irrelevant here. Asserts a served response, not merely a # start, so a config that loads but breaks compression still fails. # # Needs the estimator, so only a -DZSTD_STATIC_LINKING_ONLY build runs # it; TEST 23b is the same config on the release-shape build. --- skip_eval: 3: !$::static_linking --- config location /budget { zstd on; zstd_min_length 1; zstd_comp_level 1; zstd_max_cctx_memory 2m; zstd_types text/plain; default_type text/plain; return 200 "hello world padding padding padding padding padding\n"; } --- request GET /budget --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 23b: a release-shape build refuses zstd_max_cctx_memory by name at config load # The other half of TEST 23: without the memory-estimation API the # directive cannot be honoured, and the build says so at "nginx -t" # rather than loading with a silently unenforced budget. Pins the message # an operator sees (00-filter.t TEST 45 pins only the must_die). --- skip_eval: 2: $::static_linking --- config location /budget { zstd on; zstd_min_length 1; zstd_comp_level 1; zstd_max_cctx_memory 2m; zstd_types text/plain; default_type text/plain; return 200 "hello world padding padding padding padding padding\n"; } --- must_die --- error_log "zstd_max_cctx_memory" requires the module to be built with -DZSTD_STATIC_LINKING_ONLY --- no_error_log [alert] === TEST 24: filterless dict bypass warns at config load (identity trap) # The static-only deployment (this module's own reason for the split) # with zstd_static_dict_bypass on: every matching HTTPS client that # acquired an advertised dictionary bypasses its .zst sidecar into a # filter that does not exist, and is served identity permanently. The # module cannot see per-location filter state across the split, but # module-absent IS detectable (the #110 cycle->modules name scan) — one # warning at startup. Only expressible on dynamic builds, where the # filter .so can genuinely be left unloaded: skip_eval otherwise. --- skip_eval: 3: !$::static_so --- config location /t { zstd_static on; zstd_static_dict_bypass on; root html; } location /ok { return 200 "up\n"; } --- request GET /ok --- error_code: 200 --- error_log "zstd_static_dict_bypass on" but ngx_http_zstd_filter_module is not loaded --- no_error_log [emerg] === TEST 25: with the filter module loaded, the bypass warning is silent # Control for TEST 24 in both build shapes: dynamic builds load both # .so files here (the default preprocessor path), static-linked builds # have the filter compiled in — either way the module is present and # the warning must not fire. --- config location /t { zstd_static on; zstd_static_dict_bypass on; root html; } location /ok { return 200 "up\n"; } --- request GET /ok --- error_code: 200 --- no_error_log eval [qr/zstd_static_dict_bypass on. but ngx_http_zstd_filter_module/, qr/\[emerg\]/] === TEST 25b: a zstd_dict_file above the 10 MB limit is refused at config load # The size is compared as off_t before the size_t narrowing, so the refusal # is the loader's own message, not a later read or allocation failure. One # byte over the limit; the complement (a file under it loads) is every # other zstd_dict_file block in this suite. --- http_config zstd_dict_file_unsafe on; zstd_dict_file $TEST_NGINX_ZSTD_HUGEDICT; --- config location /d { zstd on; default_type text/plain; return 200 "body"; } --- must_die --- error_log dictionary file too large: 10485761 bytes (limit: 10485760 bytes) --- no_error_log [alert] === TEST 26: zstd_dict_strict_path refuses a path that names a directory # "/" is absolute, so the walk starts, vets the root, and then finds no # component left to open: there is no leaf. Refused as naming a # directory rather than opened and rejected later as not a regular # file, so the operator hears what is wrong with the PATH. --- http_config zstd_dict_file_unsafe on; zstd_dict_strict_path on; zstd_dict_file /; --- config location /d { zstd on; default_type text/plain; return 200 "body"; } --- must_die --- error_log "/" names a directory, not a dictionary file; refused by "zstd_dict_strict_path on" --- no_error_log [alert] === TEST 27: zstd_dict_strict_path refuses a trailing separator as naming a directory # open(2) on "/x/dict.zdict/" fails with ENOTDIR: the trailing separator # requires a directory. The walk used to open the last component as the # leaf anyway, accepting a regular file the kernel would have refused; # it now stops after vetting the directories before it and refuses the # path as naming a directory. The fixture's leaf IS a regular file, so # the old behaviour would have loaded it and served. --- http_config eval " zstd_dict_file_unsafe on; zstd_dict_strict_path on; zstd_dict_file \$TEST_NGINX_SERVER_ROOT/html/zstd.dict/;" --- user_files >>> zstd.dict the quick brown fox jumps over the lazy dog --- post_setup_server_root eval 'my $root = $ENV{TEST_NGINX_SERVER_ROOT} or die "TEST_NGINX_SERVER_ROOT unset"; chmod(0755, $root, "$root/html") == 2 or die "chmod strict-path fixture: $!";' --- config location /d { zstd on; default_type text/plain; return 200 "body"; } --- must_die --- error_log eval qr{"[^"]+/html/zstd\.dict/" names a directory, not a dictionary file; refused by "zstd_dict_strict_path on"} --- no_error_log [alert] === TEST 28: zstd_dict_strict_path refuses a world-writable intermediate directory # A33-F2: every directory the walk opens is vetted before it is trusted # as the base of the next openat(). A mode-0777 ancestor lets any local # user rename a file into place, so it is refused by name even though # the leaf itself is self-owned 0644. --- http_config eval " zstd_dict_file_unsafe on; zstd_dict_strict_path on; zstd_dict_file \$TEST_NGINX_SERVER_ROOT/html/loose/zstd.dict;" --- post_setup_server_root eval 'my $root = $ENV{TEST_NGINX_SERVER_ROOT} or die "TEST_NGINX_SERVER_ROOT unset"; chmod(0755, $root, "$root/html") == 2 or die "chmod strict-path fixture: $!"; my $dir = "$root/html/loose"; mkdir $dir or die "mkdir $dir: $!"; open my $fh, ">", "$dir/zstd.dict" or die "open zstd.dict: $!"; print $fh "the quick brown fox jumps over the lazy dog"; close $fh; chmod(0644, "$dir/zstd.dict") == 1 or die "chmod zstd.dict: $!"; chmod(0777, $dir) == 1 or die "chmod loose: $!";' --- config location /d { zstd on; default_type text/plain; return 200 "body"; } --- must_die --- error_log directory component "loose" of writable by group or other --- no_error_log [alert] === TEST 29: zstd_dict_strict_path grants no sticky-bit exemption to a world-writable intermediate directory # A sticky world-writable ancestor (the /tmp layout) still lets an # unprivileged user CREATE the next component; it only stops them # renaming someone else's entry away, which is not the attack. Refused # exactly like the plain 0777 case, and the diagnostic says so. --- http_config eval " zstd_dict_file_unsafe on; zstd_dict_strict_path on; zstd_dict_file \$TEST_NGINX_SERVER_ROOT/html/sticky/zstd.dict;" --- post_setup_server_root eval 'my $root = $ENV{TEST_NGINX_SERVER_ROOT} or die "TEST_NGINX_SERVER_ROOT unset"; chmod(0755, $root, "$root/html") == 2 or die "chmod strict-path fixture: $!"; my $dir = "$root/html/sticky"; mkdir $dir or die "mkdir $dir: $!"; open my $fh, ">", "$dir/zstd.dict" or die "open zstd.dict: $!"; print $fh "the quick brown fox jumps over the lazy dog"; close $fh; chmod(0644, "$dir/zstd.dict") == 1 or die "chmod zstd.dict: $!"; chmod(01777, $dir) == 1 or die "chmod sticky: $!";' --- config location /d { zstd on; default_type text/plain; return 200 "body"; } --- must_die --- error_log directory component "sticky" of no sticky-bit exemption --- no_error_log [alert] === TEST 30: zstd_dict_strict_path refuses a group-writable intermediate directory # The group bit alone is enough: the rule is S_IWGRP | S_IWOTH, the # same one the leaf check applies, not world-writable only. --- http_config eval " zstd_dict_file_unsafe on; zstd_dict_strict_path on; zstd_dict_file \$TEST_NGINX_SERVER_ROOT/html/shared/zstd.dict;" --- post_setup_server_root eval 'my $root = $ENV{TEST_NGINX_SERVER_ROOT} or die "TEST_NGINX_SERVER_ROOT unset"; chmod(0755, $root, "$root/html") == 2 or die "chmod strict-path fixture: $!"; my $dir = "$root/html/shared"; mkdir $dir or die "mkdir $dir: $!"; open my $fh, ">", "$dir/zstd.dict" or die "open zstd.dict: $!"; print $fh "the quick brown fox jumps over the lazy dog"; close $fh; chmod(0644, "$dir/zstd.dict") == 1 or die "chmod zstd.dict: $!"; chmod(0775, $dir) == 1 or die "chmod shared: $!";' --- config location /d { zstd on; default_type text/plain; return 200 "body"; } --- must_die --- error_log directory component "shared" of writable by group or other --- no_error_log [alert] === TEST 31: zstd_dict_strict_path loads a real nested path and serves # The complement every refusal above needs: two self-owned 0755 # directories below html and a 0644 leaf walk clean, the dictionary # loads, and the location serves. Without this block a walk that # refused everything would pass TESTs 21, 22 and 26-30. --- http_config eval " zstd_dict_file_unsafe on; zstd_dict_strict_path on; zstd_dict_file \$TEST_NGINX_SERVER_ROOT/html/releases/7/zstd.dict;" --- post_setup_server_root eval 'my $root = $ENV{TEST_NGINX_SERVER_ROOT} or die "TEST_NGINX_SERVER_ROOT unset"; chmod(0755, $root, "$root/html") == 2 or die "chmod strict-path fixture: $!"; for my $dir ("$root/html/releases", "$root/html/releases/7") { mkdir $dir or die "mkdir $dir: $!"; chmod(0755, $dir) == 1 or die "chmod $dir: $!"; } open my $fh, ">", "$root/html/releases/7/zstd.dict" or die "open zstd.dict: $!"; print $fh "the quick brown fox jumps over the lazy dog"; close $fh; chmod(0644, "$root/html/releases/7/zstd.dict") == 1 or die "chmod zstd.dict: $!";' --- config location /d { zstd on; zstd_min_length 1; zstd_types text/plain; default_type text/plain; return 200 "the quick brown fox jumps over the lazy dog, again and again"; } --- request GET /d --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- no_error_log [emerg] [error] [alert] === TEST 32: dcz dictionaries at a high zstd_comp_level warn at config load # A33-F1 advisory. A dcz response rebuilds its dictionary's match tables # via ZSTD_CCtx_refPrefix() on every request, at a cost set by dictionary # size and level and independent of the body. Level 9 is the first level # whose strategy builds those tables (measured: 1 MB dict 0.75 ms at # level 3 against 4.5 ms at level 9), so it is where the advisory starts. # Config-load only: the warning must not change whether the location # serves, so the request below still succeeds. --- config location /t { zstd on; zstd_comp_level 9; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; zstd_min_length 1; zstd_types text/plain; default_type text/plain; return 200 "the quick brown fox jumps over the lazy dog, again"; } --- request GET /t --- error_code: 200 --- error_log re-references its dictionary with ZSTD_CCtx_refPrefix() --- no_error_log [emerg] [error] [alert] === TEST 33: dcz dictionaries at the default level stay silent # Negative control for TEST 32, and the one that matters: the common # web-serving profile configures dcz dictionaries at the default level # and must not be warned at. Identical to TEST 32 except for the level, # so a warning appearing here is the gate firing on dictionary presence # alone rather than on the compressor profile. --- config location /t { zstd on; zstd_comp_level 3; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; zstd_min_length 1; zstd_types text/plain; default_type text/plain; return 200 "the quick brown fox jumps over the lazy dog, again"; } --- request GET /t --- error_code: 200 --- no_error_log eval [qr/ZSTD_CCtx_refPrefix/, qr/\[emerg\]/, qr/\[error\]/, qr/\[alert\]/] === TEST 34: "zstd_long on" warns at the default level and names itself # The second lever: long-distance matching builds its own per-request # tables, so it earns the advisory independently of the level. Pinned at # level 3, where TEST 33 proves the level arm is silent -- so only the # long_mode arm can be firing here. Also pins that the message names the # directive, which is what the operator acts on. --- config location /t { zstd on; zstd_comp_level 3; zstd_long on; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; zstd_min_length 1; zstd_types text/plain; default_type text/plain; return 200 "the quick brown fox jumps over the lazy dog, again"; } --- request GET /t --- error_code: 200 --- error_log with "zstd_long on" disable "zstd_long" --- no_error_log [emerg] [error] [alert] === TEST 35b: the advisory renders the dictionary count and the LARGEST size # Pins the formatted region itself, which the other arms match around: the # count, the plural branch, and which dictionary the size comes from. Two # fixtures of different sizes in one location, declared smallest-last so a # "keep the first" or "keep the last" bug both read wrong -- the message # must name 59738 (suite/test), not 2660 (suite/dcz-dict). Also the only # arm that would catch the count and size arguments being swapped. --- config location /t { zstd on; zstd_comp_level 9; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/test; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; zstd_min_length 1; zstd_types text/plain; default_type text/plain; return 200 "the quick brown fox jumps over the lazy dog, again"; } --- request GET /t --- error_code: 200 --- error_log 2 dcz dictionaries are configured at "zstd_comp_level" 9 largest configured dictionary here is 59738 bytes --- no_error_log [emerg] [error] [alert] === TEST 35: a high level WITHOUT dcz dictionaries stays silent # The other half of the gate. Level 9 alone is not the finding: without a # configured dictionary there is no refPrefix call and nothing to warn # about. Separates the advisory from the memory advisories above, which # do fire on level alone. --- config location /t { zstd on; zstd_comp_level 9; zstd_min_length 1; zstd_types text/plain; default_type text/plain; return 200 "the quick brown fox jumps over the lazy dog, again"; } --- request GET /t --- error_code: 200 --- no_error_log eval [qr/ZSTD_CCtx_refPrefix/, qr/\[emerg\]/, qr/\[error\]/, qr/\[alert\]/] myguard-labs-nginx-zstd-module-3486b5b/ci/t/03-dcz.t000066400000000000000000001421731525232545500220710ustar00rootroot00000000000000use Test::Nginx::Socket; use File::Basename; use File::Spec; use File::Temp qw(tempfile tempdir); use Digest::SHA qw(sha256); use MIME::Base64 qw(encode_base64); use lib 'lib'; my $dirname = dirname(__FILE__); # local: this process is the test run, but perlcritic is right that a bare # assignment to %ENV leaks into anything that runs after it. local $ENV{'TEST_NGINX_PERL_PATH'} = "$ENV{'PWD'}/$dirname"; my @dynamic_modules; if (defined $ENV{'TEST_NGINX_BINARY'}) { my $nginx_dir = dirname($ENV{'TEST_NGINX_BINARY'}); for my $module_name (qw(ngx_http_zstd_filter_module.so ngx_http_zstd_static_module.so)) { my $module_path = "$nginx_dir/$module_name"; push @dynamic_modules, $module_path if -f $module_path; } } add_block_preprocessor(sub { my $block = shift; if (@dynamic_modules) { my $main_config = join "\n", map { "load_module $_;" } @dynamic_modules; $block->set_value("main_config", $main_config); } # RFC 9842 section 8 restricts dcz to secure contexts, and this # harness speaks plain HTTP -- Test::Nginx::Socket has no TLS # listener. Every block below therefore runs behind the explicit # "TLS is terminated upstream" acknowledgement, which is exactly the # deployment those tests model. Blocks named "secure-context:" are # the tests OF that gate and are left alone so they see the # compiled-in default (off) or set the flag themselves. return if $block->name =~ /secure-context:/; my $http_config = $block->http_config; $http_config = '' if !defined $http_config; $block->set_value("http_config", "$http_config\nzstd_dcz_assume_secure_transport on;\n"); }); # The negotiation key is the SHA-256 of the committed dictionary fixture, # computed here rather than hardcoded so a fixture edit cannot silently # desynchronize the suite ("passing" against a stale hash). my $dict_raw = do { local $/; open my $fh, '<', "$dirname/suite/dcz-dict" or die "dcz-dict: $!"; binmode $fh; my $c = <$fh>; close $fh or die "dcz-dict: $!"; $c; }; our $dict_b64 = encode_base64(sha256($dict_raw), ""); our $bad_b64 = encode_base64("\x01" x 32, ""); # For the optional supplied-hash directive argument: the fixture's true # hash as hex, and a deliberately different well-formed hash. The # supplied literal is VERIFIED against the loaded bytes by default, so # the wrong one must abort config load (TESTs 17-18) rather than become # the negotiation key. Under trust_hashes it is used verbatim (TEST 40). # $zstd_dcz_dicts_hashed counts every loaded dictionary by default # (TESTs 21-23), but excludes trusted literals (TESTs 41-42). our $dict_hex = unpack("H*", sha256($dict_raw)); our $dict_hex_upper = uc($dict_hex); our $odd_hex = "01" x 32; our $odd_b64 = encode_base64("\x01" x 32, ""); # Generated dictionaries, never committed. File::Temp picks unpredictable # names and opens O_EXCL, so a pre-seeded symlink at a guessable path # cannot redirect a write on a shared runner; every write and close is # checked so a short fixture cannot pass silently; all go at exit. # # A dictionary above the 8 MB dcz window cap but under the 10 MB hard # limit (nobody wants an 8 MB fixture in-tree). Exposed to config blocks # via $TEST_NGINX_DCZ_BIGDICT. my ($big_fh, $big_path) = tempfile("zstd-dcz-bigdict-XXXXXX", TMPDIR => 1, UNLINK => 1); binmode $big_fh; print {$big_fh} 'A' x (8 * 1024 * 1024 + 17) or die "bigdict: write: $!"; close $big_fh or die "bigdict: close: $!"; local $ENV{'TEST_NGINX_DCZ_BIGDICT'} = $big_path; # One byte over the 10 MB hard limit, for the too-large refusal (TEST 54). # Exposed via $TEST_NGINX_DCZ_HUGEDICT. The loader refuses it at fstat() # on size alone, so it is extended with a checked truncate, not written. my ($huge_fh, $huge_path) = tempfile("zstd-dcz-hugedict-XXXXXX", TMPDIR => 1, UNLINK => 1); truncate($huge_fh, 10 * 1024 * 1024 + 1) or die "hugedict: truncate: $!"; close $huge_fh or die "hugedict: close: $!"; local $ENV{'TEST_NGINX_DCZ_HUGEDICT'} = $huge_path; # Twenty-five filler request headers: nginx's headers_in list holds # twenty entries per part, so a request carrying more than that makes the # header walks continue into a second list part (TEST 55). our $many_headers = join("\n", map { "X-Pad-$_: filler" } 1 .. 25); # Sixteen more dictionaries, generated, for the lookup-shape blocks # (TESTs 51-53): with the committed fixture they make seventeen in one # location, one past NGX_HTTP_ZSTD_DCZ_BSEARCH_THRESHOLD, so negotiation # takes the binary search over the digest-sorted array instead of the # linear scan. Exposed to config blocks via $TEST_NGINX_DCZ_MANYDIR; # $::many_cfg is the seventeen directives, $::many9_b64 the digest of a # generated entry that sorts somewhere in the middle. # The directory is a fresh private (0700) tempdir, so the fixed names # inside it cannot meet a pre-existing entry. my $many_dir = tempdir("zstd-dcz-many-XXXXXX", TMPDIR => 1, CLEANUP => 1); our @many_raw; for my $i (1 .. 16) { my $raw = "dcz filler dictionary number $i: shared boilerplate " x 8; open my $mf, '>', "$many_dir/d$i" or die "d$i: open: $!"; binmode $mf; print {$mf} $raw or die "d$i: write: $!"; close $mf or die "d$i: close: $!"; push @many_raw, $raw; } local $ENV{'TEST_NGINX_DCZ_MANYDIR'} = $many_dir; our $many_cfg = join("\n", " zstd_dcz_dict_file \$TEST_NGINX_PERL_PATH/suite/dcz-dict;", map { " zstd_dcz_dict_file \$TEST_NGINX_DCZ_MANYDIR/d$_;" } 1 .. 16); our $many9_b64 = encode_base64(sha256($many_raw[8]), ""); no_long_string(); log_level 'warn'; repeat_each(1); plan 'no_plan'; run_tests(); __DATA__ === TEST 1: dcz negotiated when hash matches and dcz is accepted # The full happy path: Available-Dictionary carries the SHA-256 of a # configured dictionary and Accept-Encoding lists dcz explicitly, so the # response commits to Content-Encoding: dcz and declares the # Available-Dictionary cache key. Wire-format/byte-exactness assertions # live in tools/test_dcz.py; this suite pins the negotiation contract. --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request GET /t --- more_headers eval "Accept-Encoding: zstd, dcz\nAvailable-Dictionary: :$::dict_b64:" --- response_headers Content-Encoding: dcz Vary: Accept-Encoding, Available-Dictionary, Sec-Fetch-Site --- no_error_log [error] === TEST 2: no Available-Dictionary falls back to zstd, Vary still set # A dictionary-less client on the same location gets plain zstd — and the # response STILL varies on Available-Dictionary: which encoding this # location serves depends on that request header for every client, so a # shared cache must key both variants on it. --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request GET /t --- more_headers Accept-Encoding: zstd, dcz --- response_headers Content-Encoding: zstd Vary: Accept-Encoding, Available-Dictionary, Sec-Fetch-Site --- no_error_log [error] === TEST 3: unknown dictionary hash falls back to zstd # A hash we do not hold must never negotiate dcz — the client would # receive a frame referencing a dictionary it cannot pair with ours. --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request GET /t --- more_headers eval "Accept-Encoding: zstd, dcz\nAvailable-Dictionary: :$::bad_b64:" --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 4: Available-Dictionary without dcz in Accept-Encoding # Advertising a dictionary is not accepting the coding. Without an # explicit dcz token the response must stay plain zstd. --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request GET /t --- more_headers eval "Accept-Encoding: zstd\nAvailable-Dictionary: :$::dict_b64:" --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 5: dcz;q=0 is an explicit refusal --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request GET /t --- more_headers eval "Accept-Encoding: zstd, dcz;q=0\nAvailable-Dictionary: :$::dict_b64:" --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 6: the "*" wildcard does not enable dcz # RFC 9110's "*" matches any coding not explicitly listed, but only a # client that actually holds the dictionary can decode dcz — the module # requires the explicit token, so a wildcard-only request stays zstd. --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request GET /t --- more_headers eval "Accept-Encoding: zstd, *\nAvailable-Dictionary: :$::dict_b64:" --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 7: Sec-Fetch-Site cross-site is refused, and still varies on it # RFC 9842 §8: dictionaries are same-origin-partitioned; compressing a # cross-site response against one leaks it. Falls back to plain zstd. # The REFUSAL path must carry "Vary: ... Sec-Fetch-Site" too: it is the # variant a shared cache would otherwise hand to a same-origin client # (dcz silently suppressed), and its counterpart is the dcz body handed # to a cross-site client, bypassing this gate entirely. Proven end to # end against a real proxy_cache in # ci/tools/test_dcz_cache_partition.py. --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request GET /t --- more_headers eval "Accept-Encoding: zstd, dcz\nAvailable-Dictionary: :$::dict_b64:\nSec-Fetch-Site: cross-site" --- response_headers Content-Encoding: zstd Vary: Accept-Encoding, Available-Dictionary, Sec-Fetch-Site --- no_error_log [error] === TEST 8: Sec-Fetch-Site same-origin is allowed --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request GET /t --- more_headers eval "Accept-Encoding: zstd, dcz\nAvailable-Dictionary: :$::dict_b64:\nSec-Fetch-Site: same-origin" --- response_headers Content-Encoding: dcz --- no_error_log [error] === TEST 9: malformed Available-Dictionary (not a byte sequence) # RFC 8941 byte sequences are colon-delimited; anything else is treated # as "no usable dictionary", never an error. --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request GET /t --- more_headers Accept-Encoding: zstd, dcz Available-Dictionary: not-a-structured-field --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 10: Available-Dictionary decoding to the wrong length # ":aGk=:" is valid base64 ("hi") but not a SHA-256; must decline. --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request GET /t --- more_headers Accept-Encoding: zstd, dcz Available-Dictionary: :aGk=: --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 11: a location's own dictionary list replaces the inherited one # Standard nginx array-directive semantics: the child declares its own # zstd_dcz_dict_file, so the server-level dictionary the client # advertises is no longer in scope and negotiation falls back to zstd. --- config zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; location /inherited { default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } location /own { zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/test; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request eval ["GET /inherited", "GET /own"] --- more_headers eval [ "Accept-Encoding: zstd, dcz\nAvailable-Dictionary: :$::dict_b64:", "Accept-Encoding: zstd, dcz\nAvailable-Dictionary: :$::dict_b64:", ] --- response_headers eval [ "Content-Encoding: dcz", "Content-Encoding: zstd", ] --- no_error_log [error] === TEST 12: identity fallback still varies on Available-Dictionary # PR #92 review: a client sending "Accept-Encoding: dcz" (no zstd) with # a hash we do not hold gets identity — but the SAME client holding a # dictionary we DO hold would get dcz, so the identity variant is not # invariant in Available-Dictionary. Without the Vary a shared cache # keeps serving the stored identity body after the client acquires the # right dictionary (silent compression loss). Guards the hoisting of # the Vary push above the acceptance gate. --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request GET /t --- more_headers eval "Accept-Encoding: dcz\nAvailable-Dictionary: :$::bad_b64:" --- response_headers !Content-Encoding Vary: Accept-Encoding, Available-Dictionary, Sec-Fetch-Site --- no_error_log [error] === TEST 13: a dictionary above the 8 MB window cap warns at config load # The frame stays well-formed (the RFC client guarantee is a floor of # max(8 MB, 1.25 x dict)), but dictionary bytes beyond the window cannot # be referenced — a silent ratio cliff the operator should hear about at # load time, not discover in telemetry. --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_DCZ_BIGDICT; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request GET /t --- more_headers Accept-Encoding: zstd --- response_headers Content-Encoding: zstd --- error_log larger than the 8 MB dcz compression window --- no_error_log [error] === TEST 14: an empty dictionary file is a config-load error --- config location /t { zstd on; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-empty; default_type text/plain; return 200 "unreachable\n"; } --- request GET /t --- must_die --- error_log is empty --- no_error_log [alert] === TEST 15: two dictionaries with the same hash are a config-load error # The negotiation lookup would be ambiguous; for computed hashes that # means identical content — almost certainly a copy that was meant to be # a new version. Refuse to start rather than match the first silently. --- config location /t { zstd on; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "unreachable\n"; } --- request GET /t --- must_die --- error_log has the same hash as --- no_error_log [alert] === TEST 16: supplied hash argument — correct value negotiates dcz --- config eval " location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file \$TEST_NGINX_PERL_PATH/suite/dcz-dict $::dict_hex; default_type text/plain; return 200 \"dcz negotiation body: shared-boilerplate compute render\n\"; }" --- request GET /t --- more_headers eval "Accept-Encoding: zstd, dcz\nAvailable-Dictionary: :$::dict_b64:" --- response_headers Content-Encoding: dcz Vary: Accept-Encoding, Available-Dictionary, Sec-Fetch-Site --- no_error_log [error] === TEST 17: a supplied hash that is not the file's aborts config load # THE regression test for the trusted-literal bug: "new.dict with the # old dict's hash" used to load happily and advertise a hash no client # could decode against. The dictionary path EXISTS and its contents are # fine — only the declared hash is wrong — so nothing but the # verification can reject this config. --- config eval " location /t { zstd on; zstd_dcz_dict_file \$TEST_NGINX_PERL_PATH/suite/dcz-dict $::odd_hex; default_type text/plain; return 200 \"unreachable\n\"; }" --- request GET /t --- must_die --- error_log eval qr/does not match the supplied hash "0101010101010101010101010101010101010101010101010101010101010101": the file's SHA-256 is "$::dict_hex"/ --- no_error_log [alert] === TEST 18: the file's own true hash as the literal still loads and negotiates # Positive control for TEST 17: verification accepts the matching pair, # and the negotiation key is the file's real hash. Without this, TEST 17 # would also pass if the directive rejected EVERY supplied literal. --- config eval " location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file \$TEST_NGINX_PERL_PATH/suite/dcz-dict $::dict_hex; default_type text/plain; return 200 \"dcz negotiation body: shared-boilerplate compute render\n\"; }" --- request GET /t --- more_headers eval "Accept-Encoding: zstd, dcz\nAvailable-Dictionary: :$::dict_b64:" --- response_headers Content-Encoding: dcz --- no_error_log [error] === TEST 50: an uppercase supplied hash loads and negotiates # Pins the case-folding branch in ngx_http_zstd_hex_nibble(). --- config eval " location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file \$TEST_NGINX_PERL_PATH/suite/dcz-dict $::dict_hex_upper; default_type text/plain; return 200 \"dcz negotiation body: shared-boilerplate compute render\n\"; }" --- request GET /t --- more_headers eval "Accept-Encoding: zstd, dcz\nAvailable-Dictionary: :$::dict_b64:" --- response_headers Content-Encoding: dcz --- no_error_log [error] === TEST 19: supplied hash with the wrong length is a config-load error # The dictionary path deliberately does NOT exist: the hash error must # surface anyway, pinning that malformed literals are validated before # ngx_open_file() rather than shadowed by the file error. --- config location /t { zstd on; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/no-such-dict abc123; default_type text/plain; return 200 "unreachable\n"; } --- request GET /t --- must_die --- error_log invalid dcz dictionary hash --- no_error_log [alert] === TEST 20: supplied hash with non-hex characters is a config-load error # Nonexistent path for the same reason as TEST 19. --- config location /t { zstd on; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/no-such-dict zz23456789012345678901234567890123456789012345678901234567890123; default_type text/plain; return 200 "unreachable\n"; } --- request GET /t --- must_die --- error_log non-hex character --- no_error_log [alert] === TEST 21: a supplied hash does NOT skip the load-time hashing pass # $zstd_dcz_dicts_hashed counts ngx_http_zstd_sha256() calls from the # dictionary loader this config cycle. The literal is verified, not # trusted, so the file must be hashed even when one is supplied: this # is "1". Restoring the old "if (!have_hash)" skip makes it "0" and # fails here — the counter is what distinguishes verifying the literal # from substituting it, since both produce the same dict->hash on a # MATCHING pair. --- config eval " location /t { zstd on; zstd_dcz_dict_file \$TEST_NGINX_PERL_PATH/suite/dcz-dict $::dict_hex; default_type text/plain; return 200 \"hashed=\$zstd_dcz_dicts_hashed\n\"; }" --- request GET /t --- response_body hashed=1 --- no_error_log [error] === TEST 22: without a supplied hash the loader hashes the file once # Positive control for TEST 21: the counter is not stuck at zero. --- config location /t { zstd on; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "hashed=$zstd_dcz_dicts_hashed\n"; } --- request GET /t --- response_body hashed=1 --- no_error_log [error] === TEST 23: mixed supplied and computed entries count BOTH # Every dictionary is hashed from its own bytes regardless of whether # a literal was declared, so two dictionaries are two hashes. --- config eval " location /t { zstd on; zstd_dcz_dict_file \$TEST_NGINX_PERL_PATH/suite/dcz-dict $::dict_hex; zstd_dcz_dict_file \$TEST_NGINX_PERL_PATH/suite/test; default_type text/plain; return 200 \"hashed=\$zstd_dcz_dicts_hashed\n\"; }" --- request GET /t --- response_body hashed=2 --- no_error_log [error] === TEST 24: duplicate Sec-Fetch-Site fails closed # The §8.3 cross-origin gate must not be decided by whichever line sorts # first. Sec-Fetch-Site is not in nginx's headers_in table, so duplicates # are chained rather than rejected and reach the module; a proxy that # merges a client-supplied duplicate, or a smuggling desync, could # otherwise prepend an agreeable value and switch the gate off. A browser # never sends two, so more than one falls back to plain zstd. --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request GET /t --- more_headers eval "Accept-Encoding: zstd, dcz\nAvailable-Dictionary: :$::dict_b64:\nSec-Fetch-Site: same-origin\nSec-Fetch-Site: cross-site" --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 25: duplicate Sec-Fetch-Site fails closed regardless of order # The cross-site line first: still plain zstd, so the result does not # depend on which duplicate the lookup happens to reach. --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request GET /t --- more_headers eval "Accept-Encoding: zstd, dcz\nAvailable-Dictionary: :$::dict_b64:\nSec-Fetch-Site: cross-site\nSec-Fetch-Site: same-origin" --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 26: duplicate Available-Dictionary fails closed # Same reasoning: a single-valued structured field, not deduplicated by # nginx. Two of them is not a browser, and picking the first would let a # second dictionary hash be smuggled past the operator's expectation. --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request GET /t --- more_headers eval "Accept-Encoding: zstd, dcz\nAvailable-Dictionary: :$::dict_b64:\nAvailable-Dictionary: :$::dict_b64:\nSec-Fetch-Site: same-origin" --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 27: secure-context: plain HTTP falls back to zstd with no directive # RFC 9842 section 8: compression dictionary transport MUST only be used # in a secure context. This block is exempt from the suite's # assume-secure preprocessor, so it exercises the COMPILED-IN default # (zstd_dcz_assume_secure_transport off) with no directive anywhere in # the configuration -- a block that opted in explicitly would pass even # if the default were wrong. Every other dcz gate here is satisfied: the # only reason this is not dcz is the cleartext connection. --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request GET /t --- more_headers eval "Accept-Encoding: zstd, dcz\nAvailable-Dictionary: :$::dict_b64:\nSec-Fetch-Site: same-origin" --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 28: secure-context: explicit off is the same fail-closed answer # Pins the directive's off value rather than only its absence, so a # future default flip cannot silently make TEST 27 vacuous. --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_assume_secure_transport off; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request GET /t --- more_headers eval "Accept-Encoding: zstd, dcz\nAvailable-Dictionary: :$::dict_b64:" --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 29: secure-context: the acknowledgement re-enables dcz over cleartext # The supported TLS-terminating-proxy deployment. Same request as TEST # 27, one directive different, opposite outcome -- so the pair isolates # the gate rather than some other difference. --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_assume_secure_transport on; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request GET /t --- more_headers eval "Accept-Encoding: zstd, dcz\nAvailable-Dictionary: :$::dict_b64:\nSec-Fetch-Site: same-origin" --- response_headers Content-Encoding: dcz Vary: Accept-Encoding, Available-Dictionary, Sec-Fetch-Site --- no_error_log [error] === TEST 30: secure-context: a client-supplied scheme header does not enable dcz # X-Forwarded-Proto (and every sibling spelling) is settable by anyone # who can reach a cleartext listener. If the module inferred "secure" # from one, the gate would be a suggestion. Nothing here is trusted: # all four requests stay plain zstd. --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request eval ["GET /t", "GET /t", "GET /t", "GET /t"] --- more_headers eval [ "Accept-Encoding: zstd, dcz\nAvailable-Dictionary: :$::dict_b64:\nX-Forwarded-Proto: https", "Accept-Encoding: zstd, dcz\nAvailable-Dictionary: :$::dict_b64:\nForwarded: proto=https", "Accept-Encoding: zstd, dcz\nAvailable-Dictionary: :$::dict_b64:\nX-Forwarded-Ssl: on", "Accept-Encoding: zstd, dcz\nAvailable-Dictionary: :$::dict_b64:\nFront-End-Https: on", ] --- response_headers eval [ "Content-Encoding: zstd", "Content-Encoding: zstd", "Content-Encoding: zstd", "Content-Encoding: zstd", ] --- no_error_log [error] === TEST 31: secure-context: the acknowledgement inherits into locations # http/server/location context: set once at server level, every location # under it is covered. Pins the merge, which a per-location-only test # would leave unproven. --- config zstd on; zstd_min_length 16; zstd_dcz_assume_secure_transport on; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; location /child { default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request GET /child --- more_headers eval "Accept-Encoding: zstd, dcz\nAvailable-Dictionary: :$::dict_b64:" --- response_headers Content-Encoding: dcz --- no_error_log [error] === TEST 32: secure-context: a location can opt back out of an inherited on # The merge is a real inheritance, not a one-way latch: a location that # is reachable directly over cleartext can turn the acknowledgement off # again even when the server block set it. --- config zstd on; zstd_min_length 16; zstd_dcz_assume_secure_transport on; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; location /direct { zstd_dcz_assume_secure_transport off; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request GET /direct --- more_headers eval "Accept-Encoding: zstd, dcz\nAvailable-Dictionary: :$::dict_b64:" --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 33: $zstd_bytes_out / $zstd_ratio resolve without fault on a dcz response # Regression for the dcz 40-byte prefix being double-counted: # ngx_http_zstd_filter_get_buf() writes the prefix into out_buf and # advances ->last past it, then the emit path in # ngx_http_zstd_filter_compress() (`ctx->bytes_out += ngx_buf_size(b)`) # already counts that whole buffer -- prefix included -- when it queues # the buffer downstream. A second, separate `ctx->bytes_out += # NGX_HTTP_ZSTD_DCZ_HEADER_LEN` in get_buf() double-counted the prefix on # every dcz response. # # Exact-value correctness (bytes_out equal to the real wire byte count, # and the ratio derived from it) is the province of the byte-oracle in # tools/test_dcz.py, which can read the response bytes and the access # log; the compressed size otherwise depends on the linked libzstd # version and is deliberately not pinned here (see TEST 1's own note on # wire-format assertions). This test instead pins the same contract # TEST 32/39 in 00-filter.t pin for the plain zstd path: referencing # both log-phase variables via `set` on a dcz response must resolve # without the get_handler faulting. --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; set $unused_bytes_out $zstd_bytes_out; set $unused_ratio $zstd_ratio; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request GET /t --- more_headers eval "Accept-Encoding: zstd, dcz\nAvailable-Dictionary: :$::dict_b64:" --- response_headers Content-Encoding: dcz --- no_error_log [error] === TEST 34: chained Accept-Encoding — "zstd" then "dcz" negotiates dcz # RFC 9110 section 5.3: repeated field lines are the single comma-joined # field, so these lines ARE "zstd, dcz" and advertise dcz. The dcz gate # used to read only the first line and fall back to plain zstd. --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request GET /t --- more_headers eval "Accept-Encoding: zstd\nAccept-Encoding: dcz\nAvailable-Dictionary: :$::dict_b64:" --- response_headers Content-Encoding: dcz --- no_error_log [error] === TEST 35: chained Accept-Encoding — "dcz" then "zstd" negotiates dcz --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request GET /t --- more_headers eval "Accept-Encoding: dcz\nAccept-Encoding: zstd\nAvailable-Dictionary: :$::dict_b64:" --- response_headers Content-Encoding: dcz --- no_error_log [error] === TEST 36: chained Accept-Encoding — "dcz;q=0" on a later line falls back # An explicit dcz refusal is the latest matching token, so this falls back # to plain zstd. --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request GET /t --- more_headers eval "Accept-Encoding: zstd\nAccept-Encoding: dcz;q=0\nAvailable-Dictionary: :$::dict_b64:" --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 37: chained Accept-Encoding — "dcz;q=0" first, "dcz;q=1" later accepts # Duplicate field lines are comma-joined in received order, so the latest # explicit dcz token decides just as it does in a single field line. --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request GET /t --- more_headers eval "Accept-Encoding: zstd, dcz;q=0\nAccept-Encoding: dcz;q=1\nAvailable-Dictionary: :$::dict_b64:" --- response_headers Content-Encoding: dcz --- no_error_log [error] === TEST 37b: chained Accept-Encoding — "dcz;q=1" first, "dcz;q=0" later falls back # Negative control for TEST 37: reversing the same fields must reverse the # dcz decision and leave the plain zstd fallback selected. --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request GET /t --- more_headers eval "Accept-Encoding: zstd, dcz;q=1\nAccept-Encoding: dcz;q=0\nAvailable-Dictionary: :$::dict_b64:" --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 38: chained Accept-Encoding — a "*" on a later line still does not turn dcz on # The wildcard gate survives the chain walk: only a client that actually # holds the dictionary can decode dcz, so "*" must never enable it no # matter which field line carries it. --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request GET /t --- more_headers eval "Accept-Encoding: zstd\nAccept-Encoding: *\nAvailable-Dictionary: :$::dict_b64:" --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 39: chained Accept-Encoding — three field lines, dcz on the third --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request GET /t --- more_headers eval "Accept-Encoding: gzip\nAccept-Encoding: zstd\nAccept-Encoding: dcz\nAvailable-Dictionary: :$::dict_b64:" --- response_headers Content-Encoding: dcz --- no_error_log [error] === TEST 40: trust_hashes on — the declared literal IS the negotiation key # The opt-out's contract, proven the only observable way: the literal # deliberately does NOT match the file, and the client advertising the # DECLARED value gets dcz. Under the default this exact config is TEST # 17's must_die; under trust the operator's pipeline is the authority. --- http_config zstd_dcz_dict_trust_hashes on; --- config eval " location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file \$TEST_NGINX_PERL_PATH/suite/dcz-dict $::odd_hex; default_type text/plain; return 200 \"dcz negotiation body: shared-boilerplate compute render\n\"; }" --- request GET /t --- more_headers eval "Accept-Encoding: zstd, dcz\nAvailable-Dictionary: :$::odd_b64:" --- response_headers Content-Encoding: dcz --- no_error_log [error] === TEST 41: trust_hashes on — the hashing pass is actually skipped # The perf contract, witnessed by the same counter that pinned the # verify pass in TEST 21: a trusted literal must contribute ZERO to # $zstd_dcz_dicts_hashed. Substituting the literal after hashing # anyway (trust as a no-op) produces the same negotiation key but # reads "1" here. --- http_config zstd_dcz_dict_trust_hashes on; --- config eval " location /t { zstd on; zstd_dcz_dict_file \$TEST_NGINX_PERL_PATH/suite/dcz-dict $::dict_hex; default_type text/plain; return 200 \"hashed=\$zstd_dcz_dicts_hashed\n\"; }" --- request GET /t --- response_body hashed=0 --- no_error_log [error] === TEST 42: trust_hashes on — a line without a literal is still hashed # Trust changes what a SUPPLIED literal means, nothing else: an # unhashed line has nothing to trust, so it is computed as always and # negotiates on the file's real hash. --- http_config zstd_dcz_dict_trust_hashes on; --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "hashed=$zstd_dcz_dicts_hashed\n"; } --- request GET /t --- response_body hashed=1 --- no_error_log [error] === TEST 43: trust_hashes declared AFTER a literal line is a config error # Same ordering trap and same remedy as zstd_dict_strict_path: the # flag is read at parse time, so a literal above the "on" line was # verified — correct bytes, but the full hashing pass the directive # exists to skip was silently paid. Reject rather than be quietly # position-dependent. --- http_config eval " zstd_dcz_dict_file \$TEST_NGINX_PERL_PATH/suite/dcz-dict $::dict_hex; zstd_dcz_dict_trust_hashes on;" --- config location /t { return 200 "x"; } --- must_die --- error_log "zstd_dcz_dict_trust_hashes on" was declared AFTER --- no_error_log [alert] === TEST 44: trust_hashes on does not excuse a malformed literal # Trust changes what a well-formed literal means, not what a malformed # one does: syntax is validated before the file is opened under either # policy. --- http_config zstd_dcz_dict_trust_hashes on; --- config location /t { zstd on; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict zz11; default_type text/plain; return 200 "unreachable\n"; } --- must_die --- error_log invalid dcz dictionary hash --- no_error_log [alert] === TEST 45: static sidecar keeps priority when dictionary bypass is absent # Default-off compatibility control. Both dictionary negotiation inputs are # present, but the existing sidecar remains the selected representation. --- config location /test { zstd on; zstd_types *; zstd_min_length 1; zstd_static on; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; root $TEST_NGINX_PERL_PATH/suite; } --- request GET /test --- more_headers eval "Accept-Encoding: zstd, dcz\nAvailable-Dictionary: :$::dict_b64:" --- response_headers Content-Length: 3717 ETag: "5be17d33-e85" Content-Encoding: zstd Vary: Accept-Encoding --- no_error_log [error] === TEST 46: static dictionary bypass lets the filter negotiate dcz # The weak origin ETag and missing sidecar Content-Length distinguish this # response from TEST 45's precompressed representation, while the combined # Vary proves the static handler declined before emitting its own header. --- config location /test { zstd on; zstd_types *; zstd_min_length 1; zstd_static on; zstd_static_dict_bypass on; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; root $TEST_NGINX_PERL_PATH/suite; } --- request GET /test --- more_headers eval "Accept-Encoding: zstd, dcz\nAvailable-Dictionary: :$::dict_b64:" --- response_headers ! Content-Length ETag: W/"5be17d33-e95a" Content-Encoding: dcz Vary: Accept-Encoding, Available-Dictionary, Sec-Fetch-Site --- no_error_log [error] === TEST 47: no explicit dcz gate fires before the header-collect gate # Pins the negotiation gate ORDER, not just the accept/reject outcome. # This request fails BOTH the cheap "no explicit dcz in Accept-Encoding" # predicate (Accept-Encoding carries no dcz coding at all) AND the more # expensive duplicate-Available-Dictionary gate that used to run first # (behind ngx_http_zstd_collect_dcz_headers()'s header-list walk). The # gates are pure predicates that only return NULL, so the accept/reject # answer (fall back to zstd) is identical either way -- but WHICH debug # line is emitted depends on gate order, and that is what this test # pins. If a future change reorders the cheap dcz-coding gate back below # the collect call, this flips: the duplicate-Available-Dictionary line # would reappear and the "no explicit dcz" line would vanish. --- log_level: debug --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request GET /t --- more_headers eval "Accept-Encoding: zstd\nAvailable-Dictionary: :$::dict_b64:\nAvailable-Dictionary: :$::dict_b64:\nSec-Fetch-Site: same-origin" --- response_headers Content-Encoding: zstd --- error_log eval qr/zstd dcz: skip, no explicit dcz in Accept-Encoding/ --- no_error_log eval [ qr/zstd dcz: skip, \d+ Available-Dictionary headers/, qr/\[error\]/, ] === TEST 48: vary_dcz folds an already-present token instead of duplicating it # ngx_http_zstd_vary_dcz() detects both tokens in a single pass # (ngx_http_zstd_vary_has_two_tokens()) rather than two independent # ngx_http_zstd_vary_has_token() calls. Coverage for the fold: the # upstream response already carries "Vary: Available-Dictionary" (only # ONE of the two tokens), so vary_dcz must detect has_available_dictionary # = 1 / has_sec_fetch_site = 0 from that single combined pass and push # exactly "Sec-Fetch-Site" -- not the "both absent" 2-token line every # other test in this file exercises, and not a duplicated # Available-Dictionary. A fold that mixed up which flag belongs to which # token, or that only checked one of the two in the single pass, flips # this response's Vary line. --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/t-upstream; } location /t-upstream { add_header Vary "Available-Dictionary"; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request GET /t --- more_headers eval "Accept-Encoding: zstd, dcz\nAvailable-Dictionary: :$::dict_b64:" --- response_headers Content-Encoding: dcz Vary: Available-Dictionary, Accept-Encoding, Sec-Fetch-Site --- no_error_log [error] === TEST 49: dcz window log is computed exactly once per request # ngx_http_zstd_dcz_window_log() used to run twice on a cold dcz request: # once in acquire_cctx() to key the CCtx ring slot, once again in # init_cctx() to set ZSTD_c_windowLog. Both calls take the same four # inputs and cannot observably differ, so a plain response-body test # cannot tell one computation from two -- exactly why TEST 46 above passes # either way. The debug line at the single memoisation site # (ctx->dcz_window_log_cache) is the witness that the value is computed # once, not read twice from a shared source that happens to agree: it is # logged ONLY on a cache miss, so it appears in the log exactly once per # request regardless of how many times the cached value is subsequently # read. # # Falsifiability: the witness is emitted at BOTH memoisation sites # (acquire_cctx() and init_cctx()), each inside its own cache-miss guard, # so it is logged once per ACTUAL computation rather than once per # request. Defeating the memoisation (forcing both guards true) makes the # line appear twice and this test goes red on grep_error_log_out, while # every byte-level assertion in this suite (TEST 46 included) stays green # -- verified by mutation, not assumed. Emitting the witness at only one # of the two sites would make this test pass whether the value is computed # once or twice; do not "simplify" it that way. --- config error_log logs/error.log debug; location /test { zstd on; zstd_types *; zstd_min_length 1; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "dcz window-log single-compute witness body\n"; } --- request GET /test --- more_headers eval "Accept-Encoding: zstd, dcz\nAvailable-Dictionary: :$::dict_b64:" --- response_headers Content-Encoding: dcz --- grep_error_log eval qr/zstd: dcz window log computed once: \d+/ --- grep_error_log_out eval qr/^zstd: dcz window log computed once: \d+\n?$/ --- no_error_log [error] === TEST 50: missing Available-Dictionary emits its dcz fallback trace # The response contract matches TEST 2 (ordinary zstd fallback), but that # outcome alone cannot distinguish a missing header from any other dcz # negotiation rejection. The debug line is the operator-facing witness for # this specific branch. A valid dcz coding is required here: without it the # earlier Accept-Encoding gate correctly wins and this assertion would be # vacuous. --- log_level: debug --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "dcz missing Available-Dictionary trace body\n"; } --- request GET /t --- more_headers Accept-Encoding: zstd, dcz --- response_headers Content-Encoding: zstd --- error_log zstd dcz: skip, no Available-Dictionary header --- no_error_log [error] === TEST 51: seventeen dictionaries: a mid-list one negotiates through the binary search # One past NGX_HTTP_ZSTD_DCZ_BSEARCH_THRESHOLD the lookup is a binary search # over the pointer array sorted by digest at merge time. The ninth # generated dictionary is neither first nor last of anything once sorted, # so a search over an unsorted array, or one that stops a step early, # would miss it. ci/tools/test_dcz_dict_lookup_unit.sh proves the lookup # against a linear oracle; this pins the config-to-wire path through it. --- config eval " location /t { zstd on; zstd_min_length 16; $::many_cfg default_type text/plain; return 200 \"dcz negotiation body: shared-boilerplate compute render\\n\"; }" --- request GET /t --- more_headers eval qq{Accept-Encoding: zstd, dcz Available-Dictionary: :$::many9_b64:} --- response_headers Content-Encoding: dcz --- no_error_log [error] === TEST 52: seventeen dictionaries: the first-declared one still negotiates # Declaration order is not lookup order any more; the committed fixture, # declared first, must be found wherever the sort put it. --- config eval " location /t { zstd on; zstd_min_length 16; $::many_cfg default_type text/plain; return 200 \"dcz negotiation body: shared-boilerplate compute render\\n\"; }" --- request GET /t --- more_headers eval qq{Accept-Encoding: zstd, dcz Available-Dictionary: :$::dict_b64:} --- response_headers Content-Encoding: dcz --- no_error_log [error] === TEST 53: seventeen dictionaries: an unknown digest falls through to plain zstd # The search's miss path: no entry has this digest, so negotiation # declines dcz and plain zstd serves. --- config eval " location /t { zstd on; zstd_min_length 16; $::many_cfg default_type text/plain; return 200 \"dcz negotiation body: shared-boilerplate compute render\\n\"; }" --- request GET /t --- more_headers eval qq{Accept-Encoding: zstd, dcz Available-Dictionary: :$::odd_b64:} --- response_headers Content-Encoding: zstd --- no_error_log [error] === TEST 54: a dcz dictionary above the 10 MB limit is refused at config load # TEST 13 covers the 8 MB window warning; this is the hard limit one byte # over, refused by the loader's own message before any read. --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_DCZ_HUGEDICT; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- must_die --- error_log too large: 10485761 bytes (limit: 10485760 bytes) --- no_error_log [alert] === TEST 55: Available-Dictionary beyond the first headers_in list part still negotiates # The dcz header collect walks r->headers_in.headers part by part. With # twenty-five filler headers ahead of it, Available-Dictionary lands in # the second part, so the walk's part-advance arm is the one that finds # it; a walk that stopped at the first part would serve plain zstd. --- config location /t { zstd on; zstd_min_length 16; zstd_dcz_dict_file $TEST_NGINX_PERL_PATH/suite/dcz-dict; default_type text/plain; return 200 "dcz negotiation body: shared-boilerplate compute render\n"; } --- request GET /t --- more_headers eval qq{$::many_headers Accept-Encoding: zstd, dcz Available-Dictionary: :$::dict_b64:} --- response_headers Content-Encoding: dcz --- no_error_log [error] myguard-labs-nginx-zstd-module-3486b5b/ci/t/harness/000077500000000000000000000000001525232545500223375ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/t/harness-scenarios/000077500000000000000000000000001525232545500243235ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/t/harness-scenarios/alloc-neutral/000077500000000000000000000000001525232545500270655ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/t/harness-scenarios/alloc-neutral/driver.sh000077500000000000000000000263751525232545500307340ustar00rootroot00000000000000#!/usr/bin/env bash # # Scenario: PER-REQUEST ALLOCATION NEUTRALITY for ngx_http_zstd_filter_module. # # The module serves real compressed responses and this driver proves the # per-request work is allocation-neutral: cycle-pool counters (cycle_used, # cycle_blocks, cycle_large), worker fd count and MASTER fd count are all # identical across two post-drain QUIESCENT snapshots taken around one extra # full served request. Equal means the per-request path frees everything it # allocates. # # WHY THIS LIVES HERE AND NOT UPSTREAM. testkit ships an equivalent oracle at # ci/prober/scenarios/consumer-zstd, but it load_modules a consumer .so out of # testkit's gitignored consumers/ dir, so it SKIPs on every remote leg there -- # permanently, by its own requires-gate comment: "Nothing in CI will tell you a # consumer regressed... a SKIP reads as a pass." In THIS repo the .so is our own # build product, so the same oracle actually runs. That is the whole point of # re-homing it: a skipping oracle is not coverage. # # DESIGN NOTE (inherited from testkit's consumer-cache-turbo, the hand-written # reference): a per-request-leak oracle canNOT use a COLD pre-request baseline # -- that carries the module's startup one-off -- nor a mid-work snapshot, which # flakes on live per-request buffers. Two post-drain quiescent snapshots around # one extra request is the shape that works. # # The module's config-time state (parsed dictionaries, hashed dcz strings) is # allocated once at parse time, NOT per request, so it must not appear as a # per-request delta; that absence is exactly the property under test. set -euo pipefail # shellcheck source=lib.sh . "$PROBER_LIB" HOST=127.0.0.1 PORT="$PROBER_RESOLVED_PORT" ELOG="$PROBER_PREFIX/logs/error.log" MASTER="$PROBER_SERVER_PID" export PROBER_ERROR_LOG="$ELOG" FAILED=0 # The docroot the conf's `root @PREFIX@/www` points at. Compressible content, # and large enough that the filter takes its streaming path (several # ZSTD_compressStream2 calls and at least one out_buf) rather than a single # trivial frame -- a one-buffer response would exercise far less of the # per-request allocation surface this scenario is measuring. mkdir -p "$PROBER_PREFIX/www" { printf '\n' i=0 while [ "$i" -lt 400 ]; do printf '

the quick brown fox jumps over the lazy dog %d

\n' "$i" i=$((i + 1)) done printf '\n' } >"$PROBER_PREFIX/www/index.html" # master_fds: open descriptor count of the MASTER process. /proc/$MASTER/fd is # not readable on every host, so a failed read is "cannot observe" -- surfaced # as a visible SKIP on oracle 6, never as a passing zero. master_fds() { [ -r "/proc/$MASTER/fd" ] || return 1 # shellcheck disable=SC2012 # /proc fd names are decimal ints, ls|wc is exact ls "/proc/$MASTER/fd" 2>/dev/null | wc -l } snapshot() { # read one probe snapshot into SNAP_* globals local body body="$(prober_probe_body "$HOST" "$PORT")" || return 1 SNAP_USED="$(prober_probe_field "$body" cycle_used)" || return 1 SNAP_BLOCKS="$(prober_probe_field "$body" cycle_blocks)" || return 1 SNAP_LARGE="$(prober_probe_field "$body" cycle_large)" || return 1 SNAP_FDS="$(prober_probe_field "$body" fds)" || return 1 } # one_request OUTFILE [EXTRA_HEADER]: drive ONE full GET / to completion under # a bounded deadline. A hung fetch must not hang the scenario, and a truncated # capture must not be trusted as a completed request -- that would make the # quiescent snapshot around it meaningless. one_request() { local out="$1" extra="${2:-}" pid dl timed_out=0 ( exec 3<>"/dev/tcp/$HOST/$PORT" || exit 1 printf 'GET / HTTP/1.1\r\nHost: prober\r\n' >&3 [ -n "$extra" ] && printf '%s\r\n' "$extra" >&3 printf 'Connection: close\r\n\r\n' >&3 cat <&3 2>/dev/null || true ) >"$out" 2>/dev/null & pid=$! dl=$(( SECONDS + 10 )) while kill -0 "$pid" 2>/dev/null; do if [ "$SECONDS" -ge "$dl" ]; then pkill -P "$pid" 2>/dev/null || true kill "$pid" 2>/dev/null || true timed_out=1 break fi sleep 0.05 done wait "$pid" 2>/dev/null || true # The deadline kill leaves the status line already in the capture, so the # grep below cannot tell a completed response from one cut off mid-body. # Without this the snapshot pair could straddle a request the worker still # holds buffers for, and oracles 3-6 would compare against a mid-flight # reading. [ "$timed_out" -eq 0 ] || return 1 grep -q '^HTTP/1.1 200' "$out" } # TAP plan: # 1 warm-up request serves 200 (readiness / non-vacuity) # 2 the filter demonstrably COMPRESSED a response (anti-vacuity) # 3 cycle_used equal across the two post-drain quiescent snapshots # 4 cycle_blocks + cycle_large equal across the same two snapshots # 5 worker fds equal across the same two snapshots # 6 master fd count flat across the same window # 7 no signal-death in the error log + a strict clean 200 on a final request echo "1..7" # --- 1: warm-up -- settles the worker so request 2 onward is steady-state ---- # Deliberately a COMPRESSING warm-up (Accept-Encoding: zstd): the compressor's # first-use one-off allocations must land BEFORE the measured window, or # oracle 3 would attribute that startup cost to the measured request. WARMUP="$PROBER_PREFIX/warmup.out" WARMUP_OK=0 if one_request "$WARMUP" "Accept-Encoding: zstd"; then WARMUP_OK=1 fi if [ "$WARMUP_OK" -eq 1 ]; then echo "ok 1 - the warm-up request served a clean 200 (server is ready)" else echo "not ok 1 - the warm-up request did not complete as a clean 200" head -5 "$WARMUP" 2>/dev/null | sed 's/^/# /' || true FAILED=$((FAILED + 1)) fi # --- 2: anti-vacuity --------------------------------------------------------- # REAL ANTI-VACUITY. nginx core never emits Content-Encoding: zstd by itself -- # only this module does. Take `zstd on;` out of the conf and THIS oracle fails, # which is what makes oracles 3-6 worth reading: they are measuring a request # the module demonstrably participated in, not an unfiltered static file. HITCHECK="$PROBER_PREFIX/hitcheck.out" SIG_SEEN=0 if [ "$WARMUP_OK" -eq 1 ] && one_request "$HITCHECK" "Accept-Encoding: zstd"; then if grep -qiE '^Content-Encoding:[[:space:]]*zstd' "$HITCHECK"; then SIG_SEEN=1 fi fi if [ "$SIG_SEEN" -eq 1 ]; then echo "ok 2 - the zstd filter COMPRESSED the response and set Content-Encoding: zstd (nginx never emits that header by itself)" else echo "not ok 2 - no signature that the zstd filter ran was observed" head -20 "$HITCHECK" 2>/dev/null | sed 's/^/# /' || true FAILED=$((FAILED + 1)) fi # --- oracle-3..6 measurement: two QUIESCENT snapshots around one more request # # A silent probe endpoint is a BROKEN tree, not an unmet environmental # requirement: @PROBE@ is in this scenario's conf unconditionally and the # requires gate has already established the .so carries the probe symbols. If # oracles 3-6 all SKIP, the only surviving oracles are 1, 2 and 7 -- none of # which touch an allocation metric -- so the scenario would exit 0 having # asserted nothing about the property in its own title. That is a vacuous # green, so it reds instead. BASE_OK=1 if ! snapshot; then BASE_OK=0 echo "# the probe endpoint did not answer for the first post-drain snapshot" fi if [ "$BASE_OK" -eq 1 ]; then BASE_USED="$SNAP_USED"; BASE_BLOCKS="$SNAP_BLOCKS"; BASE_LARGE="$SNAP_LARGE"; BASE_FDS="$SNAP_FDS" fi BASE_MFDS="$(master_fds || true)" STIM="$PROBER_PREFIX/stimulus.out" STIM_OK=0 if [ "$BASE_OK" -eq 1 ] && one_request "$STIM" "Accept-Encoding: zstd"; then STIM_OK=1 fi FINAL_OK=0 if [ "$STIM_OK" -eq 1 ] && snapshot; then FINAL_OK=1 FINAL_USED="$SNAP_USED"; FINAL_BLOCKS="$SNAP_BLOCKS"; FINAL_LARGE="$SNAP_LARGE"; FINAL_FDS="$SNAP_FDS" fi FINAL_MFDS="$(master_fds || true)" if [ "$STIM_OK" -ne 1 ]; then echo "# the extra measured request did not complete -- oracles 3-6 will SKIP rather than compare against a vacuous reading" fi if [ "$BASE_OK" -eq 0 ] || [ "$STIM_OK" -eq 0 ] || [ "$FINAL_OK" -eq 0 ]; then echo "# oracles 3-6 cannot be evaluated (base=$BASE_OK stim=$STIM_OK final=$FINAL_OK); failing rather than banking a green" FAILED=$((FAILED + 1)) fi # --- 3: cycle_used flat ------------------------------------------------------- if [ "$BASE_OK" -eq 0 ] || [ "$FINAL_OK" -eq 0 ]; then echo "ok 3 - cycle_used allocation-neutrality # SKIP a post-drain snapshot did not answer" elif [ "$BASE_USED" = "$FINAL_USED" ]; then echo "ok 3 - cycle_used was flat across the extra compressed request ($BASE_USED)" else echo "not ok 3 - cycle_used grew across the extra compressed request (before=$BASE_USED after=$FINAL_USED)" FAILED=$((FAILED + 1)) fi # --- 4: cycle_blocks + cycle_large flat -------------------------------------- if [ "$BASE_OK" -eq 0 ] || [ "$FINAL_OK" -eq 0 ]; then echo "ok 4 - cycle_blocks/cycle_large allocation-neutrality # SKIP a post-drain snapshot did not answer" elif [ "$BASE_BLOCKS" = "$FINAL_BLOCKS" ] && [ "$BASE_LARGE" = "$FINAL_LARGE" ]; then echo "ok 4 - cycle_blocks ($BASE_BLOCKS) and cycle_large ($BASE_LARGE) were flat across the extra compressed request" else echo "not ok 4 - cycle_blocks/cycle_large grew across the extra compressed request (before blocks=$BASE_BLOCKS large=$BASE_LARGE, after blocks=$FINAL_BLOCKS large=$FINAL_LARGE)" FAILED=$((FAILED + 1)) fi # --- 5: worker fds flat ------------------------------------------------------- if [ "$BASE_OK" -eq 0 ] || [ "$FINAL_OK" -eq 0 ]; then echo "ok 5 - worker fds allocation-neutrality # SKIP a post-drain snapshot did not answer" elif [ "$BASE_FDS" = "$FINAL_FDS" ]; then echo "ok 5 - worker fd count was flat across the extra compressed request ($BASE_FDS)" else echo "not ok 5 - worker fd count grew across the extra compressed request (before=$BASE_FDS after=$FINAL_FDS)" FAILED=$((FAILED + 1)) fi # --- 6: master fd count flat -------------------------------------------------- if [ -z "$BASE_MFDS" ] || [ -z "$FINAL_MFDS" ]; then echo "ok 6 - master descriptor count # SKIP /proc/$MASTER/fd not readable on this host" elif [ "$BASE_OK" -eq 0 ] || [ "$FINAL_OK" -eq 0 ]; then echo "ok 6 - master descriptor count allocation-neutrality # SKIP a post-drain snapshot did not answer" elif [ "$BASE_MFDS" = "$FINAL_MFDS" ]; then echo "ok 6 - master fd count was flat across the extra compressed request ($BASE_MFDS)" else echo "not ok 6 - master fd count grew across the extra compressed request (before=$BASE_MFDS after=$FINAL_MFDS)" FAILED=$((FAILED + 1)) fi # --- 7: no signal-death + a final strict clean 200 --------------------------- SIGDEATH=0 if grep -qE 'worker process .* exited on signal|SIGSEGV|SIGABRT|SIGBUS' "$ELOG"; then SIGDEATH=1 fi FINAL_REQ="$PROBER_PREFIX/final.out" FINAL_CLEAN=0 if one_request "$FINAL_REQ" "Accept-Encoding: zstd"; then FINAL_CLEAN=1 fi if [ "$SIGDEATH" -eq 0 ] && [ "$FINAL_CLEAN" -eq 1 ]; then echo "ok 7 - no worker died by signal, and a final request still served a clean 200" else echo "not ok 7 - server health check failed (signal-death=$SIGDEATH, final-clean=$FINAL_CLEAN)" if [ "$SIGDEATH" -eq 1 ]; then grep -nE 'exited on signal|SIGSEGV|SIGABRT|SIGBUS' "$ELOG" | sed 's/^/# /' fi FAILED=$((FAILED + 1)) fi exit "$(( FAILED > 0 ? 1 : 0 ))" myguard-labs-nginx-zstd-module-3486b5b/ci/t/harness-scenarios/alloc-neutral/nginx.conf000066400000000000000000000031601525232545500310570ustar00rootroot00000000000000@LOAD@ # Local scenario (NOT vendored, NOT gitignored). Same shape as fault-arms: # this repo IS the module under test, so PROBER_MODULE is # ngx_http_zstd_filter_module.so itself and @LOAD@ already load_modules it. # There is no separate probe .so and no @PROBE_ZONE@ -- this module registers # ZONELESS hooks (ngx_test_probe_register_module), so the probe document's # pool/fd counters reach the driver through the module_render path that is # emitted BEFORE the zone early-return (ngx_test_probe.c:551). # # Why this exists next to testkit's own consumer-zstd scenario: that one is # generated into the testkit tree, load_modules a SEPARATE consumer .so out of # a gitignored consumers/ dir, and therefore SKIPs on every remote leg there -- # permanently, by its own requires-gate comment ("Nothing in CI will tell you a # consumer regressed"). Here the .so is the repo's own build product, so the # identical oracle actually RUNS instead of skipping. daemon off; pid @PREFIX@/nginx.pid; error_log @PREFIX@/logs/error.log info; worker_processes 1; events { worker_connections 16; } http { server { listen 127.0.0.1:@PORT@; # A REAL STATIC ROOT, not `return 200`. A return directive fires at the # REWRITE phase and short-circuits later phases, so the body filter # chain this module lives in would never run on a synthetic response # and the scenario would measure nothing. The driver writes # @PREFIX@/www/index.html before booting. root @PREFIX@/www; location / { zstd on; zstd_min_length 0; } location /__probe { @PROBE@ } } } myguard-labs-nginx-zstd-module-3486b5b/ci/t/harness-scenarios/alloc-neutral/requires000077500000000000000000000030611525232545500306520ustar00rootroot00000000000000#!/usr/bin/env bash # # Gate: the tree run-scenario.sh is about to boot must be a TEST_HARNESS build. # This scenario reads pool/fd counters out of the probe document, and without # both TEST_HARNESS=1 and -DNGX_TEST_HARNESS the probe endpoint compiles out # entirely -- `nginx -t` would then reject the `zstd_probe;` in nginx.conf and # the scenario would red for a build reason rather than a module one. # # Derived the same way run-scenario.sh/lib.sh resolve the build tree, so this # gate can never SKIP a tree the engine would boot fine or pass one it would # reject -- identical reasoning to fault-arms/requires and to testkit's own # consumer-zstd/requires. set -euo pipefail FLAVOR="${2:-nginx}" VERSION="${3:-1.31.3}" BUILD="${PROBER_BUILD:-${PROBER_ROOT:-$(cd ../.. && pwd)}/.build/${FLAVOR}-${VERSION}}" SO="$BUILD/objs/ngx_http_zstd_filter_module.so" if [ ! -f "$SO" ]; then echo "ngx_http_zstd_filter_module.so not found at $SO -- build it first" exit 1 fi # Same non-vacuity gate fault-arms carries: the env var ALONE compiles the # sources and still yields a probe-free .so (verified live in this repo -- # packaged=0 symbols, TEST_HARNESS=1 alone=0, both switches=18). Gate on the # symbol count, not on the env var, so that trap cannot silently return. PROBE_SYMBOLS="$(nm "$SO" 2>/dev/null | grep -c ' ngx_test_probe_' || true)" if [ "$PROBE_SYMBOLS" -eq 0 ]; then echo "$SO carries no ngx_test_probe_* symbols -- this is a packaged (non-harness) build; configure with TEST_HARNESS=1 CFLAGS=\"\$CFLAGS -DNGX_TEST_HARNESS\" first" exit 1 fi exit 0 myguard-labs-nginx-zstd-module-3486b5b/ci/t/harness-scenarios/codec-call-count/000077500000000000000000000000001525232545500274375ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/t/harness-scenarios/codec-call-count/driver.sh000077500000000000000000000410241525232545500312720ustar00rootroot00000000000000#!/usr/bin/env bash # # Scenario: ZSTD_compressStream2() CALL COUNT AT THE END-OF-FRAME SITE. # # The filter picks its ZSTD_EndDirective once per compress call. END is # selected as soon as `ctx->last && ctx->in == NULL` holds -- i.e. on the call # that carries the response's final input -- instead of one iteration later, # after a ZSTD_e_continue call had already drained that input and the state # machine transitioned COMPRESS->END. libzstd accepts pending input under # ZSTD_e_end, consumes what it can and closes the frame in the same call, so # the earlier selection removes one ZSTD_compressStream2() call from every # completed response. # # WHAT THIS SCENARIO ASSERTS, AND WHY IT IS NOT VACUOUS. # # A correct response proves nothing about the call count: the two-call and the # one-call sequences both produce a valid frame that decodes to the same # plaintext. That is the whole difficulty of covering this change -- the # observable it turns on is the COUNT, not the bytes. So every oracle here is # a PAIR: # # (a) the response decodes byte-for-byte to the origin file -- correctness, # which must hold under both the optimized and the unoptimized path; and # (b) codec_end_calls for that one request equals the expected count -- # the marker unique to the new path, which goes red the moment the extra # ZSTD_e_continue iteration comes back. # # Dropping (b) leaves a suite that a revert passes. Dropping (a) leaves a # suite that a fast, truncating filter passes. Both halves are load-bearing. # # HOW A PER-REQUEST CALL COUNT IS READ. ngx_http_zstd_probe_codec_fault() # advances a per-site counter on every call and fault_set_global ZEROES that # counter whenever the site is armed OR disarmed. So `fault_codec_end=-1` # (a disarm) is also a counter RESET that leaves no fault armed. Reset, issue # exactly ONE request, read /__probe: the counter is that request's call count # at that site. Both sites are reset before each measurement so neither can # carry a previous case's traffic into this one. # # The probe request itself never runs the body filter over a compressible # response (the probe handler renders its own JSON and the /__probe location # has no zstd directive), so reading the counter does not perturb it. set -euo pipefail # shellcheck source=lib.sh . "$PROBER_LIB" HOST=127.0.0.1 PORT="$PROBER_RESOLVED_PORT" ELOG="$PROBER_PREFIX/logs/error.log" export PROBER_ERROR_LOG="$ELOG" FAILED=0 N=0 ok() { N=$((N + 1)); echo "ok $N - $1"; } notok() { N=$((N + 1)); echo "not ok $N - $1"; FAILED=1; } diag() { echo "# $1"; } WWW="$PROBER_PREFIX/www" mkdir -p "$WWW" "$PROBER_PREFIX/tmp" # Compressible body, large enough that the filter takes its streaming path # (several ZSTD_compressStream2 calls, more than one out_buf) rather than a # single trivial frame. A one-buffer response would leave the multi-call END # epilogue -- the part most at risk from selecting END earlier -- untested. { printf '\n' i=0 while [ "$i" -lt 4000 ]; do printf '

the quick brown fox jumps over the lazy dog %d

\n' "$i" i=$((i + 1)) done printf '\n' } >"$WWW/index.html" # A genuinely EMPTY body: the terminal chain link carries last_buf and no # data whatsoever, so the response's only codec call is the END one. : >"$WWW/empty.txt" # The dcz and trained-dictionary arms use a dictionary this scenario's # `requires` gate stages into the build's objs/ dir (see there for why that # is the only hook early enough). The dcz arm additionally has to NEGOTIATE: # the client must send Available-Dictionary carrying the dictionary's raw # SHA-256 in the "::" structured-field form, or the module answers # plain zstd and the arm would measure the wrong path -- which the # Content-Encoding assertion inside measure() catches rather than passing. # PROBER_SERVER_BIN is the exported /objs/nginx, so its dirname is # the same objs/ dir @BUILD_OBJS@ renders into the conf. DICT_FILE="$(dirname "$PROBER_SERVER_BIN")/codec-call-count.dict" # openssl(1) presence is already proven by this scenario's `requires` gate, so # an empty result here means the dictionary FILE is unreadable, not that the # tool is missing -- which is why the failure branch below says so. DICT_SHA_B64="$(openssl dgst -sha256 -binary "$DICT_FILE" 2>/dev/null \ | openssl base64 -A 2>/dev/null || true)" # reset_counters: disarm BOTH codec sites. A disarm zeroes that site's call # counter (fault_set_global does it on every arm and every disarm alike), so # this is the "start counting from here" primitive. Returns non-zero if the # probe did not answer, so a measurement can never silently proceed on a # counter that was not actually reset. reset_counters() { curl -fsS --max-time 5 "http://$HOST:$PORT/__probe?fault_codec=-1" \ -o /dev/null || return 1 curl -fsS --max-time 5 "http://$HOST:$PORT/__probe?fault_codec_end=-1" \ -o /dev/null || return 1 } # read_counter NAME: echo one integer field from a fresh probe document. read_counter() { local body body="$(prober_probe_body "$HOST" "$PORT")" || return 1 prober_probe_field "$body" "$1" || return 1 } # fetch PATH OUTFILE [EXTRA_HEADER...]: one bounded GET that asks for zstd and # writes the raw (still compressed) body to OUTFILE. --fail is deliberately # NOT used: an arm that ends up serving 200-but-identity must be caught by the # encoding assertion below, not silently by curl. # # A caller-supplied Accept-Encoding REPLACES the default rather than adding a # second one. curl sends every -H it is given, and two Accept-Encoding header # lines are not the same request as one: RFC 9842 SS8.3 makes the module # decline dcz on a duplicated negotiation header, so the dcz arm silently fell # back to plain zstd and measured the wrong path. (Caught by measure()'s # Content-Encoding assertion, which is exactly why that assertion is there.) fetch() { local path="$1" out="$2"; shift 2 local hdrs="$out.hdrs" rc=0 local -a extra=() local h have_ae=0 for h in "$@"; do case "$h" in [Aa]ccept-[Ee]ncoding:*) have_ae=1 ;; esac extra+=(-H "$h") done [ "$have_ae" -eq 1 ] || extra=(-H 'Accept-Encoding: zstd' "${extra[@]}") curl -sS --max-time 15 -D "$hdrs" -o "$out" "${extra[@]}" \ "http://$HOST:$PORT$path" 2>"$out.err" || rc=$? return "$rc" } # encoded_as HDRFILE ENC: did the response actually carry Content-Encoding: # ENC? Guards every oracle against the vacuous case where the filter declined # and the body is plain -- a call count of 0 would then be "correct" for a # response the scenario is not measuring at all. encoded_as() { grep -qi "^content-encoding:[[:space:]]*$2" "$1" } # decodes_to RAWFILE ORIGINFILE: the compressed body decodes byte-for-byte to # the origin. This is half (a) of every oracle pair. decodes_to() { local raw="$1" origin="$2" dec="$1.dec" zstd -d -q -f -o "$dec" "$raw" 2>/dev/null || return 1 cmp -s "$dec" "$origin" } # measure LABEL PATH ORIGIN ENC EXPECT_TOTAL_CALLS [EXTRA_HEADER...] # # The whole oracle pair in one place: reset both counters, issue exactly one # request, then assert (a) it was encoded as ENC and decodes to ORIGIN and # (b) the response made EXPECT_TOTAL_CALLS ZSTD_compressStream2() calls in # total, summed across BOTH sites. # # THE SUM, NOT codec_end_calls. This was measured, not assumed, and the # distinction is the entire value of the oracle. The call this change removes # is the redundant ZSTD_e_continue that used to precede the frame close, and # a ZSTD_e_continue lands at the STREAMING site (CODEC), not the end-of-frame # site (CODEC_END) -- the fault-injection split is keyed on # `directive == ZSTD_e_end`. Both the optimized and the reverted path close # the frame in exactly one END call, so codec_end_calls is 1 either way and # an oracle asserting it passes on both: a textbook vacuous control. Reverting # the optimization and re-running showed codec_end_calls unchanged at 1 while # codec_calls moved 6 -> 7. The sum is what actually discriminates. # # EXPECT_TOTAL_CALLS is exact except for the comma-separated flush-last values # documented below. The deterministic arms remain exact because those are what # distinguish the optimized and reverted paths. measure() { local label="$1" path="$2" origin="$3" enc="$4" expect="$5"; shift 5 local raw="$PROBER_PREFIX/tmp/${label//\//_}.bin" local got if ! reset_counters; then notok "$label: probe did not answer the counter reset" notok "$label: total codec calls (not measured -- reset failed)" return fi if ! fetch "$path" "$raw" "$@"; then notok "$label: request failed or timed out" notok "$label: total codec calls (not measured -- request failed)" return fi if ! encoded_as "$raw.hdrs" "$enc"; then diag "$label: headers were: $(tr -d '\r' <"$raw.hdrs" | tr '\n' ' ')" notok "$label: response carried Content-Encoding: $enc" notok "$label: total codec calls (not measured -- wrong encoding)" return fi if decodes_to "$raw" "$origin"; then ok "$label: response decodes byte-for-byte to the origin" else notok "$label: response decodes byte-for-byte to the origin" fi local ends if ! got="$(read_counter codec_calls)" \ || ! ends="$(read_counter codec_end_calls)"; then notok "$label: codec call count (probe field absent)" return fi got=$((got + ends)) if [[ "$expect" != *,* ]] && [ "$got" -eq "$expect" ]; then ok "$label: total codec calls is $expect" elif [[ ",$expect," == *",$got,"* ]] && [ "$ends" -eq 1 ]; then ok "$label: total codec calls is ${expect//,/ or } (got $got; one END call)" else if [[ "$expect" == *,* ]]; then diag "$label: codec_calls+codec_end_calls=$got expected=$expect; codec_end_calls=$ends expected=1" else diag "$label: codec_calls+codec_end_calls=$got expected=$expect" fi notok "$label: total codec calls is ${expect//,/ or } (got $got)" fi } # --- oracle plan -------------------------------------------------------- # 1,2 default arm: many-buffer streaming body, ONE END call # 3,4 tiny configured output buffers (zstd_buffers 4 4k) # 5,6 data-less final buffer (empty body) # 7,8 dcz framing (40-byte skippable prefix in the first buffer) # 9,10 flush + last (proxy_buffering off) # 11 a completed response makes at least one NON-end codec call too, so # the split between the two sites is real and oracle 2 is not counting # the whole response # 12 client abort mid-response does not wedge the worker # 13 error log carries no crit/alert/emerg across the whole run # # There is no separate trained-dictionary arm: zstd_dict_file is MAIN_CONF # only, so enabling it would load the CDict for EVERY location here and move # every count in the table below rather than isolating one arm. The dcz arm # above is this scenario's dictionary-loaded CCtx coverage -- it goes through # the same CDict-backed compressor, plus the skippable-prefix framing. The # non-dcz trained-dictionary path keeps its existing coverage in ci/t/03-dcz.t # and tools/test_dcz.py. echo "1..13" # EVERY EXPECTED COUNT BELOW WAS MEASURED, on this conf, against both the # optimized and the reverted filter. Each one is exactly one lower with the # optimization than without it, which is the row's claim stated as a number: # # arm optimized reverted delta # default 7 8 -1 # tiny-buffers 7 8 -1 # empty-body 1 2 -1 # dcz 7 8 -1 # flush-last 56/57 57/58 -1 # # The first four rows are exact and intentionally brittle. The flush-last # count additionally depends on how the kernel segments upstream reads with # proxy_buffering off; both 56 and 57 have been observed across CI runners. # Its decode oracle and exactly-one-END requirement cover flush+last behavior, # while the four deterministic rows keep the removed call from creeping back. # # The flush-last arm's 56/57 reflects proxy_buffering off chunking the body into # many small upstream writes, each of which drives its own compress calls. # Its absolute value is an artifact of that chunking; its DELTA of one is the # part that carries meaning, and it is the arm where a pending ctx->flush # coexists with last_buf. measure "default" "/index.html" "$WWW/index.html" zstd 7 # Tiny output buffers: the compressed body still fits well inside one 4k # buffer's remaining space at the point the frame closes, so this stays 1 -- # but it reaches that point having already cycled several output buffers, # which is the property being covered. If a future libzstd needs more than # one END call here, this is the oracle that will say so, loudly, rather # than a range that hides it. measure "tiny-buffers" "/tiny/index.html" "$WWW/index.html" zstd 7 # Data-less final buffer: an empty body's terminal link carries last_buf and # nothing else. Under the old two-call sequence this response reached END # via the state machine's transition; now the directive selection picks END # on the first and only call. measure "empty-body" "/empty" "$WWW/empty.txt" zstd 1 # flush + last. proxy_buffering off makes the upstream force flushes around # chunks the encoder buffers internally, so ctx->flush can still be pending # when last_buf arrives. END must win the directive selection there -- if # the pending flush won instead, the frame would never be closed and this # request would hang until the 15s deadline rather than decode. if [ -n "$DICT_SHA_B64" ]; then measure "dcz" "/dcz/index.html" "$WWW/index.html" dcz 7 \ "Accept-Encoding: zstd, dcz" \ "Available-Dictionary: :$DICT_SHA_B64:" else # NOT a SKIP. openssl(1) is a hard requires-gate for this scenario, so # reaching here means the hash derivation failed for some other reason -- # an unreadable or truncated dictionary file. A SKIP would print # "ok ... # SKIP", which reads as a pass to anything grepping for the # oracle, so the dcz arm would assert nothing while the run stayed green. # Fail loudly instead, and say what actually broke rather than blaming a # missing tool the gate already proved is present. notok "dcz: response decodes byte-for-byte to the origin (could not derive the dictionary hash from $DICT_FILE)" notok "dcz: total codec calls is 7 (not measured -- no dictionary hash)" fi measure "flush-last" "/flushlast" "$WWW/index.html" zstd 56,57 # The two sites really are split. If the END selection had swallowed the # whole response (every call landing at the END site), codec_calls would be # 0 and oracle 2's "1" would be counting the entire compression rather than # just the frame close -- a passing number for the wrong reason. if reset_counters \ && fetch "/index.html" "$PROBER_PREFIX/tmp/split.bin" \ && encoded_as "$PROBER_PREFIX/tmp/split.bin.hdrs" zstd; then NONEND="$(read_counter codec_calls || echo -1)" if [ "$NONEND" -ge 1 ]; then ok "streaming site is still reached ($NONEND non-END calls)" else diag "codec_calls=$NONEND" notok "streaming site is still reached (got $NONEND non-END calls)" fi else notok "streaming site is still reached (request or reset failed)" fi # Client abort: tear the connection down mid-response and require the worker # to still serve a clean, decodable 200 afterwards. A terminal-frame change # that leaks a half-closed frame or wedges ctx->done would show up here as a # dead or wrong follow-up response. ( exec 3<>"/dev/tcp/$HOST/$PORT" || exit 1 printf 'GET /index.html HTTP/1.1\r\nHost: prober\r\n' >&3 printf 'Accept-Encoding: zstd\r\nConnection: close\r\n\r\n' >&3 head -c 64 <&3 >/dev/null 2>&1 || true ) >/dev/null 2>&1 || true if fetch "/index.html" "$PROBER_PREFIX/tmp/after-abort.bin" \ && encoded_as "$PROBER_PREFIX/tmp/after-abort.bin.hdrs" zstd \ && decodes_to "$PROBER_PREFIX/tmp/after-abort.bin" "$WWW/index.html"; then ok "a full response still decodes after a mid-response client abort" else notok "a full response still decodes after a mid-response client abort" fi # No crit/alert/emerg anywhere in this run. This scenario injects no faults, # so any such line is a real defect -- and the terminal-frame bug class this # change touches announces itself exactly there ("zero size buf in writer"). if grep -qE '\[(crit|alert|emerg)\]' "$ELOG" 2>/dev/null; then diag "$(grep -E '\[(crit|alert|emerg)\]' "$ELOG" | head -3)" notok "error log carries no crit/alert/emerg lines" else ok "error log carries no crit/alert/emerg lines" fi exit "$FAILED" myguard-labs-nginx-zstd-module-3486b5b/ci/t/harness-scenarios/codec-call-count/nginx.conf000066400000000000000000000113261525232545500314340ustar00rootroot00000000000000@LOAD@ # Local scenario (NOT vendored, NOT gitignored). Same shape as alloc-neutral: # this repo IS the module under test, so PROBER_MODULE is # ngx_http_zstd_filter_module.so itself and @LOAD@ already load_modules it. # The module registers ZONELESS hooks, so codec_calls / codec_end_calls reach # the driver through the module_render path. daemon off; pid @PREFIX@/nginx.pid; error_log @PREFIX@/logs/error.log info; worker_processes 1; events { worker_connections 16; } http { default_type text/plain; server { listen 127.0.0.1:@PORT@; # A REAL STATIC ROOT, not `return 200`: a return directive fires at # the REWRITE phase and short-circuits later phases, so the body # filter chain this module lives in would never run and every # call-count oracle would measure zero calls and pass vacuously. root @PREFIX@/www; # Default arm: ordinary compression, module-default output buffers. location / { zstd on; zstd_min_length 0; } # TINY configured output buffers. This is the arm that forces the # ZSTD_e_end epilogue across SEVERAL calls (zrc > 0 -> ctx->redo), # which is precisely the loop the one-call selection must keep # re-driving correctly rather than truncating. 4k matches the # smallest value 00-filter.t TEST 49 already pins as valid. location /tiny/ { alias @PREFIX@/www/; zstd on; zstd_min_length 0; zstd_buffers 4 4k; } # proxy_buffering off: the upstream forces a FLUSH around chunks the # encoder buffers internally, so this arm exercises flush-then-last # -- a pending ctx->flush arriving together with last_buf. END must # still win the directive selection there. location /flushlast { zstd on; zstd_min_length 0; proxy_buffering off; proxy_http_version 1.1; proxy_pass http://127.0.0.1:@PORT@/origin/index.html; } # dcz framing: the 40-byte skippable prefix is written into the # first compressor buffer by get_buf BEFORE the first compress call, # so this is the arm where "this call produced no output" and "the # out_buf is non-empty" disagree -- the exact distinction the # END-transition arm is keyed on. Negotiated by the driver with an # Available-Dictionary request header. The dictionary is staged into # the build's objs/ dir by this scenario's `requires` gate, which is # the only hook that runs before nginx's config test opens it. location /dcz/ { alias @PREFIX@/www/; zstd on; zstd_min_length 0; zstd_dcz_dict_file @BUILD_OBJS@/codec-call-count.dict; # The prober speaks plaintext HTTP, and dcz is refused on a # non-secure hop unless the operator asserts the client hop was # secure. Without this the module answers plain zstd and this arm # silently measures the wrong path -- caught, not hidden, by # measure()'s Content-Encoding assertion. ci/t/03-dcz.t sets the # same flag on every one of its blocks for the same reason. zstd_dcz_assume_secure_transport on; } # Plain (uncompressed) origin for the proxied arm above. # # `zstd off` here is belt-and-braces, not the mechanism: `zstd` is off # by default and every `zstd on` in this conf is location-scoped, so # this location would serve identity anyway. It is worth keeping # explicit because nginx does NOT strip Accept-Encoding on the proxy # hop, so the client's `Accept-Encoding: zstd` really does arrive # here -- and if anyone later adds a server- or http-level `zstd on`, # this line is what stops /flushlast from compressing twice and # measuring a call count for the wrong stream. location /origin/ { alias @PREFIX@/www/; zstd off; } # An EMPTY body: the terminal buffer carries last_buf and no data at # all, so the END call is the only codec call the response makes. # `location = /empty`, an EXACT match, not a prefix. A prefix # location whose alias does not end in a separator is the # alias-traversal shape (gixy alias_traversal): a request for # /emptyX would be aliased to /empty.txtX. Nothing here serves # attacker-controlled paths, but a test fixture should not model a # pattern the repo's own linters flag. location = /empty { zstd on; zstd_min_length 0; alias @PREFIX@/www/empty.txt; } location /__probe { @PROBE@ } } } myguard-labs-nginx-zstd-module-3486b5b/ci/t/harness-scenarios/codec-call-count/requires000077500000000000000000000072331525232545500312310ustar00rootroot00000000000000#!/usr/bin/env bash # # Gate: the tree run-scenario.sh is about to boot must be a TEST_HARNESS build. # This scenario reads the codec_calls / codec_end_calls counters out of the # probe document, and without both TEST_HARNESS=1 and -DNGX_TEST_HARNESS the # probe endpoint compiles out entirely -- `nginx -t` would then reject the # `zstd_probe;` in nginx.conf and the scenario would red for a build reason # rather than a module one. # # Derived the same way run-scenario.sh/lib.sh resolve the build tree, so this # gate can never SKIP a tree the engine would boot fine or pass one it would # reject -- identical reasoning to fault-arms/requires and alloc-neutral's. set -euo pipefail FLAVOR="${2:-nginx}" VERSION="${3:-1.31.3}" BUILD="${PROBER_BUILD:-${PROBER_ROOT:-$(cd ../.. && pwd)}/.build/${FLAVOR}-${VERSION}}" SO="$BUILD/objs/ngx_http_zstd_filter_module.so" if [ ! -f "$SO" ]; then echo "ngx_http_zstd_filter_module.so not found at $SO -- build it first" exit 1 fi PROBE_SYMBOLS="$(nm "$SO" 2>/dev/null | grep -c ' ngx_test_probe_' || true)" if [ "$PROBE_SYMBOLS" -eq 0 ]; then echo "$SO carries no ngx_test_probe_* symbols -- this is a packaged (non-harness) build; configure with TEST_HARNESS=1 CFLAGS=\"\$CFLAGS -DNGX_TEST_HARNESS\" first" exit 1 fi # Every call-count oracle here is PAIRED with a decode of the same response # against the origin bytes, because a call count on its own is satisfied by a # response that is fast and wrong. A missing decoder would silently weaken # every one of them to "200 and some bytes". Fail here instead: a gate that # cannot run its assertion is not a gate. if ! command -v zstd >/dev/null 2>&1; then echo "zstd(1) not found -- every oracle decodes its response and compares it byte-for-byte with the origin file" exit 1 fi # openssl(1) is gated on exactly the same reasoning, and it is the CLI that is # required, not libssl: the driver shells out to `openssl dgst`/`openssl base64` # to derive the Available-Dictionary hash the dcz arm negotiates with. Without # it that arm cannot run. # # It is a hard gate rather than a driver-side SKIP deliberately. A SKIP prints # an "ok ... # SKIP" line, which reads as a pass to every consumer that greps # for the oracle -- so the dcz arm would assert nothing while the run stayed # green, which is the false green this scenario exists to remove. Failing the # whole scenario here is loud and impossible to misread. if ! command -v openssl >/dev/null 2>&1; then echo "openssl(1) not found -- the dcz arm derives its Available-Dictionary hash with \"openssl dgst -sha256 | openssl base64\"; the CLI is required, not just libssl" exit 1 fi # Stage the dictionary fixture the dcz and trained-dict arms need. # # It has to exist before nginx runs its CONFIG TEST -- zstd_dcz_dict_file and # zstd_dict_file open their file at configuration time, so a driver-written # fixture is far too late (the prefix the driver writes into is not created # until prober_render_conf, immediately before that config test). This gate is # the only hook that runs earlier, so it stages the file and the conf # references it through @BUILD_OBJS@, the one absolute path the renderer # substitutes that exists this early. # # Copied from the repo's own checked-in ci/t/suite/dcz-dict rather than # generated, so the dcz arm here negotiates against exactly the dictionary # 03-dcz.t already pins the wire contract against. DICT_SRC="$(cd "$(dirname "$0")/../../suite" && pwd)/dcz-dict" if [ ! -f "$DICT_SRC" ]; then echo "dictionary fixture not found at $DICT_SRC" exit 1 fi if ! cp -f "$DICT_SRC" "$BUILD/objs/codec-call-count.dict"; then echo "cannot stage the dictionary fixture into $BUILD/objs" exit 1 fi exit 0 myguard-labs-nginx-zstd-module-3486b5b/ci/t/harness-scenarios/fault-arms/000077500000000000000000000000001525232545500263765ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/t/harness-scenarios/fault-arms/driver.sh000077500000000000000000000545171525232545500302440ustar00rootroot00000000000000#!/usr/bin/env bash # # Scenario: CODEC FAULT-INJECTION ARMS for ngx_http_zstd_filter_module.so. # # Reachable only through the harness's TEST_HARNESS build (see # plan-testkit-integration.md Ground truth + Phase 3): the module's single # ZSTD_compressStream2() call site at filter_module.c is preceded by a fault # check keyed on `directive == ZSTD_e_end` (site CODEC_END) vs. every other # directive (site CODEC). Armed via GET /__probe?fault_codec= or # fault_codec_end=: 1..999 injects a ZSTD_isError()-true return on the # n-th call at that site since the arm; 1000+n injects success-with- # zero-output on the n-th call. # # The same scenario owns the dedicated dcz setup site immediately before # ZSTD_CCtx_refPrefix(). GET /__probe?fault_refprefix= injects an error on # its n-th call; the separately rendered refprefix_calls counter proves a # plain zstd request cannot consume that dcz-only arm. # # Non-vacuity discipline copied from consumer-zstd/driver.sh: if the basis a # set of oracles needs (a probe reading, a completed request) is missing, # those oracles FAIL rather than SKIP-to-green. See "if every oracle SKIPs, # red" in Phase 4's task list. set -euo pipefail # shellcheck source-path=SCRIPTDIR # shellcheck source=../../harness/ci/prober/lib.sh . "$PROBER_LIB" HOST=127.0.0.1 PORT="$PROBER_RESOLVED_PORT" ELOG="$PROBER_PREFIX/logs/error.log" VLOG="$PROBER_PREFIX/logs/zstd-vars.log" FAILED=0 mkdir -p "$PROBER_PREFIX/www" # 200 KB body: big enough that a single ZSTD_compressStream2 call does not # necessarily drain it in one shot (module's own buffer sizing), which is # what makes the CODEC (non-end) site reachable at all on a real request -- # not just the CODEC_END site every response also passes through. head -c 204800 /dev/zero | tr '\0' 'a' >"$PROBER_PREFIX/www/body.bin" printf 'irrelevant\n' >"$PROBER_PREFIX/www/index.html" # arm SITE NTH: GET /__probe?fault_=, discard the body, return 0 # only if the probe answered at all (arming and reading are the same request # here -- see ngx_http_zstd_probe_handler's "arm-then-render" comment). arm() { local site="$1" nth="$2" curl -fsS --max-time 5 \ "http://$HOST:$PORT/__probe?fault_${site}=${nth}" -o /dev/null } # disarm SITE: same shape, negative nth. disarm() { local site="$1" curl -fsS --max-time 5 \ "http://$HOST:$PORT/__probe?fault_${site}=-1" -o /dev/null } # fetch PATH OUTFILE [EXTRA_HEADER...]: bounded GET, writes headers and body # separately, and returns the exit status of curl. Uses -D to capture # headers separately so an ERROR outcome (nginx tears the connection down # with no body) is still observable via headers-vs-no-headers, not just via # curl's exit status, which a transfer-encoding mismatch can also trip. fetch() { local path="$1" out="$2" hdrs="$2.hdrs"; shift 2 local -a extra=(-H 'Accept-Encoding: zstd') if [ "$#" -gt 0 ]; then extra=() while [ "$#" -gt 0 ]; do extra+=(-H "$1") shift done fi curl -sS --max-time 10 -D "$hdrs" -o "$out" "${extra[@]}" \ "http://$HOST:$PORT$path" 2>"$out.stderr" } # Wait for the last /body.bin log record to change. The completed warm-up is # the negative control: it must expose all three values. The faulted request # then has to append a distinct all-not-found record; reading a stale warm-up # line would otherwise let an unreachable fault path pass. wait_body_var_line() { local previous="$1" line deadline deadline=$((SECONDS + 5)) while [ "$SECONDS" -lt "$deadline" ]; do line="$(awk '$1 == "/body.bin" { value = $0 } END { print value }' "$VLOG" 2>/dev/null || true)" if [ -n "$line" ] && [ "$line" != "$previous" ]; then printf '%s\n' "$line" return 0 fi sleep 0.05 done return 1 } read_probe_field() { local field="$1" local body body="$(prober_probe_body "$HOST" "$PORT")" || return 1 prober_probe_field "$body" "$field" } DICT_FILE="$(dirname "$PROBER_SERVER_BIN")/fault-arms.dict" DICT_SHA_B64="$(openssl dgst -sha256 -binary "$DICT_FILE" 2>/dev/null \ | openssl base64 -A 2>/dev/null || true)" DCZ_HEADERS=( 'Accept-Encoding: zstd, dcz' "Available-Dictionary: :$DICT_SHA_B64:" ) # --- oracle plan -------------------------------------------------------- # 1 warm-up: plain request serves 200 and emits completed log variables # 2 CODEC ERROR (fault_codec=1) on a plain compressing request: the request # fails closed (no 200, connection torn down) rather than serving # corrupt/truncated compressed bytes and emits no completed log variables # -- proves the central ZSTD_isError arm is reachable, fails safely, and # cannot fabricate a final ratio from its partial counters # 3 CODEC_END ERROR (fault_codec_end=1) on the SAME plain request: same # fail-closed assertion at the end-of-frame site specifically # 4 CODEC_END ZERO on the 2nd END call (fault_codec_end=1002): forces the # empty-out_buf suppression arm and the response still completes fully # -- not a hang, not a torn-down connection. Deliberately nth=2, not 1: # see the oracle's own comment for the separate, real defect nth=1 hits # on this body (ledgered, not fixed here) # 5 CODEC ERROR on the 2nd call (fault_codec=2): proves the non-END # streaming site is reached MORE THAN ONCE per response (a body this # size needs several ZSTD_compressStream2 calls to drain), by failing # that 2nd call closed the same way oracle 2 proves the 1st call fails # closed # 6 disarming restores normal (uncorrupted) compression -- proves the # fault state is per-arm, not sticky # 8 an unfaulted negotiated dcz request succeeds (normal-path control) # 9 the upper valid boundary (999) arms, while malformed/out-of-range values # leave that arm unchanged # 10 negative disarm clears that boundary arm # 11 REFPREFIX armed at its lower boundary does not affect plain zstd and # the dedicated counter remains zero (dcz-only negative control) # 12 the SAME still-armed fault is consumed by a negotiated dcz request, # which fails closed and logs the existing refPrefix error branch # 13 arm-then-disarm before consumption leaves the first dcz call clean # 14 no worker died by signal across the whole run # # Mutation accounting for the deliberately broad mutant-parity matcher. It # counts both halves of each TAP verdict (success and failure echoes) as # separate gates even though they share one condition. The named manual # controls below were run against this scenario on 2026-08-28. # mutant-exempt: oracle 8 is the positive dcz setup control, not a regression assertion # mutant-exempt: oracle 8's not-ok echo is the other half of that one setup control # mutant-exempt: oracle 9 failed when invalid input was mutated to clear the boundary-999 arm # mutant-exempt: oracle 9's not-ok echo shares the same observed parser mutant # mutant-exempt: oracle 10 failed when negative-value disarming was removed # mutant-exempt: oracle 10's not-ok echo shares the same observed disarm mutant # mutant-exempt: oracle 11 failed when the fault decision was moved above the dcz guard # mutant-exempt: oracle 11's not-ok echo shares the same observed dcz-only mutant # mutant-exempt: oracle 12 failed when the armed outcome returned REFPREFIX_NONE # mutant-exempt: oracle 12's not-ok echo shares the same observed outcome mutant # mutant-exempt: oracle 13 failed when the pre-consumption disarm was removed # mutant-exempt: oracle 13's not-ok echo shares the same observed disarm mutant # mutant-exempt: oracle 14 is the pre-existing signal-death assertion, renumbered only echo "1..14" # --- 1: warm-up ----------------------------------------------------------- WARMUP="$PROBER_PREFIX/warmup.out" WARMUP_VARS="" if fetch /body.bin "$WARMUP" \ && grep -q '^HTTP/1.1 200' "$WARMUP.hdrs" \ && WARMUP_VARS="$(wait_body_var_line '')" \ && [[ "$WARMUP_VARS" =~ ^/body.bin\ [0-9]+\.[0-9]{3}\ [1-9][0-9]*\ [1-9][0-9]*$ ]] then echo "ok 1 - warm-up request served a clean 200 with completed zstd log variables" else echo "not ok 1 - warm-up request did not serve a clean 200 with completed zstd log variables (vars=${WARMUP_VARS:-missing})" FAILED=$((FAILED + 1)) fi # --- 2: CODEC ERROR fails closed ------------------------------------------- # # arm must succeed on its own -- a curl failure to even reach /__probe means # nothing was armed, and the request that follows would then simply succeed # for an unrelated reason (never armed), which must NOT read as "failed # closed, ok 2". Armed-then-either(no-200-headers, fetch itself errored) is # the fail-closed signature. ERR_OUT="$PROBER_PREFIX/codec-err.out" ARM2_OK=0 GOT200_2=0 FAULT_VARS="" if arm codec 1; then ARM2_OK=1 if fetch /body.bin "$ERR_OUT" && grep -q '^HTTP/1.1 200' "$ERR_OUT.hdrs" 2>/dev/null; then GOT200_2=1 fi FAULT_VARS="$(wait_body_var_line "$WARMUP_VARS" || true)" fi if [ "$ARM2_OK" -eq 1 ] && [ "$GOT200_2" -eq 0 ] \ && [ "$FAULT_VARS" = '/body.bin - - -' ] then echo "ok 2 - CODEC ERROR (fault_codec=1) failed closed and exposed no completed zstd log variables" else echo "not ok 2 - CODEC ERROR did not fail closed without fabricated log variables (arm_ok=$ARM2_OK; vars=${FAULT_VARS:-missing})" FAILED=$((FAILED + 1)) fi disarm codec || true # --- 3: CODEC_END ERROR fails closed --------------------------------------- END_ERR_OUT="$PROBER_PREFIX/codec-end-err.out" ARM3_OK=0 GOT200_3=0 if arm codec_end 1; then ARM3_OK=1 if fetch /body.bin "$END_ERR_OUT" && grep -q '^HTTP/1.1 200' "$END_ERR_OUT.hdrs" 2>/dev/null; then GOT200_3=1 fi fi if [ "$ARM3_OK" -eq 1 ] && [ "$GOT200_3" -eq 0 ]; then echo "ok 3 - CODEC_END ERROR (fault_codec_end=1) made the terminal-frame call fail closed" else echo "not ok 3 - CODEC_END ERROR did not fail the request (arm_ok=$ARM3_OK)" FAILED=$((FAILED + 1)) fi disarm codec_end || true # --- 4: CODEC_END ZERO (2nd END call) completes cleanly --------------------- # # nth MUST be 1002 (2nd call at the END site), never 1001 (the 1st): with # this body size, forcing zero output on the very FIRST ZSTD_e_end call # reaches an existing, independent defect -- a genuinely empty terminal # buffer that is emitted with last_buf set trips nginx core's # "zero size buf in writer" guard and the connection is torn down mid- # response (reproduced manually, filed as issues.md: forcing zero output on # the first END call truncates the frame). That is real and ledgered # separately; it is NOT what this oracle exists to assert, and using nth=1 # here would make Phase 4 ship a permanently-red gate for an unrelated, # unfixed bug. nth=2 lands on a LATER END call, once real streaming has # already produced output, which is the suppression arm this oracle # actually targets (documented ok-for-3+ manually: 1002-1999 all complete # cleanly on this body). # # "Completes with a 200" alone would be vacuous -- a compressible 200-KB body # of identical bytes and an unfaulted request both give the same ~26-byte # frame -- so the response must also DECODE: a complete, parseable zstd frame # whose plaintext is byte-for-byte the origin file. A truncated or aborted # transfer fails `zstd -d`, and a frame that decodes to the wrong bytes fails # the cmp. Checking only `[ -s ]` here would accept both. END_ZERO_OUT="$PROBER_PREFIX/codec-end-zero.out" END_ZERO_OK=0 if arm codec_end 1002 && fetch /body.bin "$END_ZERO_OUT"; then if grep -q '^HTTP/1.1 200' "$END_ZERO_OUT.hdrs" \ && ! grep -qE '^(curl:|$)' "$END_ZERO_OUT.stderr" 2>/dev/null \ && [ -s "$END_ZERO_OUT" ] \ && zstd -d -q -f -o "$END_ZERO_OUT.plain" "$END_ZERO_OUT" 2>/dev/null \ && cmp -s "$END_ZERO_OUT.plain" "$PROBER_PREFIX/www/body.bin" then END_ZERO_OK=1 fi fi if [ "$END_ZERO_OK" -eq 1 ]; then echo "ok 4 - CODEC_END ZERO (fault_codec_end=1002, 2nd END call) still completed with a full response (suppression arm reached past the first-call defect, no hang/truncation)" else echo "not ok 4 - CODEC_END ZERO (2nd call) did not complete cleanly" FAILED=$((FAILED + 1)) fi disarm codec_end || true # --- 5: CODEC site is called MORE THAN ONCE -- fail-closed on the 2nd call - # # A single ZSTD_compressStream2() call rarely drains a 200-KB body (the # module's own output buffer is much smaller), so the non-END (CODEC) site # is called repeatedly across one response. fault_codec=2 (ERROR outcome, # 2nd call) proves that second call is genuinely reached: if the loop # somehow only ever ran the site once (e.g. a regression that emits # everything from the first call, or that reroutes subsequent iterations # around the fault check), nth=2 would never match and the request would # complete normally instead of failing closed. Same fail-closed assertion # shape as oracle 2 (which already covers nth=1); this is the nth-2+ # reachability half of the same arm. ERR2_OUT="$PROBER_PREFIX/codec-err2.out" ARM5_OK=0 GOT200_5=0 if arm codec 2; then ARM5_OK=1 if fetch /body.bin "$ERR2_OUT" && grep -q '^HTTP/1.1 200' "$ERR2_OUT.hdrs" 2>/dev/null; then GOT200_5=1 fi fi if [ "$ARM5_OK" -eq 1 ] && [ "$GOT200_5" -eq 0 ]; then echo "ok 5 - CODEC ERROR on the 2nd call (fault_codec=2) failed closed, proving the streaming site is reached more than once per response" else echo "not ok 5 - CODEC ERROR on the 2nd call did not fail the request (arm_ok=$ARM5_OK; site called fewer than 2 times, or the fault check does not re-run)" FAILED=$((FAILED + 1)) fi disarm codec || true # --- 6: disarm restores normal compression ---------------------------------- RESTORE_OUT="$PROBER_PREFIX/restore.out" RESTORE_OK=0 if fetch /body.bin "$RESTORE_OUT" && grep -q '^HTTP/1.1 200' "$RESTORE_OUT.hdrs"; then RESTORE_OK=1 fi if [ "$RESTORE_OK" -eq 1 ]; then echo "ok 6 - after disarming, a compressing request serves a clean 200 again (fault state is per-arm, not sticky)" else echo "not ok 6 - request after disarm did not serve a clean 200" FAILED=$((FAILED + 1)) fi # --- 7: terminal zero-output END emits a WRITER-LEGAL zero-size last_buf ---- # # fault_codec_end=1001 is the FIRST end-of-frame call producing no output -- # the arm oracle 4 deliberately avoids. The module must still emit the # zero-length last_buf (the suppression arm lets it through on purpose), and # nginx's writer only tolerates a zero-size buffer that ngx_buf_special() # accepts: `(flush||last_buf||sync) && !ngx_buf_in_memory(b) && !b->in_file`. # out_buf comes from ngx_http_zstd_create_temp_buf(), so `temporary` stays set # and # ngx_buf_in_memory() stays true even when the buffer is fully drained. That # made ngx_http_write_filter log "zero size buf in writer" and abort the # response mid-stream. # # The assertion is on the ALERT and the transfer, NOT on frame validity: this # arm suppresses the real terminal call, so the body legitimately is not a # complete zstd frame. Asserting `zstd -d` here would be asserting that an # injected fault did not happen. What must hold is that the fault degrades the # BODY without provoking a writer-level abort or a worker alert. # # Non-vacuity: the alert grep is scoped to lines logged after this point in # the run, and the fetch must genuinely reach the module -- an arm that never # fired would leave a full decodable frame, which the size check below rejects. END_ZERO1_OUT="$PROBER_PREFIX/codec-end-zero1.out" ELOG_MARK_7="$(wc -l < "$ELOG")" ARM7_OK=0 if arm codec_end 1001 && fetch /body.bin "$END_ZERO1_OUT"; then ARM7_OK=1 fi ZEROBUF_7=0 if [ "$(tail -n +$((ELOG_MARK_7 + 1)) "$ELOG" | grep -c 'zero size buf in writer' || true)" -gt 0 ]; then ZEROBUF_7=1 fi if [ "$ARM7_OK" -eq 1 ] \ && [ "$ZEROBUF_7" -eq 0 ] \ && grep -q '^HTTP/1.1 200' "$END_ZERO1_OUT.hdrs" 2>/dev/null then echo "ok 7 - CODEC_END ZERO on the FIRST end call emitted a writer-legal zero-size last_buf (no \"zero size buf in writer\", no aborted transfer)" else echo "not ok 7 - first-call CODEC_END ZERO tripped the writer or aborted the response" tail -n +$((ELOG_MARK_7 + 1)) "$ELOG" | grep -n 'zero size buf in writer' | sed 's/^/# /' FAILED=$((FAILED + 1)) fi disarm codec_end || true # --- 8: unfaulted dcz normal path ------------------------------------------- DCZ_BASE_OUT="$PROBER_PREFIX/dcz-base.out" DCZ_BASE_OK=0 if [ -n "$DICT_SHA_B64" ] \ && fetch /dcz/body.bin "$DCZ_BASE_OUT" "${DCZ_HEADERS[@]}" \ && grep -q '^HTTP/1.1 200' "$DCZ_BASE_OUT.hdrs" \ && grep -qi '^Content-Encoding:[[:space:]]*dcz' "$DCZ_BASE_OUT.hdrs" then DCZ_BASE_OK=1 fi if [ "$DCZ_BASE_OK" -eq 1 ]; then echo "ok 8 - an unfaulted request negotiated dcz before the refPrefix error arm" else echo "not ok 8 - dcz normal-path control did not negotiate a clean 200" FAILED=$((FAILED + 1)) fi # --- 9: parser boundary and malformed-input controls ----------------------- # Manual mutation control (2026-08-28): changing the invalid-input branch to # clear refprefix_armed made this named oracle turn red. PARSER9_OK=1 if ! arm refprefix 999 \ || [ "$(read_probe_field refprefix_armed || echo invalid)" != 999 ] then PARSER9_OK=0 fi for invalid in '' '-' 0 1000 10000 1junk; do if ! curl -fsS --max-time 5 \ "http://$HOST:$PORT/__probe?fault_refprefix=${invalid}" -o /dev/null \ || [ "$(read_probe_field refprefix_armed || echo invalid)" != 999 ] then PARSER9_OK=0 fi done if ! curl -fsS --max-time 5 \ "http://$HOST:$PORT/__probe?xfault_refprefix=1" -o /dev/null \ || [ "$(read_probe_field refprefix_armed || echo invalid)" != 999 ] then PARSER9_OK=0 fi if [ "$PARSER9_OK" -eq 1 ]; then echo "ok 9 - fault_refprefix accepts boundary 999 and rejects empty, malformed, out-of-range, and substring keys without changing the arm" else echo "not ok 9 - fault_refprefix parser changed or lost the boundary-999 arm on invalid input" FAILED=$((FAILED + 1)) fi # --- 10: negative disarm clears the boundary arm --------------------------- # Manual mutation control (2026-08-28): removing the negative-value state # reset made this named oracle and oracle 13 turn red. Use a value longer than # the positive arm range to pin the "any non-empty negative" parser contract. curl -fsS --max-time 5 \ "http://$HOST:$PORT/__probe?fault_refprefix=-10000" -o /dev/null || true if [ "$(read_probe_field refprefix_armed || echo invalid)" = 0 ]; then echo "ok 10 - a negative fault_refprefix value disarmed the boundary arm" else echo "not ok 10 - negative fault_refprefix did not clear the armed state" FAILED=$((FAILED + 1)) fi # --- 11: dedicated site is dcz-only ---------------------------------------- # Manual mutation control (2026-08-28): moving the fault decision above the # ctx->dcz_dict guard made this named oracle turn red on the plain request. PLAIN_CONTROL_OUT="$PROBER_PREFIX/refprefix-plain-control.out" ARM11_OK=0 PLAIN11_OK=0 REFPREFIX_AFTER_PLAIN=-1 if arm refprefix 1; then ARM11_OK=1 if fetch /body.bin "$PLAIN_CONTROL_OUT" \ && grep -q '^HTTP/1.1 200' "$PLAIN_CONTROL_OUT.hdrs" then PLAIN11_OK=1 fi REFPREFIX_AFTER_PLAIN="$(read_probe_field refprefix_calls || echo -1)" fi if [ "$ARM11_OK" -eq 1 ] && [ "$PLAIN11_OK" -eq 1 ] \ && [ "$REFPREFIX_AFTER_PLAIN" -eq 0 ] then echo "ok 11 - fault_refprefix=1 left plain zstd untouched and its dedicated counter at zero" else echo "not ok 11 - plain zstd consumed or was affected by the dcz-only refPrefix arm (arm_ok=$ARM11_OK; plain_ok=$PLAIN11_OK; calls=$REFPREFIX_AFTER_PLAIN)" FAILED=$((FAILED + 1)) fi # --- 12: negotiated dcz consumes REFPREFIX ERROR and fails closed ----------- # Manual mutation control (2026-08-28): returning REFPREFIX_NONE at the armed # event made this named oracle turn red (got200=1, calls=1, log=0). # Do not re-arm here. Oracle 11 proved the plain request did not consume the # lower-bound nth=1 arm; this request must consume that same pending event. REF_ERR_OUT="$PROBER_PREFIX/refprefix-err.out" ELOG_MARK_12="$(wc -l < "$ELOG")" GOT200_12=0 # Manual mutation control (2026-08-28): inverting the empty-body check made # this named oracle turn red. A reset or truncated response is acceptable # here only when no representation bytes escaped before the connection failed. if (fetch /dcz/body.bin "$REF_ERR_OUT" "${DCZ_HEADERS[@]}" \ && grep -q '^HTTP/1.1 200' "$REF_ERR_OUT.hdrs" 2>/dev/null) \ || [ -s "$REF_ERR_OUT" ] then GOT200_12=1 fi REFPREFIX_AFTER_DCZ="$(read_probe_field refprefix_calls || echo -1)" REFPREFIX_LOG_12=0 if tail -n +$((ELOG_MARK_12 + 1)) "$ELOG" \ | grep -q 'zstd: ZSTD_CCtx_refPrefix() failed:' then REFPREFIX_LOG_12=1 fi if [ "$GOT200_12" -eq 0 ] && [ "$REFPREFIX_AFTER_DCZ" -eq 1 ] \ && [ "$REFPREFIX_LOG_12" -eq 1 ] then echo "ok 12 - negotiated dcz consumed fault_refprefix=1 and the existing refPrefix error branch failed closed" else echo "not ok 12 - refPrefix fault was not reached or did not fail closed (got200=$GOT200_12; calls=$REFPREFIX_AFTER_DCZ; log=$REFPREFIX_LOG_12)" FAILED=$((FAILED + 1)) fi # --- 13: arm-then-disarm before consumption restores normal dcz ------------ # Re-arm first: the previous nth=1 event has already fired, so merely # disarming it here would be vacuous. Deleting this disarm must make the first # dcz call below fail. That mutation was run on 2026-08-28 and made this named # oracle turn red. ARM13_OK=0 DISARM13_OK=0 if arm refprefix 1 \ && [ "$(read_probe_field refprefix_armed || echo invalid)" = 1 ] then ARM13_OK=1 fi if [ "$ARM13_OK" -eq 1 ] && disarm refprefix \ && [ "$(read_probe_field refprefix_armed || echo invalid)" = 0 ] then DISARM13_OK=1 fi DCZ_RESTORE_OUT="$PROBER_PREFIX/dcz-restore.out" if [ "$ARM13_OK" -eq 1 ] && [ "$DISARM13_OK" -eq 1 ] \ && fetch /dcz/body.bin "$DCZ_RESTORE_OUT" "${DCZ_HEADERS[@]}" \ && grep -q '^HTTP/1.1 200' "$DCZ_RESTORE_OUT.hdrs" \ && grep -qi '^Content-Encoding:[[:space:]]*dcz' "$DCZ_RESTORE_OUT.hdrs" then echo "ok 13 - arm-then-disarm left the first negotiated dcz request clean" else echo "not ok 13 - arm/disarm setup failed or dcz failed afterwards (arm_ok=$ARM13_OK; disarm_ok=$DISARM13_OK)" FAILED=$((FAILED + 1)) fi # --- 14: no signal-death across the run ------------------------------------ if grep -qE 'worker process .* exited on signal|SIGSEGV|SIGABRT|SIGBUS' "$ELOG"; then echo "not ok 14 - a worker died by signal during fault injection" grep -nE 'exited on signal|SIGSEGV|SIGABRT|SIGBUS' "$ELOG" | sed 's/^/# /' FAILED=$((FAILED + 1)) else echo "ok 14 - no worker died by signal across the whole fault-injection run" fi if [ "$FAILED" -gt 0 ]; then exit 1 fi exit 0 myguard-labs-nginx-zstd-module-3486b5b/ci/t/harness-scenarios/fault-arms/nginx.conf000066400000000000000000000027701525232545500303760ustar00rootroot00000000000000@LOAD@ # Local scenario (NOT vendored, NOT gitignored): unlike testkit's # consumer-zstd, this repo IS the module under test, so PROBER_MODULE is # ngx_http_zstd_filter_module.so itself -- @LOAD@ already load_modules it, # there is no separate probe .so to add. No requires-gate skip-forever # hazard either. See plan-testkit-integration.md Phase 4. daemon off; pid @PREFIX@/nginx.pid; error_log @PREFIX@/logs/error.log info; worker_processes 1; events { worker_connections 16; } http { # The CODEC-error oracle below reads these after nginx enters log phase. # A completed frame has all three values; an aborted one must expose none. log_format zstd_fault_vars "$uri $zstd_ratio $zstd_bytes_in $zstd_bytes_out"; access_log @PREFIX@/logs/zstd-vars.log zstd_fault_vars; server { listen 127.0.0.1:@PORT@; root @PREFIX@/www; location / { zstd on; zstd_min_length 0; } # Dedicated dcz arm for the ZSTD_CCtx_refPrefix() fault site. The # prober speaks cleartext, so explicitly acknowledge that test-only # transport; the driver still has to negotiate dcz with the fixture's # exact SHA-256 and asserts Content-Encoding before trusting the arm. location /dcz/ { alias @PREFIX@/www/; zstd on; zstd_min_length 0; zstd_dcz_dict_file @BUILD_OBJS@/fault-arms.dict; zstd_dcz_assume_secure_transport on; } location /__probe { @PROBE@ } } } myguard-labs-nginx-zstd-module-3486b5b/ci/t/harness-scenarios/fault-arms/requires000077500000000000000000000051301525232545500301620ustar00rootroot00000000000000#!/usr/bin/env bash # # Gate: the tree run-scenario.sh is about to boot must be a TEST_HARNESS # build -- the probe endpoint and codec fault sites compile out entirely # otherwise (plan-testkit-integration.md Ground truth: without both # TEST_HARNESS=1 and -DNGX_TEST_HARNESS, the module builds and answers, but # every fault-injection oracle here would pass by testing nothing). # # Checked the same way run-scenario.sh/lib.sh resolve the build tree, so this # gate can never SKIP a tree the engine would boot fine or pass one it would # reject -- see consumer-zstd/requires's identical reasoning for why the # derivation must match lib.sh exactly. set -euo pipefail FLAVOR="${2:-nginx}" VERSION="${3:-1.31.3}" BUILD="${PROBER_BUILD:-${PROBER_ROOT:-$(cd ../.. && pwd)}/.build/${FLAVOR}-${VERSION}}" SO="$BUILD/objs/ngx_http_zstd_filter_module.so" if [ ! -f "$SO" ]; then echo "ngx_http_zstd_filter_module.so not found at $SO -- build it first" exit 1 fi PROBE_SYMBOLS="$(nm "$SO" 2>/dev/null | grep -c ' ngx_test_probe_' || true)" if [ "$PROBE_SYMBOLS" -eq 0 ]; then echo "$SO carries no ngx_test_probe_* symbols -- this is a packaged (non-harness) build; configure with TEST_HARNESS=1 CFLAGS=\"\$CFLAGS -DNGX_TEST_HARNESS\" first (see plan-testkit-integration.md Ground truth)" exit 1 fi # Oracle 4 decodes the response and compares it against the origin file, so a # missing decoder would silently weaken it to "200 and nonempty". Fail here # instead: a gate that cannot run its assertion is not a gate. if ! command -v zstd >/dev/null 2>&1; then echo "zstd(1) not found -- oracle 4 decodes the CODEC_END ZERO response and compares it byte-for-byte with the origin file" exit 1 fi # The refPrefix fault arm negotiates a real dcz response. It derives the # Available-Dictionary value from the checked-in fixture, so the CLI used for # that derivation is part of the oracle rather than an optional convenience. if ! command -v openssl >/dev/null 2>&1; then echo "openssl(1) not found -- the refPrefix arm derives the dcz negotiation hash with openssl dgst/base64" exit 1 fi # zstd_dcz_dict_file opens the dictionary during nginx -t, before driver.sh # runs. Stage the canonical dcz fixture here, the same pre-config hook used by # codec-call-count, and reference it from nginx.conf through @BUILD_OBJS@. DICT_SRC="$(cd "$(dirname "$0")/../../suite" && pwd)/dcz-dict" if [ ! -f "$DICT_SRC" ]; then echo "dictionary fixture not found at $DICT_SRC" exit 1 fi if ! cp -f "$DICT_SRC" "$BUILD/objs/fault-arms.dict"; then echo "cannot stage the dictionary fixture into $BUILD/objs" exit 1 fi exit 0 myguard-labs-nginx-zstd-module-3486b5b/ci/t/harness-scenarios/fault-palloc/000077500000000000000000000000001525232545500267065ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/t/harness-scenarios/fault-palloc/driver.sh000077500000000000000000000372601525232545500305500ustar00rootroot00000000000000#!/usr/bin/env bash # # Scenario: POOL-ALLOCATION FAULT INJECTION for ngx_http_zstd_filter_module.so. # # Reachable only through the harness's TEST_HARNESS build (see the requires # gate): the per-request allocation sites in the filter go through the # ngx_http_zstd_{palloc,pnalloc,pcalloc,alloc_chain_link,list_push, # pool_cleanup_add} wrappers, which consult the PALLOC fault site # before delegating to the real allocator. Armed via # GET /__probe?fault_palloc=: the n-th WRAPPED allocation since the arm # returns NULL. Negative disarms. # # WHY THIS SCENARIO EXISTS. Pool allocation failure is unprovokable in a # normal test -- nginx's pool only fails when the OS refuses memory, and an # ulimit big enough to force that kills the worker and every other oracle # with it. So the NULL-check branch on each of those sites had no test at # all: an unchecked NULL, or a half-mutated header list, would stay green # forever. That is issues.md row m8. # # THE ORDINAL -> SITE MAP. Which wrapped allocation a given nth reaches is # a property of this request (200 KB, zstd on, default buffer sizing), not a # constant of the module. Measured on nginx 1.31.4 by logging __LINE__ at # each wrapper call, the 6 wrapped allocations of one such request are: # # nth site bytes on the wire? # 1 ctx pcalloc, header filter no # 2 Content-Encoding ngx_list_push, header filter no # 3 CCtx cleanup ngx_pool_cleanup_add no # 4 out_buf combined allocation no # 5 emit chain link no # 6 emit chain link YES # # WHERE "COMMITTED" ACTUALLY IS. Note that the boundary is NOT the header # filter / body filter split, which is where one would expect it. nginx # buffers the response, so ngx_http_send_header() returning does not put # bytes on the wire; the status line only reaches the client once enough # output has accumulated to flush. Measured on this request that happens # between the 5th and 6th wrapped allocation -- so allocations 3..5 are in # the body filter yet still fail with nothing delivered, and only 6 fails # with a response already committed. # # That is why the two oracles below are split at 5/6 rather than at 2/3: # the split has to follow the OBSERVABLE (has the client been promised a # zstd stream yet?), because that is what decides which behaviour is # correct. Splitting it where the code structure suggests would have made # oracle 4 assert "200 was on the wire" for allocations where it never # is, and the oracle would have been red for a reason that is not a defect. # # The oracles aim at 1..5 (uncommitted; every distinct site) and 6 (committed # emit link). Incoming links are intentionally absent: the synchronously # drained path now walks caller-owned links and allocates no wrappers. If the # ordering ever changes, the counter assertions # still hold but an oracle could silently start testing a different site -- # so each oracle names the site it believes it is hitting, and the mutation # testing recorded in the PR is what ties an oracle to its guard. # # NON-VACUITY DISCIPLINE. Every oracle below asserts the fault ACTUALLY # FIRED, not merely that the request misbehaved. A request can fail for # reasons unrelated to the arm, and an arm the module silently DECLINEs # (which is exactly what this site did before this change) leaves the # request succeeding normally. Both would be indistinguishable from a real # pass if the oracle only looked at the response. So each oracle checks the # probe's palloc_calls counter advanced to the armed ordinal -- the # observable that exists only when the wrapper ran and the site was armed. set -euo pipefail # shellcheck source=../../harness/ci/prober/lib.sh . "$PROBER_LIB" HOST=127.0.0.1 PORT="$PROBER_RESOLVED_PORT" ELOG="$PROBER_PREFIX/logs/error.log" FAILED=0 mkdir -p "$PROBER_PREFIX/www" # 200 KB: large enough to need several output buffers and several chain # links, which is what puts allocation sites AFTER the header commit on the # call sequence at all. A small body commits headers and finishes in one # body-filter pass, and oracle 3 would have nothing to aim at. head -c 204800 /dev/zero | tr '\0' 'a' >"$PROBER_PREFIX/www/body.bin" # arm NTH / disarm: GET /__probe?fault_palloc=. Arming and reading are # the same request (see ngx_http_zstd_probe_handler's arm-then-render # comment), and arming RESETS the counter, so the counter read must come # from a LATER, key-less /__probe request -- never from the arming one. arm() { curl -fsS --max-time 5 \ "http://$HOST:$PORT/__probe?fault_palloc=${1}" -o /dev/null } disarm() { arm -1; } # palloc_calls: the module's wrapped-allocation counter, since the arm. # A key-less /__probe neither arms nor resets. palloc_calls() { curl -fsS --max-time 5 "http://$HOST:$PORT/__probe" \ | sed 's/.*"palloc_calls":\([0-9]*\).*/\1/' } chain_links_allocated() { curl -fsS --max-time 5 "http://$HOST:$PORT/__probe" \ | sed 's/.*"chain_links_allocated":\([0-9]*\).*/\1/' } # fetch: bounded GET of the compressible body. Headers to $2.hdrs, body to # $2, curl's own diagnostic to $2.stderr. Returns curl's exit status -- # which matters here: the two failure classes this scenario separates are # distinguished precisely by whether any response line was written before # the connection went away. fetch() { curl -sS --max-time 10 -D "$2.hdrs" -o "$2" \ -H 'Accept-Encoding: zstd' \ "http://$HOST:$PORT$1" 2>"$2.stderr" } # --- oracle plan -------------------------------------------------------- # 1 warm-up: a plain compressing request serves 200 (readiness), and # establishes how many wrapped allocations one such request makes # 2 the PALLOC site is ARMABLE at all -- the counter advances. Before this # change fault_set_global returned NGX_DECLINED for PALLOC, so this # oracle is the one that fails outright on the old module # 3 BEFORE anything is committed to the wire (nth=1 ctx pcalloc, 2 # Content-Encoding list_push, 3 cleanup_add, 4 out_buf, 5 emit chain link # -- every distinct site on the uncommitted side): the # request fails closed with NO response line at all # 4 AFTER the response is committed (nth=6 emit chain link): the 200 and # Content-Encoding: zstd ARE on the wire, and then the # connection is torn down mid-body rather than completing a truncated or # corrupt zstd stream. Completing the response is the defect this oracle # exists to catch # 5 the fault is one-shot and per-arm: after disarming, a compressing # request serves a clean 200 again # 6 an out-of-range nth is REFUSED rather than stored as an arm that can # never fire (the codec ZERO_BASE encoding does not apply to this site). # Observed via the counter, not the HTTP status -- see the oracle # 7 no worker died by signal across the whole run echo "1..7" # --- 1: warm-up + establish the wrapped-allocation count ------------------ disarm || true WARM="$PROBER_PREFIX/warm.out" BASE_CALLS=0 if fetch /body.bin "$WARM" && grep -q '^HTTP/1.1 200' "$WARM.hdrs"; then BASE_CALLS=$(palloc_calls) BASE_INPUT_LINKS=$(chain_links_allocated) if [ "${BASE_CALLS:-0}" -gt 0 ] && [ "${BASE_INPUT_LINKS:-1}" -eq 0 ]; then echo "ok 1 - warm-up served a clean 200, made $BASE_CALLS wrapped pool allocations, and copied zero fully-drained input links" elif [ "${BASE_CALLS:-0}" -le 0 ]; then echo "not ok 1 - warm-up served 200 but the module reported ZERO wrapped allocations (palloc_calls=$BASE_CALLS) -- the wrappers are not on the request path, so every oracle below would assert nothing" FAILED=$((FAILED + 1)) else echo "not ok 1 - warm-up copied $BASE_INPUT_LINKS input chain links even though it drained synchronously; expected zero lazy tail copies" FAILED=$((FAILED + 1)) fi else echo "not ok 1 - warm-up request did not serve a clean 200" FAILED=$((FAILED + 1)) fi # --- 2: the PALLOC site is armable ---------------------------------------- # # THE load-bearing non-vacuity oracle. fault_set_global used to answer # NGX_DECLINED for NGX_TEST_PROBE_FAULT_PALLOC, which the testkit reports as # "no hook at all": the arm silently did nothing and every request stayed # normal. Asserting the counter advances to exactly the armed ordinal proves # the arm was accepted AND that the wrapper consulted the site. if arm 3; then fetch /body.bin "$PROBER_PREFIX/arm.out" || true SEEN=$(palloc_calls) if [ "${SEEN:-0}" -ge 3 ]; then echo "ok 2 - PALLOC site accepted the arm and the wrapped-allocation counter advanced (palloc_calls=$SEEN)" else echo "not ok 2 - PALLOC arm did not take effect (palloc_calls=$SEEN after arming nth=3) -- the site is refusing the arm, so no fault below can fire" FAILED=$((FAILED + 1)) fi else echo "not ok 2 - /__probe refused the fault_palloc arm outright" FAILED=$((FAILED + 1)) fi disarm || true # --- 3: allocation failure BEFORE anything is committed ------------------- # # Every distinct site on the uncommitted side of the flush boundary: nth=1 # ctx pcalloc and nth=2 Content-Encoding ngx_list_push (header filter), # nth=3 CCtx cleanup_add, nth=4 out_buf temp buffer, and nth=5 emit chain # link (body filter, but still before the response # flushes -- see the map above). In all of these the client has been # promised nothing yet, so the correct outcome is fail-closed with nothing # on the wire: NGX_ERROR finalizes the request and no status line is ever # produced. # # nth=2 matters on its own. A half-mutated header list -- the push # succeeding but its fields left unset, or the failure ignored -- is one of # the concrete regressions this row was filed about, and it is invisible # unless the allocation can actually be failed there. # # The counter check is what makes this non-vacuous: "no response line" is # also what a dead worker or a wedged port produces, and neither has # anything to do with the arm. PRE_FAILED=0 PRE_DETAIL="" for n in 1 2 3 4 5; do OUT="$PROBER_PREFIX/precommit.$n" rm -f "$OUT" "$OUT.hdrs" armed=0 st=1 if arm "$n"; then armed=1 fetch /body.bin "$OUT" && st=0 || st=$? fi calls=$(palloc_calls) line=$(head -1 "$OUT.hdrs" 2>/dev/null | tr -d '\r' || true) if [ "$armed" -eq 1 ] && [ "${calls:-0}" -ge "$n" ] \ && [ "$st" -ne 0 ] && [ -z "$line" ] then PRE_DETAIL="$PRE_DETAIL nth=$n(rc=$st,calls=$calls,ok)" else PRE_DETAIL="$PRE_DETAIL nth=$n(armed=$armed,calls=$calls,rc=$st,line='$line',BAD)" PRE_FAILED=1 fi disarm || true done if [ "$PRE_FAILED" -eq 0 ]; then echo "ok 3 - allocation failure before the response is committed failed closed with no response line at all:$PRE_DETAIL" else echo "not ok 3 - an uncommitted-side allocation failure did not fail closed as expected:$PRE_DETAIL" FAILED=$((FAILED + 1)) fi # --- 4: allocation failure AFTER the response is committed ---------------- # # The emit-link site reached past the flush boundary at nth=6. By this point # "200 OK" and "Content-Encoding: zstd" really are on the wire, so it CANNOT be # retracted # -- the client has been promised a zstd stream. The only correct behaviour # left is to abort the connection, so the client sees a truncated transfer # (a hard, detectable error) instead of a complete-looking body that is not # a valid zstd frame. # # So this oracle asserts BOTH halves: the 200 really was committed, AND the # transfer did not complete. Asserting only the second half would also pass # for a request that failed before ever sending headers -- that is oracle # 3's case, a different branch, and conflating the two would let a # regression in either one hide behind the other. POST_FAILED=0 POST_DETAIL="" n=6 OUT="$PROBER_PREFIX/postcommit.$n" rm -f "$OUT" "$OUT.hdrs" armed=0 st=1 if arm "$n"; then armed=1 fetch /body.bin "$OUT" && st=0 || st=$? fi calls=$(palloc_calls) got200=0 gotce=0 grep -q '^HTTP/1.1 200' "$OUT.hdrs" 2>/dev/null && got200=1 grep -qi '^Content-Encoding: *zstd' "$OUT.hdrs" 2>/dev/null && gotce=1 if [ "$armed" -eq 1 ] && [ "${calls:-0}" -ge "$n" ] \ && [ "$got200" -eq 1 ] && [ "$gotce" -eq 1 ] && [ "$st" -ne 0 ] then POST_DETAIL="$POST_DETAIL nth=$n(rc=$st,calls=$calls,ok)" else POST_DETAIL="$POST_DETAIL nth=$n(armed=$armed,calls=$calls,200=$got200,ce=$gotce,rc=$st,BAD)" POST_FAILED=1 fi disarm || true if [ "$POST_FAILED" -eq 0 ]; then echo "ok 4 - allocation failure after the response is committed aborted the connection mid-body instead of completing a corrupt zstd stream:$POST_DETAIL" else echo "not ok 4 - a post-commit allocation failure did not abort as expected (a completed transfer here means a truncated body was served as if valid):$POST_DETAIL" FAILED=$((FAILED + 1)) fi # --- 5: disarming restores normal service --------------------------------- CLEAN="$PROBER_PREFIX/clean.out" if fetch /body.bin "$CLEAN" && grep -q '^HTTP/1.1 200' "$CLEAN.hdrs"; then echo "ok 5 - after disarming, a compressing request serves a clean 200 again (fault state is per-arm, not sticky)" else echo "not ok 5 - a disarmed request did not serve a clean 200 -- fault state leaked past the arm" FAILED=$((FAILED + 1)) fi # --- 6: an out-of-range nth is refused, not stored ------------------------- # # 1000+ is the codec site's ZERO-outcome encoding and means nothing here. # Storing it would arm allocation call 1000+, which no request ever reaches: # the arm would look accepted and never fire, which is exactly the # false-green class this surface exists to prevent. # # HOW REFUSAL IS OBSERVED. Not by HTTP status -- ngx_http_zstd_probe_handler # discards ngx_test_probe_arm()'s return value on purpose (it renders the # post-arm state either way), so a declined arm still answers 200 and # `curl -f` cannot see it. The observable that DOES differ is the counter: # accepting an arm resets palloc_calls to 0, refusing it leaves the counter # untouched. So drive the counter to a known non-zero value first, then # offer the bad arm and require the counter to have survived. disarm || true fetch /body.bin "$PROBER_PREFIX/pre-range.out" || true BEFORE_BAD=$(palloc_calls) arm 1001 || true AFTER_BAD=$(palloc_calls) if [ "${BEFORE_BAD:-0}" -gt 0 ] && [ "${AFTER_BAD:-0}" -eq "${BEFORE_BAD:-0}" ]; then echo "ok 6 - fault_palloc=1001 (out of this site's range) was refused, not stored: the counter was left untouched at $AFTER_BAD" elif [ "${BEFORE_BAD:-0}" -le 0 ]; then echo "not ok 6 - could not establish a non-zero counter before offering the out-of-range arm (palloc_calls=$BEFORE_BAD), so refusal is unobservable and this oracle would assert nothing" FAILED=$((FAILED + 1)) else echo "not ok 6 - fault_palloc=1001 was ACCEPTED (counter reset $BEFORE_BAD -> $AFTER_BAD); the codec ZERO_BASE encoding does not apply to this site and storing it arms a call no request ever reaches" FAILED=$((FAILED + 1)) fi disarm || true # --- 7: no worker died by signal ------------------------------------------- # # Every oracle above deliberately makes an allocation fail. If any of those # branches dereferenced the NULL instead of handling it, the worker would # have taken a SIGSEGV -- and oracles 3 and 4 would still have "passed", # because a crashed worker also produces a torn-down connection. This is the # oracle that separates "handled the failure" from "crashed on it". if grep -qE 'worker process .* exited on signal|SIGSEGV|SIGABRT|SIGBUS' "$ELOG"; then echo "not ok 7 - a worker died by signal -- an allocation-failure branch dereferenced NULL rather than handling it" grep -nE 'exited on signal|SIGSEGV|SIGABRT|SIGBUS' "$ELOG" | sed 's/^/# /' FAILED=$((FAILED + 1)) else echo "ok 7 - no worker died by signal across the whole allocation-fault run" fi if [ "$FAILED" -gt 0 ]; then exit 1 fi exit 0 myguard-labs-nginx-zstd-module-3486b5b/ci/t/harness-scenarios/fault-palloc/nginx.conf000066400000000000000000000012641525232545500307030ustar00rootroot00000000000000@LOAD@ # Local scenario (NOT vendored, NOT gitignored): unlike testkit's # consumer-zstd, this repo IS the module under test, so PROBER_MODULE is # ngx_http_zstd_filter_module.so itself -- @LOAD@ already load_modules it, # there is no separate probe .so to add. No requires-gate skip-forever # hazard either. See plan-testkit-integration.md Phase 4. daemon off; pid @PREFIX@/nginx.pid; error_log @PREFIX@/logs/error.log info; worker_processes 1; events { worker_connections 16; } http { server { listen 127.0.0.1:@PORT@; root @PREFIX@/www; location / { zstd on; zstd_min_length 0; } location /__probe { @PROBE@ } } } myguard-labs-nginx-zstd-module-3486b5b/ci/t/harness-scenarios/fault-palloc/requires000077500000000000000000000024621525232545500304770ustar00rootroot00000000000000#!/usr/bin/env bash # # Gate: the tree run-scenario.sh is about to boot must be a TEST_HARNESS # build -- the probe endpoint and the pool-allocation fault site compile # out entirely otherwise (plan-testkit-integration.md Ground truth: without # both TEST_HARNESS=1 and -DNGX_TEST_HARNESS, the module builds and answers, # but every fault-injection oracle here would pass by testing nothing). # # Checked the same way run-scenario.sh/lib.sh resolve the build tree, so this # gate can never SKIP a tree the engine would boot fine or pass one it would # reject -- see consumer-zstd/requires's identical reasoning for why the # derivation must match lib.sh exactly. set -euo pipefail FLAVOR="${2:-nginx}" VERSION="${3:-1.31.3}" BUILD="${PROBER_BUILD:-${PROBER_ROOT:-$(cd ../.. && pwd)}/.build/${FLAVOR}-${VERSION}}" SO="$BUILD/objs/ngx_http_zstd_filter_module.so" if [ ! -f "$SO" ]; then echo "ngx_http_zstd_filter_module.so not found at $SO -- build it first" exit 1 fi PROBE_SYMBOLS="$(nm "$SO" 2>/dev/null | grep -c ' ngx_test_probe_' || true)" if [ "$PROBE_SYMBOLS" -eq 0 ]; then echo "$SO carries no ngx_test_probe_* symbols -- this is a packaged (non-harness) build; configure with TEST_HARNESS=1 CFLAGS=\"\$CFLAGS -DNGX_TEST_HARNESS\" first (see plan-testkit-integration.md Ground truth)" exit 1 fi exit 0 myguard-labs-nginx-zstd-module-3486b5b/ci/t/harness-scenarios/setparam-call-count-nodict/000077500000000000000000000000001525232545500314545ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/t/harness-scenarios/setparam-call-count-nodict/driver.sh000077500000000000000000000111441525232545500333070ustar00rootroot00000000000000#!/usr/bin/env bash # # Scenario: ZSTD_CCtx_setParameter() CALL COUNT, NO-DICTIONARY MODE. # # Pins the per-mode instrumentation this item's done criterion asks for. The # no-dict mode takes NEITHER the zlcf->dict (CDict) NOR the ctx->dcz_dict # (raw prefix) branch in ngx_http_zstd_filter_init_cctx(), so every # parameter the location configures is a live CCtx parameter and none of # them is superseded-by-cdict: level, windowLog, and (with zstd_long on) # enableLongDistanceMatching are all genuinely applied -> 3 calls. # # This is the reference count the dictionary-mode oracles in the sibling # setparam-call-count scenario are measured AGAINST. Under a trained CDict # only level drops out (ZSTD_CCtx_refCDict() supersedes compressionLevel # from the CDict's own baked-in ZSTD_compressionParameters -- zstd.h). # windowLog is NOT superseded: ZSTD_resetCCtx_byAttachingCDict()/ # ZSTD_resetCCtx_byCopyingCDict() (zstd_compress.c) both restore windowLog # from the CCtx's own applied params right after copying the CDict's other # cParams, and assert it is nonzero -- so init_cctx must keep setting it # even in CDict mode, or the operator's zstd_window_log cap silently stops # applying. This arm is what proves level and windowLog are both genuinely # set when nothing supersedes either of them. set -euo pipefail # shellcheck source=lib.sh . "$PROBER_LIB" HOST=127.0.0.1 PORT="$PROBER_RESOLVED_PORT" ELOG="$PROBER_PREFIX/logs/error.log" FAILED=0 N=0 ok() { N=$((N + 1)); echo "ok $N - $1"; } notok() { N=$((N + 1)); echo "not ok $N - $1"; FAILED=1; } diag() { echo "# $1"; } WWW="$PROBER_PREFIX/www" mkdir -p "$WWW" # Larger than 2**20 (zstd_window_log 20 in nginx.conf): a body smaller than # the window cap lets zstd shrink the FRAME's encoded window to fit the # content regardless of what ZSTD_c_windowLog was set to, which would make # the window-cap assertion below pass whether or not the cap was actually # applied. Random bytes, same construction as the sibling # setparam-call-count scenario's fixture and for the same reason. awk 'BEGIN { srand(43) n = 1200000 for (i = 0; i < n; i++) { printf "%c", 97 + int(rand() * 26) } }' >"$WWW/index.html" reset_counter() { curl -fsS --max-time 5 "http://$HOST:$PORT/__probe?setparam_reset=1" \ -o /dev/null } read_counter() { local body body="$(prober_probe_body "$HOST" "$PORT")" || return 1 prober_probe_field "$body" "setparam_calls" || return 1 } echo "1..4" if ! reset_counter; then notok "reset: probe did not answer setparam_reset" notok "response decodes byte-for-byte to the origin" notok "setParameter call count is 3" notok "frame window size is 1048576 B" exit 1 fi RAW="$PROBER_PREFIX/tmp" mkdir -p "$RAW" HDRS="$RAW/nodict.hdrs" BODY="$RAW/nodict.bin" if curl -sS --max-time 15 -D "$HDRS" -o "$BODY" \ -H 'Accept-Encoding: zstd' \ "http://$HOST:$PORT/index.html" 2>"$RAW/nodict.err"; then ok "request succeeded" else notok "request succeeded" diag "curl stderr: $(cat "$RAW/nodict.err" 2>/dev/null || true)" fi if grep -qi '^content-encoding:[[:space:]]*zstd' "$HDRS" 2>/dev/null; then : else diag "headers: $(tr -d '\r' <"$HDRS" 2>/dev/null | tr '\n' ' ')" fi if zstd -d -q -f -o "$BODY.dec" "$BODY" 2>/dev/null \ && cmp -s "$BODY.dec" "$WWW/index.html"; then ok "response decodes byte-for-byte to the origin" else notok "response decodes byte-for-byte to the origin" fi GOT="$(read_counter || echo -1)" if [ "$GOT" -eq 3 ]; then ok "setParameter call count is 3 (level, windowLog, LDM -- none superseded)" else diag "setparam_calls=$GOT expected=3" notok "setParameter call count is 3 (got $GOT)" fi # Direct frame-level check, same reasoning as the sibling # setparam-call-count scenario's assert_window_cap: the call count above # proves windowLog was SET on the CCtx, not that the resulting frame # carries the operator's actual cap. This arm establishes the reference # window (1 MiB, 2**20) the dictionary-mode oracles are compared against. WANT_WINDOW=$((1 << 20)) GOT_WINDOW="$(zstd -l -v "$BODY" 2>/dev/null \ | grep -oP 'Window Size:.*\(\K[0-9]+(?= B\))' | head -1)" if [ -z "$GOT_WINDOW" ]; then notok "frame window size is $WANT_WINDOW B (zstd -l -v produced no Window Size line)" elif [ "$GOT_WINDOW" -eq "$WANT_WINDOW" ]; then ok "frame window size is $WANT_WINDOW B" else diag "frame Window Size=$GOT_WINDOW B expected=$WANT_WINDOW B" notok "frame window size is $WANT_WINDOW B (got $GOT_WINDOW B)" fi if grep -qE '\[(crit|alert|emerg)\]' "$ELOG" 2>/dev/null; then diag "$(grep -E '\[(crit|alert|emerg)\]' "$ELOG" | head -3)" fi exit "$FAILED" myguard-labs-nginx-zstd-module-3486b5b/ci/t/harness-scenarios/setparam-call-count-nodict/nginx.conf000066400000000000000000000021611525232545500334460ustar00rootroot00000000000000@LOAD@ # Sibling of setparam-call-count, isolated because zstd_dict_file is an # http{}-context directive: any conf that loads one gives every "zstd on" # location a non-NULL conf->dict (see the sibling scenario's comment), so a # genuinely dict-free arm needs its own conf with no such directive anywhere. daemon off; pid @PREFIX@/nginx.pid; error_log @PREFIX@/logs/error.log info; worker_processes 1; events { worker_connections 16; } http { default_type text/plain; server { listen 127.0.0.1:@PORT@; root @PREFIX@/www; # No dictionary anywhere in this conf: init_cctx takes neither the # zlcf->dict (CDict) nor the ctx->dcz_dict (raw prefix) branch, so # every configured parameter is a live CCtx parameter, none # superseded: # level set # windowLog set # LDM set (zstd_long on) # -> 3 setParameter calls. location / { alias @PREFIX@/www/; zstd on; zstd_min_length 0; zstd_window_log 20; zstd_long on; } location /__probe { @PROBE@ } } } myguard-labs-nginx-zstd-module-3486b5b/ci/t/harness-scenarios/setparam-call-count-nodict/requires000066400000000000000000000032171525232545500332410ustar00rootroot00000000000000#!/usr/bin/env bash # # Gate: the tree run-scenario.sh is about to boot must be a TEST_HARNESS # build. This scenario reads the setparam_calls counter out of the probe # document, and without both TEST_HARNESS=1 and -DNGX_TEST_HARNESS the probe # endpoint compiles out entirely -- `nginx -t` would then reject the # `zstd_probe;` in nginx.conf and the scenario would red for a build reason # rather than a module one. # # No dictionary fixture to stage here -- this scenario's whole point is a # conf with no zstd_dict_file / zstd_dcz_dict_file anywhere in it, so the # dict-staging step the sibling setparam-call-count scenario needs does not # apply. # # Derived the same way run-scenario.sh/lib.sh resolve the build tree, so this # gate can never SKIP a tree the engine would boot fine or pass one it would # reject -- identical reasoning to fault-arms/requires and alloc-neutral's. set -euo pipefail FLAVOR="${2:-nginx}" VERSION="${3:-1.31.3}" BUILD="${PROBER_BUILD:-${PROBER_ROOT:-$(cd ../.. && pwd)}/.build/${FLAVOR}-${VERSION}}" SO="$BUILD/objs/ngx_http_zstd_filter_module.so" if [ ! -f "$SO" ]; then echo "ngx_http_zstd_filter_module.so not found at $SO -- build it first" exit 1 fi PROBE_SYMBOLS="$(nm "$SO" 2>/dev/null | grep -c ' ngx_test_probe_' || true)" if [ "$PROBE_SYMBOLS" -eq 0 ]; then echo "$SO carries no ngx_test_probe_* symbols -- this is a packaged (non-harness) build; configure with TEST_HARNESS=1 CFLAGS=\"\$CFLAGS -DNGX_TEST_HARNESS\" first" exit 1 fi if ! command -v zstd >/dev/null 2>&1; then echo "zstd(1) not found -- the oracle decodes its response and compares it byte-for-byte with the origin file" exit 1 fi exit 0 myguard-labs-nginx-zstd-module-3486b5b/ci/t/harness-scenarios/setparam-call-count/000077500000000000000000000000001525232545500301765ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/t/harness-scenarios/setparam-call-count/driver.sh000077500000000000000000000172531525232545500320400ustar00rootroot00000000000000#!/usr/bin/env bash # # Scenario: ZSTD_CCtx_setParameter() CALL COUNT, DICTIONARY MODES. # # Pins the per-mode instrumentation this item's done criterion asks for. # Both dictionary paths live here because zstd_dict_file is http{}-context # (see nginx.conf's comment for why a genuinely dict-free arm cannot share # this conf) -- the no-dict reference count (3) is measured separately in # the sibling setparam-call-count-nodict scenario. # # /cdict/ trained CDict (zlcf->dict, refCDict): only compressionLevel is # in zstd.h's refCDict-superseded "compression parameters" block # (ZSTD_CCtx_refCDict() takes it from the CDict's own baked # ZSTD_compressionParameters). windowLog is NOT superseded -- # ZSTD_resetCCtx_byAttachingCDict()/ZSTD_resetCCtx_byCopyingCDict() # (zstd_compress.c) both restore it from the CCtx's own applied # params right after copying the CDict's other cParams, so # init_cctx keeps setting it here -> 2 calls (windowLog, LDM). # /dcz/ negotiated raw prefix (ctx->dcz_dict, refPrefix): refPrefix # does NOT override sticky parameters, so level, windowLog # (the dcz-aware value, set once, not twice), LDM and # checksumFlag(dcz) all apply -> 4 calls. # # THE OUTPUT-IDENTITY HALF. A call count on its own is satisfied by a # response that is fast and wrong, so every oracle here is paired with a # byte-for-byte decode against the origin -- the same shape as # codec-call-count/driver.sh, and for the same reason (see its header). set -euo pipefail # shellcheck source=lib.sh . "$PROBER_LIB" HOST=127.0.0.1 PORT="$PROBER_RESOLVED_PORT" ELOG="$PROBER_PREFIX/logs/error.log" FAILED=0 N=0 ok() { N=$((N + 1)); echo "ok $N - $1"; } notok() { N=$((N + 1)); echo "not ok $N - $1"; FAILED=1; } diag() { echo "# $1"; } WWW="$PROBER_PREFIX/www" mkdir -p "$WWW" "$PROBER_PREFIX/tmp" # Larger than 2**20 (zstd_window_log 20 in nginx.conf, both arms): a body # smaller than the window cap lets zstd shrink the FRAME's encoded window to # fit the content regardless of what ZSTD_c_windowLog was set to, which would # make the cap assertion below pass whether or not the cap was actually # applied. Random bytes so the compressor cannot fold the whole body into a # handful of long matches and shrink the decode/measure cost. awk 'BEGIN { srand(42) n = 1200000 for (i = 0; i < n; i++) { printf "%c", 97 + int(rand() * 26) } }' >"$WWW/index.html" DICT_FILE="$(dirname "$PROBER_SERVER_BIN")/setparam-call-count.dict" DICT_SHA_B64="$(openssl dgst -sha256 -binary "$DICT_FILE" 2>/dev/null \ | openssl base64 -A 2>/dev/null || true)" reset_counter() { curl -fsS --max-time 5 "http://$HOST:$PORT/__probe?setparam_reset=1" \ -o /dev/null } read_counter() { local body body="$(prober_probe_body "$HOST" "$PORT")" || return 1 prober_probe_field "$body" "setparam_calls" || return 1 } # measure LABEL PATH ENC EXPECT [EXTRA_HEADER...] measure() { local label="$1" path="$2" enc="$3" expect="$4"; shift 4 local hdrs="$PROBER_PREFIX/tmp/${label}.hdrs" local body="$PROBER_PREFIX/tmp/${label}.bin" local -a extra=(-H 'Accept-Encoding: zstd') local h for h in "$@"; do extra+=(-H "$h"); done if ! reset_counter; then notok "$label: probe did not answer setparam_reset" notok "$label: response decodes byte-for-byte to the origin" notok "$label: setParameter call count is $expect" return fi if ! curl -sS --max-time 15 -D "$hdrs" -o "$body" "${extra[@]}" \ "http://$HOST:$PORT$path" 2>"$body.err"; then notok "$label: request failed or timed out" notok "$label: response decodes byte-for-byte to the origin" notok "$label: setParameter call count is $expect" return fi if ! grep -qi "^content-encoding:[[:space:]]*$enc" "$hdrs"; then diag "$label: headers were: $(tr -d '\r' <"$hdrs" | tr '\n' ' ')" notok "$label: response carried Content-Encoding: $enc" notok "$label: response decodes byte-for-byte to the origin" notok "$label: setParameter call count is $expect" return fi ok "$label: response carried Content-Encoding: $enc" if zstd -d -q -f -o "$body.dec" "$body" -D "$DICT_FILE" 2>/dev/null \ && cmp -s "$body.dec" "$WWW/index.html"; then ok "$label: response decodes byte-for-byte to the origin" else notok "$label: response decodes byte-for-byte to the origin" fi local got got="$(read_counter || echo -1)" if [ "$got" -eq "$expect" ]; then ok "$label: setParameter call count is $expect" else diag "$label: setparam_calls=$got expected=$expect" notok "$label: setParameter call count is $expect (got $got)" fi } # assert_window_cap LABEL FRAME_FILE WINDOW_LOG: parse the compressed # frame's own Window_Descriptor via `zstd -l -v` and require it to be # EXACTLY 2**WINDOW_LOG. This is the direct regression check for the # windowLog-is-NOT-superseded-by-cdict finding: the setParameter call # count above proves windowLog was SET on the CCtx, but not that the # resulting FRAME actually carries the operator's cap -- a build that # silently dropped the CCtx-side set (the exact bug ZSTD_resetCCtx_ # byAttachingCDict()/ZSTD_resetCCtx_byCopyingCDict() would produce, see # nginx.conf's comment) still passes the call-count oracle if something # else pushes the count back to 2 by coincidence, but the frame itself # would then carry libzstd's level-derived window instead of 1 MiB. The # fixture is 1200000 bytes, larger than 2**20, specifically so the window # cannot shrink to fit content and mask an uncapped default. assert_window_cap() { local label="$1" frame="$2" wlog="$3" local want=$((1 << wlog)) local got got="$(zstd -l -v "$frame" 2>/dev/null \ | grep -oP 'Window Size:.*\(\K[0-9]+(?= B\))' | head -1)" if [ -z "$got" ]; then notok "$label: frame window size is $want B (zstd -l -v produced no Window Size line)" return fi if [ "$got" -eq "$want" ]; then ok "$label: frame window size is $want B" else diag "$label: frame Window Size=$got B expected=$want B" notok "$label: frame window size is $want B (got $got B)" fi } echo "1..8" # /cdict/: no dcz negotiation header, so init_cctx takes the zlcf->dict # branch. level is superseded-by-cdict and skipped; windowLog is NOT # superseded (restored from the CCtx's own params by # ZSTD_resetCCtx_by{Attaching,Copying}CDict()) and stays set alongside LDM. measure "cdict" "/cdict/index.html" "zstd" 2 assert_window_cap "cdict" "$PROBER_PREFIX/tmp/cdict.bin" 20 if [ -n "$DICT_SHA_B64" ]; then measure "dcz" "/dcz/index.html" "dcz" 4 \ "Accept-Encoding: zstd, dcz" \ "Available-Dictionary: :$DICT_SHA_B64:" else # NOT a SKIP -- see codec-call-count/driver.sh's identical reasoning: # openssl(1) is a hard requires-gate, so reaching here means the hash # derivation failed for some other reason (unreadable/truncated # dictionary), and a SKIP would read as a pass to anything grepping # for the oracle. notok "dcz: response carried Content-Encoding: dcz (could not derive the dictionary hash from $DICT_FILE)" notok "dcz: response decodes byte-for-byte to the origin (no dictionary hash)" notok "dcz: setParameter call count is 4 (no dictionary hash)" fi # error log carries no crit/alert/emerg: this scenario injects no faults. if grep -qE '\[(crit|alert|emerg)\]' "$ELOG" 2>/dev/null; then diag "$(grep -E '\[(crit|alert|emerg)\]' "$ELOG" | head -3)" notok "error log carries no crit/alert/emerg lines" else ok "error log carries no crit/alert/emerg lines" fi exit "$FAILED" myguard-labs-nginx-zstd-module-3486b5b/ci/t/harness-scenarios/setparam-call-count/nginx.conf000066400000000000000000000067131525232545500321770ustar00rootroot00000000000000@LOAD@ # Pins the per-mode ZSTD_CCtx_setParameter() call count init_cctx makes. # Same shape as codec-call-count: this repo IS the module under test, so # PROBER_MODULE is ngx_http_zstd_filter_module.so and @LOAD@ already # load_modules it, reaching setparam_calls through the same zoneless # module_render path. daemon off; pid @PREFIX@/nginx.pid; error_log @PREFIX@/logs/error.log info; worker_processes 1; events { worker_connections 16; } http { default_type text/plain; # Trained CDict, http{}-context only (NGX_HTTP_MAIN_CONF). Applies to # EVERY location below unless a dcz negotiation on that request takes # priority (dcz_dict != NULL wins in init_cctx -- see the else-if there). zstd_dict_file_unsafe on; zstd_dict_file @BUILD_OBJS@/setparam-call-count.dict; server { listen 127.0.0.1:@PORT@; root @PREFIX@/www; # No-dcz-negotiation arm: init_cctx takes the zlcf->dict (CDict) # branch. Only compressionLevel is superseded by the CDict's own # baked ZSTD_compressionParameters (zstd.h). windowLog is NOT # superseded -- ZSTD_resetCCtx_byAttachingCDict()/ # ZSTD_resetCCtx_byCopyingCDict() (zstd_compress.c) both restore it # from the CCtx's own applied params right after copying the # CDict's other cParams, and assert it nonzero, so init_cctx must # keep setting it here or zstd_window_log's cap silently stops # applying under a dictionary. The fixed set is: # level SKIPPED (superseded-by-cdict) # windowLog set (NOT superseded-by-cdict) # LDM set (zstd_long on; not superseded) # checksumFlag NOT set (dcz-only) # -> 2 setParameter calls (windowLog, enableLongDistanceMatching). location /cdict/ { alias @PREFIX@/www/; zstd on; zstd_min_length 0; zstd_window_log 20; zstd_long on; } # dcz negotiation arm: init_cctx takes the ctx->dcz_dict (raw # prefix) branch. refPrefix does NOT supersede sticky parameters, # so: # level set # windowLog set ONCE, with the dcz-aware value (not the # generic zlcf->window_log -- see the skip # condition in init_cctx) # LDM set (zstd_long on) # checksumFlag set (dcz-only) # -> 4 setParameter calls. location /dcz/ { alias @PREFIX@/www/; zstd on; zstd_min_length 0; zstd_window_log 20; zstd_long on; zstd_dcz_dict_file @BUILD_OBJS@/setparam-call-count.dict; zstd_dcz_assume_secure_transport on; } # No-dictionary arm lives in the SEPARATE setparam-call-count-nodict # scenario, not here: zstd_dict_file is an http{}-context # (NGX_HTTP_MAIN_CONF) directive, and any "zstd on" location in an # http{} block where it is set gets conf->dict populated from the # shared registry (see the location-merge code around # zmcf->dict_registry in ngx_http_zstd_merge_loc_conf) -- there is # no per-location opt-out short of a second http{} block, which one # nginx.conf cannot express. Splitting into two confs keeps each one # a single, unambiguous arm rather than a location whose comment # promises "no dict" while conf->dict is silently non-NULL. location /__probe { @PROBE@ } } } myguard-labs-nginx-zstd-module-3486b5b/ci/t/harness-scenarios/setparam-call-count/requires000077500000000000000000000072161525232545500317710ustar00rootroot00000000000000#!/usr/bin/env bash # # Gate: the tree run-scenario.sh is about to boot must be a TEST_HARNESS build. # This scenario reads the setparam_calls counter out of the probe document, # and without both TEST_HARNESS=1 and -DNGX_TEST_HARNESS the probe endpoint # compiles out entirely -- `nginx -t` would then reject the `zstd_probe;` in # nginx.conf and the scenario would red for a build reason rather than a # module one. # # Derived the same way run-scenario.sh/lib.sh resolve the build tree, so this # gate can never SKIP a tree the engine would boot fine or pass one it would # reject -- identical reasoning to fault-arms/requires and alloc-neutral's. set -euo pipefail FLAVOR="${2:-nginx}" VERSION="${3:-1.31.3}" BUILD="${PROBER_BUILD:-${PROBER_ROOT:-$(cd ../.. && pwd)}/.build/${FLAVOR}-${VERSION}}" SO="$BUILD/objs/ngx_http_zstd_filter_module.so" if [ ! -f "$SO" ]; then echo "ngx_http_zstd_filter_module.so not found at $SO -- build it first" exit 1 fi PROBE_SYMBOLS="$(nm "$SO" 2>/dev/null | grep -c ' ngx_test_probe_' || true)" if [ "$PROBE_SYMBOLS" -eq 0 ]; then echo "$SO carries no ngx_test_probe_* symbols -- this is a packaged (non-harness) build; configure with TEST_HARNESS=1 CFLAGS=\"\$CFLAGS -DNGX_TEST_HARNESS\" first" exit 1 fi # Every call-count oracle here is PAIRED with a decode of the same response # against the origin bytes, because a call count on its own is satisfied by a # response that is fast and wrong. A missing decoder would silently weaken # every one of them to "200 and some bytes". Fail here instead: a gate that # cannot run its assertion is not a gate. if ! command -v zstd >/dev/null 2>&1; then echo "zstd(1) not found -- every oracle decodes its response and compares it byte-for-byte with the origin file" exit 1 fi # openssl(1) is gated on exactly the same reasoning, and it is the CLI that is # required, not libssl: the driver shells out to `openssl dgst`/`openssl base64` # to derive the Available-Dictionary hash the dcz arm negotiates with. Without # it that arm cannot run. # # It is a hard gate rather than a driver-side SKIP deliberately. A SKIP prints # an "ok ... # SKIP" line, which reads as a pass to every consumer that greps # for the oracle -- so the dcz arm would assert nothing while the run stayed # green, which is the false green this scenario exists to remove. Failing the # whole scenario here is loud and impossible to misread. if ! command -v openssl >/dev/null 2>&1; then echo "openssl(1) not found -- the dcz arm derives its Available-Dictionary hash with \"openssl dgst -sha256 | openssl base64\"; the CLI is required, not just libssl" exit 1 fi # Stage the dictionary fixture the dcz and trained-dict arms need. # # It has to exist before nginx runs its CONFIG TEST -- zstd_dcz_dict_file and # zstd_dict_file open their file at configuration time, so a driver-written # fixture is far too late (the prefix the driver writes into is not created # until prober_render_conf, immediately before that config test). This gate is # the only hook that runs earlier, so it stages the file and the conf # references it through @BUILD_OBJS@, the one absolute path the renderer # substitutes that exists this early. # # Copied from the repo's own checked-in ci/t/suite/dcz-dict rather than # generated, so the dcz arm here negotiates against exactly the dictionary # 03-dcz.t already pins the wire contract against. DICT_SRC="$(cd "$(dirname "$0")/../../suite" && pwd)/dcz-dict" if [ ! -f "$DICT_SRC" ]; then echo "dictionary fixture not found at $DICT_SRC" exit 1 fi if ! cp -f "$DICT_SRC" "$BUILD/objs/setparam-call-count.dict"; then echo "cannot stage the dictionary fixture into $BUILD/objs" exit 1 fi exit 0 myguard-labs-nginx-zstd-module-3486b5b/ci/t/suite/000077500000000000000000000000001525232545500220255ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/t/suite/dcz-dict000066400000000000000000000051441525232545500234550ustar00rootroot00000000000000function widget_0() { return compute(0); } // shared-boilerplate function widget_1() { return compute(1); } // shared-boilerplate function widget_2() { return compute(2); } // shared-boilerplate function widget_3() { return compute(3); } // shared-boilerplate function widget_4() { return compute(4); } // shared-boilerplate function widget_5() { return compute(5); } // shared-boilerplate function widget_6() { return compute(6); } // shared-boilerplate function widget_7() { return compute(7); } // shared-boilerplate function widget_8() { return compute(8); } // shared-boilerplate function widget_9() { return compute(9); } // shared-boilerplate function widget_10() { return compute(10); } // shared-boilerplate function widget_11() { return compute(11); } // shared-boilerplate function widget_12() { return compute(12); } // shared-boilerplate function widget_13() { return compute(13); } // shared-boilerplate function widget_14() { return compute(14); } // shared-boilerplate function widget_15() { return compute(15); } // shared-boilerplate function widget_16() { return compute(16); } // shared-boilerplate function widget_17() { return compute(17); } // shared-boilerplate function widget_18() { return compute(18); } // shared-boilerplate function widget_19() { return compute(19); } // shared-boilerplate function widget_20() { return compute(20); } // shared-boilerplate function widget_21() { return compute(21); } // shared-boilerplate function widget_22() { return compute(22); } // shared-boilerplate function widget_23() { return compute(23); } // shared-boilerplate function widget_24() { return compute(24); } // shared-boilerplate function widget_25() { return compute(25); } // shared-boilerplate function widget_26() { return compute(26); } // shared-boilerplate function widget_27() { return compute(27); } // shared-boilerplate function widget_28() { return compute(28); } // shared-boilerplate function widget_29() { return compute(29); } // shared-boilerplate function widget_30() { return compute(30); } // shared-boilerplate function widget_31() { return compute(31); } // shared-boilerplate function widget_32() { return compute(32); } // shared-boilerplate function widget_33() { return compute(33); } // shared-boilerplate function widget_34() { return compute(34); } // shared-boilerplate function widget_35() { return compute(35); } // shared-boilerplate function widget_36() { return compute(36); } // shared-boilerplate function widget_37() { return compute(37); } // shared-boilerplate function widget_38() { return compute(38); } // shared-boilerplate function widget_39() { return compute(39); } // shared-boilerplate myguard-labs-nginx-zstd-module-3486b5b/ci/t/suite/dcz-empty000066400000000000000000000000001525232545500236520ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/t/suite/test000066400000000000000000001645321525232545500227420ustar00rootroot00000000000000 zstd-nginx-module test fixture

zstd-nginx-module Test Fixture

This file is used as a test fixture for the zstd-nginx-module test suite. It is original content, authored for this purpose, and licensed under the BSD 2-Clause License. It exists solely to provide a realistic-sized HTML document for compression tests.

About zstd Compression

Zstandard (zstd) is a fast lossless compression algorithm developed at Meta. It targets real-time compression scenarios at zlib-level and better compression ratios. It also offers a special mode for small data, called dictionary compression, that can be used to dramatically improve the compression ratio on small data (less than 1 KB).

Key properties of zstd:

  • Compression ratio comparable to zlib at much higher speed.
  • Decompression speed exceeding 1 GB/s on modern hardware.
  • Adjustable compression levels from 1 (fastest) to 22 (best ratio).
  • Negative compression levels for ultra-fast, lower-ratio use cases.
  • Dictionary training for small, repetitive payloads.
  • Frame format with content-size and checksum fields.

About This Module

The ngx_http_zstd_filter_module compresses HTTP responses on the fly using zstd. It integrates into the nginx filter chain and is activated when a client sends Accept-Encoding: zstd.

The ngx_http_zstd_static_module serves pre-compressed .zst files in place of originals when they exist on disk, without spending CPU on compression at request time.

Configuration Example

http {
    zstd             on;
    zstd_comp_level  3;
    zstd_min_length  1000;
    zstd_types       text/plain text/css application/json
                     application/javascript text/xml;
    gzip_vary        on;

    server {
        listen 80;

        location /static/ {
            zstd_static on;
            root /var/www;
        }
    }
}

Directives

zstd

Syntax: zstd on | off;
Default: zstd off;
Context: http, server, location, if in location

Enables or disables on-the-fly zstd compression of responses.

zstd_comp_level

Syntax: zstd_comp_level level;
Default: zstd_comp_level 3;
Context: http, server, location

Sets the compression level. Range: ZSTD_minCLevel() to ZSTD_maxCLevel(). Level 0 means the library default. Negative levels trade ratio for speed.

zstd_min_length

Syntax: zstd_min_length length;
Default: zstd_min_length 20;
Context: http, server, location

Minimum response size for compression to apply. Responses smaller than this value are passed through unmodified.

zstd_max_length

Syntax: zstd_max_length length;
Default: (no limit)
Context: http, server, location

Maximum response size to compress. Larger responses bypass compression. Note: not enforced for chunked or streaming responses without Content-Length.

zstd_buffers

Syntax: zstd_buffers number size;
Default: zstd_buffers 32 4k; or 16 8k;
Context: http, server, location

Number and size of output buffers used during compression. Total buffer space is number * size, defaulting to 128 KB regardless of page size.

zstd_types

Syntax: zstd_types mime-type ...;
Default: zstd_types text/html;
Context: http, server, location

MIME types eligible for compression in addition to text/html. Use * to match all types. Binary formats (JPEG, PNG, MP4) should be excluded as they are already compressed.

zstd_dict_file

Syntax: zstd_dict_file /path/to/dict;
Default: (none)
Context: http

Path to a pre-trained zstd dictionary. Dictionaries can significantly improve compression of small, structurally similar responses. Both server and client must use the same dictionary.

zstd_static

Syntax: zstd_static on | off | always;
Default: zstd_static off;
Context: http, server, location

Controls serving of pre-compressed .zst files:

  • off: disabled, always serve original.
  • on: serve .zst if client supports zstd and file exists.
  • always: serve .zst unconditionally if it exists.

Variables

$zstd_ratio

The compression ratio for the current response (original / compressed). For example 3.42 means the output is about 29% of the input. Only set after the filter module has compressed a response.

log_format main '$remote_addr - $request - ratio: $zstd_ratio';

Building

./configure --add-dynamic-module=/path/to/zstd-nginx-module
make && make install

Or point at the subdirectories for separate modules:

./configure \
  --add-dynamic-module=/path/to/zstd-nginx-module/filter \
  --add-dynamic-module=/path/to/zstd-nginx-module/static

Load in nginx.conf:

load_module modules/ngx_http_zstd_filter_module.so;
load_module modules/ngx_http_zstd_static_module.so;

Pre-compressing Static Files

find /var/www/static -name "*.js" -o -name "*.css" | \
    xargs -I{} zstd -3 -k {}

This creates file.js.zst alongside file.js. nginx will serve the .zst variant to clients that support zstd when zstd_static on; is configured.

Notes on gzip_vary

Always enable gzip_vary on; alongside this module. When compression is applied the module sets r->gzip_vary = 1, which causes nginx to emit Vary: Accept-Encoding — but only when gzip_vary is enabled. Without it, proxies and CDNs may cache the compressed response and serve it to clients that cannot decode zstd, resulting in broken responses.

ETag Behaviour

When a response is compressed, nginx automatically weakens the ETag value ("abc" becomes W/"abc"). This is correct per HTTP semantics — a compressed variant is a different representation — but means strong ETag validation will not match across compressed and uncompressed responses.

Padding

U0G%QSt ̩KTi,(MFS41T2Kd@49h&LE@te"QteB]DHwPH#U'DL$ˣ ˄b:uD\~DBDɣ XdX,(d>#i.$,RĄL(((;0+%UHS);&.RBHwLD]&Hd*k0Hq O$}$|`(IC]0W.ObP42# "^*&Iz\PDAͧ,5\e8 7۪ ;CnA+'O8⎰fčB^[uz #7;_5{jngDm7gr;`֊-v1ޢǚ5;R!o[ȓjyL{۰l\  B7p`pB.-Rg福9@)r$h'܊-?@䑼GS ~jrcՖoR,V~Gȃ빱r(wkx͈RL{jd D$=V4.fj)ײ|:#Gnnf66Y*jn KjkozZ|,t}sj]? T$Byoq38`hcsԲX= n7xB+gO[L:=ΨMd2b )e폭A+͐;+npAvVd: _u=\Ln[3+}&bkyn<+>s'\ EQ|.$![֍ap.(Y*yֳmBZL:q 2d]; =ՂT.c^{Y;8<znO,JT ';0~B"& ۭ8 @8C=FYufcA|ֲ鬸f;jOW+crj7 FKh!jq!m/"\yǺX{GbUZ=>/\D|`|o[PHuМcܴ:V18D_@wbD/O>|f7Tx^0><3"L ATO"Àol r恈qc|J66ߐvh'Ch٣Tq qAv p9؎J 8d0*mK;yXLj?qSn]mi#?^<nfF;Hl֪A.aFW윮|Wqy"6 Sq%sDY@5X՚=;-}6|G{<_fRVtR`84!;E (Rb^6:% hʸM`d@dRQ#}z5 }}FKfy&L$%_8RȋsOt FdF/ioG:bXLٷM*8+! IvS*D1tї5{@uDL\fUl#oDBs"%k4us[b.1"XxfתE f (6$_/"y{㡲BPLO/rjZBIk˟;NGtnר?x{jخONtn#FAf.? TŬJCu(^{h14Ryvl^ESHR%lvcǛªaXf6Wri-NW9'k#m!NO[40rAR. +ZdĕAU!H|(4>rڞTSj ye4l'z 2#V)LsYx @I^/Z @(K7?ٸն@@ FߠR`-oM%\ {Jo4`.#]H k'jry';˸:$q tȑ%ˊrEyè]liUp Ipg \nчwǏ9 3-\ ׏ 9fnJ Z8aoPś 2dv]T@(=֟PV_!) NQ~&-ր`^ʰr.4SSlUgb2ͧ*'yfJMfH*1pXb1CF7~0*YgB2NV[bE`h*V:nnt͜qc Lë8X*3I%Zڰsed V1Lg+1? myguard-labs-nginx-zstd-module-3486b5b/ci/tests/000077500000000000000000000000001525232545500215735ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/tests/unit/000077500000000000000000000000001525232545500225525ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/tests/unit/run.sh000077500000000000000000000143051525232545500237200ustar00rootroot00000000000000#!/usr/bin/env bash # Copyright (C) 2026 Thijs Eilander # SPDX-License-Identifier: BSD-2-Clause # # ci/tests/unit/run.sh -- build and run the pure-function unit tests: the # Accept-Encoding parser (src/ngx_http_zstd_common.h) and the .zst # frame-header probe (src/ngx_http_zstd_frame_probe.h). # # ci/tests/unit/run.sh # regenerate parser slice, build, run # ci/tests/unit/run.sh clean # remove build products # COVERAGE=1 ci/tests/unit/run.sh # also instrument, so ci/tools/coverage.sh # # can gcov src/ afterwards # # Env: # CC compiler (default cc). Takes a full driver line, so CC="gcc -m32" # runs the suite as a 32-bit binary. # # Exit: 0 all checks passed, 1 a check failed or the parser-extraction # guard tripped. # # Runs in well under a second, needs no nginx build tree, no network and no # root: unlike the skeleton's scan core, the Accept-Encoding parser is pure, # self-contained ASCII walking (see the header comment atop # src/ngx_http_zstd_common.h) that needs only ngx_strncasecmp() / # ngx_strcasestrn(), which ci/fuzz/ngx_shim.h reproduces as faithful, cited # copies of the upstream src/core/ngx_string.c implementations. There is # therefore no dependency on ci/tools/nginx-tree.sh or a .build/ tree here -- # this layer is deliberately the cheapest one, so it is the one a derived # change can afford to run on every save. # # WHAT IT DOES *NOT* DO: it does not re-implement the parser. This script # runs ci/fuzz/extract_parser.sh first, exactly as the fuzz build does, so # the tests always link the SHIPPED src/ngx_http_zstd_common.h -- never a # hand-copied version that can drift from production. # # -Werror applies throughout: unlike the skeleton's scan-core layer, there is # no upstream translation unit linked in here to exempt. # # See test_accept_encoding.c's header comment for the list of mutations this # suite has been observed to catch (each named there, with the check that # went red). set -euo pipefail DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT="$(cd "$DIR/../../.." && pwd)" FUZZ_DIR="$ROOT/ci/fuzz" BIN="$DIR/test_accept_encoding" LEGACY_BIN="$DIR/test_accept_encoding_legacy" CHAIN_BIN="$DIR/test_accept_encoding_chain" PROBE_BIN="$DIR/test_static_probe" VERSION_BIN="$DIR/test_version_policy" if [ "${1:-}" = "clean" ]; then rm -f "$BIN" "$LEGACY_BIN" "$CHAIN_BIN" "$PROBE_BIN" "$VERSION_BIN" "$DIR"/*.o "$DIR"/*.gcda "$DIR"/*.gcno echo "unit test binary removed" exit 0 fi # Regenerate the extracted parser slice so this binary always links the # shipped src/ngx_http_zstd_common.h, never a stale copy. # test_accept_encoding.c includes ci/fuzz/ headers by RELATIVE path, so a static # analyser invoked from anywhere parses it without -I flags. That is deliberate: # an analyser that cannot resolve an include skips the whole TU and reports no # findings, which is indistinguishable from a clean result. bash "$FUZZ_DIR/extract_parser.sh" CC="${CC:-cc}" OWN_CFLAGS=(-g -O1 -Wall -Wextra -Wshadow -Werror) if [ "${COVERAGE:-0}" = 1 ]; then OWN_CFLAGS+=(--coverage) LINK_EXTRA=(--coverage) else LINK_EXTRA=() fi echo "==> Building $BIN with ${CC}" # shellcheck disable=SC2086 # $CC may legitimately carry flags (e.g. "gcc -m32") $CC "${OWN_CFLAGS[@]}" -I"$FUZZ_DIR" -c "$DIR/test_accept_encoding.c" \ -o "$DIR/test_accept_encoding.o" # shellcheck disable=SC2086 $CC "${LINK_EXTRA[@]}" -o "$BIN" "$DIR/test_accept_encoding.o" echo "==> Running" # Bounded: a regression in the quoted-string walk can hang the scanner outright # (one of the documented mutations does exactly that). Without a local timeout # the job burns its whole 15-minute budget and the log never says which layer # failed. timeout 60s "$BIN" echo "==> Building $LEGACY_BIN with ${CC} (nginx 1.22.1 header shape)" # The shim omits ngx_table_elt_t.next under this define, so this compilation # catches an accidental legacy ->next access as well as exercising the list walk. # shellcheck disable=SC2086 $CC "${OWN_CFLAGS[@]}" -DNGX_ZSTD_LEGACY_SHIM -I"$FUZZ_DIR" \ -c "$DIR/test_accept_encoding_legacy.c" -o "$DIR/test_accept_encoding_legacy.o" # shellcheck disable=SC2086 $CC "${LINK_EXTRA[@]}" -o "$LEGACY_BIN" "$DIR/test_accept_encoding_legacy.o" echo "==> Running nginx 1.22.1-shaped Accept-Encoding checks" timeout 60s "$LEGACY_BIN" echo "==> Building $CHAIN_BIN with ${CC} (nginx 1.23+ chained headers)" # This reuses the legacy fixture with the modern shim shape, proving the # ordered duplicate-field contract in both nginx storage layouts. # shellcheck disable=SC2086 $CC "${OWN_CFLAGS[@]}" -I"$FUZZ_DIR" \ -c "$DIR/test_accept_encoding_legacy.c" -o "$DIR/test_accept_encoding_chain.o" # shellcheck disable=SC2086 $CC "${LINK_EXTRA[@]}" -o "$CHAIN_BIN" "$DIR/test_accept_encoding_chain.o" echo "==> Running nginx 1.23+-shaped Accept-Encoding checks" timeout 60s "$CHAIN_BIN" # --------------------------------------------------------------------------- # The .zst frame-header probe (src/ngx_http_zstd_frame_probe.h: # ngx_http_zstd_static_probe_frame). Unlike the parser suite above there # is no extraction step: the probe lives in its own shipped header (#270) # and the TU includes it directly, so the binary can never link a # hand-copied drifted version. The probe is pure arithmetic over a fixed # 18-byte buffer -- no nginx tree, no libzstd, no filesystem -- so it belongs # in this cheapest layer too. # --------------------------------------------------------------------------- echo "==> Building $PROBE_BIN with ${CC}" # shellcheck disable=SC2086 $CC "${OWN_CFLAGS[@]}" -c "$DIR/test_static_probe.c" \ -o "$DIR/test_static_probe.o" # shellcheck disable=SC2086 $CC "${LINK_EXTRA[@]}" -o "$PROBE_BIN" "$DIR/test_static_probe.o" echo "==> Running" # Bounded for the same reason as above; the probe has no loop that can run # away, but a timeout costs nothing and keeps the layer's failure mode # uniform. timeout 60s "$PROBE_BIN" echo "==> Building $VERSION_BIN with ${CC}" # shellcheck disable=SC2086 $CC "${OWN_CFLAGS[@]}" -c "$DIR/test_version_policy.c" \ -o "$DIR/test_version_policy.o" # shellcheck disable=SC2086 $CC "${LINK_EXTRA[@]}" -o "$VERSION_BIN" "$DIR/test_version_policy.o" echo "==> Running libzstd version policy checks" timeout 60s "$VERSION_BIN" myguard-labs-nginx-zstd-module-3486b5b/ci/tests/unit/test_accept_encoding.c000066400000000000000000001474021525232545500270720ustar00rootroot00000000000000/* * Copyright (C) 2026 Thijs Eilander * SPDX-License-Identifier: BSD-2-Clause * * Unit tests for the Accept-Encoding parser * (src/ngx_http_zstd_common.h: ngx_http_zstd_skip_quoted(), * ngx_http_zstd_eval_qvalue(), ngx_http_zstd_coding_weight(), * ngx_http_zstd_accept_encoding()). * * WHY THIS EXISTS ALONGSIDE ci/t/ AND ci/fuzz/ * * ci/t, Test::Nginx drives the parser through a live nginx request. It * proves the header actually reaches the module, but * every boundary (an exact q-value digit count, the * "*" vs explicit-token precedence, a malformed * weight) has to be reached by crafting a raw header * string and is awkward to name. * ci/fuzz targets drive the same code with random bytes plus an * independent reference oracle. That proves the * parser never reads out of bounds and agrees with a * second implementation on the UNAMBIGUOUS cases -- * but it does not state "q=0.1 must be treated as * weight 100", it only states agreement or disagreement * on inputs the oracle is confident about. * this file states values, at named boundaries, with no nginx * process, no network, in well under a second. * * This file, like the fuzz target, does NOT link the real ngx_string.c: the * production parser is fully self-contained ASCII walking (see the comment * atop ngx_http_zstd_common.h) and needs only ngx_strncasecmp/ngx_strcasestrn, * which ci/fuzz/ngx_shim.h reproduces as faithful, cited copies of the * upstream implementations -- the same shim the fuzz target links, so this * layer and the fuzz layer test the exact same compiled bytes. There is no * shim of the DECISION logic itself: run.sh regenerates * ci/fuzz/generated_parser.inc from the real src/ngx_http_zstd_common.h via * ci/fuzz/extract_parser.sh before every build, so this binary always links * the shipped parser, never a copy. * * SEEN RED -- every mutation below was APPLIED to * src/ngx_http_zstd_common.h and the named check was observed failing * (see ci/adoption-findings.md for the exact commands). Re-run them after * touching the parser or this file: a check that has never failed is not * known to be a check. * * * wildcard precedence flipped -- in ngx_http_zstd_coding_weight(), return * star_q first when both are set * -> "explicit zstd;q=0 overrides a permissive *" FAILS. * * q-value scale dropped a digit -- in ngx_http_zstd_eval_qvalue(), * `scale /= 10` removed so every fractional digit is worth 100 instead * of decaying by a factor of ten per digit * -> "a fourth q decimal digit makes the element non-matching" FAILS * (the loop no longer stops contributing after 3 digits at * decreasing weight -- the value only exists to bound the digit * count via the `scale > 0` loop guard, so removing the decrement * also breaks the qvalue=0*3DIGIT length limit, caught by the * same check as the malformed-fourth-digit mutation below rather * than by a magnitude check on a 2-digit value). * * malformed weight silently defaults to q=1 -- `if (q < 0) q = 1000;` * inserted in ngx_http_zstd_coding_weight() * -> "a fourth q decimal digit makes the element non-matching" FAILS. * * quote skip disabled -- ngx_http_zstd_skip_quoted() changed to * `return p;` unconditionally (before its own bounds check) * -> the suite does not FAIL, it HANGS: case_skip_quoted_unterminated's * unterminated `"..."` value makes the caller's `while (*p != ',') * skip-quoted-else-advance` loop call skip_quoted() at the same `p` * forever, since it now always returns its input unchanged. Verified * with a 5s `timeout`, observed exit 124 (see * ci/adoption-findings.md for the exact command). This is itself * the reason case_skip_quoted_unterminated exists: an unterminated * quote is exactly the input that turns "quote skip is a no-op" * from a wrong-answer bug into a hang, which a FAIL-only check * would never distinguish from a slow pass. * * boundary off by one -- in ngx_http_zstd_eval_qvalue(), `p <= end` * instead of `p < end` in the parameter-name scan loop condition * -> "'q' with no '=value' makes the element non-matching, not an * implicit q=1" FAILS, both under plain cc and under * clang -fsanitize=address,undefined (no heap-buffer-overflow was * raised by ASan for this specific mutation -- the outer `while (p * < end && *p == ';')` guard means the widened inner condition is * only ever reached with p == end already inside the caller's * bound, so this mutation is a logic bug the tests catch, not a * memory-safety one; recorded here rather than assumed, since * "off by one" does not automatically imply "overread"). No * sanitizer trap fired and no crash occurred; only the checked * assertion moved. * * single-pass cursor: quoted-name fallback dropped -- in * ngx_http_zstd_eval_qvalue(), `*pp = p` unconditionally instead of * only when no DQUOTE was seen inside a parameter name * -> "differential: every hand-written boundary input agrees with * the pre-single-pass parser" FAILS on `gzip;x"y=1,zstd` * (zstd new=1000 old=-1), plus generated divergences, and * "single-pass: DQUOTE inside a parameter name rescans from ';'" * FAILS. * * single-pass cursor: caller's element skip made quote-blind -- in * ngx_http_zstd_coding_weight(), the post-eval skip to the next ',' * reduced to a plain `p++` walk * -> the same differential check FAILS on `gzip;q=0.5"a,zstd` * (zstd new=1000 old=-1) and on generated inputs. * * single-pass cursor: caller's element skip removed outright * (`if (0)` around it) * -> the suite HANGS (run.sh's 60s timeout, exit 124): a malformed * element leaves p on its junk byte with nothing advancing it. * * (equivalent mutant, recorded so nobody re-runs it) forcing the * fallback on every -1 return (`quoted_name = 1` under `malformed:`) * survives: a rescan from the ';' and a resume at the junk byte * produce the same element boundary, so the -1 path's single pass is * a cost property, not an observable one. * * Extend: add a CASE() function and one line in main(). */ /* Path-relative so any analyser parses this TU without -I flags: an * unparsed TU is silently skipped, which reads as clean. */ #include "../../fuzz/ngx_shim.h" #include "../../fuzz/generated_parser.inc" #include #include #include #include #include static int failures; static int checks; static void check(int ok, const char *what) { checks++; if (ok) { printf("ok %s\n", what); return; } printf("FAIL %s\n", what); failures++; } /* Helper: parse `s` (a NUL-terminated C string used only to size the * buffer -- the underlying ae.len is exact, no NUL reliance) through the * full decision entry point. */ static ngx_int_t decide(const char *s) { ngx_str_t ae; ae.data = (u_char *) s; ae.len = strlen(s); return ngx_http_zstd_accept_encoding(&ae); } /* Helper: the generic weight walker, parameterized like the "dcz" caller * in the fuzz harness -- exercises the wildcard-not-allowed branch that * ngx_http_zstd_accept_encoding() (allow_wildcard=1) never reaches. */ static ngx_int_t weight_no_wildcard(const char *s, const char *coding) { ngx_str_t ae; ae.data = (u_char *) s; ae.len = strlen(s); return ngx_http_zstd_coding_weight(&ae, coding, strlen(coding), 0); } static void case_empty_header(void) { check(decide("") == NGX_DECLINED, "empty header declines"); } static void case_explicit_zstd_no_params(void) { check(decide("zstd") == NGX_OK, "bare 'zstd' accepts (implicit q=1)"); } static void case_explicit_zstd_q1(void) { check(decide("zstd;q=1") == NGX_OK, "'zstd;q=1' accepts"); } static void case_explicit_zstd_q0(void) { check(decide("zstd;q=0") == NGX_DECLINED, "'zstd;q=0' declines"); } static void case_explicit_zstd_fractional(void) { /* q=0.05 -> 50 milli-units, which is >0, so still accepted. */ check(decide("zstd;q=0.05") == NGX_OK, "'zstd;q=0.05' still accepts (>0)"); } static void case_zstd_q_exactly_zero_fraction(void) { check(decide("zstd;q=0.000") == NGX_DECLINED, "'zstd;q=0.000' declines (exactly zero)"); } static void case_wildcard_only(void) { check(decide("*") == NGX_OK, "bare '*' accepts zstd via wildcard"); } static void case_wildcard_q0(void) { check(decide("*;q=0") == NGX_DECLINED, "'*;q=0' declines"); } static void case_other_coding_only(void) { check(decide("gzip") == NGX_DECLINED, "'gzip' alone declines (no zstd, no wildcard)"); } static void case_explicit_overrides_permissive_wildcard(void) { /* This is the precedence rule the wildcard-flip mutation breaks. */ check(decide("*, zstd;q=0") == NGX_DECLINED, "explicit 'zstd;q=0' overrides a permissive '*'"); } static void case_explicit_overrides_restrictive_wildcard(void) { check(decide("*;q=0, zstd") == NGX_OK, "explicit 'zstd' overrides a restrictive '*;q=0'"); } static void case_case_insensitive_token(void) { check(decide("ZsTd") == NGX_OK, "coding name match is case-insensitive"); } static void case_multiple_codings_with_zstd(void) { check(decide("gzip;q=1, deflate, zstd;q=0.5, br") == NGX_OK, "zstd found among several codings"); } static void case_stray_commas_and_ows(void) { check(decide(" , ,zstd , ") == NGX_OK, "stray commas and OWS around the list are tolerated (RFC 9110)"); } static void case_malformed_q_fourth_digit(void) { /* qvalue = 0*3DIGIT after the decimal point; a fourth digit is * malformed, so the element must not match -- and there is no * wildcard here to fall back on, so DECLINED. */ check(decide("zstd;q=0.1234") == NGX_DECLINED, "a fourth q decimal digit makes the element non-matching"); } static void case_malformed_q_missing_value(void) { check(decide("zstd;q=") == NGX_DECLINED, "'q=' with no digits makes the element non-matching"); } static void case_malformed_q_bare_param(void) { check(decide("zstd;q") == NGX_DECLINED, "'q' with no '=value' makes the element non-matching, " "not an implicit q=1"); } static void case_quoted_param_hides_comma(void) { /* * The comma inside the quoted x="..." parameter value must NOT be * read as an element separator -- if it were, the bytes after it * ("zstd") would be misparsed as a fresh, matching coding name. * This is exactly the skip_quoted seam: disabling it makes this * case wrongly accept. */ check(decide("gzip;x=\"a, zstd\"") == NGX_DECLINED, "a comma inside a quoted parameter value is not an element " "boundary (no phantom zstd token)"); } static void case_quoted_name_position_not_a_token(void) { /* A quoted-string can never be a valid coding name; the whole quoted * blob plus its trailing params must be skipped as one non-matching * element, not split on the interior comma. */ check(decide("\"a,zstd\"") == NGX_DECLINED, "a quoted string in name position is not a coding token"); } static void case_duplicate_explicit_token_last_wins(void) { check(decide("zstd;q=0, zstd;q=1") == NGX_OK, "a later duplicate explicit zstd token wins over an earlier one"); } static void case_end_of_buffer_no_trailing_nul_reliance(void) { /* Exercise a q= right at the exact end of the buffer -- ae.len is * exact with no allocated trailing NUL (mirrors the fuzz harness's * malloc(exact size) contract), so any read at or past ae->len is a * bounds violation the boundary-off-by-one mutation introduces. */ static const char raw[] = "zstd;q="; ngx_str_t ae; ae.data = (u_char *) raw; ae.len = sizeof(raw) - 1; /* exclude the C-string NUL from the walk */ check(ngx_http_zstd_accept_encoding(&ae) == NGX_DECLINED, "'q=' ending exactly at buffer end is malformed, not an overread"); } static void case_generic_weight_wildcard_not_allowed(void) { /* dcz's own contract: a bare "*" must NOT turn dcz on. */ check(weight_no_wildcard("*", "dcz") == -1, "generic weight walker: wildcard does not apply when the caller " "disallows it (dcz contract)"); check(weight_no_wildcard("dcz;q=1", "dcz") == 1000, "generic weight walker: explicit token still applies with " "allow_wildcard=0"); } static void case_nonq_param_is_skipped_trailing_q_honoured(void) { /* * Deliberate, specified divergence from nginx core. Core's * ngx_http_gzip_accept_encoding() NGX_DECLINEs on any post-';' byte * that is not 'q'/'Q'/OWS, so it drops the whole element on * "gzip;foo=bar" -- losing any weight the client did express. * * This parser instead skips an unrecognized parameter's value to the * next top-level delimiter and keeps parsing, so a trailing "q" on * the SAME element is still found and honoured. That is strictly more * faithful to what the client asked for, in both directions: * q=0 must still suppress, q=1 must still accept. Ignoring an * unrecognized parameter is the standard HTTP robustness posture; the * leniency is a superset that only ever compresses for a client that * explicitly named zstd. */ check(decide("zstd;foo=bar;q=0") == NGX_DECLINED, "a skipped non-q parameter must not swallow the trailing q=0 " "(client explicitly refused zstd)"); check(decide("zstd;foo=bar;q=1") == NGX_OK, "a skipped non-q parameter must not swallow the trailing q=1"); check(decide("zstd;foo=bar") == NGX_OK, "an unrecognized parameter with no q defaults to q=1, rather " "than dropping an element the client explicitly asked for"); check(decide("zstd;foo") == NGX_OK, "a bare unrecognized parameter is likewise ignored, not fatal"); } static void case_nonq_param_quoted_value_delimiter_scan(void) { /* * The quoted-string arm of the skip loop: a top-level ';' or ',' that * appears INSIDE a quoted parameter value must not be mistaken for the * end of the value, or the trailing q would be parsed out of the wrong * position (or missed entirely). */ check(decide("zstd;foo=\"a,b;c\";q=0") == NGX_DECLINED, "a ';' and ',' inside a quoted non-q value do not end the value; " "the real trailing q=0 is still the one honoured"); check(decide("zstd;foo=a\"b\";q=0") == NGX_DECLINED, "a quoted run starting mid-value is skipped as one unit and " "parsing resumes at the ';', finding the trailing q=0"); check(decide("zstd;foo=\"unterm") == NGX_OK, "an unterminated quoted non-q value runs to end without an OOB " "read and leaves the element at the default q=1"); } static void case_skip_quoted_unterminated(void) { /* ngx_http_zstd_skip_quoted() must still terminate (advance past at * least the opening DQUOTE) on an unterminated quote, or the caller's * loop stalls. Exercised indirectly: an unterminated quote must not * hang the whole decision walk. */ check(decide("gzip;x=\"unterminated, zstd") == NGX_DECLINED, "an unterminated quoted parameter value does not hang the walk " "and does not fabricate a matching zstd token"); } /* * Direct oracle on ngx_http_zstd_parse_q_fraction() itself -- every case * above only ever asks "did the whole header accept or decline", which is * satisfied by any weight > 0 (or, for a q=0 case, is satisfied by any * weight == 0). Neither shape can tell an exact milliweight, or a wrong * cursor, from a correct one: a tens-digit arithmetic mistake that still * lands on "some positive number" or "exactly zero" is invisible to * decide(). These cases call the pure digit-walk helper directly and * assert BOTH of its documented outputs -- the returned weight, and the * cursor position it leaves in *p -- which is the only way to observe * that "10s digit worth 10" specifically (not just "worth something") and * that "a 4th digit is left unconsumed" (the caller's trailing-junk * contract, ngx_http_zstd_eval_qvalue()'s q += ... callers all depend on * *p pointing AT the offending byte, not past it). * * OBSERVED: mutating the tens weight from `* 10` to `* 11` in * ngx_http_zstd_parse_q_fraction() left all 31 pre-existing checks above * still green (every accept/decline decision the header drives is * unaffected by an 11-vs-10 tens digit, since every fixture above uses at * most one nonzero fractional digit or crosses the q>0/q==0 boundary by a * wide margin). case_fraction_exhaustive_three_digit() below is what * catches it -- see ci/adoption-findings.md for the exact command and * its failing output. */ static void run_one_fraction(const char *digits, ngx_int_t want_frac, ptrdiff_t want_consumed, const char *what) { u_char *p; const u_char *end; ngx_int_t frac; ptrdiff_t consumed; p = (u_char *) digits; end = (const u_char *) digits + strlen(digits); frac = ngx_http_zstd_parse_q_fraction(end, &p); consumed = p - (u_char *) digits; if (frac != want_frac || consumed != want_consumed) { printf("FAIL %s (got frac=%ld consumed=%ld, want frac=%ld " "consumed=%ld)\n", what, (long) frac, (long) consumed, (long) want_frac, (long) want_consumed); checks++; failures++; return; } check(1, what); } static void case_fraction_empty_input(void) { /* "" -- end == p already, no digit consumed at all. */ run_one_fraction("", 0, 0, "fraction: empty input -> frac=0, cursor=0"); } static void case_fraction_one_digit(void) { run_one_fraction("5", 500, 1, "fraction: one digit '5' -> frac=500, " "cursor advances 1"); } static void case_fraction_two_digit(void) { /* This is exactly the boundary the auditor's tens-weight mutation * (10 -> 11) breaks: '5' contributes 50 only if the tens digit is * genuinely worth 10 per unit. */ run_one_fraction("05", 50, 2, "fraction: two digits '05' -> frac=50 " "(tens digit worth exactly 10/unit), " "cursor advances 2"); } static void case_fraction_three_digit(void) { run_one_fraction("123", 123, 3, "fraction: three digits '123' -> " "frac=123, cursor advances 3"); } static void case_fraction_four_digit_leaves_fourth_unconsumed(void) { /* * The documented contract this row exists to pin: after three digits * the cursor stops WITHOUT consuming a fourth digit byte, so the * caller's trailing-junk check (ngx_http_zstd_eval_qvalue() / * ngx_http_zstd_coding_weight()'s post-qvalue scan) sees the '4' and * rejects the element. Assert the exact stop position, not just "some * prefix was consumed". */ run_one_fraction("1234", 123, 3, "fraction: four digits '1234' -> only the first three are " "consumed (frac=123), the 4th digit is LEFT UNCONSUMED"); } static void case_fraction_truncated_mid_walk(void) { /* `end` reached after one digit: the walk must stop at `end`, not * read past it, and the cursor must sit exactly at `end`. */ u_char buf[1] = { '7' }; u_char *p = buf; const u_char *end = buf + 1; ngx_int_t frac = ngx_http_zstd_parse_q_fraction(end, &p); check(frac == 700 && p == end, "fraction: input truncated after one digit -- cursor stops " "exactly at end, frac=700"); } static void case_fraction_truncated_mid_walk_two_digits(void) { u_char buf[2] = { '4', '2' }; u_char *p = buf; const u_char *end = buf + 2; ngx_int_t frac = ngx_http_zstd_parse_q_fraction(end, &p); check(frac == 420 && p == end, "fraction: input truncated after two digits -- cursor stops " "exactly at end, frac=420"); } static void case_fraction_non_digit_terminator_not_consumed(void) { /* A non-digit byte (here ';', the real-world terminator after * "q=0.5") must stop the walk WITHOUT consuming it -- the caller * needs *p pointing AT that byte, e.g. to continue scanning * parameters. */ run_one_fraction("5;q=1", 500, 1, "fraction: non-digit terminator ';' is not consumed, " "cursor sits on it"); } static void case_fraction_non_digit_terminator_at_start(void) { /* Not even one digit: the very first byte is non-digit, so nothing * is consumed and frac stays 0. */ run_one_fraction("x", 0, 0, "fraction: non-digit terminator at position 0 -- frac=0, " "cursor=0 (nothing consumed)"); } static void case_fraction_exhaustive_three_digit(void) { /* * Exhaustive 000..999: every three-digit fraction, asserting the * EXACT returned weight (hundreds*100 + tens*10 + units*1) and that * the cursor always advances by exactly 3. This is the check that * actually catches an arithmetic-weight mutation on any single digit * position -- a spot check on a handful of values can miss a * mutation whose effect happens to cancel out on those particular * inputs, but summing every input in the space cannot. */ /* * NUL-terminated for the "%s" diagnostic below; the parser end pointer * still targets exactly the first 3 bytes, so the '\0' at digits[3] is * never itself in the parsed slice. */ char digits[4]; int n, h, t, u; int local_failures = 0; digits[3] = '\0'; for (n = 0; n < 1000; n++) { h = n / 100; t = (n / 10) % 10; u = n % 10; digits[0] = (char) ('0' + h); digits[1] = (char) ('0' + t); digits[2] = (char) ('0' + u); { u_char *p = (u_char *) digits; const u_char *end = (const u_char *) digits + 3; ngx_int_t frac = ngx_http_zstd_parse_q_fraction(end, &p); ngx_int_t want = h * 100 + t * 10 + u; ptrdiff_t consumed = p - (u_char *) digits; checks++; if (frac != want || consumed != 3) { if (local_failures < 5) { printf("FAIL fraction: exhaustive \"%s\" -> got " "frac=%ld consumed=%ld, want frac=%ld " "consumed=3\n", digits, (long) frac, (long) consumed, (long) want); } local_failures++; failures++; } } } if (local_failures == 0) { printf("ok fraction: exhaustive 000..999 all match " "hundreds*100+tens*10+units*1, cursor always advances 3 " "(1000 cases)\n"); } else { printf("FAIL fraction: exhaustive 000..999 -- %d/1000 mismatched " "(first 5 shown above)\n", local_failures); } } /* ------------------------------------------------------------------ * * Chained Accept-Encoding field lines (ngx_http_zstd_chain_coding_weight) * * nginx chains duplicate Accept-Encoding request header lines on * ->next. RFC 9110 section 5.3 makes that identical to the single field * whose value is the lines joined with commas, so * * Accept-Encoding: gzip * Accept-Encoding: zstd * * IS "gzip, zstd" and accepts zstd. Every case below states one such * request as its line list and asserts the decision the joined value * would have produced. * * Duplicate coding values retain the order of the comma-joined field: the * later explicit value decides, just as it does within one field line. * ------------------------------------------------------------------ */ /* * Build a ->next chain over up to 4 literal field values and run the * chain walker across it. Storage is static per call site depth, which is * fine: each case builds, decides, and discards before the next runs. */ static ngx_int_t chain_weight(const char *coding, ngx_uint_t allow_wildcard, const char *l0, const char *l1, const char *l2, const char *l3) { static ngx_table_elt_t e[4]; const char *lines[4]; ngx_uint_t n; lines[0] = l0; lines[1] = l1; lines[2] = l2; lines[3] = l3; for (n = 0; n < 4 && lines[n] != NULL; n++) { e[n].value.data = (u_char *) lines[n]; e[n].value.len = strlen(lines[n]); e[n].next = NULL; if (n > 0) { e[n - 1].next = &e[n]; } } if (n == 0) { return -1; } return ngx_http_zstd_chain_coding_weight(&e[0], coding, strlen(coding), allow_wildcard); } /* zstd decision over a chain: same predicate as ngx_http_zstd_accepts(). */ static ngx_int_t chain_zstd(const char *l0, const char *l1, const char *l2, const char *l3) { return chain_weight("zstd", 1, l0, l1, l2, l3) > 0 ? NGX_OK : NGX_DECLINED; } /* dcz decision over a chain: explicit token only, no wildcard. */ static ngx_int_t chain_dcz(const char *l0, const char *l1, const char *l2, const char *l3) { return chain_weight("dcz", 0, l0, l1, l2, l3) > 0 ? NGX_OK : NGX_DECLINED; } /* * THE WITNESS. Two lines, the acceptable coding on the SECOND. Before the * chain walker only the first line was parsed, so this declined. */ static void case_chain_gzip_then_zstd(void) { check(chain_zstd("gzip", "zstd", NULL, NULL) == NGX_OK, "chain: 'gzip' then 'zstd' accepts (joined: \"gzip, zstd\")"); } static void case_chain_zstd_then_gzip(void) { check(chain_zstd("zstd", "gzip", NULL, NULL) == NGX_OK, "chain: 'zstd' then 'gzip' accepts"); } /* A single line is unchanged: the walker must not alter the common case. */ static void case_chain_single_line_unchanged(void) { check(chain_zstd("gzip", NULL, NULL, NULL) == NGX_DECLINED, "chain: single 'gzip' line still declines"); check(chain_zstd("zstd", NULL, NULL, NULL) == NGX_OK, "chain: single 'zstd' line still accepts"); } /* Duplicate explicit values follow the received (comma-joined) order. */ static void case_chain_q0_first_line(void) { check(chain_zstd("zstd;q=0", "gzip", NULL, NULL) == NGX_DECLINED, "chain: 'zstd;q=0' then 'gzip' declines"); } static void case_chain_q0_second_line(void) { check(chain_zstd("gzip", "zstd;q=0", NULL, NULL) == NGX_DECLINED, "chain: 'gzip' then 'zstd;q=0' declines"); } static void case_chain_q0_then_q1_accepts(void) { check(chain_zstd("zstd;q=0", "zstd;q=1", NULL, NULL) == NGX_OK, "chain: later 'zstd;q=1' overrides an earlier refusal"); } static void case_chain_q1_then_q0_stays_declined(void) { check(chain_zstd("zstd;q=1", "zstd;q=0", NULL, NULL) == NGX_DECLINED, "chain: 'zstd;q=1' then 'zstd;q=0' declines"); } /* The later value wins across field lines, as it does within one line. */ static void case_chain_two_nonzero_weights(void) { check(chain_weight("zstd", 1, "zstd;q=0.5", "zstd;q=1", NULL, NULL) == 1000, "chain: later explicit weight wins (0.5 then 1 -> 1000)"); check(chain_zstd("zstd;q=0.5", "zstd;q=1", NULL, NULL) == NGX_OK, "chain: two non-zero weights still accept"); } /* A wildcard on a LATER line must still be seen. */ static void case_chain_wildcard_later_line(void) { check(chain_zstd("gzip", "*", NULL, NULL) == NGX_OK, "chain: '*' on the second line accepts"); } /* An explicit q=0 anywhere overrides a permissive '*' on another line. */ static void case_chain_wildcard_vs_explicit_q0(void) { check(chain_zstd("*", "zstd;q=0", NULL, NULL) == NGX_DECLINED, "chain: explicit 'zstd;q=0' overrides a '*' on an earlier line"); check(chain_zstd("zstd;q=0", "*", NULL, NULL) == NGX_DECLINED, "chain: explicit 'zstd;q=0' overrides a '*' on a later line"); } /* '*;q=0' on a later line is a refusal too, when nothing names zstd. */ static void case_chain_wildcard_q0_later(void) { check(chain_zstd("gzip", "*;q=0", NULL, NULL) == NGX_DECLINED, "chain: '*;q=0' on the second line declines"); } /* Three and four lines: the walk must not stop at two. */ static void case_chain_three_lines(void) { check(chain_zstd("gzip", "br", "zstd", NULL) == NGX_OK, "chain: three lines, 'zstd' on the third, accepts"); } static void case_chain_four_lines(void) { check(chain_zstd("gzip", "br", "deflate", "zstd") == NGX_OK, "chain: four lines, 'zstd' on the fourth, accepts"); check(chain_zstd("gzip", "br", "deflate", "identity") == NGX_DECLINED, "chain: four lines with no zstd and no '*' declines"); } /* An empty later line contributes nothing and must not break the walk. */ static void case_chain_empty_line(void) { check(chain_zstd("", "zstd", NULL, NULL) == NGX_OK, "chain: empty first line then 'zstd' accepts"); check(chain_zstd("zstd", "", NULL, NULL) == NGX_OK, "chain: 'zstd' then an empty line accepts"); } /* * Splitting a list mid-way across lines is the whole point of RFC 9110 * section 5.3: these lines joined are "gzip, zstd;q=0.5, br". */ static void case_chain_split_list(void) { check(chain_zstd("gzip", "zstd;q=0.5, br", NULL, NULL) == NGX_OK, "chain: a list split across lines negotiates as the joined value"); } /* --- dcz variants: explicit token only, '*' must never turn dcz on --- */ static void case_chain_dcz_second_line(void) { check(chain_dcz("zstd", "dcz", NULL, NULL) == NGX_OK, "chain/dcz: 'zstd' then 'dcz' accepts"); } static void case_chain_dcz_first_line(void) { check(chain_dcz("dcz", "zstd", NULL, NULL) == NGX_OK, "chain/dcz: 'dcz' then 'zstd' accepts"); } static void case_chain_dcz_q0_second_line(void) { check(chain_dcz("zstd", "dcz;q=0", NULL, NULL) == NGX_DECLINED, "chain/dcz: 'zstd' then 'dcz;q=0' declines"); } static void case_chain_dcz_q0_first_line(void) { check(chain_dcz("dcz;q=0", "zstd", NULL, NULL) == NGX_DECLINED, "chain/dcz: 'dcz;q=0' then 'zstd' declines"); } static void case_chain_dcz_q0_then_q1_accepts(void) { check(chain_dcz("dcz;q=0", "dcz;q=1", NULL, NULL) == NGX_OK, "chain/dcz: later 'dcz;q=1' overrides an earlier refusal"); } static void case_chain_dcz_q1_then_q0_declines(void) { check(chain_dcz("dcz;q=1", "dcz;q=0", NULL, NULL) == NGX_DECLINED, "chain/dcz: later 'dcz;q=0' overrides an earlier allowance"); } /* * The wildcard gate survives the chain: a '*' on ANY line must not turn * dcz on, because only a client holding the dictionary can decode it. */ static void case_chain_dcz_wildcard_never_matches(void) { check(chain_dcz("zstd", "*", NULL, NULL) == NGX_DECLINED, "chain/dcz: '*' on a later line must NOT turn dcz on"); check(chain_dcz("*", "zstd", NULL, NULL) == NGX_DECLINED, "chain/dcz: '*' on an earlier line must NOT turn dcz on"); } static void case_chain_dcz_three_lines(void) { check(chain_dcz("gzip", "zstd", "dcz", NULL) == NGX_OK, "chain/dcz: three lines, 'dcz' on the third, accepts"); } /* * --------------------------------------------------------------------------- * Differential test: single-pass parameter scan vs the pre-single-pass parser * * ngx_http_zstd_eval_qvalue() used to take its cursor by value, so * ngx_http_zstd_coding_weight() rescanned every parameter byte (quoted * strings included) to find the next top-level ','. It now advances the * caller's cursor. The two functions below are the VERBATIM pre-change * bodies (renamed; they share ngx_http_zstd_skip_quoted() and * ngx_http_zstd_parse_q_fraction() with the shipped slice, which the change * did not touch), kept here as the oracle: for every generated and * hand-written input the shipped walker must return the same weight for * "zstd" (wildcard allowed), for "dcz" (wildcard refused) and the same * accept/decline verdict. Any divergence is a negotiation change, which the * single-pass rework is not allowed to make. * * Every input is copied into a malloc'd buffer of EXACTLY ae.len bytes with * no trailing NUL, so an ASan build of this suite traps a read past the * field on either implementation. */ static ngx_int_t ref_eval_qvalue(const ngx_str_t *ae, u_char *p) { const u_char *end = ae->data + ae->len; ngx_int_t q = 1000; /* no q parameter → q=1 */ ngx_int_t q_seen = 0; /* reject a second "q" parameter (RFC 9110) */ while (p < end && *p == ';') { const u_char *nstart, *nend; ngx_int_t is_q; p++; /* skip ';' */ while (p < end && (*p == ' ' || *p == '\t')) { p++; } /* parameter name */ nstart = p; while (p < end && *p != '=' && *p != ';' && *p != ',' && *p != ' ' && *p != '\t') { p++; } nend = p; /* * RFC 9110 has no empty-parameter production, so "zstd;;q=1" is * malformed rather than "a skipped parameter followed by q=1". * Reject it instead of silently resolving the element to q=1. */ if (nend == nstart) { return -1; } is_q = (nend - nstart == 1 && (nstart[0] == 'q' || nstart[0] == 'Q')); while (p < end && (*p == ' ' || *p == '\t')) { p++; } if (p < end && *p == '=') { p++; while (p < end && (*p == ' ' || *p == '\t')) { p++; } if (is_q) { /* * Strict qvalue grammar. Leading digit must be 0 or 1. */ if (q_seen) { return -1; /* repeated "q" parameter */ } q_seen = 1; if (p >= end) { return -1; /* "q=" with no value */ } if (*p == '0') { p++; q = 0; if (p < end && *p == '.') { p++; q += ngx_http_zstd_parse_q_fraction(end, &p); } } else if (*p == '1') { p++; q = 1000; if (p < end && *p == '.') { int i = 0; p++; while (p < end && *p == '0' && i < 3) { p++; i++; } } } else { return -1; /* leading digit not 0 or 1 */ } /* * After a valid qvalue only OWS / ';' / ',' / end may * follow. A fourth decimal digit or trailing junk * (q=1x, q=0.0001) lands here as a non-delimiter byte and * is rejected. */ if (p < end && *p != ' ' && *p != '\t' && *p != ';' && *p != ',') { return -1; } } else { /* * non-q parameter: skip its value to the next top-level ';' * (another parameter) or ',' (next element), stepping over a * quoted-string so an embedded delimiter is not mistaken for * the value's end. */ while (p < end && *p != ';' && *p != ',') { if (*p == '"') { p = ngx_http_zstd_skip_quoted(p, end); } else { p++; } } } } else { /* parameter present without a value */ if (is_q) { return -1; /* "q" with no "=value" is malformed */ } } while (p < end && (*p == ' ' || *p == '\t')) { p++; } /* * After the OWS that may trail any parameter (its q-specific * check above only catches junk BEFORE this OWS skip, e.g. * "q=1x"), only ';' (another parameter), ',' (next element), or * end may follow -- anything else is trailing junk the loop's own * "while (*p == ';')" condition would otherwise silently accept * as "no more parameters" instead of rejecting (e.g. * "zstd;q=1 garbage" previously parsed as zstd;q=1). */ if (p < end && *p != ';' && *p != ',') { return -1; } } return q; } static ngx_int_t ref_coding_weight(const ngx_str_t *ae, const char *coding, size_t coding_len, ngx_uint_t allow_wildcard) { u_char *p = ae->data; const u_char *end = ae->data + ae->len; ngx_int_t coding_q = -1; /* explicit `coding` weight, -1 = absent */ ngx_int_t star_q = -1; /* "*" wildcard weight, -1 = absent */ while (p < end) { u_char *tok; const u_char *name_end; ngx_int_t is_coding, is_star, q; /* Skip OWS and empty list elements (RFC 9110 allows stray * commas, e.g. ", ,zstd"). */ while (p < end && (*p == ' ' || *p == '\t' || *p == ',')) { p++; } if (p >= end) { break; } /* The coding name runs until OWS, ';' (params), ',' (next * element), or a DQUOTE. A '"' can never be part of a valid coding * token; stopping here keeps a quoted-string that opens in * name position (e.g. `"a,zstd "`) from being split on a comma * inside the quotes — the quote-aware element-skip below then * swallows the whole quoted blob and the element declines. Without * this stop, the bytes after an in-quote comma are mis-read as a * fresh coding name and can fabricate a phantom "zstd" token. */ tok = p; while (p < end && *p != ' ' && *p != '\t' && *p != ';' && *p != ',' && *p != '"') { p++; } name_end = p; is_coding = ((size_t) (name_end - tok) == coding_len && ngx_strncasecmp(tok, (u_char *) coding, coding_len) == 0); is_star = (name_end - tok == 1 && tok[0] == '*'); /* Step over any OWS between the name and its ';' or ','. */ while (p < end && (*p == ' ' || *p == '\t')) { p++; } /* * Only ';' (parameters), ',' (next element) or end of field may * follow a coding name. RFC 9110 12.5.3 makes `codings` a token, * so anything else means this element is not the coding it looked * like: `zstd"x` and `zstd "x` advertise nothing, and neither does * `zstd x`. nginx's own ngx_http_gzip_accept_encoding() applies * the same rule, so accepting these diverged from the sibling * filter and compressed for clients that never offered zstd. * * The check must sit AFTER the OWS skip: the name scan stops on * OWS as well as on '"', so testing the stopping byte alone * catches `zstd"x` and misses everything hiding behind a space. * The quote-aware element-skip below still swallows the rest of * the element, so the phantom-token guard is unchanged. */ if (p < end && *p != ';' && *p != ',') { is_coding = 0; is_star = 0; } q = 1000; /* no parameters → q=1 */ if (p < end && *p == ';') { q = ref_eval_qvalue(ae, p); } if (q >= 0) { if (is_coding) { coding_q = q; /* a later duplicate explicit token wins */ } else if (is_star) { star_q = q; } } /* q < 0 → malformed weight: leave this element non-matching. */ /* * Skip the remainder of this element up to the next top-level comma, * stepping over any quoted-string so a ',' inside quotes is not * mistaken for an element boundary (which would otherwise let a * quoted comma fabricate a phantom coding token from the bytes that * follow it, e.g. `gzip;x="a, zstd";q=1`). */ while (p < end && *p != ',') { if (*p == '"') { p = ngx_http_zstd_skip_quoted(p, end); } else { p++; } } } /* * An explicit token decides the result (even q=0, which then * overrides a permissive "*"). With no explicit token, the "*" * wildcard applies if present and permitted by the caller. */ if (coding_q >= 0) { return coding_q; } if (allow_wildcard && star_q >= 0) { return star_q; } return -1; } static ngx_int_t ref_accept_encoding(const ngx_str_t *ae) { return ref_coding_weight(ae, "zstd", 4, 1) > 0 ? NGX_OK : NGX_DECLINED; } /* xorshift64*: fixed seed, so a divergence is reproducible by index. */ static uint64_t diff_rng_state = 0x9e3779b97f4a7c15ULL; static uint32_t diff_rng(void) { uint64_t x = diff_rng_state; x ^= x >> 12; x ^= x << 25; x ^= x >> 27; diff_rng_state = x; return (uint32_t) ((x * 0x2545f4914f6cdd1dULL) >> 32); } /* * Fragment alphabet. Weighted toward the delimiters and q spellings the * parser branches on; bytes 0x80..0xff and NUL come from the "raw byte" * fragment so non-ASCII and embedded-NUL headers are covered too. */ static const char *const diff_frag[] = { "zstd", "ZSTD", "Zstd", "gzip", "dcz", "DCZ", "br", "*", "identity", ",", ",", ",", ", ", " ,", " , ", ",,", ";", ";", "; ", " ;", ";;", " ", "\t", " ", "=", "= ", " =", "q=", "q=1", "q=1.", "q=1.0", "q=1.00", "q=1.000", "q=1.0000", "q=1.001", "q=0", "q=0.", "q=0.5", "q=0.05", "q=0.005", "q=0.0001", "q=0.999", "q=2", "q=-1", "q=.5", "q= 1", "q =1", "Q=0.3", "q=1x", "q=0.5x", "q", "q=\"1\"", "q=1\"", "qq=1", "x=1", "x", "x=", "level=9", "a=b=c", "\"", "\"", "\\", "\\\"", "\\,", "\\;", "\"a,b\"", "\"a;b\"", "\"a\\\"b\"", "\"a\\,b\"", "\"unterminated", "\"zstd\"", "\"a,zstd\"", "\", zstd;q=1\"", "x=\"a,zstd\"", "\"\"", "\"\\\"", "\"\\", "zstd;q=0", "zstd;q=1", "*;q=0", "*;q=0.5", "zstd;x=\"a,b\";q=0.5", "\r", "\n", "\x7f", NULL /* raw byte marker */ }; #define DIFF_NFRAG (sizeof(diff_frag) / sizeof(diff_frag[0])) #define DIFF_MAXLEN 8192 static size_t diff_generate(uint8_t *buf, size_t cap) { size_t n = 0, i; size_t frags = 1 + diff_rng() % 12; /* one case in 64 is a very long list */ if (diff_rng() % 64 == 0) { frags = 200 + diff_rng() % 600; } for (i = 0; i < frags && n < cap; i++) { const char *f = diff_frag[diff_rng() % DIFF_NFRAG]; size_t l; if (f == NULL) { uint32_t r = diff_rng(); /* raw byte: NUL, controls, or high bytes 0x80..0xff */ buf[n++] = (uint8_t) (r & 1 ? 0x80 | (r >> 8) : (r >> 8) & 0x1f); continue; } l = strlen(f); if (l > cap - n) { l = cap - n; } memcpy(buf + n, f, l); n += l; } return n; } static void diff_hex(const uint8_t *d, size_t n, char *out, size_t cap) { size_t i, o = 0; for (i = 0; i < n && o + 4 < cap; i++) { if (d[i] >= 0x20 && d[i] < 0x7f && d[i] != '\\') { out[o++] = (char) d[i]; } else { o += (size_t) snprintf(out + o, cap - o, "\\x%02x", d[i]); } } out[o] = '\0'; } /* * Returns 1 when old and new agree on every observable for this input, * 0 (and prints the input) otherwise. */ static int diff_one(const uint8_t *src, size_t n, const char *label) { ngx_str_t ae; uint8_t *exact; ngx_int_t nz, oz, ndcz, odcz, na, oa; int ok; exact = malloc(n ? n : 1); /* exact-size: ASan bounds the field */ if (exact == NULL) { printf("FAIL %s: malloc\n", label); return 0; } memcpy(exact, src, n); ae.data = exact; ae.len = n; nz = ngx_http_zstd_coding_weight(&ae, "zstd", 4, 1); oz = ref_coding_weight(&ae, "zstd", 4, 1); ndcz = ngx_http_zstd_coding_weight(&ae, "dcz", 3, 0); odcz = ref_coding_weight(&ae, "dcz", 3, 0); na = ngx_http_zstd_accept_encoding(&ae); oa = ref_accept_encoding(&ae); ok = (nz == oz && ndcz == odcz && na == oa); if (!ok) { char hex[512]; diff_hex(src, n < 120 ? n : 120, hex, sizeof(hex)); printf("FAIL %s: zstd new=%ld old=%ld dcz new=%ld old=%ld " "accept new=%ld old=%ld len=%zu input=\"%s\"%s\n", label, (long) nz, (long) oz, (long) ndcz, (long) odcz, (long) na, (long) oa, n, hex, n > 120 ? "..." : ""); } free(exact); return ok; } /* * Hand-written boundary inputs. Each is a case the generator might reach * only by chance; naming them here keeps the mutation controls stable. */ static const char *const diff_hand[] = { "", ",", ",,,", " ", ";", "zstd;", "zstd; ", "zstd ;", "zstd;;q=1", "zstd;q=", "zstd;q=1.000", "zstd;q=0.0001", "zstd;q=2", "zstd;q", "zstd;x=1;q=0.5", "zstd;q=0.5;x=1", "zstd;q=0.5;q=1", "zstd;x;q=0.5", "zstd;x=\"a;b\";q=0.5", "zstd;x=\"a,b\";q=0.5", "zstd;x=\"a\\\"b\";q=0", "zstd;x=\"a\\,b\";q=0", "zstd;x=\"a\\;b\";q=0", "zstd;x=\"unterminated", "zstd;x=\"a\\", "zstd;x=\"", "zstd;x=\"\\\"", "gzip;x=\"a,zstd\"", "gzip;x=\"a, zstd;q=1\"", "gzip;\"a,zstd,b\"", "gzip;\"a,zstd\"=1", "gzip;x\"y=1,zstd", "gzip;x\"y=1,zstd\"", "zstd;x\"y=1", "zstd;x\"=1", "gzip;q=2 zstd", "gzip;q=1x,zstd", "gzip;q=1 x,zstd", "gzip;q=0.5\"a,zstd", "gzip;q=1.0000\"a,zstd\",br", "gzip;q=\"1\",zstd", "gzip;q= ,zstd", "gzip;q=1 \"a,zstd", "gzip;q ,zstd", "gzip;x ,zstd", "gzip;x=,zstd", "gzip;x=a\"b,zstd\"c,zstd", "gzip;=1,zstd", "gzip; =1,zstd", "zstd;Q=0.3", "zstd;q=1.", "zstd;q=0.", "zstd;q=.5", "zstd;q= 1", "zstd;q =1", "zstd;q=-1", "zstd; q=1 ", "zstd;\tq=1\t", "zstd\t;q=1", "*;q=0,zstd;x=\"a,b\"", "dcz;q=0.5,zstd", "*,dcz", "dcz;x=\"a\",*", "zstd\"x", "zstd \"x", "zstd x", "\"a,zstd \"", "\"zstd\"", "zstd;x=\xff\xfe;q=0.5", "zstd;q=0.5\xff", "zstd\xff;q=1", "zstd;\xff=1;q=0.5", "zstd;x=\"\xff,\";q=0.5", "zstd;q=0.5;\x80", "gzip;x=\"a\", zstd;q=0.5, br;q=1", "gzip;x=\"a\",zstd;q=0.5;y=\"b,c\"", }; #define DIFF_NHAND (sizeof(diff_hand) / sizeof(diff_hand[0])) #define DIFF_CASES 100000 static void case_differential_hand_written(void) { size_t i; int bad = 0; for (i = 0; i < DIFF_NHAND; i++) { if (!diff_one((const uint8_t *) diff_hand[i], strlen(diff_hand[i]), "differential/hand")) { bad++; } } /* embedded NUL: strlen() cannot express it, so build it by hand */ if (!diff_one((const uint8_t *) "gzip;x=\0,zstd", 13, "differential/nul")) { bad++; } check(bad == 0, "differential: every hand-written boundary input agrees " "with the pre-single-pass parser"); } static void case_differential_generated(void) { static uint8_t buf[DIFF_MAXLEN]; size_t i, n, bad = 0, total = 0; for (i = 0; i < DIFF_CASES; i++) { char label[48]; n = diff_generate(buf, sizeof(buf)); total += n; snprintf(label, sizeof(label), "differential/gen#%zu", i); if (!diff_one(buf, n, label)) { bad++; if (bad > 10) { printf("... more than 10 divergences, stopping\n"); break; } } } printf("info differential: %d generated inputs, %zu bytes total\n", DIFF_CASES, total); check(bad == 0, "differential: 100000 generated inputs agree with the " "pre-single-pass parser (zstd weight, dcz weight, verdict)"); } /* * Three named pins on the cursor contract, so a mutation that survives the * differential (it cannot -- but a pin is cheaper to read than a diff * dump) has a value-stating check as well. */ static void case_single_pass_cursor_contract(void) { /* * A DQUOTE inside a parameter NAME is the one input where the * single-pass cursor must fall back to a rescan from the ';': a * quote-aware walk from there swallows `"y=1,zstd` as one quoted * blob, so zstd is never advertised. A cursor left after the value * would let ",zstd" surface as a second element. */ check(decide("gzip;x\"y=1,zstd") == NGX_DECLINED, "single-pass: DQUOTE inside a parameter name rescans from ';' " "(no phantom zstd after the quote)"); /* * A malformed weight leaves the cursor on the offending byte; the * caller must still skip to the next top-level ',' rather than start * a new element there (which would read "zstd" as a fresh token). */ check(decide("gzip;q=2 zstd") == NGX_DECLINED, "single-pass: malformed weight still skips the rest of the " "element (no phantom zstd after junk)"); /* well-formed: the cursor lands on the ',' and the next element is * read exactly once. */ check(weight_no_wildcard("gzip;x=\"a,b\";q=0.5, zstd;q=0.25", "zstd") == 250, "single-pass: cursor resumes at the top-level ',' after a quoted " "parameter value"); } int main(void) { case_empty_header(); case_explicit_zstd_no_params(); case_explicit_zstd_q1(); case_explicit_zstd_q0(); case_explicit_zstd_fractional(); case_zstd_q_exactly_zero_fraction(); case_wildcard_only(); case_wildcard_q0(); case_other_coding_only(); case_explicit_overrides_permissive_wildcard(); case_explicit_overrides_restrictive_wildcard(); case_case_insensitive_token(); case_multiple_codings_with_zstd(); case_stray_commas_and_ows(); case_malformed_q_fourth_digit(); case_malformed_q_missing_value(); case_malformed_q_bare_param(); case_quoted_param_hides_comma(); case_quoted_name_position_not_a_token(); case_duplicate_explicit_token_last_wins(); case_end_of_buffer_no_trailing_nul_reliance(); case_generic_weight_wildcard_not_allowed(); case_skip_quoted_unterminated(); case_nonq_param_is_skipped_trailing_q_honoured(); case_nonq_param_quoted_value_delimiter_scan(); case_fraction_empty_input(); case_fraction_one_digit(); case_fraction_two_digit(); case_fraction_three_digit(); case_fraction_four_digit_leaves_fourth_unconsumed(); case_fraction_truncated_mid_walk(); case_fraction_truncated_mid_walk_two_digits(); case_fraction_non_digit_terminator_not_consumed(); case_fraction_non_digit_terminator_at_start(); case_fraction_exhaustive_three_digit(); /* Chained Accept-Encoding field lines (RFC 9110 section 5.3). */ case_chain_gzip_then_zstd(); case_chain_zstd_then_gzip(); case_chain_single_line_unchanged(); case_chain_q0_first_line(); case_chain_q0_second_line(); case_chain_q0_then_q1_accepts(); case_chain_q1_then_q0_stays_declined(); case_chain_two_nonzero_weights(); case_chain_wildcard_later_line(); case_chain_wildcard_vs_explicit_q0(); case_chain_wildcard_q0_later(); case_chain_three_lines(); case_chain_four_lines(); case_chain_empty_line(); case_chain_split_list(); case_chain_dcz_second_line(); case_chain_dcz_first_line(); case_chain_dcz_q0_second_line(); case_chain_dcz_q0_first_line(); case_chain_dcz_q0_then_q1_accepts(); case_chain_dcz_q1_then_q0_declines(); case_chain_dcz_wildcard_never_matches(); case_chain_dcz_three_lines(); /* Single-pass parameter scan vs the pre-change parser. */ case_differential_hand_written(); case_differential_generated(); case_single_pass_cursor_contract(); printf("\n%d/%d checks passed\n", checks - failures, checks); return failures ? 1 : 0; } myguard-labs-nginx-zstd-module-3486b5b/ci/tests/unit/test_accept_encoding_legacy.c000066400000000000000000000166311525232545500304150ustar00rootroot00000000000000/* Copyright (C) 2026 Thijs Eilander * SPDX-License-Identifier: BSD-2-Clause * * nginx 1.22.1-shaped regression checks. Its ngx_table_elt_t has no next * member; repeated request headers remain in headers_in.headers. * * This TU is compiled twice by ci/tests/unit/run.sh: once with * -DNGX_ZSTD_LEGACY_SHIM (the pre-1.23.0 shape, exercising the #else branch * of ngx_http_zstd_request_coding_weight() that walks headers_in.headers), * and once without it (the >= 1.23.0 shape, exercising the ->next chain via * ngx_http_zstd_chain_coding_weight()). Every case below therefore runs * under both branches and must agree -- branch-equivalence is the oracle * for A30-F1 ("repeated Accept-Encoding fields must not be dropped on * old-nginx"), since a real second implementation of RFC 9110 comma-joining * does not exist to diff against. */ #include "../../fuzz/ngx_shim.h" #define NGX_HTTP_ZSTD_TEST_COUNT_CODING_WEIGHT static ngx_uint_t ngx_http_zstd_test_coding_weight_calls; #include "../../fuzz/generated_parser.inc" #include #include #include #define MAX_HEADERS 8 static int failures; static int checks; static void check(int ok, const char *what) { checks++; if (ok) { printf("ok %s\n", what); } else { printf("FAIL %s\n", what); failures++; } } /* * Build a headers_in.headers list from `n` (name, value) pairs and decide * ngx_http_zstd_accepts() over it. A NULL name defaults to "Accept-Encoding" * -- pass an explicit non-AE name to interleave an unrelated header, or a * case-varied spelling ("accept-encoding" / "ACCEPT-ENCODING") to prove the * match is case-insensitive (ngx_strncasecmp, matching upstream). */ static ngx_int_t request_decide_named(const char *names[], const char *values[], size_t n) { ngx_table_elt_t headers[MAX_HEADERS]; ngx_http_request_t r; size_t i; #if !defined(NGX_ZSTD_LEGACY_SHIM) ngx_table_elt_t *first_ae = NULL; ngx_table_elt_t *prev_ae = NULL; #endif if (n > MAX_HEADERS) { /* * A fixture bug, not a parser result: silently truncating would let * a future over-sized table test fewer headers than the case * claims and still report a pass. Fail loudly instead. */ fprintf(stderr, "test fixture error: %zu headers exceeds " "MAX_HEADERS (%d)\n", n, MAX_HEADERS); abort(); } memset(&r, 0, sizeof(r)); memset(headers, 0, sizeof(headers)); for (i = 0; i < n; i++) { const char *name = names[i] ? names[i] : "Accept-Encoding"; headers[i].key.data = (u_char *) name; headers[i].key.len = strlen(name); headers[i].value.data = (u_char *) values[i]; headers[i].value.len = strlen(values[i]); #if !defined(NGX_ZSTD_LEGACY_SHIM) headers[i].next = NULL; if (headers[i].key.len == sizeof("Accept-Encoding") - 1 && ngx_strncasecmp(headers[i].key.data, (u_char *) "Accept-Encoding", sizeof("Accept-Encoding") - 1) == 0) { if (first_ae == NULL) { first_ae = &headers[i]; } else { prev_ae->next = &headers[i]; } prev_ae = &headers[i]; } #endif } r.main = &r; #if !defined(NGX_ZSTD_LEGACY_SHIM) r.headers_in.accept_encoding = first_ae; #endif r.headers_in.headers.part.elts = headers; r.headers_in.headers.part.nelts = n; r.headers_in.headers.part.next = NULL; return ngx_http_zstd_accepts(&r); } static ngx_int_t request_decide(const char *first, const char *second) { const char *names[2] = { NULL, NULL }; const char *values[2] = { first, second }; return request_decide_named(names, values, 2); } int main(void) { ngx_http_zstd_test_coding_weight_calls = 0; check(request_decide("gzip", "br") == NGX_DECLINED, "two non-matching fields decline"); check(ngx_http_zstd_test_coding_weight_calls == 2, "each field is parsed exactly once"); check(request_decide("gzip", "zstd") == NGX_OK, "legacy list: gzip then zstd accepts"); check(request_decide("*", "zstd;q=0") == NGX_DECLINED, "legacy list: explicit zstd;q=0 overrides wildcard"); check(request_decide("zstd;q=0", "zstd;q=1") == NGX_OK, "later explicit zstd allowance wins"); check(request_decide("zstd;q=1", "zstd;q=0") == NGX_DECLINED, "later explicit zstd refusal wins"); /* A30-F1: reversed order of the named case. */ check(request_decide("zstd;q=0", "*") == NGX_DECLINED, "A30-F1 reversed: explicit zstd;q=0 first, wildcard second, still declines"); /* A30-F1: partial q rather than exact zero. */ check(request_decide("zstd;q=0.5", "zstd;q=0") == NGX_DECLINED, "A30-F1: zstd;q=0.5 then zstd;q=0 -- later explicit weight wins, declines"); check(request_decide("zstd;q=0", "zstd;q=0.5") == NGX_OK, "A30-F1: zstd;q=0 then zstd;q=0.5 -- later explicit weight wins, accepts"); /* Wildcard only, no explicit token anywhere. */ check(request_decide("*", "gzip") == NGX_OK, "A30-F1: wildcard-only field (no explicit zstd token) accepts via '*'"); /* Explicit token only, no wildcard anywhere. */ check(request_decide("zstd", "gzip") == NGX_OK, "A30-F1: explicit-token-only field accepts, no wildcard involved"); /* Three occurrences. */ { const char *names3[3] = { NULL, NULL, NULL }; const char *values3[3] = { "*", "gzip", "zstd;q=0" }; check(request_decide_named(names3, values3, 3) == NGX_DECLINED, "A30-F1: three occurrences, explicit zstd;q=0 last overrides " "earlier wildcard"); } /* Four occurrences, zstd only on the last one. */ { const char *names4[4] = { NULL, NULL, NULL, NULL }; const char *values4[4] = { "gzip", "br", "identity", "zstd" }; check(request_decide_named(names4, values4, 4) == NGX_OK, "A30-F1: four occurrences, explicit zstd on the fourth accepts"); } /* An occurrence with an empty value interleaved. */ { const char *namesE[3] = { NULL, NULL, NULL }; const char *valuesE[3] = { "", "zstd;q=0", "*" }; check(request_decide_named(namesE, valuesE, 3) == NGX_DECLINED, "A30-F1: empty-value occurrence contributes nothing; explicit " "zstd;q=0 still overrides the later wildcard"); } /* A non-Accept-Encoding header interleaved between two AE occurrences. */ { const char *namesX[3] = { NULL, "X-Requested-With", NULL }; const char *valuesX[3] = { "*", "XMLHttpRequest", "zstd;q=0" }; check(request_decide_named(namesX, valuesX, 3) == NGX_DECLINED, "A30-F1: named case with an unrelated header interleaved " "between the two Accept-Encoding lines still declines"); } /* Case-varied header names must still be recognised as Accept-Encoding. */ { const char *namesC[2] = { "accept-encoding", "ACCEPT-ENCODING" }; const char *valuesC[2] = { "*", "zstd;q=0" }; check(request_decide_named(namesC, valuesC, 2) == NGX_DECLINED, "A30-F1: case-varied header names ('accept-encoding' / " "'ACCEPT-ENCODING') still match and the named case declines"); } printf("\n%d/%d checks passed\n", checks - failures, checks); return failures ? 1 : 0; } myguard-labs-nginx-zstd-module-3486b5b/ci/tests/unit/test_static_probe.c000066400000000000000000000572721525232545500264500ustar00rootroot00000000000000/* * Copyright (C) 2026 Thijs Eilander * SPDX-License-Identifier: BSD-2-Clause * * Unit tests for the .zst frame-header probe * (src/ngx_http_zstd_frame_probe.h: ngx_http_zstd_static_probe_frame()). * * WHY THIS EXISTS ALONGSIDE ci/t/01-static.t * * ci/t/01-static.t drives the probe through a live nginx serving real * files from ci/t/suite/. It proves the probe is wired * into the request path and that a declined file falls * back correctly -- but every boundary it wants to * reach has to be reached by materialising a file on * disk with exactly the right bytes, and a probe * verdict is only observable through the resulting * status code plus an error-log line. * this file states the verdict directly, at named byte * boundaries, for buffers the on-disk fixtures cannot * conveniently express (a 4-byte file, an exactly-18- * byte file, a Single_Segment header whose content-size * field is one byte short of the buffer), with no * nginx process and no filesystem, in well under a * second. * * The probe is deliberately PURE -- no I/O, no logging, no pool, no * request -- precisely so this layer can exist. The handler keeps the * pread(2), the directio alignment and every ngx_log_error() call; this * function is the arithmetic over the bytes that read returned. That split * is what makes the bounds below directly assertable: an off-by-one in the * "did I get enough bytes for this layout" checks is the entire risk in * this code, and it is invisible from an end-to-end test that only ever * feeds it complete frames. * * Like ci/tests/unit/test_accept_encoding.c this does NOT re-implement the * function: it includes src/ngx_http_zstd_frame_probe.h directly (#270), * so this binary always links the SHIPPED probe -- the header IS the * production code, with no extraction step left to drift or skip. * * Extend: add a case_*() function and one line in main(). */ #include #include #include #include typedef intptr_t ngx_int_t; typedef uintptr_t ngx_uint_t; typedef unsigned char u_char; #define ngx_memcpy memcpy /* * Included by RELATIVE path, not via -I, for the same reason * test_accept_encoding.c does: a static analyser invoked from anywhere * must be able to resolve it. An analyser that cannot resolve an include * SKIPS THE WHOLE TRANSLATION UNIT and reports no findings, which is * indistinguishable from a clean result. The header self-defines the * RFC-frozen constants and the verdict set, so the mirrored copies this * shim used to carry are gone -- one authoritative spelling (#270). */ #include "../../../src/ngx_http_zstd_frame_probe.h" static int failures; static int checks; static void check(int ok, const char *what) { checks++; if (ok) { printf("ok %s\n", what); return; } printf("FAIL %s\n", what); failures++; } /* * Poisoned, deliberately unaligned probe buffer: 18 usable bytes preceded * and followed by a guard region of 0xA5. The backing array is uint64_t- * aligned and the one-byte leading guard offsets every probe address. A read * past either end that happens to land on a guard * byte changes the verdict (0xA5A5A5A5 is neither magic), and the trailing * guard is what would absorb an off-by-one in the "enough bytes for this * layout" checks -- so a mutant that reads hdr[n] instead of stopping is * reading a defined, wrong value rather than heap garbage that might * happen to be correct. */ #define GUARD 1 static _Alignas(uint64_t) u_char buf[GUARD + 18 + GUARD]; static const u_char * frame(const u_char *bytes, size_t len) { memset(buf, 0xA5, sizeof(buf)); if (len) { memcpy(buf + GUARD, bytes, len); } return buf + GUARD; } static ngx_int_t probe(const u_char *bytes, size_t len, uint64_t *window) { *window = 0; return ngx_http_zstd_static_probe_frame(frame(bytes, len), len, window); } /* Little-endian ZSTD_MAGICNUMBER, the leading 4 bytes of a regular frame. */ #define MAGIC 0x28, 0xB5, 0x2F, 0xFD /* * A file shorter than the 18-byte probe buffer: only `n` bytes are valid * and the probe must never look past them. Here n == 4 -- a valid magic * and nothing else. A regular frame needs at least a Frame_Header_ * Descriptor at hdr[4], which is not present, so the verdict is TRUNCATED * and NOT a window decision made from the guard byte. */ static void case_shorter_than_probe_buffer(void) { static const u_char f[] = { MAGIC }; uint64_t w; check(probe(f, sizeof(f), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_TRUNCATED, "a 4-byte file (magic only, no descriptor) is TRUNCATED, not " "parsed from bytes past the read"); /* * n == 5: descriptor present, Single_Segment clear, so a * Window_Descriptor is required at hdr[5] and is past the read. * This is the exact off-by-one the `n < 6` guard exists for. */ static const u_char g[] = { MAGIC, 0x00 }; check(probe(g, sizeof(g), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_TRUNCATED, "a 5-byte streaming frame (descriptor but no Window_Descriptor) " "is TRUNCATED"); } /* * n == 6 is the smallest COMPLETE streaming frame header the probe can * decide on: descriptor 0x00 (no dict id, no content size, Single_Segment * clear) plus one Window_Descriptor byte. 0x00 => exponent 10, mantissa 0 * => a 1 KB window, far under the cap. */ static void case_smallest_complete_streaming_header(void) { static const u_char f[] = { MAGIC, 0x00, 0x00 }; uint64_t w; check(probe(f, sizeof(f), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_OK, "a 6-byte streaming frame declaring a 1 KB window is OK"); } /* * A file exactly the size of the probe buffer: all 18 bytes valid, no * short-read path involved. Descriptor 0xE0 sets Single_Segment (0x20) and * Frame_Content_Size_flag 3 (0xC0 => an 8-byte field), dict-id flag 0, so * the content size occupies hdr[5..12] -- entirely inside 18 bytes. Value * 4 MB, under the cap. */ static void case_exactly_probe_buffer_size(void) { static const u_char f[] = { MAGIC, 0xE0, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, /* 4 MB, LE64 */ 0x11, 0x22, 0x33, 0x44, 0x55 /* trailing payload */ }; uint64_t w; check(sizeof(f) == 18, "the exact-size fixture really is 18 bytes"); check(probe(f, sizeof(f), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_OK, "an exactly-18-byte file with a 4 MB single-segment window is OK"); } /* * A valid, ordinary .zst frame -- the shape every real precompressed asset * has: streaming frame, no dictionary, a modest window. Window_Descriptor * 0x68 => exponent 10 + 13 = 23 (8 MB base), mantissa 0 => exactly the * 8 MB cap, which is ACCEPTED (the check is `> MAX`, not `>=`). */ static void case_valid_zst_frame(void) { static const u_char f[] = { MAGIC, 0x00, 0x68 }; uint64_t w; check(probe(f, sizeof(f), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_OK, "a frame declaring exactly the 8 MB window cap is OK " "(the limit is inclusive)"); /* One mantissa step above the cap must be rejected, and must report * the value it computed so the operator's log line is actionable. */ static const u_char g[] = { MAGIC, 0x00, 0x69 }; check(probe(g, sizeof(g), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_WINDOW_BIG, "one mantissa step above the 8 MB cap is WINDOW_BIG"); check(w == (uint64_t) NGX_HTTP_ZSTD_STATIC_MAX_WINDOW + (NGX_HTTP_ZSTD_STATIC_MAX_WINDOW / 8), "WINDOW_BIG reports the declared window (8 MB + one eighth)"); } /* * Zstd reserves Frame_Header_Descriptor bit 0x08. A static sidecar with that * bit set is not a valid frame; the handler must decline it and fall back to * the uncompressed file rather than serving undecodable bytes. */ static void case_reserved_descriptor_bit(void) { static const u_char f[] = { MAGIC, 0x08, 0x00 }; uint64_t w; check(probe(f, sizeof(f), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_RESERVED, "a frame descriptor with reserved bit 0x08 is RESERVED"); } /* * A skippable leading frame is a legal start to a zstd stream, but the * probe itself never certifies one OK: it returns SKIP with the declared * skip length, and it is the CALLER's job (the handler's bounded walk, * covered in ci/t/01-static.t, not here) to resolve the skip and probe * whatever frame follows before deciding. All 16 skippable magics must * take this path, and none of them may fall through into the * regular-frame window parse (which would read a Window_Descriptor out of * what is actually a length field) or -- the bug this replaces -- be * certified OK on the magic alone. */ static void case_skippable_frame_reports_skip_not_ok(void) { uint64_t w; int i, ok = 1; for (i = 0; i < 16; i++) { /* ZSTD_MAGIC_SKIPPABLE_START | i, little-endian, then a * declared skip length of 0xFFFFFFFF -- the caller must reject * this against of.size, not the probe. */ u_char f[] = { 0x50 + (u_char) i, 0x2A, 0x4D, 0x18, 0xFF, 0xFF, 0xFF, 0xFF }; w = 0; if (probe(f, sizeof(f), &w) != NGX_HTTP_ZSTD_STATIC_FRAME_SKIP || w != 0xFFFFFFFFU) { ok = 0; } } check(ok, "all 16 skippable-frame magics report SKIP with the declared " "skip length, never OK on the magic alone"); } /* * A skippable frame's 8-byte header (magic + Frame_Size) is the outermost * bound the probe can check without file-level knowledge: fewer than 8 * bytes and it cannot even read the length field. These fixtures mirror * the "widest header, one byte short" boundary test above, but for the * skip-header layout. */ static void case_skippable_frame_header_truncation(void) { uint64_t w; static const u_char magic_only[] = { 0x50, 0x2A, 0x4D, 0x18 }; check(probe(magic_only, sizeof(magic_only), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_TRUNCATED, "a 4-byte skippable magic with no Frame_Size field is TRUNCATED"); static const u_char seven[] = { 0x50, 0x2A, 0x4D, 0x18, 0x00, 0x00, 0x00 }; check(probe(seven, sizeof(seven), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_TRUNCATED, "a 7-byte skippable header (Frame_Size one byte short) is " "TRUNCATED"); static const u_char eight[] = { 0x50, 0x2A, 0x4D, 0x18, 0x00, 0x00, 0x00, 0x00 }; w = 123; check(probe(eight, sizeof(eight), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_SKIP && w == 0, "an exact 8-byte skippable header with a zero-length payload " "is SKIP with skip length 0"); } /* * A file whose magic does not match -- the `cp foo.txt foo.zst` case the * probe exists for. It must be NOT_ZSTD regardless of what follows, and * must not be mistaken for a truncation. */ static void case_magic_mismatch(void) { static const u_char f[] = { 'h', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', '\n' }; uint64_t w; check(probe(f, sizeof(f), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_NOT_ZSTD, "plain text is NOT_ZSTD"); /* * One bit off the real magic, and one bit off the skippable range -- * the two near-misses a naive mask comparison would wave through. */ static const u_char g[] = { 0x29, 0xB5, 0x2F, 0xFD, 0x00, 0x00 }; check(probe(g, sizeof(g), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_NOT_ZSTD, "a one-bit-off zstd magic is NOT_ZSTD"); /* 0x184D2A40: one nibble below ZSTD_MAGIC_SKIPPABLE_START. */ static const u_char h[] = { 0x40, 0x2A, 0x4D, 0x18, 0x00, 0x00 }; check(probe(h, sizeof(h), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_NOT_ZSTD, "the value one below the skippable magic range is NOT_ZSTD"); /* 0x184D2A60: one nibble above the range. */ static const u_char k[] = { 0x60, 0x2A, 0x4D, 0x18, 0x00, 0x00 }; check(probe(k, sizeof(k), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_NOT_ZSTD, "the value one above the skippable magic range is NOT_ZSTD"); } /* * A zero-length file never reaches the probe: the handler rejects * of.size < 4 with its own "too small to be a zstd frame" line, and * rejects a pread returning fewer than 4 bytes before calling. The probe's * contract is therefore n >= 4, and this case pins the boundary that * contract rests on -- the smallest n the probe is ever handed is exactly * the 4 bytes the magic compare needs, so the compare itself never reads * past the read even at the minimum. * * Asserting it here rather than trusting the comment: the 4-byte call is * the one that would read hdr[4] if the `n < 5` guard were dropped, and * the poisoned buffer makes that read a guard byte rather than a lucky * zero. */ static void case_zero_length_file_never_reaches_the_probe(void) { static const u_char f[] = { MAGIC }; uint64_t w; check(probe(f, 4, &w) == NGX_HTTP_ZSTD_STATIC_FRAME_TRUNCATED, "the minimum n the probe accepts (4) decides on the magic alone " "and never indexes hdr[4]"); } /* * Single_Segment content-size widths. Frame_Content_Size_flag selects a * 1, 2, 4 or 8-byte field (flag 0 means ONE byte when Single_Segment is * set -- the flag only means "absent" when it is clear), and the 2-byte * form is offset by 256 per RFC 8878. Each width's truncation boundary is * one byte before the field ends. */ static void case_single_segment_content_size_widths(void) { uint64_t w; /* flag 0 => 1-byte field at hdr[5]. 0x40 = 64 bytes. */ static const u_char a[] = { MAGIC, 0x20, 0x40 }; check(probe(a, sizeof(a), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_OK, "Single_Segment with a 1-byte content size is OK"); /* ...and one byte short of that field is TRUNCATED. */ static const u_char a0[] = { MAGIC, 0x20 }; check(probe(a0, sizeof(a0), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_TRUNCATED, "Single_Segment missing its 1-byte content size is TRUNCATED"); /* flag 1 (0x40) => 2-byte field, +256 offset. 0x0000 => 256. */ static const u_char b[] = { MAGIC, 0x60, 0x00, 0x00 }; check(probe(b, sizeof(b), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_OK, "Single_Segment with a 2-byte content size is OK"); static const u_char b0[] = { MAGIC, 0x60, 0x00 }; check(probe(b0, sizeof(b0), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_TRUNCATED, "Single_Segment one byte short of its 2-byte content size is " "TRUNCATED"); /* * The +256 offset is load-bearing at the cap: 0xFFFF00 would be * (8 MB - 256) without it and (8 MB) with it -- both accepted -- so * pin it where it changes the verdict instead. 2-byte field * 0xFFFF = 65535, +256 = 65791, comfortably OK; the offset is * observable through WINDOW_BIG's reported value below. */ static const u_char b1[] = { MAGIC, 0x60, 0xFF, 0xFF }; check(probe(b1, sizeof(b1), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_OK, "a 2-byte content size of 65535 (+256) is under the cap"); /* flag 2 (0x80) => 4-byte field. 16 MB, over the cap. */ static const u_char c[] = { MAGIC, 0xA0, 0x00, 0x00, 0x00, 0x01 }; check(probe(c, sizeof(c), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_WINDOW_BIG, "Single_Segment declaring a 16 MB content size is WINDOW_BIG"); check(w == 16777216, "the 4-byte content size is decoded little-endian"); static const u_char c0[] = { MAGIC, 0xA0, 0x00, 0x00, 0x00 }; check(probe(c0, sizeof(c0), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_TRUNCATED, "Single_Segment one byte short of its 4-byte content size is " "TRUNCATED"); /* flag 3 (0xC0) => 8-byte field. */ static const u_char d0[] = { MAGIC, 0xE0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; check(probe(d0, sizeof(d0), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_TRUNCATED, "Single_Segment one byte short of its 8-byte content size is " "TRUNCATED"); } /* * The Dictionary_ID_flag widens the offset of the content-size field by * 0, 1, 2 or 4 bytes. Getting that table wrong reads the content size * from the dictionary id (or from payload), so each width gets a case * whose content-size bytes sit at a DIFFERENT offset from the previous * one, with dictionary-id bytes chosen so that misreading them would * produce a different verdict. */ static void case_dictionary_id_shifts_the_content_size(void) { uint64_t w; /* * Single_Segment + 1-byte content size, dict-id flag 1 => a 1-byte * dict id at hdr[5], content size at hdr[6]. The dict-id byte is 0xFF * and the content size 0x01: reading the wrong one is 255 vs 1, both * OK, so pin it via the 4-byte-content-size form instead where the * shift changes the magnitude by orders of magnitude. * * Descriptor 0xA1: Single_Segment(0x20) + FCS flag 2 (0x80 => 4 * bytes) + dict-id flag 1 (0x01 => 1 byte). * Layout: hdr[5] = dict id, hdr[6..9] = content size = 16 MB. */ static const u_char a[] = { MAGIC, 0xA1, 0xFF, /* dict id */ 0x00, 0x00, 0x00, 0x01 }; /* 16 MB LE32 */ check(probe(a, sizeof(a), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_WINDOW_BIG, "a 1-byte dictionary id shifts the content-size field by one"); check(w == 16777216, "the content size is read from behind the 1-byte dictionary id"); /* dict-id flag 2 => 2 bytes; content size at hdr[7..10]. */ static const u_char b[] = { MAGIC, 0xA2, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x01 }; check(probe(b, sizeof(b), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_WINDOW_BIG && w == 16777216, "the content size is read from behind a 2-byte dictionary id"); /* dict-id flag 3 => 4 bytes; content size at hdr[9..12]. */ static const u_char c[] = { MAGIC, 0xA3, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x01 }; check(probe(c, sizeof(c), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_WINDOW_BIG && w == 16777216, "the content size is read from behind a 4-byte dictionary id"); /* * The widest layout the probe must handle: 4-byte dict id + 8-byte * content size ends at hdr[16], inside the 18-byte buffer. One byte * short of it is TRUNCATED -- this is the outermost bound in the * function and the one an off-by-one would most plausibly break. */ static const u_char d[] = { MAGIC, 0xE3, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; /* 7 of 8 bytes */ check(sizeof(d) == 16, "the widest-layout truncation fixture is 16 bytes"); check(probe(d, sizeof(d), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_TRUNCATED, "the widest header (4-byte dict id + 8-byte content size) one " "byte short is TRUNCATED"); static const u_char e[] = { MAGIC, 0xE3, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; /* all 8 */ check(sizeof(e) == 17, "the widest-layout complete fixture is 17 bytes"); check(probe(e, sizeof(e), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_OK, "the widest header, complete at 17 bytes, is decided (OK)"); } /* * Window_Descriptor arithmetic on streaming frames: RFC 8878 3.1.1.1.2 * defines window = (1 << (10 + exponent)) + ((1 << (10 + exponent)) >> 3) * * mantissa, where exponent = byte >> 3 and mantissa = byte & 7. The * verdict boundary sits between two adjacent byte values, so the pair * below is exactly the mutation-sensitive point. */ static void case_window_descriptor_arithmetic(void) { uint64_t w; /* exponent 12 (0x60 >> 3 = 12) => 1 << 22 = 4 MB, mantissa 0. */ static const u_char a[] = { MAGIC, 0x00, 0x60 }; check(probe(a, sizeof(a), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_OK, "Window_Descriptor 0x60 (4 MB) is OK"); /* exponent 13 => 8 MB exactly, mantissa 0 -- the inclusive cap. */ static const u_char b[] = { MAGIC, 0x00, 0x68 }; check(probe(b, sizeof(b), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_OK, "Window_Descriptor 0x68 (exactly 8 MB) is OK"); /* exponent 14 => 16 MB. */ static const u_char c[] = { MAGIC, 0x00, 0x70 }; check(probe(c, sizeof(c), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_WINDOW_BIG, "Window_Descriptor 0x70 (16 MB) is WINDOW_BIG"); check(w == 16777216, "the exponent-only window is 1 << (10 + exponent)"); /* * Mantissa contribution: exponent 13, mantissa 7 => 8 MB + 7/8 MB. * A mutant that drops the mantissa term reports exactly 8 MB and * flips this to OK. */ static const u_char d[] = { MAGIC, 0x00, 0x6F }; check(probe(d, sizeof(d), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_WINDOW_BIG, "the mantissa pushes an 8 MB-exponent frame over the cap"); check(w == (uint64_t) NGX_HTTP_ZSTD_STATIC_MAX_WINDOW + (NGX_HTTP_ZSTD_STATIC_MAX_WINDOW / 8) * 7, "the mantissa contributes (window >> 3) * mantissa"); /* * The largest Window_Descriptor, 0xFF: exponent 31 => 1 << 41, plus * mantissa. This must not overflow the 64-bit accumulator into a * small value that reads as OK -- the reason the computation is * uint64_t rather than the ngx_uint_t the surrounding code uses. */ static const u_char e[] = { MAGIC, 0x00, 0xFF }; check(probe(e, sizeof(e), &w) == NGX_HTTP_ZSTD_STATIC_FRAME_WINDOW_BIG, "the maximum Window_Descriptor (0xFF) is WINDOW_BIG, not " "wrapped to a small window"); check(w == ((uint64_t) 1 << 41) + (((uint64_t) 1 << 41) >> 3) * 7, "the maximum window is computed in 64 bits without wrapping"); } /* * The window out-param is written ONLY on WINDOW_BIG. The handler reads it * only in that branch, but a probe that scribbled on it unconditionally * would make the handler's other log lines depend on uninitialised-looking * state if that ever changed, and it is free to pin here. */ static void case_window_untouched_unless_oversized(void) { uint64_t w; static const u_char ok[] = { MAGIC, 0x00, 0x00 }; w = 0xDEADBEEF; ngx_http_zstd_static_probe_frame(frame(ok, sizeof(ok)), sizeof(ok), &w); check(w == 0xDEADBEEF, "window is untouched on OK"); static const u_char bad[] = { 'n', 'o', 'p', 'e' }; w = 0xDEADBEEF; ngx_http_zstd_static_probe_frame(frame(bad, sizeof(bad)), sizeof(bad), &w); check(w == 0xDEADBEEF, "window is untouched on NOT_ZSTD"); static const u_char trunc[] = { MAGIC, 0x00 }; w = 0xDEADBEEF; ngx_http_zstd_static_probe_frame(frame(trunc, sizeof(trunc)), sizeof(trunc), &w); check(w == 0xDEADBEEF, "window is untouched on TRUNCATED"); } int main(void) { case_shorter_than_probe_buffer(); case_smallest_complete_streaming_header(); case_exactly_probe_buffer_size(); case_valid_zst_frame(); case_reserved_descriptor_bit(); case_skippable_frame_reports_skip_not_ok(); case_skippable_frame_header_truncation(); case_magic_mismatch(); case_zero_length_file_never_reaches_the_probe(); case_single_segment_content_size_widths(); case_dictionary_id_shifts_the_content_size(); case_window_descriptor_arithmetic(); case_window_untouched_unless_oversized(); printf("\n%d/%d checks passed\n", checks - failures, checks); return failures ? 1 : 0; } myguard-labs-nginx-zstd-module-3486b5b/ci/tests/unit/test_vary_fusion.c000066400000000000000000000351651525232545500263330ustar00rootroot00000000000000/* * Copyright (C) 2026 Thijs Eilander * SPDX-License-Identifier: BSD-2-Clause * * Unit tests for the fused Vary walk in src/ngx_http_zstd_common.h * (ngx_http_zstd_vary_ae_dcz(), which replaces calling * ngx_http_zstd_vary_accept_encoding() then ngx_http_zstd_vary_dcz() back * to back at a call site that always wants both). * * This links the REAL, shipped src/ngx_http_zstd_common.h against the * configured nginx headers supplied to ci/tools/test_vary_fusion.sh -- * not a hand-copied decision-logic shim. ngx_list_push()/ngx_palloc() are * stubbed as trivial arena allocators and ngx_strncasecmp() uses the same * faithful upstream copy as the parser fixture, so the test needs no full * nginx process, ngx_cycle or config parse: the Vary helpers only touch * r->headers_out.headers, r->gzip_vary and one loc_conf pointer. * * WHAT THIS PINS: the ae_dcz fusion must produce byte-identical Vary * header lines to calling the two original (still-shipped, still used * elsewhere) helpers back to back, in every combination of: * - clcf->gzip_vary on vs off * - want_dcz 1 vs 0 (the dcz_dicts-configured guard) * - a pre-existing "Vary: Accept-Encoding" line already in * headers_out.headers (the zstd_bypass_vary case) -- must still * produce exactly ONE Accept-Encoding line, never two. * * At least one assertion below FAILS if the want_dcz=0 guard is * dropped (case_no_dcz_when_disabled): with want_dcz=0 the helper must * push no Available-Dictionary/Sec-Fetch-Site line even though the * dcz tokens are absent from the request's Vary state -- a fused * helper that always looks for and pushes the dcz tokens regardless of * the flag makes this case FAIL by producing an extra header line. */ #include #include #include #include #include static int failures = 0; static int checks = 0; static int casefold_calls = 0; #define OK(desc, cond) \ do { \ checks++; \ if (cond) { \ printf("ok %s\n", desc); \ } else { \ printf("FAIL %s\n", desc); \ failures++; \ } \ } while (0) /* ------------------------------------------------------------------- */ /* Minimal arena: ngx_palloc()/ngx_list_push() need no real ngx_pool_t */ /* internals for this test -- only bump-pointer allocation. */ /* ------------------------------------------------------------------- */ #define ARENA_SIZE (1 << 20) static u_char arena[ARENA_SIZE]; static size_t arena_used; static void * arena_alloc(size_t size) { void *p; /* 8-byte align, matching real ngx_palloc's small-alloc behaviour * closely enough for POD structs. */ arena_used = (arena_used + 7) & ~(size_t) 7; if (arena_used + size > ARENA_SIZE) { fprintf(stderr, "test arena exhausted\n"); abort(); } p = &arena[arena_used]; arena_used += size; return p; } void * ngx_palloc(ngx_pool_t *pool, size_t size) { (void) pool; return arena_alloc(size); } void * ngx_list_push(ngx_list_t *l) { void *elt; ngx_list_part_t *last; last = l->last; if (last->nelts == l->nalloc) { last = arena_alloc(sizeof(ngx_list_part_t)); if (last == NULL) { return NULL; } last->elts = arena_alloc(l->nalloc * l->size); if (last->elts == NULL) { return NULL; } last->nelts = 0; last->next = NULL; l->last->next = last; l->last = last; } elt = (char *) last->elts + l->size * last->nelts; last->nelts++; return elt; } /* * src/core/ngx_string.c: ngx_strncasecmp() -- faithful upstream copy, * identical to the one ci/fuzz/ngx_shim.h ships for the same reason: * linking the real ngx_string.o here pulls in ngx_cycle/ngx_alloc/ * ngx_pnalloc from unrelated functions in that translation unit. This * is not the code under test (the Vary token walk is); it is the same * few-line primitive the parser suite already trusts a shimmed copy of. */ ngx_int_t ngx_strncasecmp(u_char *s1, u_char *s2, size_t n) { ngx_uint_t c1, c2; casefold_calls++; while (n) { c1 = (ngx_uint_t) *s1++; c2 = (ngx_uint_t) *s2++; c1 = (c1 >= 'A' && c1 <= 'Z') ? (c1 | 0x20) : c1; c2 = (c2 >= 'A' && c2 <= 'Z') ? (c2 | 0x20) : c2; if (c1 == c2) { if (c1) { n--; continue; } return 0; } return c1 - c2; } return 0; } /* Now pull in the real, shipped Vary helpers under test. */ #include "../../../src/ngx_http_zstd_common.h" /* ------------------------------------------------------------------- */ /* Fixture plumbing */ /* ------------------------------------------------------------------- */ /* * ngx_http_get_module_loc_conf(r, module) expands to * r->loc_conf[module.ctx_index] -- only .ctx_index is ever read, so a * zero-initialized definition of the real extern global (declared in * ngx_http_core_module.h) is enough; no other field of ngx_module_t is * touched by anything this test exercises. */ ngx_module_t ngx_http_core_module = { 0 }; static ngx_http_request_t * make_request(ngx_flag_t gzip_vary) { ngx_http_request_t *r; ngx_http_core_loc_conf_t *clcf; void **loc_conf; r = arena_alloc(sizeof(ngx_http_request_t)); memset(r, 0, sizeof(*r)); clcf = arena_alloc(sizeof(ngx_http_core_loc_conf_t)); memset(clcf, 0, sizeof(*clcf)); clcf->gzip_vary = gzip_vary; loc_conf = arena_alloc(sizeof(void *) * 1); loc_conf[0] = clcf; r->loc_conf = loc_conf; if (ngx_list_init(&r->headers_out.headers, NULL, 8, sizeof(ngx_table_elt_t)) != NGX_OK) { abort(); } return r; } /* Push a raw "Vary: " line directly, as zstd_bypass_vary does. */ static void push_vary_line(ngx_http_request_t *r, const char *value) { ngx_table_elt_t *v; v = ngx_list_push(&r->headers_out.headers); if (v == NULL) { abort(); } v->hash = 1; #if (nginx_version >= 1023000) v->next = NULL; #endif ngx_str_set(&v->key, "Vary"); v->value.data = (u_char *) value; v->value.len = ngx_strlen(value); } /* Collect every "Vary" line's value into one comma-joined buffer, in * header-list order, skipping hash==0 (deleted) lines -- mirrors what a * client/cache actually sees. */ static void collect_vary(ngx_http_request_t *r, char *out, size_t out_len) { ngx_uint_t i; ngx_list_part_t *part; ngx_table_elt_t *h; size_t used; out[0] = '\0'; used = 0; for (part = &r->headers_out.headers.part, h = part->elts, i = 0; /* void */; i++) { if (i >= part->nelts) { if (part->next == NULL) { break; } part = part->next; h = part->elts; i = 0; } if (h[i].hash == 0 || h[i].key.len != sizeof("Vary") - 1 || ngx_strncasecmp(h[i].key.data, (u_char *) "Vary", sizeof("Vary") - 1) != 0) { continue; } if (used != 0 && used + 2 < out_len) { memcpy(out + used, "; ", 2); used += 2; } if (used + h[i].value.len < out_len) { memcpy(out + used, h[i].value.data, h[i].value.len); used += h[i].value.len; out[used] = '\0'; } } } static int count_token_occurrences(const char *haystack, const char *token) { int n; const char *p; size_t tlen; n = 0; tlen = strlen(token); p = haystack; while ((p = strstr(p, token)) != NULL) { n++; p += tlen; } return n; } static void check_differential(const char *name, ngx_flag_t gzip_vary, ngx_uint_t want_dcz, const char *initial_vary) { ngx_http_request_t *fused, *reference; ngx_int_t fused_rc, reference_rc; char fused_out[512], reference_out[512]; int fused_calls, reference_calls; fused = make_request(gzip_vary); reference = make_request(gzip_vary); if (initial_vary != NULL) { push_vary_line(fused, initial_vary); push_vary_line(reference, initial_vary); } casefold_calls = 0; fused_rc = ngx_http_zstd_vary_ae_dcz(fused, want_dcz); fused_calls = casefold_calls; casefold_calls = 0; reference_rc = ngx_http_zstd_vary_accept_encoding(reference); if (reference_rc == NGX_OK && want_dcz) { reference_rc = ngx_http_zstd_vary_dcz(reference); } reference_calls = casefold_calls; collect_vary(fused, fused_out, sizeof(fused_out)); collect_vary(reference, reference_out, sizeof(reference_out)); OK(name, fused_rc == reference_rc && fused->gzip_vary == reference->gzip_vary && strcmp(fused_out, reference_out) == 0); OK("fused walk performs no extra case-fold comparisons", fused_calls <= reference_calls); } int main(void) { char buf[512]; check_differential("gzip_vary on matches the two-helper oracle", 1, 1, "X-Existing"); check_differential("gzip_vary on skips the unnecessary AE comparison", 1, 1, "123456789012345"); check_differential("gzip_vary on with dcz disabled skips the whole walk", 1, 0, "123456789012345"); check_differential("gzip_vary off matches the two-helper oracle", 0, 1, "X-Existing"); check_differential("dcz disabled matches Accept-Encoding oracle", 0, 0, "X-Existing"); check_differential("pre-existing AE matches the two-helper oracle", 0, 1, "X-No-Compression, Accept-Encoding"); check_differential("no pre-existing Vary matches the two-helper oracle", 0, 1, NULL); check_differential("pre-existing Sec-Fetch-Site matches the oracle", 0, 1, "Sec-Fetch-Site"); check_differential("both dcz tokens present match the oracle", 0, 1, "Available-Dictionary, Sec-Fetch-Site"); check_differential("case-insensitive tokens match the oracle", 0, 1, "accept-encoding, AVAILABLE-DICTIONARY, " "sec-fetch-site"); /* ---- gzip_vary ON, want_dcz 1, no pre-existing Vary ---- */ { ngx_http_request_t *r = make_request(1); OK("gzip_vary=on/want_dcz=1: returns NGX_OK", ngx_http_zstd_vary_ae_dcz(r, 1) == NGX_OK); OK("gzip_vary=on/want_dcz=1: r->gzip_vary latched", r->gzip_vary == 1); collect_vary(r, buf, sizeof(buf)); OK("gzip_vary=on/want_dcz=1: no Accept-Encoding line pushed " "(nginx emits it from r->gzip_vary)", strstr(buf, "Accept-Encoding") == NULL); OK("gzip_vary=on/want_dcz=1: combined dcz line pushed", strstr(buf, "Available-Dictionary, Sec-Fetch-Site") != NULL); } /* ---- gzip_vary OFF, want_dcz 1, no pre-existing Vary ---- */ { ngx_http_request_t *r = make_request(0); OK("gzip_vary=off/want_dcz=1: returns NGX_OK", ngx_http_zstd_vary_ae_dcz(r, 1) == NGX_OK); collect_vary(r, buf, sizeof(buf)); OK("gzip_vary=off/want_dcz=1: exactly one Accept-Encoding line", count_token_occurrences(buf, "Accept-Encoding") == 1); OK("gzip_vary=off/want_dcz=1: combined dcz line pushed", strstr(buf, "Available-Dictionary, Sec-Fetch-Site") != NULL); } /* * ---- want_dcz 0 (dcz_dicts NOT configured): the guard this whole * item exists to preserve. A fused helper that drops the guard and * always emits the dcz tokens makes this FAIL. * ---- */ { ngx_http_request_t *r = make_request(0); OK("want_dcz=0: returns NGX_OK", ngx_http_zstd_vary_ae_dcz(r, 0) == NGX_OK); collect_vary(r, buf, sizeof(buf)); OK("want_dcz=0: Accept-Encoding line still pushed", strstr(buf, "Accept-Encoding") != NULL); OK("case_no_dcz_when_disabled: no Available-Dictionary token " "when want_dcz=0", strstr(buf, "Available-Dictionary") == NULL); OK("case_no_dcz_when_disabled: no Sec-Fetch-Site token " "when want_dcz=0", strstr(buf, "Sec-Fetch-Site") == NULL); } /* * ---- pre-existing "Vary: Accept-Encoding" (the zstd_bypass_vary * shape, or any upstream/operator-pushed line naming the same * token): must not double it. * ---- */ { ngx_http_request_t *r = make_request(0); push_vary_line(r, "X-No-Compression, Accept-Encoding"); OK("pre-existing AE line: returns NGX_OK", ngx_http_zstd_vary_ae_dcz(r, 1) == NGX_OK); collect_vary(r, buf, sizeof(buf)); OK("pre-existing AE line: exactly one Accept-Encoding token " "across all Vary lines", count_token_occurrences(buf, "Accept-Encoding") == 1); OK("pre-existing AE line: dcz tokens still pushed", strstr(buf, "Available-Dictionary, Sec-Fetch-Site") != NULL); } /* ---- want_dcz=1 but Available-Dictionary already present ---- */ { ngx_http_request_t *r = make_request(0); push_vary_line(r, "Available-Dictionary"); OK("Available-Dictionary already present: returns NGX_OK", ngx_http_zstd_vary_ae_dcz(r, 1) == NGX_OK); collect_vary(r, buf, sizeof(buf)); OK("Available-Dictionary already present: only Sec-Fetch-Site " "pushed for the dcz half, not the combined line", strstr(buf, "Available-Dictionary, Sec-Fetch-Site") == NULL && strstr(buf, "Sec-Fetch-Site") != NULL); OK("Available-Dictionary already present: token still appears " "exactly once", count_token_occurrences(buf, "Available-Dictionary") == 1); } /* ---- hash==0 (deleted) Vary line must be ignored, not counted ---- */ { ngx_http_request_t *r = make_request(0); push_vary_line(r, "Accept-Encoding"); { ngx_table_elt_t *h = r->headers_out.headers.part.elts; h[0].hash = 0; } OK("deleted Vary line: returns NGX_OK", ngx_http_zstd_vary_ae_dcz(r, 0) == NGX_OK); collect_vary(r, buf, sizeof(buf)); OK("deleted Vary line: a fresh Accept-Encoding line is pushed " "(the deleted one does not count as present)", count_token_occurrences(buf, "Accept-Encoding") == 1); } printf("\n%d/%d checks passed\n", checks - failures, checks); return failures ? 1 : 0; } myguard-labs-nginx-zstd-module-3486b5b/ci/tests/unit/test_version_policy.c000066400000000000000000000013621525232545500270230ustar00rootroot00000000000000#include #include "../../../src/ngx_http_zstd_version.h" int main(void) { assert(ngx_http_zstd_version_policy(10507, 10507, 1, 1) == NGX_HTTP_ZSTD_VERSION_OK); assert(ngx_http_zstd_version_policy(10507, 10506, 0, 0) == NGX_HTTP_ZSTD_VERSION_WARN); assert(ngx_http_zstd_version_policy(10507, 10505, 0, 0) == NGX_HTTP_ZSTD_VERSION_WARN); assert(ngx_http_zstd_version_policy(10507, 10505, 1, 0) == NGX_HTTP_ZSTD_VERSION_REFUSE_TARGET_CBLOCK); assert(ngx_http_zstd_version_policy(10507, 10309, 0, 1) == NGX_HTTP_ZSTD_VERSION_REFUSE_NEGATIVE_LEVEL); assert(ngx_http_zstd_version_policy(10309, 10308, 1, 1) == NGX_HTTP_ZSTD_VERSION_WARN); return 0; } myguard-labs-nginx-zstd-module-3486b5b/ci/tools/000077500000000000000000000000001525232545500215715ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/tools/ab_bench.py000077500000000000000000002043621525232545500236760ustar00rootroot00000000000000#!/usr/bin/env python3 """Request-path A/B benchmark harness: drives a real nginx over the zstd filter under `wrk` load, interleaved between two arms to cancel machine drift, and reports throughput/latency/RSS per (arm, concurrency, body size). This is the gating dependency for the CCtx-ring, per-profile-slot, one-reset-per-request and first-buffer-sizing TODO rows, all of which say "do NOT guess -- measure". It exists to answer "did this change help, on what body sizes, at what concurrency" with a number, not a guess. What it measures ----------------- - requests/sec, p50 and p99 latency, and peak WORKER (not master) RSS, per (arm, concurrency, body-size class), under a real nginx + `wrk` load. - Two arms (two nginx binaries, or one binary run under two labelled configs) run in INTERLEAVED rounds -- not sequentially -- because a sequential A-then-B run confounds the comparison with machine drift (thermal throttling, background load, cache warmth). The existing worker-CCtx-cache +79% measurement used exactly this: 3 interleaved rounds. Reported numbers are the per-arm MEDIAN across rounds, plus the delta. - Body-size mix matters and is never collapsed into one number: the +79% win above was measured on 8 KB bodies and was ZERO on 64 KB, where compression itself dominates. Defaults to both sizes, reported separately, specifically so nobody can quote a blanket speedup that doesn't hold at every size. Why bodies are proxied through a paced backend, not served static ------------------------------------------------------------------- The worker-lifetime CCtx loan (acquire_cctx sets the busy flag; only request CLEANUP clears it -- src/ngx_http_zstd_filter_module.c:1847 and :2896) is held for the FULL lifetime of one compression, so two compressions only contend when their lifetimes actually overlap on one worker. A location serving a static, fully-buffered, known-length body from page cache completes inside a single event-loop turn; `wrk` connections then queue rather than overlap, and every concurrency in the sweep reports the same meaningless 100% hit rate with worker RSS that never moves -- indistinguishable, from the debug pass's own numbers, from a real single-slot cache never actually contending. An earlier version of this harness did exactly that and it was caught in review before the numbers were trusted. Every body is therefore served by PacedBackend: a mock upstream, proxied with `proxy_buffering off`, that sends chunked-transfer headers immediately and then the body in small pieces with a short sleep between each. The filter's compression session for one response stays open across many event-loop turns while other connections are being accepted, so a second concurrent request can reach acquire_cctx while the first still holds the loan -- the only way this workload can demonstrate the contention the ring-sizing row is about. The harness self-checks this rather than trusting it: the debug pass requires the hit rate at the LOWEST swept concurrency to be measurably higher than at the HIGHEST -- a relative fall, because "decays toward 1/N" is itself a relative claim -- plus an absolute floor on `created` witnesses at the highest concurrency, and raises a RuntimeError (harness error, non-zero exit) if either does not hold. A flat 100% hit rate at every concurrency is never printed as a result -- it is either real contention whose hit rate genuinely falls with concurrency, which the self-check has independently confirmed, or the self-check itself fails loudly instead. What it deliberately does NOT do --------------------------------- - It does NOT report a cache-engagement hit rate alongside release throughput. The reuse/create witnesses ("zstd: reusing worker cctx" / "zstd: created cctx") are ngx_log_debug lines that only exist in a --with-debug build's debug-level log, and emitting them under 128-connection load would distort the very throughput being measured. The harness therefore runs the RELEASE pass (throughput/latency/RSS, error_log at `warn`) and the DEBUG pass (witness counts and hit rate, error_log at `debug`, only if a --with-debug binary is supplied) as two structurally separate passes with separate result sections -- see BenchResult vs WitnessResult below -- so it is not possible to accidentally print one pass's numbers as if measured together with the other's. If only a release-mode binary is given, the hit rate is reported as unmeasured, never inferred. - It does NOT trust a flat hit rate. See "Why bodies are proxied" above -- the debug pass's own self-check must independently prove contention happened before its numbers are printed at all. - It is not a pass/fail gate on the MODULE's performance. Exit non-zero only on a harness error (missing wrk/nginx, nginx failed to start, zero successful requests for an arm, or the contention self-check failing) -- same contract as ci/tools/benchmark.py. A "slow" result is a result, not a failure; a workload that cannot contend IS a harness failure, because it cannot back up any hit-rate number it would otherwise print. - It is not wired into any CI workflow. This is a manual measurement tool the TODO rows above call out as their prerequisite; running it is a deliberate step a human (or a grind worker sizing the ring) takes, not something every PR pays for. Usage ----- # A/B: two release binaries, default concurrency sweep (8, 32, 128) python3 ci/tools/ab_bench.py \\ --arm-a .build/nginx-baseline/objs/nginx:baseline \\ --arm-b .build/nginx-ring/objs/nginx:ring \\ --json results.json # Single binary, release pass only (no A/B, still gives a baseline table) python3 ci/tools/ab_bench.py --arm-a .build/nginx-1.31.4/objs/nginx # Add the debug pass (separate, clearly-labelled section) for one arm python3 ci/tools/ab_bench.py \\ --arm-a .build/nginx-1.31.4/objs/nginx:release \\ --debug-binary .build/nginx-1.31.4/objs/nginx # Focus one RFC 9842 lookup cell: 16 configured dictionaries, miss path python3 ci/tools/ab_bench.py \\ --arm-a .build/nginx-1.31.4/objs/nginx \\ --workload dcz --dcz-dictionaries 16 --dcz-case miss Exit non-zero only on harness error, never on a "slow" result. """ from __future__ import annotations import argparse import base64 import dataclasses import hashlib import http.client import json import os import pathlib import re import shutil import signal import socket import statistics import subprocess import sys import tempfile import threading import time REPO = pathlib.Path(__file__).resolve().parent.parent.parent DEFAULT_CONCURRENCY = [8, 32, 128] DEFAULT_ROUNDS = 3 DEFAULT_DURATION = "10s" WARMUP_DURATION = "3s" # nginx retries bind() several times on "Address already in use" before # giving up and exiting -- measured at ~2.5s from spawn to exit on this # build. require_own_nginx_ready() waits at least this long past a # successful socket connect before trusting it, so a foreign listener # already on the port (which answers instantly, well before our nginx # would exit) cannot be mistaken for readiness. See that function's # docstring for the reproduction that caught the original 0.2s window # being too short. NGINX_BIND_RETRY_GRACE_S = 4.0 # Body-size classes. 8 KB is where the worker-CCtx-cache win showed up # (+79%); 64 KB is where it was measured as zero because the compression # work itself dominates over context-creation overhead. Keeping both by # default is deliberate -- see the module docstring. BODY_SIZES: dict[str, int] = { "8kb": 8 * 1024, "64kb": 64 * 1024, } DCZ_DICTIONARY_COUNTS = (1, 4, 16) DCZ_CASES = ("hit", "miss") # The debug pass drives ONE body-size class (the smaller one, where the # cache win is largest -- see the block above) rather than the full mix, # since it exists only to produce witness lines, not a throughput # comparison. Bound to BODY_SIZES itself, not a bare literal path, so a # future rename can't silently point this at a 404 (nginx would answer # 404, the debug pass would collect zero witnesses, and the hit rate # would misleadingly read "n/a" rather than erroring loudly). DEBUG_BODY = next(iter(BODY_SIZES)) assert DEBUG_BODY in BODY_SIZES CREATE_WITNESS = "zstd: created cctx" REUSE_WITNESS = "zstd: reusing worker cctx" # The loan (ngx_http_zstd_worker_cctx_busy) is held from acquire_cctx # until REQUEST CLEANUP, so two compressions only actually CONTEND when # their lifetimes overlap on one worker. A location that serves a static, # fully-buffered, known-length body from page cache completes inside one # event-loop turn -- wrk connections then queue rather than overlap, and # the harness would silently report a meaningless 100% hit rate at every # concurrency (caught in review: flat 100% at 8/32/128 conn plus # unmoving worker RSS is the signature of a workload that never # contends, not of a healthy cache). Bodies are therefore served via # PacedBackend + `proxy_buffering off` below: the backend sends headers # immediately, then the body in small chunked pieces with a sleep # between each, so the filter's compression session for one response # stays open across many event-loop turns while other connections are # being accepted -- long enough for a second concurrent request to # reach acquire_cctx while the first still holds the loan. PACE_CHUNK = 512 PACE_DELAY_S = 0.004 # Self-check thresholds for the debug pass -- see # evaluate_contention_self_check(). Three prior versions of this check # were tried and defeated by real (not hypothetical) counterexamples, # each one caught by actually reproducing the failure rather than # reasoning about it: # 1. An ABSOLUTE `created > 1` floor -- too weak. Even a near-instant # single-chunk send through the proxy hop picks up a handful of # "created" witnesses from ordinary thread-scheduling jitter, # which let a workload that barely contends pass silently. # 2. A RELATIVE rise in created-fraction (created / total) from the # lowest to the highest concurrency -- wrong because # created-fraction saturates near 1.0 once contention is heavy # (measured: 87.8% -> 96.9%, only a 1.10x rise) and compresses all # the signal into a narrow band exactly where resolution is # needed. # 3. A RELATIVE fall in hit rate (`lo_hr >= hi_hr * RISE`) with only a # floor on `created` at the HIGH end -- defeated by a workload # where the cache never engages at all: hit rate 0.0 at every # concurrency satisfies `0.0 >= 0.0 * 1.5` (True), and a total-miss # run creates a context on every request, so `created` at the high # end is enormous and clears any created-count floor trivially. # Verified in the interpreter, not merely reasoned about; see # test_contention_self_check_negative_control() below, which is # the negative control this defect should have caught before it # shipped twice. # The check that survived requires BOTH: an ABSOLUTE floor on the hit # rate at the LOWEST concurrency (the cache must demonstrably engage at # all when barely contended -- CONTENTION_MIN_LO_HIT_RATE), and a # STRICT relative fall to the highest concurrency (CONTENTION_MIN_RELATIVE_RISE, # compared with `>`, never `>=`, so equal values cannot pass). # CONTENTION_MIN_CREATED remains a trivial sanity bound on sample size # at the highest concurrency, not a second independent discriminator. CONTENTION_MIN_CREATED = 20 CONTENTION_MIN_RELATIVE_RISE = 1.5 CONTENTION_MIN_LO_HIT_RATE = 0.05 @dataclasses.dataclass class Arm: label: str binary: pathlib.Path filter_module: pathlib.Path | None = None static_module: pathlib.Path | None = None @dataclasses.dataclass class BenchSample: """One wrk run's result for one (arm, concurrency, body size).""" rps: float p50_ms: float p99_ms: float peak_rss_kb: int requests: int errors: int @dataclasses.dataclass class WitnessSample: """Cache-engagement counts from ONE debug-pass run. Never carries a throughput number -- the debug pass's own throughput is deliberately unreliable (see module docstring) and must never be read as comparable to a release-pass BenchSample. """ created: int reused: int @property def hit_rate(self) -> float | None: total = self.created + self.reused if total == 0: return None return self.reused / total @dataclasses.dataclass(frozen=True) class Workload: """Request/config contract for one benchmark run.""" mode: str headers: tuple[tuple[str, str], ...] expected_encoding: str config: str = "" dictionary_count: int = 0 dcz_case: str | None = None @dataclasses.dataclass(frozen=True) class PerfAttachment: """perf stat output contract for nginx workers during measured runs.""" output: pathlib.Path events: str def _available_dictionary(data: bytes) -> str: digest = hashlib.sha256(data).digest() return f":{base64.b64encode(digest).decode('ascii')}:" def _dcz_dictionary(index: int) -> bytes: marker = f"dictionary-{index:02d}".encode() unit = marker + body_bytes(8 * 1024) return unit[: 8 * 1024] def prepare_workload( root: pathlib.Path, mode: str, dictionary_count: int, dcz_case: str, ) -> Workload: """Create deterministic dcz fixtures and the matching request contract.""" if mode == "zstd": return Workload( mode="zstd", headers=(("Accept-Encoding", "zstd"),), expected_encoding="zstd", ) if mode != "dcz": raise ValueError(f"unsupported workload: {mode!r}") if dictionary_count not in DCZ_DICTIONARY_COUNTS: raise ValueError( f"dcz dictionary count must be one of {DCZ_DICTIONARY_COUNTS}, " f"got {dictionary_count}" ) if dcz_case not in DCZ_CASES: raise ValueError(f"dcz case must be one of {DCZ_CASES}, got {dcz_case!r}") dictionary_dir = root / "dictionaries" dictionary_dir.mkdir() configured: list[str] = [] directives: list[str] = [ " zstd_dcz_assume_secure_transport on;", ] for index in range(dictionary_count): data = _dcz_dictionary(index) path = dictionary_dir / f"dict-{index:02d}.bin" path.write_bytes(data) configured.append(_available_dictionary(data)) digest_hex = hashlib.sha256(data).hexdigest() directives.append(f" zstd_dcz_dict_file {path} {digest_hex};") if dcz_case == "hit": # The lookup walks configuration order, so the last entry exercises # the full configured table just like a miss does. advertised = configured[-1] expected_encoding = "dcz" else: advertised = _available_dictionary(b"unconfigured dcz benchmark dictionary") if advertised in configured: raise RuntimeError( "dcz miss fixture unexpectedly matches a configured hash" ) expected_encoding = "zstd" return Workload( mode="dcz", headers=( ("Accept-Encoding", "zstd, dcz"), ("Available-Dictionary", advertised), ("Sec-Fetch-Site", "same-origin"), ), expected_encoding=expected_encoding, config="\n".join(directives), dictionary_count=dictionary_count, dcz_case=dcz_case, ) def parse_arm(spec: str) -> tuple[str, str]: """ "path[:label]" -> (path, label). Label defaults to the basename.""" if ":" in spec: path, label = spec.rsplit(":", 1) if path and label: return path, label return spec, pathlib.Path(spec).name def detect_module(binary: pathlib.Path, name: str) -> pathlib.Path | None: sibling = binary.parent / name return sibling if sibling.exists() else None def require_own_nginx_ready( proc: subprocess.Popen, port: int, root: pathlib.Path, what: str ) -> None: """Block until `port` is accepting connections AND our own nginx process is still alive -- accepting readiness from the socket alone is not enough. If something else already holds `port`, our nginx exits almost immediately with "Address already in use", but a bare socket connect happily reaches the FOREIGN listener and reports ready instantly, before our process has even been reaped: the harness would then benchmark somebody else's server and report a confident number for software it never started -- reproduced while testing this exact fix, with a foreign HTTP server already bound to the target port. A single `proc.poll()` right after the socket connects is NOT enough: it does not block, and nginx failing to bind is not guaranteed to have been reaped by the OS in that exact instant, so the race can still slip through. This polls both conditions together over a short window instead of trusting one snapshot of either. """ deadline = time.time() + 10 ready = False while time.time() < deadline: if proc.poll() is not None: break try: with socket.create_connection(("127.0.0.1", port), timeout=0.5): ready = True break except OSError: time.sleep(0.05) # A socket connect succeeding is NOT proof our process is the one # answering: nginx retries bind() on "Address already in use" # several times before giving up (measured: ~2.5s of retries before # it actually exits), so a foreign listener already on the port # answers wait_for_port() immediately while our nginx is still # mid-retry, and a short grace window after "ready" is not long # enough to distinguish the two -- reproduced while testing this # exact fix. Once the socket is reachable, actively wait for our # process to EITHER exit (bind failure) or survive past nginx's own # bind-retry budget (genuine readiness), rather than trusting one # snapshot. if ready: try: proc.wait(timeout=NGINX_BIND_RETRY_GRACE_S) except subprocess.TimeoutExpired: pass # still alive past the retry window -- genuinely ours exited = proc.poll() is not None if ready and not exited: return tail = "" log = root / "stdout.log" if log.exists(): tail = log.read_text("utf-8", "replace")[-2000:] if exited: raise RuntimeError( f"{what}: our nginx process exited before/during the " f"readiness wait on port {port} (exit code " f"{proc.returncode}) -- if something else already held that " f"port, a socket connect alone would otherwise have reached " f"the FOREIGN listener and this run would have silently " f"benchmarked someone else's server. Log tail:\n{tail}" ) raise RuntimeError( f"{what}: nginx never became ready on port {port}. Log tail:\n{tail}" ) def body_bytes(size: int) -> bytes: """Realistic ~6:1-compressible HTML-ish body of the given size -- matches the fixture the existing +79% measurement used, not incompressible random bytes (which would exercise a different, less representative code path in the compressor). """ unit = b"
itemlink text here
\n" reps = size // len(unit) + 1 return (unit * reps)[:size] class PacedBackend(threading.Thread): """Mock upstream: sends chunked-transfer headers immediately, then the body either unpaced (`pace=False`, the whole body in one write) or in PACE_CHUNK-sized pieces with a PACE_DELAY_S sleep between each (`pace=True`). One handler thread per accepted connection so it scales to the harness's own concurrency sweep without becoming the bottleneck itself when unpaced. Pacing is what makes compression sessions overlap for the DEBUG pass. Proxied through nginx with `proxy_buffering off`, the filter processes each chunk as it streams in rather than compressing one fully-buffered body in a single event-loop turn -- so the worker-lifetime CCtx loan (acquired at acquire_cctx, released only at request cleanup, per src/ngx_http_zstd_filter_module.c:1847/:2896) stays held for the whole paced duration instead of being taken and returned before the next request is even accepted. A static, fully-buffered, known-length body completes inside one event-loop turn and can never contend, however high the connection count -- that was the workload bug this class exists to fix. Pacing must NEVER be used for the RELEASE pass. PACE_CHUNK/PACE_DELAY_S caps this backend at 512B/4ms = 128 KB/s PER CONNECTION -- a hard ceiling both arms share equally. A caught defect: the release pass was originally paced unconditionally, and its numbers (8 KB body: ~115/~478/~1878 rps at 8/32/128 conn) matched the backend's own theoretical ceiling (conn_count / stream_time) almost exactly -- the harness was measuring PacedBackend's sleep loop, never the zstd filter, and would have reported "no regression" for a genuinely slower build because the backend, not the filter, was always the bottleneck. The release pass now runs `pace=False`; only the debug pass (which exists for contention, not throughput -- see the module docstring) is ever paced. """ def __init__(self, port: int, bodies: dict[str, bytes], pace: bool) -> None: super().__init__(daemon=True) self.port = port self.bodies = bodies self.pace = pace self.srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) try: self.srv.bind(("127.0.0.1", port)) except OSError as exc: # A bare OSError here (e.g. "Address already in use") would # otherwise propagate uncaught out of run_release_pass / # run_debug_pass -- main() only catches RuntimeError, so a # busy backend port would crash with a traceback instead of # the documented harness-error path every other startup # failure in this file follows. self.srv.close() raise RuntimeError( f"PacedBackend: could not bind 127.0.0.1:{port}: {exc}" ) from exc self.srv.listen(256) self._stopping = False def stop(self) -> None: self._stopping = True try: self.srv.close() except OSError: pass def run(self) -> None: while not self._stopping: try: conn, _ = self.srv.accept() except OSError: return threading.Thread(target=self._serve_conn, args=(conn,), daemon=True).start() def _serve_conn(self, conn: socket.socket) -> None: try: conn.settimeout(15) buf = b"" while b"\r\n\r\n" not in buf: piece = conn.recv(4096) if not piece: return buf += piece request_line = buf.split(b"\r\n", 1)[0].decode("latin-1", "replace") path = request_line.split(" ")[1] if " " in request_line else "/" name = path.strip("/").split("/")[-1] or "8kb" body = self.bodies.get(name, next(iter(self.bodies.values()))) conn.sendall( b"HTTP/1.1 200 OK\r\n" b"Content-Type: application/octet-stream\r\n" b"Transfer-Encoding: chunked\r\n" b"Connection: close\r\n\r\n" ) if self.pace: for i in range(0, len(body), PACE_CHUNK): piece = body[i : i + PACE_CHUNK] conn.sendall(f"{len(piece):x}\r\n".encode() + piece + b"\r\n") time.sleep(PACE_DELAY_S) else: conn.sendall(f"{len(body):x}\r\n".encode() + body + b"\r\n") conn.sendall(b"0\r\n\r\n") except OSError: pass finally: try: conn.close() except OSError: pass def write_conf( root: pathlib.Path, port: int, backend_port: int, arm: Arm, workers: int, log_level: str, comp_level: int, workload: Workload, ) -> pathlib.Path: load = "" filt = arm.filter_module or detect_module( arm.binary, "ngx_http_zstd_filter_module.so" ) static = arm.static_module or detect_module( arm.binary, "ngx_http_zstd_static_module.so" ) if filt: load += f"load_module {filt};\n" if static: load += f"load_module {static};\n" conf = root / "nginx.conf" conf.write_text( f"""daemon off; master_process on; worker_processes {workers}; {load}error_log {root}/error.log {log_level}; pid {root}/nginx.pid; events {{ worker_connections 4096; }} http {{ access_log off; zstd on; # Pinned explicitly, not left at the compiled-in default: two arms # can be two different binaries with two different defaults, and an # unpinned level would mix a level change into the change under # measurement. ci/tools/test_compression_matrix.py pins it for the # same reason. zstd_comp_level {comp_level}; zstd_min_length 1; zstd_types application/octet-stream; {workload.config} server {{ listen 127.0.0.1:{port}; default_type application/octet-stream; location / {{ # Paced, chunked, unbuffered upstream -- see PacedBackend. # A static in-page-cache body completes in one event-loop # turn and can never make two compressions overlap. proxy_pass http://127.0.0.1:{backend_port}; proxy_http_version 1.1; proxy_set_header Connection ""; proxy_buffering off; }} }} }} """, encoding="utf-8", ) return conf def start_nginx(arm: Arm, conf: pathlib.Path, root: pathlib.Path) -> subprocess.Popen: log = open(root / "stdout.log", "w", encoding="utf-8") # noqa: SIM115 return subprocess.Popen( [str(arm.binary), "-p", str(root), "-c", str(conf)], stdout=log, stderr=subprocess.STDOUT, start_new_session=True, ) def stop_nginx(proc: subprocess.Popen) -> None: """Gracefully stop nginx, then kill and reap its private process group.""" proc.terminate() try: proc.wait(timeout=10) except subprocess.TimeoutExpired: try: os.killpg(proc.pid, signal.SIGKILL) except ProcessLookupError: pass proc.wait(timeout=5) def nginx_worker_pids(master_pid: int, expected: int) -> list[int]: """Wait for and return the exact worker child set of an nginx master.""" children_path = pathlib.Path(f"/proc/{master_pid}/task/{master_pid}/children") deadline = time.time() + 5 while time.time() < deadline: try: children = [int(pid) for pid in children_path.read_text().split()] except (FileNotFoundError, ProcessLookupError): children = [] if len(children) == expected: return children time.sleep(0.05) raise RuntimeError( f"expected {expected} nginx worker pid(s), found {len(children)}" ) def start_worker_perf( master_pid: int, workers: int, attachment: PerfAttachment ) -> subprocess.Popen: """Attach perf only to nginx workers, excluding harness/backend work.""" if workers != 1: raise RuntimeError("worker perf attachment requires --workers 1") pids = nginx_worker_pids(master_pid, workers) proc = subprocess.Popen( [ "/usr/bin/perf", "stat", "-p", ",".join(str(pid) for pid in pids), "-e", attachment.events, "-o", str(attachment.output), ], stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True, ) time.sleep(0.1) if proc.poll() is not None: stderr = proc.stderr.read() if proc.stderr else "" raise RuntimeError(f"perf stat failed to attach to nginx workers: {stderr}") return proc def stop_worker_perf(proc: subprocess.Popen) -> None: """Stop an attached perf process and require a usable exit.""" if proc.poll() is not None: stderr = proc.stderr.read() if proc.stderr else "" raise RuntimeError( f"perf stat exited before the measured pass completed: {stderr}" ) try: proc.send_signal(signal.SIGINT) except ProcessLookupError as exc: raise RuntimeError("perf stat exited before it could be stopped") from exc try: returncode = proc.wait(timeout=10) except subprocess.TimeoutExpired as exc: proc.kill() proc.wait(timeout=5) raise RuntimeError("perf stat did not stop after the measured pass") from exc if returncode not in (0, 128 + signal.SIGINT, -signal.SIGINT): stderr = proc.stderr.read() if proc.stderr else "" raise RuntimeError(f"perf stat failed with exit code {returncode}: {stderr}") def rss_monitor_start(master_pid: int, stop_path: pathlib.Path) -> subprocess.Popen: """A tiny standalone poller so RSS sampling continues during the wrk run without the harness process itself needing a thread juggling subprocess I/O. Writes the peak seen to `stop_path` on receipt of SIGTERM, then exits -- polled by the caller after wrk finishes. `worker_processes` is fixed for the whole run, so the worker pid set is resolved via `pgrep -P` ONCE, then reused for every 100ms /proc read -- not re-forked on every poll. Forking a shell plus `pgrep` 10 times a second inside the measured window would add CPU noise on the same machine the throughput itself is being measured on, undermining the very number this monitor exists to collect. The pid set is only re-resolved if a previously-known pid stops existing (a worker died or reloaded mid-run), not on a fixed schedule. """ script = f""" import os, signal, sys, time peak = 0 stop = False def _stop(signum, frame): global stop stop = True signal.signal(signal.SIGTERM, _stop) def resolve_pids(): try: out = os.popen("pgrep -P {master_pid}").read() return [int(x) for x in out.split() if x.strip()] except Exception: return [] pids = resolve_pids() while not stop: total = 0 stale = False for pid in pids: try: with open(f"/proc/{{pid}}/status") as fh: for line in fh: if line.startswith("VmRSS:"): total += int(line.split()[1]) break except (FileNotFoundError, ProcessLookupError): stale = True peak = max(peak, total) if stale or not pids: pids = resolve_pids() time.sleep(0.1) with open({str(stop_path)!r}, "w") as fh: fh.write(str(peak)) """ return subprocess.Popen([sys.executable, "-c", script]) def _to_ms(value: float, unit: str) -> float: return {"us": value / 1000, "ms": value, "s": value * 1000}[unit] def parse_wrk_percentiles(text: str) -> tuple[float, float]: """Extract p50 / p99 from `wrk --latency` output's distribution table.""" p50 = p99 = 0.0 for line in text.splitlines(): m = re.match(r"\s*(50|99)%\s+([\d.]+)(us|ms|s)", line) if not m: continue pct, val, unit = m.groups() ms = _to_ms(float(val), unit) if pct == "50": p50 = ms else: p99 = ms return p50, p99 def parse_wrk_duration_s(duration: str) -> float: """Parse a wrk `-d` duration string ("10s", "2m", "1h", or a bare number of seconds) into seconds. Raises ValueError on anything else -- callers treat that as a harness error, not a silent fallback, since a duration wrk itself would reject is worth failing loudly on before ever spawning the subprocess. """ m = re.fullmatch(r"(\d+(?:\.\d+)?)(s|m|h)?", duration.strip()) if not m: raise ValueError(f"unparsable wrk duration: {duration!r}") value = float(m.group(1)) unit = m.group(2) or "s" return value * {"s": 1, "m": 60, "h": 3600}[unit] def verify_workload(port: int, workload: Workload) -> str: """Fail before measurement unless the configured request path engages.""" conn = http.client.HTTPConnection("127.0.0.1", port, timeout=10) try: conn.request("GET", "/8kb", headers=dict(workload.headers)) response = conn.getresponse() response.read() finally: conn.close() encoding = response.getheader("Content-Encoding", "") if response.status != 200 or encoding != workload.expected_encoding: raise RuntimeError( f"{workload.mode} workload preflight failed: status={response.status}, " f"Content-Encoding={encoding!r}, expected " f"{workload.expected_encoding!r}" ) return encoding def run_wrk( url: str, connections: int, duration: str, threads: int, headers: tuple[tuple[str, str], ...], ) -> tuple[float, float, float, int, int]: wrk_bin = shutil.which("wrk") assert wrk_bin # The subprocess timeout must be derived from the user-supplied # duration, not a fixed constant -- a fixed 120s timeout raises an # uncaught subprocess.TimeoutExpired (main() only catches # RuntimeError) the moment someone passes --duration longer than # that, aborting the harness with a traceback instead of the # documented harness-error path. The margin covers wrk's own # connect/warmup/report overhead on top of the measured window. timeout_s = parse_wrk_duration_s(duration) + 30 command = [ wrk_bin, "-t", str(threads), "-c", str(connections), "-d", duration, "--latency", ] for name, value in headers: command.extend(("-H", f"{name}: {value}")) command.append(url) proc = subprocess.run( command, capture_output=True, text=True, check=False, timeout=timeout_s, ) out = proc.stdout rps_m = re.search(r"Requests/sec:\s*([\d.]+)", out) req_m = re.search(r"(\d+) requests in", out) err_m = re.search(r"Non-2xx or 3xx responses:\s*(\d+)", out) rps = float(rps_m.group(1)) if rps_m else 0.0 requests = int(req_m.group(1)) if req_m else 0 errors = int(err_m.group(1)) if err_m else 0 p50, p99 = parse_wrk_percentiles(out) if rps_m is None: raise RuntimeError(f"wrk produced no parseable output:\n{out}\n{proc.stderr}") return rps, p50, p99, requests, errors def bench_one( arm: Arm, master_pid: int, root: pathlib.Path, port: int, body: str, conc: int, duration: str, workload: Workload, ) -> BenchSample: # The config sets `daemon off; master_process on;`, so the Popen # object's own pid already IS the master -- no need to re-derive it # via `pgrep -f`, which depended on `pgrep` being installed and on # arm.binary/root containing no regex metacharacters; a miss there # silently left the RSS monitor never started and the table reported # a false peakRSS(MB) of 0.0 instead of failing loudly. stop_file = root / f"rss.{body}.{conc}.peak" monitor = rss_monitor_start(master_pid, stop_file) url = f"http://127.0.0.1:{port}/{body}" threads = min(conc, os.cpu_count() or 4) rps, p50, p99, requests, errors = run_wrk( url, conc, duration, threads, workload.headers ) peak_rss = 0 monitor.send_signal(signal.SIGTERM) try: monitor.wait(timeout=5) except subprocess.TimeoutExpired: monitor.kill() if stop_file.exists(): try: peak_rss = int(stop_file.read_text().strip() or "0") except ValueError: peak_rss = 0 return BenchSample( rps=rps, p50_ms=p50, p99_ms=p99, peak_rss_kb=peak_rss, requests=requests, errors=errors, ) def run_release_pass( arms: list[Arm], concurrencies: list[int], rounds: int, duration: str, workers: int, base_port: int, comp_level: int, workload_mode: str, dictionary_count: int, dcz_case: str, perf_attachment: PerfAttachment | None = None, ) -> tuple[dict, dict[str, str], int]: """Run interleaved rounds and return samples, engagement, and setup count.""" samples: dict[str, dict[int, dict[str, list[BenchSample]]]] = { arm.label: {c: {b: [] for b in BODY_SIZES} for c in concurrencies} for arm in arms } bodies = {name: body_bytes(size) for name, size in BODY_SIZES.items()} procs: dict[str, tuple[subprocess.Popen, pathlib.Path, int]] = {} workloads: dict[str, Workload] = {} engagement: dict[str, str] = {} setup_successful_requests = 0 backends: list[PacedBackend] = [] workdirs: list[pathlib.Path] = [] perf_proc: subprocess.Popen | None = None try: for i, arm in enumerate(arms): root = pathlib.Path(tempfile.mkdtemp(prefix=f"ab-bench-{arm.label}-")) # mkdtemp gives 0700: run as root, workers drop to the # compiled-in nginx user and cannot enter it -> 403s. os.chmod(root, 0o755) workdirs.append(root) port = base_port + i * 2 backend_port = port + 1 # pace=False: the RELEASE pass measures real filter # throughput. Pacing the backend would cap both arms at the # same artificial ceiling and could mask a genuine # regression -- see PacedBackend's docstring. backend = PacedBackend(backend_port, bodies, pace=False) backend.start() backends.append(backend) workload = prepare_workload(root, workload_mode, dictionary_count, dcz_case) workloads[arm.label] = workload conf = write_conf( root, port, backend_port, arm, workers, "warn", comp_level, workload, ) proc = start_nginx(arm, conf, root) procs[arm.label] = (proc, root, port) require_own_nginx_ready(proc, port, root, f"arm {arm.label!r}") engagement[arm.label] = verify_workload(port, workload) setup_successful_requests += 1 # Warmup: one short untimed-ish run per arm/body so the cache is # warm before the FIRST measured round -- otherwise round 1 is # biased cold for every arm equally, which would still be fair for # an A/B delta but would understate absolute rps for both. for arm in arms: _proc, root, port = procs[arm.label] for body in BODY_SIZES: _rps, _p50, _p99, requests, errors = run_wrk( f"http://127.0.0.1:{port}/{body}", 8, WARMUP_DURATION, 4, workloads[arm.label].headers, ) setup_successful_requests += max(requests - errors, 0) if perf_attachment is not None: if len(arms) != 1: raise RuntimeError("worker perf attachment requires exactly one arm") perf_proc = start_worker_perf( procs[arms[0].label][0].pid, workers, perf_attachment ) for _rnd in range(rounds): for conc in concurrencies: for body in BODY_SIZES: # Interleave arms WITHIN each (round, conc, body) cell: # this is what actually cancels drift, since it puts # each arm's measurement for the same condition # adjacent in wall-clock time rather than in two # separate blocks. for arm in arms: proc, root, port = procs[arm.label] sample = bench_one( arm, proc.pid, root, port, body, conc, duration, workloads[arm.label], ) samples[arm.label][conc][body].append(sample) if perf_proc is not None: stop_worker_perf(perf_proc) perf_proc = None finally: if perf_proc is not None: perf_proc.kill() perf_proc.wait(timeout=5) for proc, _root, _port in procs.values(): stop_nginx(proc) for backend in backends: backend.stop() for root in workdirs: shutil.rmtree(root, ignore_errors=True) return samples, engagement, setup_successful_requests def evaluate_contention_self_check( results: dict[int, WitnessSample], concurrencies: list[int] ) -> None: """The debug pass's negative control on ITSELF, not on the module. Raises RuntimeError (a harness error) unless the paced workload demonstrably (a) makes the cache engage at all at the lowest swept concurrency, and (b) shows that engagement fall STRICTLY as concurrency rises to the highest. Both conditions are required -- see the CONTENTION_MIN_* comment block for the three prior versions of this check and the real counterexample that defeated each one. Pure and side-effect-free by design specifically so it can be exercised directly by test_contention_self_check_negative_control() below with synthetic data, instead of only being reachable by actually running nginx. """ lo_conc, hi_conc = min(concurrencies), max(concurrencies) lo, hi = results.get(lo_conc), results.get(hi_conc) lo_hr = lo.hit_rate if lo is not None else None hi_hr = hi.hit_rate if hi is not None else None hi_created_ok = hi is not None and hi.created >= CONTENTION_MIN_CREATED # ABSOLUTE floor: the cache must demonstrably engage at ALL when # barely contended. Without this, an all-miss run (hit rate 0.0 at # every concurrency) satisfies a purely relative "0.0 >= 0.0 * 1.5" # comparison -- that is not a benign edge case, it is the single # most important failure this harness exists to catch (the cache # never engaging) passing as a clean result. lo_engages = lo_hr is not None and lo_hr > CONTENTION_MIN_LO_HIT_RATE # STRICT relative fall: `>`, never `>=`, so two equal hit rates # (including two equal zeros) cannot be read as "falling." degrades = ( lo_hr is not None and hi_hr is not None and lo_hr > hi_hr * CONTENTION_MIN_RELATIVE_RISE ) if len(concurrencies) < 2 or not hi_created_ok or not lo_engages or not degrades: raise RuntimeError( "harness self-check FAILED: the paced workload did not " f"demonstrate genuine single-slot contention. At {lo_conc} " f"conn the hit rate was " f"{'n/a' if lo_hr is None else f'{lo_hr * 100:.1f}%'}; " f"at {hi_conc} conn it was " f"{'n/a' if hi_hr is None else f'{hi_hr * 100:.1f}%'} " f"(need the {lo_conc}-conn rate > " f"{CONTENTION_MIN_LO_HIT_RATE * 100:.0f}% -- the cache must " f"engage at all when barely contended -- AND strictly > " f"{CONTENTION_MIN_RELATIVE_RISE}x the {hi_conc}-conn rate, " f"and >= {CONTENTION_MIN_CREATED} 'created cctx' witnesses " f"at {hi_conc} conn). A single-slot worker cache can only " "show a real hit rate when concurrent compressions " "genuinely overlap and that overlap actually WORSENS as " "concurrency rises -- a hit rate that never engages, or " "that is flat or barely falling across the sweep, means " "this run never demonstrated that, so any hit rate it " "reported would be a workload artifact, not a measurement. " "This is the harness's own negative control failing, not a " "benign 100% (or 0%) hit rate." ) def run_debug_pass( arm: Arm, concurrencies: list[int], duration: str, workers: int, base_port: int, comp_level: int, workload_mode: str, dictionary_count: int, dcz_case: str, ) -> dict[int, WitnessSample]: """Separate pass, debug-level logging, only for engagement witnesses. Its own throughput is NOT reported -- see module docstring. Runs the FULL concurrency sweep against ONE long-lived nginx instance (never restarted between concurrencies) so `created` and `reused` accumulate across the whole sweep and the ring-sizing row's actual question -- does the hit rate DEGRADE as concurrency rises -- is something this pass can show directly, not just a single snapshot. Delegates the self-check to evaluate_contention_self_check(), which raises RuntimeError (a harness error, not a "slow result") unless the paced workload demonstrates BOTH that the cache engages at all at the lowest swept concurrency (an ABSOLUTE floor, CONTENTION_MIN_LO_HIT_RATE) and that engagement STRICTLY falls to the highest (CONTENTION_MIN_RELATIVE_RISE). Both are required: a purely relative check alone was defeated by a workload where the cache never engages at all (0.0 hit rate at every concurrency trivially satisfies a relative "no worse than" comparison) -- see the CONTENTION_MIN_* comment block for the full history of three rejected check designs, each one defeated by a real counterexample. See PacedBackend's docstring for why the workload needs pacing to make genuine overlap possible at all. """ root = pathlib.Path(tempfile.mkdtemp(prefix=f"ab-bench-debug-{arm.label}-")) # mkdtemp gives 0700: run as root, workers drop to the compiled-in # nginx user and cannot enter it -> 403s. os.chmod(root, 0o755) bodies = {name: body_bytes(size) for name, size in BODY_SIZES.items()} backend_port = base_port + 1 # pace=True: the DEBUG pass exists for CONTENTION (witness counts), # not throughput, so the pacing that creates that contention is # correct here -- see PacedBackend's docstring. backend = PacedBackend(backend_port, bodies, pace=True) results: dict[int, WitnessSample] = {} try: backend.start() port = base_port workload = prepare_workload(root, workload_mode, dictionary_count, dcz_case) conf = write_conf( root, port, backend_port, arm, workers, "debug", comp_level, workload, ) proc = start_nginx(arm, conf, root) try: require_own_nginx_ready(proc, port, root, "debug pass") verify_workload(port, workload) elog_path = root / "error.log" prev_created = 0 prev_reused = 0 for conc in concurrencies: # Throughput-not-reported run purely to generate witness # lines under realistic concurrent load. run_wrk( f"http://127.0.0.1:{port}/{DEBUG_BODY}", conc, duration, min(conc, 4), workload.headers, ) time.sleep(0.3) # let the debug log flush elog = elog_path.read_text("utf-8", "replace") total_created = len(re.findall(re.escape(CREATE_WITNESS), elog)) total_reused = len(re.findall(re.escape(REUSE_WITNESS), elog)) # Per-cell delta, not the running total, so each concurrency's # own hit rate is reported rather than a cumulative blend. created = total_created - prev_created reused = total_reused - prev_reused prev_created, prev_reused = total_created, total_reused results[conc] = WitnessSample(created=created, reused=reused) finally: stop_nginx(proc) finally: backend.stop() shutil.rmtree(root, ignore_errors=True) evaluate_contention_self_check(results, concurrencies) return results def check_with_debug(binary: pathlib.Path) -> None: v = subprocess.run([str(binary), "-V"], capture_output=True, text=True, check=False) if "--with-debug" not in v.stderr: raise RuntimeError( f"{binary}: debug pass needs an nginx built --with-debug " f"(the reuse witnesses are ngx_log_debug lines); got: " f"{v.stderr.strip()}" ) def print_release_table( arms: list[Arm], concurrencies: list[int], samples: dict ) -> None: """Prints the release-pass table, including an `errors` column so a partially-failing run (non-2xx/3xx responses, which wrk itself counts inside "N requests in" alongside genuine successes) is visible instead of silently averaged into the rps/latency numbers. Any nonzero error count anywhere in the sweep gets a loud warning after the table -- a confident-looking table measured on error responses is the exact failure class this tool exists to avoid. """ header = ( f"{'arm':<14}{'conc':>6}{'body':>7}{'rps(med)':>11}" f"{'p50(ms)':>10}{'p99(ms)':>10}{'peakRSS(MB)':>13}" f"{'errors':>8}{'rounds':>8}" ) print(header) print("-" * len(header)) medians: dict[str, dict[int, dict[str, float]]] = {} any_errors = False for arm in arms: medians[arm.label] = {} for conc in concurrencies: medians[arm.label][conc] = {} for body in BODY_SIZES: sset = samples[arm.label][conc][body] rps_med = statistics.median(s.rps for s in sset) p50_med = statistics.median(s.p50_ms for s in sset) p99_med = statistics.median(s.p99_ms for s in sset) rss_peak = max((s.peak_rss_kb for s in sset), default=0) / 1024 total_errors = sum(s.errors for s in sset) if total_errors > 0: any_errors = True medians[arm.label][conc][body] = rps_med print( f"{arm.label:<14}{conc:>6}{body:>7}{rps_med:>11.1f}" f"{p50_med:>10.2f}{p99_med:>10.2f}{rss_peak:>13.1f}" f"{total_errors:>8}{len(sset):>8}" ) print() if any_errors: print( "WARNING: non-2xx/3xx responses occurred somewhere in this " "sweep (see the 'errors' column above). wrk counts those " "inside its own request total, so the rps/p50/p99 numbers " "for the affected cell(s) are measured PARTLY on error " "responses, not on successful compression -- do not treat " "them as a clean throughput measurement." ) print() if len(arms) == 2: a, b = arms[0].label, arms[1].label print(f"delta ({b} vs {a}), rps median:") dheader = f"{'conc':>6}{'body':>7}{a + ' rps':>14}{b + ' rps':>14}{'delta':>10}" print(dheader) print("-" * len(dheader)) for conc in concurrencies: for body in BODY_SIZES: ra = medians[a][conc][body] rb = medians[b][conc][body] delta = f"{((rb - ra) / ra * 100):+.1f}%" if ra > 0 else "n/a" print(f"{conc:>6}{body:>7}{ra:>14.1f}{rb:>14.1f}{delta:>10}") print() print( "NOTE: a delta at one body size does not generalize to another --" " report each size on its own, never a blanket speedup." ) def print_debug_table( label: str, results: dict[int, WitnessSample], concurrencies: list[int] ) -> None: print() print("=== DEBUG PASS: cache-engagement witnesses ===") print( "Throughput numbers from this pass are NOT reported and are NOT " "comparable to the release pass above -- debug-level logging on " "the hot path would distort them. This section is witness counts " "only. The harness self-check already confirmed the hit rate " "falls materially from the lowest to the highest swept " "concurrency; the table below is the actual per-cell evidence " "for the ring-sizing question -- does the hit rate DEGRADE as " "concurrency rises." ) header = f"{'arm':<14}{'conc':>6}{'created':>9}{'reused':>8}{'hit rate':>10}" print(header) print("-" * len(header)) prev_hr: float | None = None degraded = False for conc in concurrencies: w = results[conc] hr = w.hit_rate hr_s = f"{hr * 100:.1f}%" if hr is not None else "n/a" print(f"{label:<14}{conc:>6}{w.created:>9}{w.reused:>8}{hr_s:>10}") if hr is not None and prev_hr is not None and hr < prev_hr: degraded = True if hr is not None: prev_hr = hr print() if degraded: print( "hit rate DECREASES somewhere in this sweep as concurrency " "rises -- the single-slot contention the ring-sizing row " "expects. Use these per-conc numbers, not just the " "lowest-vs-highest self-check comparison, to size the ring." ) else: print( "hit rate did NOT decrease anywhere across this per-cell " "sweep despite the self-check proving it fell materially " "from the lowest to the highest concurrency -- re-check " "before using these numbers to size the ring; a single-slot " "cache should degrade as concurrency rises." ) def test_contention_self_check_negative_control() -> None: """The self-check's OWN negative control -- exercises evaluate_contention_self_check() directly with synthetic WitnessSample data, no nginx involved. This exists because two prior self-check designs each individually looked correct under careful reading and both turned out to pass on real non-contending input; "read the code carefully" is not a substitute for a test that actually runs, so this is that test. Asserts three things: 1. An all-zero-hit-rate run (the cache never engages at any concurrency) is REJECTED. This is the specific defect that defeated the second self-check design: `0.0 >= 0.0 * 1.5` is True, so a purely relative "does the rate fall" comparison passes on a workload that measured nothing at all. 2. A run whose hit rate is IDENTICAL at every concurrency (a nonzero constant, not falling at all) is REJECTED -- proves the inequality is strict, not `>=`. 3. A run with a genuine, large absolute-and-relative fall (the real numbers measured on this harness: 12.2% -> 3.1%) is ACCEPTED -- proves the check does not reject real contention along with the broken cases above. Called from `--self-test` (see main()), which runs this with no nginx/wrk dependency and exits before touching any of that -- cheap enough to run on every invocation of this file as a smoke check, and independent of whatever machine happens to run it. """ concurrencies = [8, 32] all_zero = { 8: WitnessSample(created=500, reused=0), 32: WitnessSample(created=2000, reused=0), } try: evaluate_contention_self_check(all_zero, concurrencies) except RuntimeError: pass else: raise AssertionError( "self-check negative control FAILED: an all-zero hit rate " "(the cache never engaged at ANY concurrency) was accepted " "as a valid contention result. This is the exact defect " "reported against the second self-check design -- fix " "evaluate_contention_self_check(), do not weaken this test." ) flat_nonzero = { 8: WitnessSample(created=100, reused=100), 32: WitnessSample(created=100, reused=100), } try: evaluate_contention_self_check(flat_nonzero, concurrencies) except RuntimeError: pass else: raise AssertionError( "self-check negative control FAILED: an identical (flat, " "non-falling) hit rate at every concurrency was accepted. " "The relative-fall comparison must be a STRICT inequality " "so two equal values cannot pass." ) genuine_decay = { 8: WitnessSample(created=309, reused=2224), # hit rate 87.8% 32: WitnessSample(created=1439, reused=47), # hit rate 3.2% } evaluate_contention_self_check(genuine_decay, concurrencies) # must not raise def run_self_test() -> int: """Runs the in-process negative control above and reports the result. No nginx, no wrk, no network -- exists so the self-check's own correctness can be verified on every invocation of this file, not just reasoned about. Exit code follows the harness-error contract: 0 on pass, 1 on failure. """ try: test_contention_self_check_negative_control() except AssertionError as exc: print(f"SELF-TEST FAILED: {exc}", file=sys.stderr) return 1 print( "SELF-TEST OK: evaluate_contention_self_check() correctly " "rejects an all-zero hit rate, correctly rejects a flat " "non-falling hit rate, and correctly accepts a genuine decay." ) return 0 def main() -> int: ap = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) ap.add_argument( "--self-test", action="store_true", help="run the self-check's own negative control (no nginx/wrk " "needed) and exit -- verifies evaluate_contention_self_check() " "correctly rejects a non-contending workload before trusting " "it against real hardware.", ) ap.add_argument( "--arm-a", help="path/to/nginx[:label] for arm A (required unless --self-test)" ) ap.add_argument( "--arm-b", help="path/to/nginx[:label] for arm B (optional; enables A/B)" ) ap.add_argument( "--debug-binary", help="path to a --with-debug nginx binary for the SEPARATE debug " "pass (cache hit-rate witnesses). If omitted, hit rate is " "reported as unmeasured.", ) ap.add_argument( "--concurrency", default=",".join(str(c) for c in DEFAULT_CONCURRENCY), help="comma-separated connection counts to sweep", ) ap.add_argument( "--rounds", type=int, default=DEFAULT_ROUNDS, help="interleaved A/B rounds per (concurrency, body size) cell", ) ap.add_argument("--duration", default=DEFAULT_DURATION, help="wrk -d per run") ap.add_argument( "--workers", type=int, default=1, help="nginx worker_processes (default 1: the TODO rows this " "harness gates are about PER-WORKER cache behaviour)", ) ap.add_argument( "--comp-level", type=int, default=6, help="zstd_comp_level pinned in the generated config for both " "arms (default 6, matches the existing +79%% measurement). " "Pinned explicitly rather than left at each binary's compiled-in " "default so an A/B between two different binaries never mixes a " "level change into the change actually under measurement.", ) ap.add_argument( "--workload", choices=("zstd", "dcz"), default="zstd", help="request workload (default zstd preserves the existing harness)", ) ap.add_argument( "--dcz-dictionaries", type=int, choices=DCZ_DICTIONARY_COUNTS, default=1, help="configured dictionary count for --workload dcz", ) ap.add_argument( "--dcz-case", choices=DCZ_CASES, default="hit", help="advertise the last configured dictionary or an absent one", ) ap.add_argument("--base-port", type=int, default=18400) ap.add_argument("--json", help="write machine-readable results here") ap.add_argument( "--perf-stat-output", help="attach perf only to nginx workers during measured release runs", ) ap.add_argument( "--perf-events", help="comma-separated perf events (required with --perf-stat-output)", ) args = ap.parse_args() if args.self_test: return run_self_test() if not args.arm_a: print("error: --arm-a is required unless --self-test is given", file=sys.stderr) return 2 if bool(args.perf_stat_output) != bool(args.perf_events): print( "error: --perf-stat-output and --perf-events must be used together", file=sys.stderr, ) return 2 if args.perf_stat_output and args.workers != 1: print("error: worker perf attachment requires --workers 1", file=sys.stderr) return 2 # Everything the scratch roots below create must stay readable by the # workers when the harness runs as root (they drop to the compiled-in # nginx user): a restrictive inherited umask would strip group/other # bits from every fixture created here, 403ing the workers even with # each scratch root itself chmod'd open -- same trap and same fix as # ci/tools/test_compression_matrix.py. os.umask(0o022) if shutil.which("wrk") is None: print("error: wrk not found on PATH", file=sys.stderr) return 2 try: parse_wrk_duration_s(args.duration) except ValueError as exc: print(f"error: {exc}", file=sys.stderr) return 2 concurrencies = [int(x) for x in args.concurrency.split(",") if x.strip()] arms: list[Arm] = [] for _i, spec in enumerate((args.arm_a, args.arm_b)): if spec is None: continue path, label = parse_arm(spec) binary = pathlib.Path(path).resolve() if not binary.exists(): print(f"error: arm binary not found: {binary}", file=sys.stderr) return 2 arms.append(Arm(label=label, binary=binary)) if len({a.label for a in arms}) != len(arms): print("error: arm labels must be distinct", file=sys.stderr) return 2 workload_label = args.workload if args.workload == "dcz": workload_label += f"/{args.dcz_dictionaries}/{args.dcz_case}" print( f"=== RELEASE PASS: rps/p50/p99/RSS, {args.rounds} interleaved " f"round(s), workload={workload_label}, " "NO cache witnesses in this section ===" ) try: perf_attachment = None if args.perf_stat_output: perf_attachment = PerfAttachment( pathlib.Path(args.perf_stat_output).resolve(), args.perf_events ) samples, engagement, setup_successful_requests = run_release_pass( arms, concurrencies, args.rounds, args.duration, args.workers, args.base_port, args.comp_level, args.workload, args.dcz_dictionaries, args.dcz_case, perf_attachment, ) except (RuntimeError, ValueError) as exc: print(f"error: {exc}", file=sys.stderr) return 1 # Gate on SUCCESSFUL requests, not raw request count -- wrk's own # "N requests in" total includes non-2xx/3xx responses, so an # all-403 or all-502 run would otherwise clear this gate and print # a full throughput table measured on nothing. total_successful = sum( max(s.requests - s.errors, 0) for arm in arms for conc in concurrencies for body in BODY_SIZES for s in samples[arm.label][conc][body] ) if total_successful == 0: print( "error: zero successful (non-error) requests across the whole sweep", file=sys.stderr, ) return 1 completed_successful_requests = total_successful + setup_successful_requests print_release_table(arms, concurrencies, samples) debug_results: dict[int, WitnessSample] = {} if args.debug_binary: debug_bin = pathlib.Path(args.debug_binary).resolve() if not debug_bin.exists(): print(f"error: debug binary not found: {debug_bin}", file=sys.stderr) return 2 try: check_with_debug(debug_bin) except RuntimeError as exc: print(f"error: {exc}", file=sys.stderr) return 1 debug_arm = Arm(label="debug", binary=debug_bin) try: debug_results = run_debug_pass( debug_arm, concurrencies, args.duration, args.workers, args.base_port + len(arms) * 2 + 2, args.comp_level, args.workload, args.dcz_dictionaries, args.dcz_case, ) except RuntimeError as exc: print(f"error: {exc}", file=sys.stderr) return 1 print_debug_table(debug_arm.label, debug_results, concurrencies) else: print() print( "=== DEBUG PASS: not run (no --debug-binary given) -- cache " "hit rate is UNMEASURED, not inferred. ===" ) if args.json: out = { "meta": { "concurrencies": concurrencies, "rounds": args.rounds, "duration": args.duration, "workers": args.workers, "arms": [a.label for a in arms], "workload": args.workload, "dcz_dictionaries": args.dcz_dictionaries if args.workload == "dcz" else None, "dcz_case": args.dcz_case if args.workload == "dcz" else None, "engagement": engagement, "successful_requests": total_successful, "completed_successful_requests": completed_successful_requests, "perf_scope": "nginx-workers-measured-release" if perf_attachment is not None else None, "commit": subprocess.run( ["git", "-C", str(REPO), "rev-parse", "--short", "HEAD"], capture_output=True, text=True, check=False, ).stdout.strip(), }, "release": { arm.label: { str(conc): { body: [ dataclasses.asdict(s) for s in samples[arm.label][conc][body] ] for body in BODY_SIZES } for conc in concurrencies } for arm in arms }, "debug": { str(conc): dataclasses.asdict(w) for conc, w in debug_results.items() } if debug_results else None, } pathlib.Path(args.json).write_text(json.dumps(out, indent=2)) print(f"wrote {args.json}") return 0 if __name__ == "__main__": raise SystemExit(main()) myguard-labs-nginx-zstd-module-3486b5b/ci/tools/benchmark.py000077500000000000000000000122121525232545500240760ustar00rootroot00000000000000#!/usr/bin/env python3 """Reproducible compression benchmark: zstd levels vs gzip. Measures compression ratio and single-threaded CPU throughput on representative web payloads, using the same libzstd / zlib the host nginx links against (the `zstd` and `gzip` CLIs). This is a *library* benchmark — it characterises the codecs the module drives, not nginx request overhead, so the numbers are stable and reproducible across machines (throughput scales with CPU; ratio does not). Usage: python3 ci/tools/benchmark.py # human table python3 ci/tools/benchmark.py --json results.json python3 ci/tools/benchmark.py --levels 1,3,9,19 --repeat 5 Exit non-zero only on harness error (missing zstd/gzip), never on a "slow" result — this is measurement, not a pass/fail gate. """ import argparse import json import pathlib import shutil import subprocess import sys import time # ci/tools/benchmark.py -> ci/tools -> ci -> repo root: THREE parents # since the ci/ move (was tools/ at the root, where two sufficed). REPO = pathlib.Path(__file__).resolve().parent.parent.parent def build_payloads() -> dict[str, bytes]: """Representative, deterministic web payloads.""" payloads: dict[str, bytes] = {} fixture = REPO / "ci" / "t" / "suite" / "test" if fixture.is_file(): payloads["html-58k (test fixture)"] = fixture.read_bytes() # Highly compressible JSON-ish API response. row = b'{"id":%d,"name":"item-%d","active":true,"tags":["a","b","c"]}' payloads["json-api-256k"] = ( b"[" + b",".join(row % (i, i) for i in range(4000)) + b"]" ) # JS-like text with realistic redundancy. js_line = b"function handler_%d(req,res){return res.json({ok:true,n:%d});}\n" payloads["js-512k"] = b"".join(js_line % (i, i) for i in range(8000)) # Low-entropy worst case (already-compressed-ish): pseudo-random. import random random.seed(1) # deterministic payloads["random-256k (incompressible)"] = bytes( random.getrandbits(8) for _ in range(256 * 1024) ) return payloads def run_codec(cmd: list[str], data: bytes, repeat: int) -> tuple[int, float]: """Return (compressed_size, best wall-seconds over `repeat` runs).""" best = float("inf") out = b"" for _ in range(repeat): t0 = time.perf_counter() proc = subprocess.run(cmd, input=data, capture_output=True, check=True) dt = time.perf_counter() - t0 best = min(best, dt) out = proc.stdout return len(out), best def main() -> int: ap = argparse.ArgumentParser() ap.add_argument( "--levels", default="1,3,6,9,19", help="comma-separated zstd levels" ) ap.add_argument( "--repeat", type=int, default=3, help="runs per measurement (best time kept)" ) ap.add_argument("--json", help="write machine-readable results here") args = ap.parse_args() zstd = shutil.which("zstd") gzip = shutil.which("gzip") if not zstd or not gzip: print("error: need both `zstd` and `gzip` on PATH", file=sys.stderr) return 2 levels = [int(x) for x in args.levels.split(",") if x.strip()] payloads = build_payloads() results: list[dict] = [] header = f"{'payload':<30}{'codec':<10}{'in':>9}{'out':>9}{'ratio':>8}{'MB/s':>9}" print(header) print("-" * len(header)) for name, data in payloads.items(): n = len(data) gz_size, gz_t = run_codec([gzip, "-6", "-c"], data, args.repeat) gz_mbps = (n / gz_t) / 1e6 results.append( { "payload": name, "codec": "gzip-6", "in": n, "out": gz_size, "ratio": n / gz_size, "mbps": gz_mbps, } ) print( f"{name:<30}{'gzip-6':<10}{n:>9}{gz_size:>9}" f"{n / gz_size:>8.2f}{gz_mbps:>9.1f}" ) for lv in levels: sz, t = run_codec([zstd, f"-{lv}", "-c", "-q"], data, args.repeat) mbps = (n / t) / 1e6 results.append( { "payload": name, "codec": f"zstd-{lv}", "in": n, "out": sz, "ratio": n / sz, "mbps": mbps, } ) print( f"{'':<30}{'zstd-' + str(lv):<10}{n:>9}{sz:>9}" f"{n / sz:>8.2f}{mbps:>9.1f}" ) print() if args.json: meta = { "zstd_version": subprocess.run( [zstd, "--version"], capture_output=True, text=True, check=False, ).stdout.strip(), "commit": subprocess.run( ["git", "-C", str(REPO), "rev-parse", "--short", "HEAD"], capture_output=True, text=True, check=False, ).stdout.strip(), "repeat": args.repeat, } pathlib.Path(args.json).write_text( json.dumps({"meta": meta, "results": results}, indent=2) ) print(f"wrote {args.json}") return 0 if __name__ == "__main__": raise SystemExit(main()) myguard-labs-nginx-zstd-module-3486b5b/ci/tools/build-input-hash.sh000077500000000000000000000021251525232545500253050ustar00rootroot00000000000000#!/usr/bin/env bash # Print a stable digest of every path in ci/build-inputs.manifest. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DEFAULT_MODULE_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")" MODULE_DIR="${1:-$DEFAULT_MODULE_DIR}" MANIFEST="$MODULE_DIR/ci/build-inputs.manifest" if [ ! -f "$MANIFEST" ]; then echo "ERROR: build-input manifest not found: $MANIFEST" >&2 exit 1 fi hash_file() { local relative="$1" local digest digest="$(sha256sum "$MODULE_DIR/$relative" | cut -d' ' -f1)" printf '%s %s\n' "$digest" "$relative" } while IFS= read -r input || [ -n "$input" ]; do case "$input" in '' | \#*) continue ;; esac if [ -f "$MODULE_DIR/$input" ]; then hash_file "$input" elif [ -d "$MODULE_DIR/$input" ]; then while IFS= read -r -d '' file; do hash_file "${file#"$MODULE_DIR/"}" done < <(find "$MODULE_DIR/$input" -type f -print0 | sort -z) else echo "ERROR: build input does not exist: $input" >&2 exit 1 fi done < "$MANIFEST" | sha256sum | cut -d' ' -f1 myguard-labs-nginx-zstd-module-3486b5b/ci/tools/build-windows.sh000077500000000000000000000612021525232545500247200ustar00rootroot00000000000000#!/bin/bash # # Windows/MSVC build of nginx with this repo's modules: the zstd module # pair by default, and/or the unified nginx-compression module and # ngx_brotli, plus optionally headers-more. Produces a static nginx.exe # comparable to the nginx.org Windows binary, plus the modules. # # Prerequisites: # 1. Visual Studio (or Build Tools) with the C++ workload. # 2. MSYS2 (https://www.msys2.org/), with git and cmake available in # the shell (pacman -S git mingw-w64-x86_64-cmake). # 3. Strawberry Perl (https://strawberryperl.com/). OpenSSL's Windows # build requires a NATIVE perl; the MSYS perl generates paths the # MSVC toolchain cannot use. Auto-detected below; override with # STRAWBERRY_PERL=/c/path/to/perl/bin if installed elsewhere. # # Launching the build shell (the part every old guide gets wrong): # 1. Open "x64 Native Tools Command Prompt for VS" (runs vcvars64, # which sets PATH/INCLUDE/LIB/LIBPATH for cl and link). # 2. From THAT prompt, start MSYS2 inheriting the environment: # set MSYS2_PATH_TYPE=inherit # C:\msys64\msys2_shell.cmd -mingw64 -use-full-path -here # 3. cd to an empty BUILD WORKSPACE directory and run this script # from there (it may live anywhere — a repo checkout, a copy, a # curl); all sources and the build land in the workspace. # # Module toggles (env or edit below). Each toggle names a MODULE; the # libraries a module needs are derived, so any combination works: # WITH_ZSTD=1 the zstd module pair (the default). # WITH_BROTLI=1 the standalone ngx_brotli module. # WITH_COMPRESSION=1 the unified nginx-compression module # (compression/ in the zstd-module checkout). # WITH_HEADERS_MORE=1 headers-more. # Library builds follow the modules: BUILDLIB_ZSTD / BUILDLIB_BROTLI # default to 1 when any enabled module can use the library. The # standalone modules REQUIRE theirs (their configure fails without it), # so those defaults are refused when overridden; the unified module # treats a missing library as a build shape (that backend is simply # absent), so BUILDLIB_*=0 is a legal way to build it without one. # A standalone module and the unified one may be built together; the # unified module's static filter anchor orders it first so it gets the # first shot at every response. Examples: # ./ci/tools/build-windows.sh (the default) # WITH_ZSTD=1 WITH_BROTLI=1 ./ci/tools/build-windows.sh # WITH_COMPRESSION=1 REPO_ZSTD_MODULE= REF_ZSTD_MODULE= \ # ./ci/tools/build-windows.sh # (until compression/ lands on master, WITH_COMPRESSION needs a ref # that carries it; the script checks and says so) # # Every downloaded tarball is pinned by version and SHA-256 in # ci/tools/windows-pins.sh, the one source of those pins (a bump edits # that file and nothing else). Every pin is verified up front, before # any compile (verify() prints the actual digest on a mismatch, so a # stale pin is a copy-paste away from fixed). # # KNOW WHAT YOU ARE BUILDING — Windows nginx caveats (nginx.org's own # position; this script does not change them): # * Effectively single-worker: the win32 event core services # connections from one worker via select() (hence the FD_SETSIZE # define below). This build is for a LOCAL DEV BOX, not production # traffic. # * No HTTP/3: deliberately absent from the configure list, not an # oversight — nginx's QUIC stack needs UDP capabilities the win32 # event core does not provide. # # Running the result as a Windows service (optional): nginx.exe has no # service mode of its own; a thin wrapper such as shawl # (https://github.com/mtkennerly/shawl) works well. From an ADMIN shell: # shawl.exe add --name "Local Nginx" --no-log-cmd \ # --cwd C:/path/to/nginx-1.x.x -- nginx.exe -c conf/nginx.conf # sc config "Local Nginx" start=auto # sc start "Local Nginx" # and to remove: sc stop "Local Nginx" & sc delete "Local Nginx" set -euo pipefail ### module toggles ##################################################### WITH_ZSTD=${WITH_ZSTD:-1} WITH_BROTLI=${WITH_BROTLI:-0} WITH_COMPRESSION=${WITH_COMPRESSION:-0} WITH_HEADERS_MORE=${WITH_HEADERS_MORE:-1} for t in WITH_COMPRESSION WITH_ZSTD WITH_BROTLI WITH_HEADERS_MORE; do case "${!t}" in 0 | 1) ;; *) echo "ERROR: $t must be 0 or 1 (got '${!t}')" >&2 exit 1 ;; esac done # Library builds default to "some enabled module can use it"; override # to 0 only for the unified module, whose backends are optional. BUILDLIB_ZSTD=${BUILDLIB_ZSTD:-$((WITH_ZSTD || WITH_COMPRESSION))} BUILDLIB_BROTLI=${BUILDLIB_BROTLI:-$((WITH_BROTLI || WITH_COMPRESSION))} for t in BUILDLIB_ZSTD BUILDLIB_BROTLI; do case "${!t}" in 0 | 1) ;; *) echo "ERROR: $t must be 0 or 1 (got '${!t}')" >&2 exit 1 ;; esac done if [ "$WITH_ZSTD" = 1 ] && [ "$BUILDLIB_ZSTD" = 0 ]; then echo "ERROR: the standalone zstd module pair requires libzstd (its" \ "auto/zstd probe is fatal); BUILDLIB_ZSTD=0 only fits WITH_ZSTD=0" >&2 exit 1 fi if [ "$WITH_BROTLI" = 1 ] && [ "$BUILDLIB_BROTLI" = 0 ]; then echo "ERROR: the standalone ngx_brotli module requires libbrotli (its" \ "config errors without it); BUILDLIB_BROTLI=0 only fits WITH_BROTLI=0" >&2 exit 1 fi if [ "$WITH_COMPRESSION" = 1 ] && { [ "$WITH_ZSTD" = 1 ] || [ "$WITH_BROTLI" = 1 ]; }; then echo "NOTE: building the unified module beside a standalone one; the" \ "unified module runs first in the body-filter chain and" \ "supersedes it (static filter-anchor policy)." >&2 fi ### version + SHA-256 pins ############################################# # Sourced from ci/tools/windows-pins.sh beside this script -- THE one # place the pins live (a bump edits that file and nothing else). A hard # dependency with a loud guard and no inline defaults: a fallback default # would silently build the wrong version, which is worse than stopping. # PINS_FILE= points at a different file to try a bump without editing # the tracked one. SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) PINS_FILE=${PINS_FILE:-$SCRIPT_DIR/windows-pins.sh} [ -f "$PINS_FILE" ] || { echo "ERROR: $PINS_FILE missing -- windows-pins.sh must sit beside this" \ "script (or set PINS_FILE=)" >&2 exit 1 } # shellcheck source=ci/tools/windows-pins.sh . "$PINS_FILE" for v in VER_NGINX SHA_NGINX VER_PCRE2 SHA_PCRE2 VER_OPENSSL SHA_OPENSSL \ VER_ZLIB SHA_ZLIB VER_NASM SHA_NASM VER_ZSTD SHA_ZSTD; do [ -n "${!v:-}" ] || { echo "ERROR: $PINS_FILE does not set $v" >&2 exit 1 } done # OpenSSL build options. nginx's own auto/lib/openssl/makefile.msvc # adds "no-shared no-threads" itself; the pre-build below must configure # with the same effective set so the library nginx links is the one it # would have built (duplicates are harmless to OpenSSL's Configure). OPENSSL_OPT="no-shared no-module no-threads" # Module refs — pinned by full commit id like everything else, bumped # deliberately (a floating branch here once handed a tester a module # checkout without the MSVC support and a mysterious configure # failure). Overridable so a fix branch can be tested before it lands: # REPO_ZSTD_MODULE= REF_ZSTD_MODULE= (fresh clone # required when switching repos — the existing clone's origin wins). # # The zstd module default resolves to the CURRENT CHECKOUT (this script's # own repo, HEAD), not a fixed historical commit (audit A30-F3: a static # commit pin here drifted ~154 commits behind and silently built a stale # module even though the README presents this script alongside current # behaviour). REPO_ZSTD_MODULE/REF_ZSTD_MODULE stay fully overridable for # the deliberate self-pin / fork-testing use case documented above. # # WITH_COMPRESSION=1 needs a ref whose tree carries compression/; until # that lands on master, point the override at a branch that has it. The # clone step checks and says so rather than failing inside configure. _zstd_module_repo_root() { # SCRIPT_DIR is not assigned yet this early — derive it locally so this # default resolves regardless of where DIR_PROJECT/SCRIPT_DIR end up. local script_dir script_dir=$(cd "$(dirname "$0")" && pwd) git -C "$script_dir" rev-parse --show-toplevel 2>/dev/null || true } if [ -z "${REPO_ZSTD_MODULE:-}" ] || [ -z "${REF_ZSTD_MODULE:-}" ]; then _default_zstd_root="$(_zstd_module_repo_root)" if [ -n "$_default_zstd_root" ] && [ -d "$_default_zstd_root/ci/tools" ]; then REPO_ZSTD_MODULE=${REPO_ZSTD_MODULE:-$_default_zstd_root} REF_ZSTD_MODULE=${REF_ZSTD_MODULE:-$(git -C "$_default_zstd_root" rev-parse HEAD)} else # Not running from inside a nginx-zstd-module checkout (e.g. a # standalone copy of this script) — fall back to the last commit # verified to build cleanly under this script, same as before. REPO_ZSTD_MODULE=${REPO_ZSTD_MODULE:-https://github.com/myguard-labs/nginx-zstd-module.git} REF_ZSTD_MODULE=${REF_ZSTD_MODULE:-37cf9ac6b58284ae2da95620f4905930d1277b54} fi fi REPO_BROTLI=${REPO_BROTLI:-https://github.com/mreiden/ngx_brotli.git} REF_BROTLI=${REF_BROTLI:-c0b96d6254f32ff36fbfff96e3995504a3bf5ef3} REPO_HEADERS_MORE=${REPO_HEADERS_MORE:-https://github.com/openresty/headers-more-nginx-module.git} REF_HEADERS_MORE=${REF_HEADERS_MORE:-53b0d98d1d57033b90491fd6a3b485ca536edfa7} ######################################################################## # The build workspace is wherever the script is RUN from, not where it # lives — so a repo checkout can drive a build elsewhere. Patches for # the cloned modules are applied from the script's own patches/ # directory (bundled: the headers-more MSVC include-order fix, without # which that module cannot build under the win32 precompiled header) # and additionally from $DIR_PROJECT/patches//*.patch for # local ones. A standalone copy of this script needs the bundled # patches directory alongside it (or WITH_HEADERS_MORE=0). DIR_PROJECT=$(pwd) DIR_NGINX="$DIR_PROJECT/nginx-$VER_NGINX" # Every pinned tarball is downloaded into dl/ and verified there (the # cache survives deleting the nginx tree); extraction happens later at # each use site. verify() prints both digests on a mismatch, so a stale # pin is a copy-paste away from fixed and a bad download is # distinguishable from it. DIR_DL="$DIR_PROJECT/dl" verify() { # url sha local url=$1 sha=$2 file="$DIR_DL/${1##*/}" src actual # A download lands beside the cache slot and is promoted into it only # once its digest matches, so neither an interrupted download nor a # completed wrong one (a mirror serving other bytes) is ever something # a later run trusts. A mismatching download stays as .part for a # look; the next run downloads over it. if [ -f "$file" ]; then src=$file else src=$file.part curl -fsSL -o "$src" "$url" fi actual=$(sha256sum "$src" | cut -d' ' -f1) if [ "$actual" != "$sha" ]; then echo "ERROR: sha256 mismatch for ${file##*/}" >&2 echo " expected $sha" >&2 echo " actual $actual" >&2 echo " source $url" >&2 if [ "$src" = "$file" ]; then echo " (stale pin? update SHA_*; bad cached file? delete $file and rerun)" >&2 else echo " (stale pin? update SHA_*; bad download? not cached, kept as ${src##*/})" >&2 fi exit 1 fi [ "$src" = "$file" ] || mv "$src" "$file" echo "verified ${file##*/}" } # Extract a verified tarball into the current directory. Extra args # narrow the extraction to specific archive members. extract() { # tarball-name [paths...] local file="$DIR_DL/$1" shift tar xzf "$file" "$@" } # Extract into DIR unless a completed extraction is already there. A killed # or failed tar leaves a partial tree behind, and a directory-exists guard # would then skip re-extraction and fail later on missing files. The # marker is written only after tar exits clean, and a tree without one is # extracted again IN PLACE: tar rewrites every selected member from the # archive, so the result is complete whatever was there, while what is # not an archive member (objs/, the OpenSSL pre-build, the module clones # under objs/lib) is kept. A tree from before the marker existed costs one # re-extraction, not a rebuild. extract_complete() { # dir tarball-name [paths...] local archive=$2 marker="$1/.extracted" shift 2 if [ ! -f "$marker" ]; then extract "$archive" "$@" : >"$marker" fi } # Clone at a pinned ref, applying the script's bundled patches and any # workspace-local ones from patches//*.patch, idempotently. # Submodules are synced AFTER the switch so nested pins (e.g. # ngx_brotli/deps/brotli) follow the checked-out module commit, not # whatever the clone's default branch pinned. clone_module() { # dest repo ref local dest=$1 repo=$2 ref=$3 p d [ -d "$dest" ] || git clone --recurse-submodules "$repo" "$dest" git -C "$dest" fetch --quiet origin 2>/dev/null || true git -C "$dest" switch --detach "$ref" 2>/dev/null \ || git -C "$dest" switch "$ref" git -C "$dest" submodule sync --recursive --quiet 2>/dev/null || true git -C "$dest" submodule update --init --recursive for d in "$SCRIPT_DIR/patches/$dest" "$DIR_PROJECT/patches/$dest"; do for p in "$d"/*.patch; do [ -f "$p" ] || continue if git -C "$dest" apply --check "$p" 2>/dev/null; then git -C "$dest" apply "$p" elif ! git -C "$dest" apply --reverse --check "$p" 2>/dev/null; then echo "ERROR: $p neither applies nor is already applied" >&2 exit 1 fi done done } ### toolchain sanity ################################################### command -v cl >/dev/null 2>&1 || { echo "ERROR: cl not on PATH — launch from the VS Native Tools" \ "prompt with MSYS2_PATH_TYPE=inherit (see header)" >&2 exit 1 } # OpenSSL needs a NATIVE perl (Strawberry); MSYS perl reports msys as # its OS and emits paths cl cannot consume. PERL_BIN= for d in "${STRAWBERRY_PERL:-}" /c/Strawberry/perl/bin /c/strawberry/perl/bin; do [ -n "$d" ] && [ -x "$d/perl.exe" ] && { PERL_BIN=$d break } done if [ -z "$PERL_BIN" ] && command -v perl >/dev/null 2>&1 \ && [ "$(perl -e 'print $^O')" = "MSWin32" ]; then PERL_BIN=$(dirname "$(command -v perl)") fi [ -n "$PERL_BIN" ] || { echo "ERROR: no native (Strawberry) perl found; install it or set" \ "STRAWBERRY_PERL=/c/path/to/perl/bin" >&2 exit 1 } # nasm first for OpenSSL (built under sources below; the entry is # harmless until then); native perl next; then cl's own directory ahead # of MSYS /usr/bin so MSVC's link.exe shadows coreutils' link (the # classic silent breaker of OpenSSL/nginx Windows builds). PATH="$DIR_PROJECT/nasm-$VER_NASM:$PERL_BIN:$(dirname "$(command -v cl)"):$PATH" ### downloads: every pin verified before any compile ################## # A stale pin or a bad mirror fails here, in seconds, instead of after # the OpenSSL build. Cached tarballs are re-verified (a few MB of # sha256 -- negligible). mkdir -p "$DIR_DL" verify "https://nginx.org/download/nginx-$VER_NGINX.tar.gz" "$SHA_NGINX" verify "https://www.nasm.us/pub/nasm/releasebuilds/$VER_NASM/nasm-$VER_NASM.tar.gz" "$SHA_NASM" verify "https://github.com/PCRE2Project/pcre2/releases/download/pcre2-$VER_PCRE2/pcre2-$VER_PCRE2.tar.gz" "$SHA_PCRE2" verify "https://github.com/openssl/openssl/releases/download/openssl-$VER_OPENSSL/openssl-$VER_OPENSSL.tar.gz" "$SHA_OPENSSL" verify "https://zlib.net/zlib-$VER_ZLIB.tar.gz" "$SHA_ZLIB" [ "$BUILDLIB_ZSTD" = 1 ] \ && verify "https://github.com/facebook/zstd/releases/download/v$VER_ZSTD/zstd-$VER_ZSTD.tar.gz" "$SHA_ZSTD" ### sources: extract and clone, still before any compile ############## extract_complete "nginx-$VER_NGINX" "nginx-$VER_NGINX.tar.gz" mkdir -p "$DIR_NGINX/objs/lib" # Module clones next, so a bad ref (or a ref without compression/) also # fails before the first library build. ngx_brotli is cloned whenever # libbrotli is built, module or not: the library source is its # deps/brotli submodule. cd "$DIR_NGINX/objs/lib" [ "$WITH_HEADERS_MORE" = 1 ] \ && clone_module headers-more-nginx-module "$REPO_HEADERS_MORE" "$REF_HEADERS_MORE" if [ "$WITH_BROTLI" = 1 ] || [ "$BUILDLIB_BROTLI" = 1 ]; then clone_module ngx_brotli "$REPO_BROTLI" "$REF_BROTLI" fi if [ "$WITH_ZSTD" = 1 ] || [ "$WITH_COMPRESSION" = 1 ]; then echo "nginx-zstd-module: repo=$REPO_ZSTD_MODULE ref=$REF_ZSTD_MODULE" >&2 # clone_module runs a real git-clone even when REPO_ZSTD_MODULE resolved # to a local checkout path (the default above) -- that only pulls # COMMITTED history, so uncommitted working-tree edits in that checkout # are not part of this build. Commit (even to a throwaway branch) before # building to include local changes. clone_module nginx-zstd-module "$REPO_ZSTD_MODULE" "$REF_ZSTD_MODULE" if [ "$WITH_COMPRESSION" = 1 ] && [ ! -f nginx-zstd-module/compression/config ]; then echo "ERROR: REF_ZSTD_MODULE=$REF_ZSTD_MODULE has no compression/ --" \ "WITH_COMPRESSION=1 needs a ref whose tree carries it (see the" \ "header); set REPO_ZSTD_MODULE/REF_ZSTD_MODULE accordingly" >&2 exit 1 fi fi extract_complete "pcre2-$VER_PCRE2" "pcre2-$VER_PCRE2.tar.gz" extract_complete "openssl-$VER_OPENSSL" "openssl-$VER_OPENSSL.tar.gz" extract_complete "zlib-$VER_ZLIB" "zlib-$VER_ZLIB.tar.gz" cd "$DIR_PROJECT" ### library builds ##################################################### # nasm for OpenSSL's assembly (fast crypto; drop by adding no-asm to # OPENSSL_OPT and deleting this block). Built first: the OpenSSL # pre-build below needs it on PATH. if [ ! -x "nasm-$VER_NASM/nasm" ] && [ ! -x "nasm-$VER_NASM/nasm.exe" ]; then extract_complete "nasm-$VER_NASM" "nasm-$VER_NASM.tar.gz" (cd "nasm-$VER_NASM" && ./configure >/dev/null && make -j"$(nproc)") fi # libzstd, built static with the static CRT (nginx's cl build is /MT; # CMake's default /MD would conflict at link). Extract only lib/ and # build/: the tarball's tests/ tree contains symlinks MSYS tar cannot # create, which would abort the whole extraction — and this build # needs neither tests nor programs. Extraction is marker-guarded like # every other tree; the build is guarded on its archive, so an aborted # cmake self-heals too. if [ "$BUILDLIB_ZSTD" = 1 ]; then extract_complete "zstd-$VER_ZSTD" "zstd-$VER_ZSTD.tar.gz" "zstd-$VER_ZSTD/lib" "zstd-$VER_ZSTD/build" if [ ! -f "zstd-$VER_ZSTD/out/lib/zstd_static.lib" ]; then cmake -B "zstd-$VER_ZSTD/out" -S "zstd-$VER_ZSTD/build/cmake" \ -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Release \ -DZSTD_BUILD_SHARED=OFF -DZSTD_BUILD_PROGRAMS=OFF \ -DZSTD_BUILD_TESTS=OFF -DZSTD_USE_STATIC_RUNTIME=ON cmake --build "zstd-$VER_ZSTD/out" fi fi # libbrotli from the ngx_brotli clone's submodule. Static, static CRT, # same rule as libzstd. Guarded on the archive, so an aborted cmake # (which leaves out/ present but empty) self-heals. if [ "$BUILDLIB_BROTLI" = 1 ] && [ ! -f "$DIR_NGINX/objs/lib/ngx_brotli/deps/brotli/out/brotlienc.lib" ]; then cmake -B "$DIR_NGINX/objs/lib/ngx_brotli/deps/brotli/out" \ -S "$DIR_NGINX/objs/lib/ngx_brotli/deps/brotli" \ -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Release \ -DBUILD_SHARED_LIBS=OFF -DBROTLI_DISABLE_TESTS=ON \ -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded cmake --build "$DIR_NGINX/objs/lib/ngx_brotli/deps/brotli/out" fi cd "$DIR_NGINX" ### OpenSSL, pre-built ################################################## # nginx would build OpenSSL itself during nmake, but only AFTER # configure -- too late for the compression module's EVP probe, which # then settles for portable SHA-256. Building it first, with the exact # Configure line nginx's makefile.msvc uses (same prefix, same options), # lets the probe find the installed headers and lets nmake skip its own # rule (see the touch before nmake). Guarded on a completion marker written # only after install_sw succeeds: OpenSSL installs headers before libraries, # so an installed evp.h alone cannot prove an interrupted install completed. # # CRT: no-shared makes OpenSSL compile its static libraries with /MT /Zl # (Configurations/10-main.conf, lib_cflags; the generated makefile's # LIB_CFLAGS shows it), the same static CRT as nginx's cl build # (auto/cc/msvc, LIBC="-MT"). enable-static-vcruntime is consulted only # for shared builds and would change nothing here. OPENSSL_INSTALL_MARKER="objs/lib/openssl-$VER_OPENSSL/openssl/.install-complete" if [ ! -f "$OPENSSL_INSTALL_MARKER" ]; then # shellcheck disable=SC2086 (cd "objs/lib/openssl-$VER_OPENSSL" \ && perl Configure VC-WIN64A $OPENSSL_OPT \ --prefix="$(cygpath -m "$PWD")/openssl" \ --openssldir="$(cygpath -m "$PWD")/openssl/ssl" \ && nmake && nmake install_sw) : >"$OPENSSL_INSTALL_MARKER" fi ### configure ########################################################## # Groups: paths/core; the modules the nginx.org Windows binary ships; # the optional third-party modules per the toggles. # # --with-http_v3_module is deliberately NOT here: nginx's QUIC needs # UDP support the win32 event core lacks (see header caveats). configure_args=( --with-cc=cl --prefix= --conf-path=conf/nginx.conf --pid-path=logs/nginx.pid --http-log-path=logs/access.log --error-log-path=logs/error.log --sbin-path=nginx.exe --http-client-body-temp-path=temp/client_body_temp --http-proxy-temp-path=temp/proxy_temp --http-fastcgi-temp-path=temp/fastcgi_temp --http-scgi-temp-path=temp/scgi_temp --http-uwsgi-temp-path=temp/uwsgi_temp --with-cc-opt=-DFD_SETSIZE=1024 # nginx.org's Windows binary ships with debug logging compiled in; # it costs a few hundred kB — the real size lever is the -opt:ref # dead-code elimination added below. --with-debug --with-pcre="objs/lib/pcre2-$VER_PCRE2" --with-zlib="objs/lib/zlib-$VER_ZLIB" --with-openssl="objs/lib/openssl-$VER_OPENSSL" --with-openssl-opt="$OPENSSL_OPT" --with-http_ssl_module --with-http_v2_module --with-http_auth_request_module --with-http_addition_module --with-http_dav_module --with-http_flv_module --with-http_gunzip_module --with-http_gzip_static_module --with-http_mp4_module --with-http_random_index_module --with-http_realip_module --with-http_secure_link_module --with-http_slice_module --with-http_sub_module --with-http_stub_status_module --with-mail --with-mail_ssl_module --with-stream --with-stream_ssl_module ) # Library locations for the module configs. Both the standalone zstd # pair and the unified module read ZSTD_INC/ZSTD_LIB; the unified module # also reads BROTLI_INC/BROTLI_LIB (ngx_brotli finds its own submodule). # An explicit path is a promise to those configs -- set only when the # library was actually built. A backend switched off is told so # explicitly rather than left to a failing auto-probe. if [ "$BUILDLIB_ZSTD" = 1 ]; then export ZSTD_INC="$DIR_PROJECT/zstd-$VER_ZSTD/lib" export ZSTD_LIB="$DIR_PROJECT/zstd-$VER_ZSTD/out/lib" fi if [ "$BUILDLIB_BROTLI" = 1 ]; then export BROTLI_INC="$DIR_NGINX/objs/lib/ngx_brotli/deps/brotli/c/include" export BROTLI_LIB="$DIR_NGINX/objs/lib/ngx_brotli/deps/brotli/out" fi if [ "$WITH_COMPRESSION" = 1 ]; then [ "$BUILDLIB_ZSTD" = 1 ] || export NGX_HTTP_COMPRESSION_NO_ZSTD=1 [ "$BUILDLIB_BROTLI" = 1 ] || export NGX_HTTP_COMPRESSION_NO_BROTLI=1 fi [ "$WITH_HEADERS_MORE" = 1 ] \ && configure_args+=(--add-module=objs/lib/headers-more-nginx-module) [ "$WITH_ZSTD" = 1 ] \ && configure_args+=(--add-module=objs/lib/nginx-zstd-module) [ "$WITH_BROTLI" = 1 ] \ && configure_args+=(--add-module=objs/lib/ngx_brotli) [ "$WITH_COMPRESSION" = 1 ] \ && configure_args+=(--add-module=objs/lib/nginx-zstd-module/compression) ./configure "${configure_args[@]}" # nginx's msvc makefile compiles with -WX (warnings as errors), which # third-party module trees do not survive under -W4; strip it. Also let # the linker drop unreferenced sections despite /DEBUG. sed -i 's/ -WX//; s/-link -verbose:lib -debug/-link -verbose:lib -debug -opt:ref -opt:icf/' objs/Makefile grep -q -- '-opt:ref' objs/Makefile || echo "WARN: link-opt sed did not match" # The generated rule is `openssl/include/openssl/ssl.h: objs/Makefile`, # so the pre-built header must be newer than the FINAL Makefile -- this # must stay the last thing before nmake, after every edit of # objs/Makefile above, or nmake rebuilds OpenSSL on every run. touch "objs/lib/openssl-$VER_OPENSSL/openssl/include/openssl/ssl.h" nmake echo echo "== built: $DIR_NGINX/objs/nginx.exe" ./objs/nginx.exe -V myguard-labs-nginx-zstd-module-3486b5b/ci/tools/bump-versions.sh000077500000000000000000000557201525232545500247520ustar00rootroot00000000000000#!/usr/bin/env bash # # Check the upstream release feeds for newer releases than what's pinned in # this repo, and rewrite those pins in place. Called by # .github/workflows/bump.yml on a schedule; also runnable locally to preview # a bump before it lands. # # ci/tools/bump-versions.sh [--dry-run] # # What gets bumped: # - nginx STABLE pin -- ci-deep.yml's build-flavors matrix (label: stable) # - angie pin -- ci-deep.yml's build-flavors matrix (label: angie) # - NGINX_SHA256 -- ci/tools/ci-build.sh, nginx-stable only (verified # against the tarball's OWN PGP signature via the # vendored keys in ci/tools/keys/ before the digest # is recorded, so a bad bump can't silently pin a # malicious tarball's hash) # - ANGIE_SHA256 -- ci/tools/ci-build.sh (angie.software publishes no # PGP signature, so sha256 is the only check) # - HARNESS_NGINX_VERSION -- the harness jobs' static mainline pin, in the # env: blocks of ci-deep.yml AND harness-fault-arms.yml # (the two must agree; a disagreement is fatal here # rather than silently bumped into a third value). # No digest is recorded for it: the jobs fetch # through ci/tools/fetch-verified-nginx.sh, which # PGP-verifies at run time. # - ci/tools/windows-pins.sh -- the Windows build's sources: nginx # (mainline), pcre2, zlib, OpenSSL and zstd, each # as a VER_X/SHA_X pair. nasm is NOT bumped: # nasm.us publishes no release feed this script can # query, and it changes rarely -- edit by hand. # # nginx mainline in the build-flavors matrix (label: mainline) and the # NGINX_VERSION: "" of build-test.yml / ci-deep.yml are resolved at CI run # time (build-test.yml's `resolve` job, the per-job "Resolve latest mainline # nginx" steps) through ci/tools/nginx-releases.sh, the same feed reader this # script uses, so they carry no static pin. The harness and Windows jobs pin mainline statically # -- a fault-injection harness and a Windows binary want a build that does # not change under them between runs -- and those are the mainline pins # bumped here. # # Digests: nginx tarballs (stable and mainline alike) are PGP-verified before # a digest is recorded. angie, pcre2, zlib, OpenSSL and zstd publish no # signature this repo has a vendored key for, so their digest is computed # from the tarball their release URL serves -- the same provenance a # hand-recorded pin has, minus the copy-paste step. zlib is served by both # zlib.net (build-windows.sh) and GitHub (windows-build.yml); both are # fetched and must agree before the digest is pinned. # # The GitHub release feeds serve several release lines at once (OpenSSL # ships an LTS and a current line, sometimes on the same day), so a pin # taken from a release API only moves FORWARD: a "latest" that sorts below # the pinned version is reported and left alone, never written. # # A version bump with a stale sha256 pin is worse than no pin (ci-build.sh # treats a missing pin as "print a warning", but a WRONG pin is a hard FATAL # for a pinned version; build-windows.sh refuses outright) -- so every # version edit here is paired with a digest computed from the exact tarball # that version resolves to, never carried over from a previous entry. set -euo pipefail # Command substitutions do not inherit -e by default, so a failed curl inside # `digest="$(fetch ...)"` would be ignored and the EMPTY temp file hashed and # pinned. Every fetch below also checks curl explicitly; this closes the same # hole for anything added later. shopt -s inherit_errexit DRY_RUN=0 [ "${1:-}" = "--dry-run" ] && DRY_RUN=1 # Derive every repo-relative path from the script's OWN location, never from # cwd -- a caller running this from elsewhere must not silently miss the # ci-build.sh / keys move (A30-F4: tools/ -> ci/tools/, verified dead on the # old paths). SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" cd "$REPO_ROOT" CI_BUILD_SH="ci/tools/ci-build.sh" KEYS_DIR="ci/tools/keys" MATRIX_FILE=".github/workflows/ci-deep.yml" PINS_FILE="ci/tools/windows-pins.sh" HARNESS_FILES=(".github/workflows/ci-deep.yml" ".github/workflows/harness-fault-arms.yml") # --- discover latest versions ------------------------------------------- # nginx stable/mainline come from ci/tools/nginx-releases.sh, the one reader # of the GitHub releases feed (the CI workflows and ci-build.sh run the same # script to resolve the mainline they build). gh_api_json comes from there # too and serves the other five feeds below. # shellcheck source=ci/tools/nginx-releases.sh . "$SCRIPT_DIR/nginx-releases.sh" # The version carried by OWNER/REPO's latest GitHub release tag, whatever # the tag's prefix (v1.3.2, pcre2-10.48, openssl-4.0.2). gh_latest_tag() { local repo="$1" json json="$(gh_api_json "repos/${repo}/releases/latest")" || return 1 echo "$json" | grep -m1 '"tag_name"' | grep -oE '[0-9]+(\.[0-9]+)+' | head -1 || true } latest_angie() { gh_latest_tag webserver-llc/angie; } latest_pcre2() { gh_latest_tag PCRE2Project/pcre2; } latest_zlib() { gh_latest_tag madler/zlib; } latest_openssl() { gh_latest_tag openssl/openssl; } latest_zstd() { gh_latest_tag facebook/zstd; } read -r NEW_STABLE NEW_MAINLINE < <(nginx_lines) [ "$NEW_STABLE" != "-" ] || NEW_STABLE="" [ "$NEW_MAINLINE" != "-" ] || NEW_MAINLINE="" NEW_ANGIE="$(latest_angie)" NEW_PCRE2="$(latest_pcre2)" NEW_ZLIB="$(latest_zlib)" NEW_OPENSSL="$(latest_openssl)" NEW_ZSTD="$(latest_zstd)" for v in NEW_STABLE NEW_MAINLINE NEW_ANGIE NEW_PCRE2 NEW_ZLIB NEW_OPENSSL NEW_ZSTD; do if [ -z "${!v}" ]; then echo "FATAL: could not determine $v -- refusing to bump with a blank version" >&2 exit 1 fi done echo "latest: nginx stable=$NEW_STABLE mainline=$NEW_MAINLINE angie=$NEW_ANGIE" \ "pcre2=$NEW_PCRE2 zlib=$NEW_ZLIB openssl=$NEW_OPENSSL zstd=$NEW_ZSTD" # --- discover current pins ---------------------------------------------- # Read version and label from the same YAML list entry regardless of their # order. The mutator below deliberately remains format-strict so unexpected # layout drift fails before anything is installed. matrix_version_for_label() { awk -v want="$1" ' function emit() { if (l == want && v != "") { print v; found = 1; exit } } /^[[:space:]]*-[[:space:]]/ { emit(); v = ""; l = "" } /version:/ { match($0, /"[0-9.]+"/); v = substr($0, RSTART+1, RLENGTH-2) } /label:/ { split($0, a, ":"); l = a[2]; gsub(/[ \t]/, "", l) } END { if (!found) emit() } ' "$MATRIX_FILE" } # The value of KEY=value in windows-pins.sh. The file is data-only (its # header says so), so the key's line IS the pin; a key that is not there is # fatal up front, before anything is rewritten -- never a blank "current". pin_value() { local line line="$(grep -E "^$1=" "$PINS_FILE" || true)" if [ -z "$line" ]; then echo "FATAL: $PINS_FILE does not set $1 -- refusing to bump a pin that is not there" >&2 exit 1 fi echo "${line#*=}" } harness_pin() { # FILE grep -oE '^ HARNESS_NGINX_VERSION: "[0-9.]+"' "$1" | grep -oE '[0-9.]+' || true } CUR_STABLE="$(matrix_version_for_label stable)" CUR_ANGIE="$(matrix_version_for_label angie)" CUR_HARNESS="$(harness_pin "${HARNESS_FILES[0]}")" for f in "${HARNESS_FILES[@]}"; do v="$(harness_pin "$f")" if [ -z "$v" ]; then echo "FATAL: no HARNESS_NGINX_VERSION pin found in $f" >&2 exit 1 fi if [ "$v" != "$CUR_HARNESS" ]; then echo "FATAL: $f pins HARNESS_NGINX_VERSION $v but ${HARNESS_FILES[0]} pins $CUR_HARNESS" \ "-- the harness jobs must build the same nginx; reconcile by hand first" >&2 exit 1 fi done # Every Windows pin is read (and so checked for presence) before any edit, # so a pins file missing one line fails here with nothing rewritten. nasm # is checked too: build-windows.sh requires all twelve keys, and a bump # that left the file unable to load would be worse than no bump. It has # no release feed, so it is not in the discovery above and is never # rewritten here. WINDOWS_NAMES=(NGINX PCRE2 ZLIB OPENSSL ZSTD NASM) declare -A CUR_WIN=() for name in "${WINDOWS_NAMES[@]}"; do CUR_WIN[$name]="$(pin_value "VER_$name")" pin_value "SHA_$name" >/dev/null done echo "pinned: nginx stable=$CUR_STABLE angie=$CUR_ANGIE harness=$CUR_HARNESS" \ "windows: nginx=${CUR_WIN[NGINX]} pcre2=${CUR_WIN[PCRE2]} zlib=${CUR_WIN[ZLIB]}" \ "openssl=${CUR_WIN[OPENSSL]} zstd=${CUR_WIN[ZSTD]} nasm=${CUR_WIN[NASM]}" CHANGED=0 # --- sha256 helpers --- # A fetch that fails is fatal here, explicitly: the digest of whatever is # left in the temp file (nothing) must never be the value recorded. fetch_or_die() { # url out if ! curl -fsSL "$1" -o "$2"; then rm -f "$2" echo "FATAL: could not fetch $1 -- refusing to pin a digest of what was not downloaded" >&2 exit 1 fi } sha256_for_url() ( local url="$1" tmp digest cleanup_url_digest() { [ -z "${tmp:-}" ] || rm -f -- "$tmp" } trap cleanup_url_digest EXIT tmp="$(mktemp)" fetch_or_die "$url" "$tmp" digest="$(sha256sum "$tmp" | awk '{print $1}')" echo "$digest" ) sha256_for_angie() { sha256_for_url "https://download.angie.software/files/angie-${1}.tar.gz" } # nginx (stable and mainline) also gets its detached signature checked # against the vendored keyring (ci/tools/keys/) before we trust the digest # we're about to pin -- a bump script that records a hash without checking # provenance first would just be moving the F3 trust gap here instead of # fixing it. sha256_for_nginx() ( local version="$1" url tmp digest gnupghome keyfiles cleanup_nginx_verify() { [ -z "${gnupghome:-}" ] || rm -rf "$gnupghome" [ -z "${tmp:-}" ] || rm -f "$tmp" "${tmp}.asc" } trap cleanup_nginx_verify EXIT url="https://nginx.org/download/nginx-${version}.tar.gz" tmp="$(mktemp)" fetch_or_die "$url" "$tmp" fetch_or_die "${url}.asc" "${tmp}.asc" gnupghome="$(mktemp -d)" GNUPGHOME="$gnupghome" export GNUPGHOME chmod 700 "$gnupghome" shopt -s nullglob keyfiles=("$KEYS_DIR"/*.key) shopt -u nullglob if [ "${#keyfiles[@]}" -eq 0 ]; then echo "FATAL: no keyring found under $KEYS_DIR -- refusing to pin an unverified digest" >&2 exit 1 fi for keyfile in "${keyfiles[@]}"; do gpg --quiet --import "$keyfile" 2>/dev/null done if ! gpg --quiet --verify "${tmp}.asc" "$tmp" 2>/dev/null; then echo "FATAL: PGP verification failed for nginx-${version}.tar.gz -- refusing to pin an unverified digest" >&2 exit 1 fi digest="$(sha256sum "$tmp" | awk '{print $1}')" echo "$digest" ) # The release tarball a Windows pin names -- the URL windows-build.yml # fetches (build-windows.sh uses the same, except zlib.net for zlib). release_url() { # NAME VERSION case "$1" in PCRE2) echo "https://github.com/PCRE2Project/pcre2/releases/download/pcre2-$2/pcre2-$2.tar.gz" ;; ZLIB) echo "https://github.com/madler/zlib/releases/download/v$2/zlib-$2.tar.gz" ;; OPENSSL) echo "https://github.com/openssl/openssl/releases/download/openssl-$2/openssl-$2.tar.gz" ;; ZSTD) echo "https://github.com/facebook/zstd/releases/download/v$2/zstd-$2.tar.gz" ;; *) echo "FATAL: no release URL for $1" >&2 exit 1 ;; esac } sha256_for_release() { # NAME VERSION local name="$1" version="$2" digest other digest="$(sha256_for_url "$(release_url "$name" "$version")")" if [ "$name" = ZLIB ]; then # Two hosts serve this one pin; a digest that only one of them # matches would fail loudly in the consumer that uses the other. other="$(sha256_for_url "https://zlib.net/zlib-${version}.tar.gz")" if [ "$other" != "$digest" ]; then echo "FATAL: zlib.net and GitHub serve different zlib-${version}.tar.gz" \ "(${other} vs ${digest}) -- refusing to pin either" >&2 exit 1 fi fi echo "$digest" } # is_newer NEW CUR -- NEW sorts strictly above CUR as a version. is_newer() { local sorted [ "$1" != "$2" ] || return 1 if ! sorted="$(printf '%s\n%s\n' "$1" "$2" | sort -V)"; then echo "FATAL: could not compare versions $1 and $2" >&2 return 2 fi [ "$(printf '%s\n' "$sorted" | tail -1)" = "$1" ] } # should_bump LABEL NEW CUR -- true only for a strict forward move. Equality # is quiet; an older feed value is reported as a hold; comparison failure is # propagated as a fatal status for the caller to distinguish from both. should_bump() { local label="$1" new="$2" cur="$3" rc [ "$new" != "$cur" ] || return 1 if is_newer "$new" "$cur"; then return 0 else rc=$? fi [ "$rc" -eq 1 ] || return "$rc" echo "hold $label: release feed says $new, pinned $cur is newer -- not moving a pin backwards" return 1 } # bump_wanted LABEL NEW CUR -- 0 to bump, 1 to hold. Any other comparator # status is fatal here, in one place, so no call site can turn a broken # comparison into a quiet hold by dropping its own check. bump_wanted() { local rc=0 should_bump "$@" || rc=$? [ "$rc" -eq 0 ] || [ "$rc" -eq 1 ] || exit "$rc" return "$rc" } # --- mutators ------------------------------------------------------------- # Every mutator here writes to a temp file and renames it into place only # after the edit is confirmed to have actually matched something -- a no-op # replacement (stale path, drifted format) FAILS LOUDLY instead of silently # leaving the file untouched while the script reports success (A30-F4). # The apply phase points them at STAGED COPIES of the tracked files (see # below), so a mutator failing part-way leaves every tracked file as it # was; messages name the tracked file, not the copy. STAGE="" UPDATES="" tracked() { echo "${1#"$UPDATES/"}"; } bump_matrix_pin() { local label="$1" old="$2" new="$3" tmp [ "$old" = "$new" ] && return 0 tmp="$(mktemp)" if ! python3 - "$label" "$old" "$new" "$MATRIX_FILE" "$tmp" "$(tracked "$MATRIX_FILE")" <<'PYEOF' import re, sys label, old, new, path, out, shown = sys.argv[1:7] text = open(path).read() pattern = re.compile( r'(version:\s*"' + re.escape(old) + r'"\n\s*label:\s*' + re.escape(label) + r')' ) replaced = pattern.sub(lambda m: m.group(1).replace(old, new), text) if replaced == text: print(f"FATAL: no matrix entry matched for label={label} old={old} in {shown}", file=sys.stderr) sys.exit(1) open(out, "w").write(replaced) PYEOF then rm -f "$tmp" exit 1 fi mv "$tmp" "$MATRIX_FILE" CHANGED=1 } _bump_sha256_pin() { local table="$1" old="$2" new="$3" digest="$4" tmp grep -q "\[\"${new}\"\]" "$CI_BUILD_SH" && return 0 # already pinned if ! grep -q "declare -A ${table}=(" "$CI_BUILD_SH"; then echo "FATAL: no 'declare -A ${table}=(' table found in $(tracked "$CI_BUILD_SH") -- refusing a no-op pin" >&2 exit 1 fi tmp="$(mktemp)" # Insert the new pin right after the table's opening line; leave old # entries in place (ci-build.sh keys by version, older callers still work). sed "/declare -A ${table}=(/a\\ [\"${new}\"]=\"${digest}\"" "$CI_BUILD_SH" >"$tmp" mv "$tmp" "$CI_BUILD_SH" CHANGED=1 } bump_angie_sha256_pin() { local old="$1" new="$2" digest="$3" _bump_sha256_pin ANGIE_SHA256 "$old" "$new" "$digest" } bump_nginx_sha256_pin() { local old="$1" new="$2" digest="$3" _bump_sha256_pin NGINX_SHA256 "$old" "$new" "$digest" } # Rewrite the one ` HARNESS_NGINX_VERSION: "OLD"` line of a workflow's env: # block. Anchored on the exact line so a same-looking value elsewhere in # the file (a comment, a step) is never touched. bump_harness_pin() { # FILE OLD NEW local file="$1" old="$2" new="$3" tmp tmp="$(mktemp)" sed -E "s|^( HARNESS_NGINX_VERSION: )\"${old//./\\.}\"\$|\\1\"${new}\"|" "$file" >"$tmp" if cmp -s "$tmp" "$file"; then rm -f "$tmp" echo "FATAL: no ' HARNESS_NGINX_VERSION: \"${old}\"' line in $(tracked "$file") -- refusing a no-op pin" >&2 exit 1 fi mv "$tmp" "$file" CHANGED=1 } # Rewrite one KEY=value line of windows-pins.sh. bump_pins_line() { # KEY NEW local key="$1" new="$2" tmp tmp="$(mktemp)" sed -E "s|^${key}=.*\$|${key}=${new}|" "$PINS_FILE" >"$tmp" if cmp -s "$tmp" "$PINS_FILE"; then rm -f "$tmp" echo "FATAL: ${key}=${new} changed nothing in $(tracked "$PINS_FILE") -- refusing a no-op pin" >&2 exit 1 fi mv "$tmp" "$PINS_FILE" CHANGED=1 } # A Windows pin is a VER_X/SHA_X pair; both lines were confirmed present up # front (pin_value), so the pair lands whole or the script has already died. bump_windows_pin() { # NAME NEW DIGEST bump_pins_line "VER_$1" "$2" bump_pins_line "SHA_$1" "$3" } # --- resolve: decide every bump and fetch every digest, editing nothing --- # All network work happens here, before the first file is touched, so a # download or verification failure part-way through leaves the tree # exactly as it was -- never a matrix bumped for a digest that was then # not fetched. The apply phase below is local edits only, each of which # fails loudly before writing its file (A30-F4). STABLE_DIGEST="" ANGIE_DIGEST="" WIN_NGINX_DIGEST="" BUMP_HARNESS=0 declare -A LIB_DIGEST=() if bump_wanted "nginx stable" "$NEW_STABLE" "$CUR_STABLE"; then echo "bump nginx stable: $CUR_STABLE -> $NEW_STABLE" CHANGED=1 if [ "$DRY_RUN" = 0 ]; then STABLE_DIGEST="$(sha256_for_nginx "$NEW_STABLE")" echo " sha256 $STABLE_DIGEST" fi fi if bump_wanted angie "$NEW_ANGIE" "$CUR_ANGIE"; then echo "bump angie: $CUR_ANGIE -> $NEW_ANGIE" CHANGED=1 if [ "$DRY_RUN" = 0 ]; then ANGIE_DIGEST="$(sha256_for_angie "$NEW_ANGIE")" echo " sha256 $ANGIE_DIGEST" fi fi # The two static mainline pins are compared on their own, so a hand-bump of # one does not hide the other falling behind. if bump_wanted "windows nginx" "$NEW_MAINLINE" "${CUR_WIN[NGINX]}"; then echo "bump windows nginx: ${CUR_WIN[NGINX]} -> $NEW_MAINLINE" CHANGED=1 if [ "$DRY_RUN" = 0 ]; then WIN_NGINX_DIGEST="$(sha256_for_nginx "$NEW_MAINLINE")" echo " sha256 $WIN_NGINX_DIGEST" fi fi if bump_wanted "harness nginx" "$NEW_MAINLINE" "$CUR_HARNESS"; then echo "bump harness nginx: $CUR_HARNESS -> $NEW_MAINLINE" CHANGED=1 BUMP_HARNESS=1 fi declare -A NEW_LIB=([PCRE2]="$NEW_PCRE2" [ZLIB]="$NEW_ZLIB" [OPENSSL]="$NEW_OPENSSL" [ZSTD]="$NEW_ZSTD") for name in PCRE2 ZLIB OPENSSL ZSTD; do new="${NEW_LIB[$name]}" cur="${CUR_WIN[$name]}" [ "$new" = "$cur" ] && continue bump_wanted "windows ${name,,}" "$new" "$cur" || continue echo "bump windows ${name,,}: $cur -> $new" CHANGED=1 if [ "$DRY_RUN" = 0 ]; then LIB_DIGEST[$name]="$(sha256_for_release "$name" "$new")" echo " sha256 ${LIB_DIGEST[$name]}" fi done if [ "$CHANGED" = 0 ]; then echo "everything up to date, nothing to bump" fi if [ "$DRY_RUN" = 1 ] || [ "$CHANGED" = 0 ]; then echo "CHANGED=$CHANGED" exit 0 fi # --- apply: local edits only, every input already in hand ---------------- # The mutators work on staged copies of the tracked files, installed # together at the end, so a mutator failing part-way (a drifted format in # the second file it reaches) leaves every tracked file as it was -- the # same guarantee the resolve phase gives for the network. STAGE="$(mktemp -d)" ORIGINALS="$STAGE/originals" UPDATES="$STAGE/updates" INSTALLING=0 INSTALL_TEMP="" INSTALLED=() cleanup_stage() { local rc=$? f rollback_tmp rollback_failed=0 i trap - EXIT INT TERM if [ "$INSTALLING" -eq 1 ] && [ "$rc" -ne 0 ]; then for ((i = ${#INSTALLED[@]} - 1; i >= 0; i--)); do f="${INSTALLED[i]}" rollback_tmp="$(mktemp "$(dirname "$f")/.bump-rollback.XXXXXX")" || { rollback_failed=1 continue } if ! cp -p "$ORIGINALS/$f" "$rollback_tmp" || ! mv -f "$rollback_tmp" "$f"; then rm -f -- "$rollback_tmp" rollback_failed=1 fi done [ "$rollback_failed" -eq 0 ] || echo "FATAL: rollback could not restore every tracked file" >&2 fi [ -z "$INSTALL_TEMP" ] || rm -f -- "$INSTALL_TEMP" rm -rf -- "$STAGE" [ "$rollback_failed" -eq 0 ] || rc=1 exit "$rc" } trap cleanup_stage EXIT trap 'exit 130' INT trap 'exit 143' TERM # ci-deep.yml is both the matrix and a harness workflow. Keep one copy in # the transaction so each tracked path is installed and rolled back once. TRACKED=() declare -A TRACKED_SEEN=() for f in "$MATRIX_FILE" "$CI_BUILD_SH" "$PINS_FILE" "${HARNESS_FILES[@]}"; do if [ -z "${TRACKED_SEEN[$f]:-}" ]; then TRACKED+=("$f") TRACKED_SEEN[$f]=1 fi done for f in "${TRACKED[@]}"; do mkdir -p "$ORIGINALS/$(dirname "$f")" "$UPDATES/$(dirname "$f")" cp -p "$f" "$ORIGINALS/$f" cp -p "$f" "$UPDATES/$f" done MATRIX_FILE="$UPDATES/$MATRIX_FILE" CI_BUILD_SH="$UPDATES/$CI_BUILD_SH" PINS_FILE="$UPDATES/$PINS_FILE" for i in "${!HARNESS_FILES[@]}"; do HARNESS_FILES[i]="$UPDATES/${HARNESS_FILES[i]}" done if [ -n "$STABLE_DIGEST" ]; then bump_matrix_pin stable "$CUR_STABLE" "$NEW_STABLE" bump_nginx_sha256_pin "$CUR_STABLE" "$NEW_STABLE" "$STABLE_DIGEST" fi if [ -n "$ANGIE_DIGEST" ]; then bump_matrix_pin angie "$CUR_ANGIE" "$NEW_ANGIE" bump_angie_sha256_pin "$CUR_ANGIE" "$NEW_ANGIE" "$ANGIE_DIGEST" fi if [ -n "$WIN_NGINX_DIGEST" ]; then bump_windows_pin NGINX "$NEW_MAINLINE" "$WIN_NGINX_DIGEST" fi if [ "$BUMP_HARNESS" = 1 ]; then for f in "${HARNESS_FILES[@]}"; do bump_harness_pin "$f" "$CUR_HARNESS" "$NEW_MAINLINE" done fi for name in PCRE2 ZLIB OPENSSL ZSTD; do if [ -n "${LIB_DIGEST[$name]:-}" ]; then bump_windows_pin "$name" "${NEW_LIB[$name]}" "${LIB_DIGEST[$name]}" fi done # Every edit succeeded. Replace each changed file atomically and roll back # earlier replacements if a later install fails or the process is interrupted. INSTALLING=1 INSTALL_COUNT=0 for f in "${TRACKED[@]}"; do cmp -s "$UPDATES/$f" "$f" && continue INSTALL_COUNT=$((INSTALL_COUNT + 1)) INSTALL_TEMP="$(mktemp "$(dirname "$f")/.bump-install.XXXXXX")" cp -p "$UPDATES/$f" "$INSTALL_TEMP" if [ "${BUMP_VERSIONS_TEST_FAIL_INSTALL_AT:-}" = "$INSTALL_COUNT" ]; then echo "FATAL: injected install failure at changed file $INSTALL_COUNT" >&2 false fi # Register before mv so a signal in the tiny post-rename window still # restores this path. Restoring an unchanged path if mv itself fails is safe. INSTALLED+=("$f") mv -f "$INSTALL_TEMP" "$f" INSTALL_TEMP="" if [ "${BUMP_VERSIONS_TEST_INTERRUPT_INSTALL_AT:-}" = "$INSTALL_COUNT" ]; then kill -TERM "$$" fi done INSTALLING=0 echo "CHANGED=$CHANGED" myguard-labs-nginx-zstd-module-3486b5b/ci/tools/check-zstd-dynamic-linkage.sh000077500000000000000000000013021525232545500272150ustar00rootroot00000000000000#!/usr/bin/env bash set -euo pipefail module="${1:-objs/ngx_http_zstd_filter_module.so}" if ! readelf -d "$module" | grep -E 'NEEDED.*libzstd' >/dev/null; then echo "FAIL: $module does not declare a libzstd DT_NEEDED entry" >&2 readelf -d "$module" | grep NEEDED || true exit 1 fi for sym in ZSTD_createCCtxParams ZSTD_CCtxParams_init \ ZSTD_CCtxParams_setParameter ZSTD_freeCCtxParams \ ZSTD_estimateCStreamSize_usingCCtxParams ZSTD_createCDict_advanced; do if nm -D --undefined-only "$module" | awk '{print $NF}' | grep -qx "$sym"; then echo "FAIL: $module imports static-only libzstd symbol $sym" >&2 exit 1 fi done echo "OK: $module links dynamically without ZSTDLIB_STATIC_API imports" myguard-labs-nginx-zstd-module-3486b5b/ci/tools/ci-build.sh000077500000000000000000000341621525232545500236260ustar00rootroot00000000000000#!/bin/bash # Build and test zstd-nginx-module against nginx or angie. # # ci/tools/ci-build.sh [flavor] [version] [mode] # flavor : nginx (default) | angie # version: source version, e.g. 1.31.2. Omit (or pass "" with flavor=nginx) # to resolve the current mainline release from the GitHub # releases feed (ci/tools/nginx-releases.sh). # mode : release (default) | coverage # coverage adds --coverage to cc-opt/ld-opt and builds into a # SEPARATE tree (.build/--coverage/) so a # coverage-instrumented .so is never the one a non-coverage job # picks up by cache hit. It is never bolted onto release as a # flag: gcda/gcno files left in a shared tree would poison the # next plain build's cache key without changing the key itself. # # Built tree persists at .build/-[-]/objs/{,ngx_http_zstd_*.so} # so a caller (a CI job) can find the binary and both dynamic modules after # the script exits — unlike the old single-nginx version of this script, which # built under /tmp and deleted it on exit. NO_CACHE=1 forces a from-scratch # rebuild of a given flavor/version/mode triple. # # nginx tarballs are verified via PGP against the signer keys VENDORED in # ci/tools/keys/ (committed to this repo) — never fetched from nginx.org at CI # time. Bootstrapping the verification keys from the same origin that serves # the tarball+signature gives an origin compromise the ability to substitute # all three (audit sha e289021 F3); a key rotation is a reviewed PR that adds # a new file under ci/tools/keys/, not a runtime fetch. For a statically-pinned # nginx version (nginx-stable in ci-deep.yml's matrix) the sha256 is ALSO # checked against NGINX_SHA256 below, same as angie -- floating mainline # (resolved at CI run time, see build-test.yml's `resolve` job) has no static # pin and relies on PGP alone. angie.software does not publish a PGP # signature for release tarballs, so angie tarballs are checked by sha256 # only, pinned in ANGIE_SHA256 below (computed once from an HTTPS fetch, same # approach the sibling nginx-skeleton-module repo uses for both flavors). A # version not in a pin table still builds (this script tracks moving # releases; refusing to build an unpinned version would break every future # version bump until someone updates the table first) but prints a loud # warning so the gap is visible. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # ci/tools/ -> ci/ -> repo root. TWO levels: this script lived in tools/ at # the root until the ci/ move, where one dirname was enough. A single climb # now lands in ci/, where the module config and .build/ do not exist, and the # build silently configures against the wrong tree. MODULE_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")" FLAVOR="${1:-nginx}" VERSION="${2:-}" MODE="${3:-release}" case "$FLAVOR" in nginx | angie) ;; *) echo "ERROR: unsupported flavor: $FLAVOR (want: nginx|angie)" >&2 exit 2 ;; esac case "$MODE" in release | coverage) ;; *) echo "ERROR: unsupported mode: $MODE (want: release|coverage)" >&2 exit 2 ;; esac if [ -z "$VERSION" ]; then if [ "$FLAVOR" != "nginx" ]; then echo "ERROR: a version is required for flavor=$FLAVOR" >&2 exit 2 fi # Resolve the current mainline release; nginx.org only keeps the newest # mainline tarball, so a hardcoded version eventually 404s. The same # resolver the workflows use (the GitHub releases feed), fatal by name # when the feed does not show a mainline release. VERSION="$("$SCRIPT_DIR/nginx-releases.sh" mainline)" fi # --- pinned sha256 for angie tarballs we've actually verified --- declare -A ANGIE_SHA256=( ["1.12.1"]="5f4f203be2aca6fe20770b489c720e46e51d337e521065e7e472b61e24e3d2f5" ["1.11.5"]="b5f297c6df2a74b9d0091a7cdd747fffd2d0e1d0be43632da61c1c7539db2043" ) # --- pinned sha256 for statically-pinned nginx-stable versions we've # actually verified (floating mainline, resolved at CI run time, has no # entry here and relies on the PGP check alone) --- declare -A NGINX_SHA256=( ["1.30.4"]="4261dc90e9e47c1c4041276e9aaa3d48ebe2e664f728e14fa95ae6c67d57a08b" ["1.30.3"]="e5823dc6f45610993def93ebf6cfce68264af4958c77e874b7d20f3709001b8f" ) KEYRING_DIR="$SCRIPT_DIR/keys" ROOT="${BUILD_ROOT:-$MODULE_DIR/.build}" NO_CACHE="${NO_CACHE:-0}" ZSTD_MODULE_DIR="$MODULE_DIR" case "$FLAVOR" in nginx) DIR="nginx-${VERSION}" URL="https://nginx.org/download/${DIR}.tar.gz" ;; angie) DIR="angie-${VERSION}" URL="https://download.angie.software/files/${DIR}.tar.gz" ;; esac # Coverage gets its own build tree so an instrumented .so is never served by # a plain release cache hit -- see the mode note in the file header. if [ "$MODE" = "coverage" ]; then SRCDIR="$ROOT/${FLAVOR}-${VERSION}-coverage" else SRCDIR="$ROOT/${FLAVOR}-${VERSION}" fi TARBALL="$ROOT/${DIR}.tar.gz" echo "==========================================================================" echo " zstd-nginx-module CI Build" echo "==========================================================================" echo "" echo "Module: $ZSTD_MODULE_DIR" echo "Flavor: $FLAVOR Version: $VERSION Mode: $MODE" echo "Build tree: $SRCDIR" echo "" mkdir -p "$ROOT" if [ "$NO_CACHE" = "1" ]; then rm -rf "$SRCDIR" "$TARBALL" fi # Input stamp. A build tree is keyed by flavor/version/mode only, but its objects # also depend on every path in ci/build-inputs.manifest. On a # persistent self-hosted runner the directory outlives these inputs, so without a # stamp a changed src/ silently reuses the old objects and CI reports on a binary # that no longer matches the tree. The Actions cache does not save us: a cache # MISS still finds the retained local directory. STAMP_INPUTS="$(bash "$SCRIPT_DIR/build-input-hash.sh" "$ZSTD_MODULE_DIR")" STAMP_FILE="$SRCDIR/.myguard-build-inputs" if [ -d "$SRCDIR" ] && [ "$(cat "$STAMP_FILE" 2>/dev/null || true)" != "$STAMP_INPUTS" ]; then echo "Build inputs changed since this tree was built -- rebuilding from scratch." rm -rf "$SRCDIR" fi echo "==========================================================================" echo "Phase 1: Downloading $FLAVOR $VERSION" echo "==========================================================================" echo "" if [ ! -f "$TARBALL" ]; then wget -q -O "$TARBALL" "$URL" echo "✓ Downloaded ${DIR}.tar.gz" else echo "✓ Using cached ${DIR}.tar.gz" fi # Verification always runs, cache hit or fresh download -- a cached tarball # is exactly as untrusted as a freshly-downloaded one until re-checked # (audit sha e289021 F3: "cached archives are not reverified" was a gap in # the pre-fix version of this script). if [ "$FLAVOR" = "nginx" ]; then # Always fetch over HTTPS and verify the detached PGP signature against # the nginx release-signing keys VENDORED in ci/tools/keys/ (not fetched # from nginx.org, which also serves the tarball+signature -- see the # file header). A plain HTTP download lets a network attacker swap the # source that we then configure and compile. if [ ! -f "${TARBALL}.asc" ]; then wget -q "${URL}.asc" -O "${TARBALL}.asc" fi gnupghome="$(mktemp -d)" export GNUPGHOME="$gnupghome" chmod 700 "$gnupghome" shopt -s nullglob keyfiles=("$KEYRING_DIR"/*.key) shopt -u nullglob if [ ${#keyfiles[@]} -eq 0 ]; then echo "✗ no vendored keys found in $KEYRING_DIR" >&2 rm -rf "$gnupghome" "$TARBALL" "${TARBALL}.asc" exit 1 fi for keyfile in "${keyfiles[@]}"; do gpg --quiet --import "$keyfile" 2>/dev/null done if gpg --quiet --verify "${TARBALL}.asc" "$TARBALL"; then echo "✓ PGP signature verified for ${DIR}.tar.gz" else echo "✗ PGP signature verification FAILED for ${DIR}.tar.gz" >&2 rm -rf "$gnupghome" "$TARBALL" "${TARBALL}.asc" exit 1 fi rm -rf "$gnupghome" unset GNUPGHOME EXPECTED="${NGINX_SHA256[$VERSION]:-}" if [ -n "$EXPECTED" ]; then ACTUAL="$(sha256sum "$TARBALL" | awk '{print $1}')" if [ "$ACTUAL" != "$EXPECTED" ]; then echo "✗ sha256 mismatch for ${DIR}.tar.gz" >&2 echo " expected: $EXPECTED" >&2 echo " actual: $ACTUAL" >&2 rm -f "$TARBALL" "${TARBALL}.asc" exit 1 fi echo "✓ sha256 verified for ${DIR}.tar.gz" fi else EXPECTED="${ANGIE_SHA256[$VERSION]:-}" if [ -n "$EXPECTED" ]; then ACTUAL="$(sha256sum "$TARBALL" | awk '{print $1}')" if [ "$ACTUAL" != "$EXPECTED" ]; then echo "✗ sha256 mismatch for ${DIR}.tar.gz" >&2 echo " expected: $EXPECTED" >&2 echo " actual: $ACTUAL" >&2 rm -f "$TARBALL" exit 1 fi echo "✓ sha256 verified for ${DIR}.tar.gz" else echo "WARNING: no pinned sha256 for angie $VERSION -- add one to" \ "ANGIE_SHA256 in tools/ci-build.sh (downloaded tarball is" \ "UNVERIFIED)" >&2 fi fi if [ ! -d "$SRCDIR" ]; then tar -xzf "$TARBALL" -C "$ROOT" # The tarball's top-level dir name ($DIR = "-") and our # build-tree name ($SRCDIR) are the same string, so the extracted path # IS already $SRCDIR -- moving it onto itself fails ("cannot move to a # subdirectory of itself") on a clean root / NO_CACHE=1 run. Only mv if # extraction actually landed somewhere else. if [ "$ROOT/$DIR" != "$SRCDIR" ]; then mv "$ROOT/$DIR" "$SRCDIR" fi fi printf '%s\n' "$STAMP_INPUTS" > "$STAMP_FILE" cd "$SRCDIR" echo "" echo "==========================================================================" echo "Phase 2: Configuring $FLAVOR with zstd module" echo "==========================================================================" echo "" # The flags below are what t/ requires, not preferences — build-test.yml's # nginx carries the same set, and CI Deep's binary has to match or the suite # fails on the build rather than on the module: # http_sub_module — t/00-filter.t TEST 33 uses sub_filter # http_auth_request — t/00-filter.t TEST 58 uses auth_request # (without either, config load fails with "unknown directive" and # Test::Nginx bails out of the whole TAP file) # --with-debug — TEST 91 asserts on a debug-level error.log line # no ZSTD_STATIC_LINKING_ONLY for release — this script's normal output is a # deployable dynamic module against libzstd.so, whose ABI does not # guarantee the static-only entry points. Coverage is the exception: it # exercises estimator-only config paths and is never published as the # deployable artifact. # Coverage adds --coverage to BOTH cc-opt and ld-opt via the configure # argument, not env CC/CFLAGS -- nginx's configure probes the compiler with # its own invocations (`` `$CC -v ...` ``, unquoted so it also splits on a # ccache prefix) and ignores a bare CC=/CFLAGS= override entirely: nothing in # configure or the generated Makefile ever reads it. CC_OPT="" LD_OPT="" if [ "$MODE" = "coverage" ]; then CC_OPT="$CC_OPT -DZSTD_STATIC_LINKING_ONLY --coverage -O0" LD_OPT="--coverage" fi # ccache goes through --with-cc, the one flag configure actually reads for the # compiler pathname, not env CC= (same reason as above). CCACHE_COMPILERCHECK # content -- not the mtime-based default -- so a compiler reinstalled at the # same path with the same version string still hits cache; skipped under # coverage, where --coverage recompiles from a clean tree every run anyway and # a cache lookup only adds an unproductive stat. CC_BIN="gcc" if [ "$MODE" != "coverage" ] && command -v ccache >/dev/null 2>&1; then export CCACHE_COMPILERCHECK=content CC_BIN="ccache gcc" fi # mold as the linker, cheapest cache layer of all (no key, no invalidation # logic -- just a faster ld). Skipped under coverage: --coverage's own gcov # runtime linking has known mold interop gaps, and this script has no ASan # mode to skip it for yet (asan.yml builds outside ci-build.sh entirely). if [ "$MODE" != "coverage" ] && command -v mold >/dev/null 2>&1; then LD_OPT="$LD_OPT -fuse-ld=mold" fi ./configure \ --prefix=/etc/nginx \ --sbin-path=/usr/sbin/"$FLAVOR" \ --with-debug \ --with-http_ssl_module \ --with-http_gzip_static_module \ --with-http_realip_module \ --with-http_v2_module \ --with-http_sub_module \ --with-http_auth_request_module \ --with-cc="$CC_BIN" \ --with-cc-opt="$CC_OPT" \ --with-ld-opt="$LD_OPT" \ --add-dynamic-module="$ZSTD_MODULE_DIR" \ 2>&1 | tail -5 echo "✓ Configuration complete" echo "" echo "==========================================================================" echo "Phase 3: Compiling $FLAVOR" echo "==========================================================================" echo "" make -j"$(nproc)" 2>&1 | tail -10 if [ "$MODE" != "coverage" ]; then "$ZSTD_MODULE_DIR/ci/tools/check-zstd-dynamic-linkage.sh" \ objs/ngx_http_zstd_filter_module.so fi echo "" echo "==========================================================================" echo "Phase 4: Verifying compiled modules" echo "==========================================================================" echo "" if [ -f objs/ngx_http_zstd_filter_module.so ]; then echo "✓ Filter module compiled: objs/ngx_http_zstd_filter_module.so" file objs/ngx_http_zstd_filter_module.so else echo "✗ Filter module NOT found!" exit 1 fi if [ -f objs/ngx_http_zstd_static_module.so ]; then echo "✓ Static module compiled: objs/ngx_http_zstd_static_module.so" file objs/ngx_http_zstd_static_module.so else echo "✗ Static module NOT found!" exit 1 fi if [ ! -f "objs/$FLAVOR" ]; then echo "✗ $FLAVOR binary NOT found!" exit 1 fi echo "" echo "==========================================================================" echo "✓ Build successful!" echo "==========================================================================" echo "" echo "Server + modules ready in: $SRCDIR/objs/" echo " - $FLAVOR" echo " - ngx_http_zstd_filter_module.so" echo " - ngx_http_zstd_static_module.so" myguard-labs-nginx-zstd-module-3486b5b/ci/tools/config_bench.py000077500000000000000000000751171525232545500245650ustar00rootroot00000000000000#!/usr/bin/env python3 """Config-load A/B benchmark harness: measures what `nginx -t` and a reload COST, as a curve against config scale, for two nginx binaries. This is the gating dependency for the config-time TODO rows -- caching the per-location config-merge work, caching libzstd profile work by effective tuple, and replacing the O(n^2) dcz duplicate-hash detection. Each of those rows says "benchmark before retaining the extra state". Why this is a SEPARATE tool from ci/tools/ab_bench.py ------------------------------------------------------ `ab_bench.py` measures the REQUEST path: throughput, latency and worker RSS under `wrk` load. Its oracle is "requests per second". None of the rows above touch the request path at all -- they run once per location at config-parse time, inside `nginx -t`, before a single request exists. A request-path harness structurally cannot see them: their entire cost has already been paid by the time the first connection is accepted. Hence a second tool with a different oracle (wall time and peak RSS of the config load itself), rather than a flag on the first one. What it measures ----------------- - Wall time of `nginx -t` and peak RSS of the config-load process, per (arm, scale point), where "scale" is the number of generated locations or dcz dictionary entries. - Reload time and post-reload RSS (`--mode reload`), for the rows that care about cycle memory across a live reconfigure rather than a bare syntax check. - Reported as a CURVE across scale points, never a single number. A single scale point cannot distinguish O(n) from O(n^2), which is the actual question every gated row asks. Two arms, interleaved ---------------------- Same rationale as `ab_bench.py`: the two arms alternate round by round rather than running A-then-B, so machine drift (page-cache warmth, thermal, a background job) hits both arms equally instead of being attributed to the change under test. Reported numbers are the per-arm MEDIAN across rounds. Peak RSS, not a final snapshot ------------------------------- Config load is short and its memory profile is a spike, not a plateau: the parse allocates into the cycle pool and a `nginx -t` process exits immediately after. A `/usr/bin/time -v`-style "maximum resident" read after exit can be taken after the interesting peak is already gone, and a single `/proc` read at an arbitrary moment is worse. This harness polls `/proc// status` continuously from spawn to exit and keeps the maximum, the same approach `ab_bench.py`'s `rss_monitor_start()` uses for workers. The harness's negative control is on its OWN oracle ----------------------------------------------------- PR #151 shipped three successive versions of a self-check that passed on input proving nothing (a flat 100% hit rate measured on a workload that never overlapped; a self-check where `0.0 >= 0.0` was True). The generalizable rule recorded from that: a measurement harness needs a negative control on its own oracle, not only on the thing being measured. Here that means: before any result is reported, the harness asserts that its own generated configs actually make config load MORE expensive as scale rises. If loading 4000 locations costs the same as loading 10, then this harness is not measuring config scale -- it is measuring process startup, and any "improvement" it later reports for a fix would be noise. That assertion is `evaluate_scale_self_check()`, a pure function exercised by `--self-test` with synthetic samples, so the check itself is testable without running nginx. Usage ------ # Does the current binary show a config-scale curve at all? (baseline) python3 ci/tools/config_bench.py --arm-a .build/nginx-1.31.4/objs/nginx # A/B two binaries across the dcz dictionary-count workload python3 ci/tools/config_bench.py \\ --arm-a .build/before/objs/nginx:before \\ --arm-b .build/after/objs/nginx:after \\ --workload dcz-dicts # Verify the harness's own oracle without touching nginx python3 ci/tools/config_bench.py --self-test Baseline measured 2026-08-24 (nginx 1.31.4, this host, --rounds 1) ------------------------------------------------------------------ Recorded so a later session need not re-derive them, and so the gated TODO rows can be decided on numbers instead of intuition. locations-same-profile dcz-dicts scale nginx -t scale nginx -t 100 4.7ms 600 6.7ms 4000 37.7ms 10000 91.8ms 16000 120.4ms 20000 282.9ms 32000 246.5ms 40000 1291.1ms Two conclusions, both of which bear directly on the rows this tool gates: 1. The per-location merge path is LINEAR across 320x of scale -- 100 -> 32000 locations costs 52x, not 102400x. (These numbers were measured when that path also ran an O(modules) scan per location for the gzip_vary-off warning; G5 deleted the warning and the scan, so the per-location constant is now strictly smaller and the linear shape still holds.) 2. The dcz duplicate-hash detection IS genuinely O(n^2), and this harness can now show it: 20000 -> 40000 entries is 2x the scale for 4.6x the time. But the quadratic term only dominates far above any realistic configuration. At ~600 entries -- the scale the TODO row itself names as the benchmarked workload -- the whole config load is 6.7ms, and the crossover sits somewhere past 10000 entries. So the measured answer to "replace the O(n^2) duplicate detection?" is: not on performance grounds at realistic scale. That is a refutation the row explicitly asked for ("keep the linear form if the crossover is outside realistic configurations"), not a null result. Exit non-zero only on harness error, never on a "slow" result. """ from __future__ import annotations import argparse import dataclasses import hashlib import json import os import pathlib import socket import statistics import subprocess import sys import tempfile import time DEFAULT_ROUNDS = 3 # Scale points per workload. The low point is a control (a config so small # that per-location cost cannot dominate); the high points are the scales the # TODO rows actually name -- "thousands of same-profile locations" for the # location workloads and "~600 entries" for the dcz dictionary list. WORKLOAD_SCALES: dict[str, list[int]] = { # 16000 (not 4000) so the largest point clears SCALE_MIN_HI_SECONDS with # margin on fast hardware: 4000 locations measured 27-33ms on a # contemporary dev box in 2026-08, below the 50ms floor, aborting the # self-check before either workload could report a curve. 16000 measured # 91.7ms (same-profile) / 101.2ms (multi-profile) on that same box -- # roughly 2x the floor -- while staying inside the range the module # docstring's baseline table already exercised (32000 locations, 246.5ms). "locations-same-profile": [10, 500, 4000, 16000], "locations-multi-profile": [10, 500, 4000, 16000], # 16000 (not 600) so the largest point clears SCALE_MIN_HI_SECONDS with # margin: 600 entries measured 5.8ms, below the 50ms floor. 5000 measured # 30.9ms, 10000 measured 98.7ms, and 16000 measured 182.3ms on a # contemporary dev box in 2026-08 -- the largest point ~3.6x the floor. "dcz-dicts": [10, 600, 5000, 16000], } # The distinct (level, long_mode, window_log) tuples cycled through by the # multi-profile workload. The profile-caching row's win depends on how many # DISTINCT profiles exist, not how many locations: a cache keyed by effective # tuple collapses same-profile locations and cannot collapse these. Kept small # and fixed so scale sweeps vary location count alone. PROFILE_TUPLES: list[tuple[int, str, int]] = [ (3, "off", 0), (9, "on", 27), (12, "on", 30), (1, "off", 0), ] # The multi-profile workload's per-request memory budget. # # Two of the tuples above enable long-distance matching at a large window # (level 9 / windowLog 27 and level 12 / windowLog 30). zstd_max_cctx_memory # is validated at config load against libzstd's own estimator, and those two # profiles genuinely need ~156 MB and ~1.14 GB of per-request compressor # memory respectively -- so the 32m this workload used to emit is a budget # they cannot satisfy, and nginx correctly refuses the configuration. # # This workload measures config-LOAD COST as a function of the number of # distinct profiles; the budget is fixture shape, not the thing under # measurement. So it is raised to a value every tuple fits in rather than # dropping the LDM tuples, which would change what the benchmark measures. # Keep this above the largest estimate any PROFILE_TUPLES entry produces. # (nginx size values accept only k/m suffixes, so this is spelled in MB.) MULTI_PROFILE_BUDGET = "2048m" # The self-check's floor. Between the smallest and largest scale point, the # measured config-load cost must rise by at least this factor, or the harness # has not demonstrated that it can see config scale at all. # # 1.5x (not, say, 10x) because the floor's job is only to separate "this # harness observes scale" from "this harness observes process startup" -- it # is deliberately NOT an assertion about the complexity class. A superlinear # path will clear it by a wide margin; a genuinely O(1) path will not clear it # at all, which is the outcome that must fail loudly rather than be reported # as a clean measurement. SCALE_MIN_RELATIVE_RISE = 1.5 # An absolute floor on the largest scale point's cost. Without it, two # equal-and-tiny timings (say 0.4ms vs 0.4ms, both dominated by fork+exec # jitter) can satisfy a purely relative ratio test through noise alone. This # is the direct analogue of ab_bench.py's CONTENTION_MIN_LO_HIT_RATE, added # for exactly the reason recorded there: a relative-only comparison passes on # degenerate input. SCALE_MIN_HI_SECONDS = 0.05 @dataclasses.dataclass class Arm: label: str binary: pathlib.Path filter_module: pathlib.Path | None = None static_module: pathlib.Path | None = None @dataclasses.dataclass class ConfigSample: """One config-load measurement for one (arm, workload, scale).""" seconds: float peak_rss_kb: int ok: bool detail: str = "" def parse_arm(spec: str) -> tuple[str, str]: """`path[:label]` -> (path, label). Mirrors ab_bench.py's --arm syntax.""" if ":" in spec: path, _, label = spec.rpartition(":") if path: return path, label return spec, pathlib.Path(spec).name def detect_module(binary: pathlib.Path, name: str) -> pathlib.Path | None: """Find a built dynamic module beside the binary, as ab_bench.py does.""" candidate = binary.parent / name return candidate if candidate.exists() else None def _dict_digest(index: int) -> str: """A deterministic, unique 64-hex digest for generated dcz entry N. Supplied hashes are used deliberately: the dcz row notes that supplying hashes makes the hashing pass free, which is what surfaces the O(n^2) duplicate-detection bookkeeping as the dominant config-time cost. If the harness let nginx hash 600 real dictionary files instead, hashing would swamp the very comparison loop under measurement. """ return hashlib.sha256(f"config_bench-dict-{index}".encode()).hexdigest() def pick_free_port() -> int: """An ephemeral port the kernel just confirmed is free. Closed immediately, so this is advisory rather than a reservation -- but `nginx -t` binds and releases within a second, and each measurement picks a fresh port, so a collision would show up as a failed config test (which the self-check rejects loudly) rather than as a silently wrong number. """ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.bind(("127.0.0.1", 0)) return int(sock.getsockname()[1]) def generate_config( root: pathlib.Path, arm: Arm, workload: str, scale: int, dict_file: pathlib.Path | None = None, ) -> pathlib.Path: """Write a synthetic nginx.conf of the requested workload and scale.""" load = "" filt = arm.filter_module or detect_module( arm.binary, "ngx_http_zstd_filter_module.so" ) static = arm.static_module or detect_module( arm.binary, "ngx_http_zstd_static_module.so" ) if filt: load += f"load_module {filt};\n" if static: load += f"load_module {static};\n" body = "" if workload == "locations-same-profile": # Every location carries an IDENTICAL profile. This is the # profile-caching row's best case (one distinct tuple) and the # baseline for per-location merge cost. gzip_vary off is kept as # part of the fixture's shape; since G5 it no longer triggers any # per-location scan or warning. for i in range(scale): body += ( f" location /l{i} {{\n" f" zstd on;\n" f" gzip_vary off;\n" f" zstd_comp_level 3;\n" f" zstd_max_cctx_memory 32m;\n" f" }}\n" ) elif workload == "locations-multi-profile": # Same scale, but cycling distinct (level, long_mode, window_log) # tuples. Compared against locations-same-profile at the same scale, # this isolates the part of config cost that a tuple-keyed cache # CANNOT collapse from the part it can. for i in range(scale): level, long_mode, window_log = PROFILE_TUPLES[i % len(PROFILE_TUPLES)] win = f" zstd_window_log {window_log};\n" if window_log else "" body += ( f" location /l{i} {{\n" f" zstd on;\n" f" gzip_vary off;\n" f" zstd_comp_level {level};\n" f" zstd_long {long_mode};\n" f"{win}" f" zstd_max_cctx_memory {MULTI_PROFILE_BUDGET};\n" f" }}\n" ) elif workload == "dcz-dicts": # N dcz dictionary entries with SUPPLIED hashes, all pointing at one # real file. The O(n^2) row compares each entry's digest against every # prior entry; supplying distinct hashes means no duplicate is ever # found, so the comparison loop runs to completion every time -- the # worst case, and the one the row asks about. if dict_file is None: raise RuntimeError( "harness error: the dcz-dicts workload needs a dictionary " "file and none was provided" ) entries = "" for i in range(scale): entries += f" zstd_dcz_dict_file {dict_file} {_dict_digest(i)};\n" body = ( f"{entries}" f" location / {{\n" f" zstd on;\n" f" gzip_vary on;\n" f" }}\n" ) else: raise RuntimeError(f"harness error: unknown workload {workload!r}") listen_port = pick_free_port() conf = root / "nginx.conf" conf.write_text( f"""daemon off; master_process on; worker_processes 1; {load}error_log {root}/error.log crit; pid {root}/nginx.pid; events {{ worker_connections 64; }} http {{ access_log off; server {{ # A high, unprivileged port. `nginx -t` genuinely bind()s the listen # socket as part of the test, so a privileged port (or one already in # use) makes the test FAIL after parsing succeeded -- the harness would # then be timing a config load that aborted early. Caught on the first # real run with port 1: "bind() to 127.0.0.1:1 failed (13: Permission # denied)" while the syntax check itself had passed. listen 127.0.0.1:{listen_port}; {body} }} }} """, encoding="utf-8", ) return conf def measure_config_test( arm: Arm, conf: pathlib.Path, root: pathlib.Path ) -> ConfigSample: """Run `nginx -t` once, timing it and sampling peak RSS throughout. RSS is polled from the parent rather than read once after exit, because config load is a short spike (see the module docstring). The poll interval is deliberately tight: a 4000-location parse can complete in well under a second, and a 100ms interval could sample it twice. """ cmd = [str(arm.binary), "-t", "-p", str(root), "-c", str(conf)] peak_kb = 0 # Output goes to a FILE, never to a pipe. `nginx -t` can emit one warning # per location on some of these workloads, so at 2000+ locations it may # produce far more than a 64 KB pipe buffer holds. (The gzip_vary-off # warning that originally motivated this is gone as of G5, but other # per-location warnings keep the hazard live and a file costs nothing.) With stdout=PIPE and the read deferred # until after the poll loop, nginx blocks in anon_pipe_write forever while # the harness waits for it to exit -- a deadlock that presents as an # arbitrarily slow config load, i.e. as a plausible measurement. Caught # here on the very first real run: the child sat at 0.0% CPU in # anon_pipe_write while the harness reported nothing. out_path = root / "nginx-t.out" started = time.monotonic() with open(out_path, "w", encoding="utf-8") as out_fh: proc = subprocess.Popen(cmd, stdout=out_fh, stderr=subprocess.STDOUT) status_path = pathlib.Path(f"/proc/{proc.pid}/status") while proc.poll() is None: try: for line in status_path.read_text().splitlines(): if line.startswith("VmRSS:"): peak_kb = max(peak_kb, int(line.split()[1])) break except (FileNotFoundError, ProcessLookupError, ValueError): pass time.sleep(0.002) elapsed = time.monotonic() - started output = out_path.read_text(encoding="utf-8", errors="replace") ok = proc.returncode == 0 return ConfigSample( seconds=elapsed, peak_rss_kb=peak_kb, ok=ok, detail="" if ok else output.strip()[:400], ) def evaluate_scale_self_check( results: dict[int, ConfigSample], scales: list[int] ) -> None: """The harness's negative control on ITSELF, not on the module. Raises RuntimeError (a harness error) unless the generated configs demonstrably make config load more expensive as scale rises. If they do not, this harness is measuring process startup rather than config scale, and every number it would go on to print for a candidate fix would be noise dressed as a measurement. Pure and side-effect-free by design, specifically so it can be exercised directly by the synthetic tests below instead of only being reachable by actually running nginx. This mirrors ab_bench.py::evaluate_contention_self_check(), and exists for the same reason: on that harness, three successive self-checks passed on input that proved nothing before one held. """ if len(scales) < 2: raise RuntimeError( "harness self-check FAILED: a config-scale curve needs at least " "two scale points; one point cannot show a trend." ) lo_scale, hi_scale = min(scales), max(scales) lo, hi = results.get(lo_scale), results.get(hi_scale) if lo is None or hi is None: raise RuntimeError( "harness self-check FAILED: missing a measurement at " f"{lo_scale if lo is None else hi_scale} locations/entries, so no " "curve can be established." ) if not lo.ok or not hi.ok: bad = lo if not lo.ok else hi raise RuntimeError( "harness self-check FAILED: `nginx -t` did not accept the " f"generated config ({bad.detail or 'no output'}). A rejected " "config is not a fast config -- it never did the work being " "measured." ) # ABSOLUTE floor first: two equal-and-tiny timings dominated by fork+exec # jitter can satisfy any purely relative ratio through noise alone. This # is the exact failure ab_bench.py hit when `0.0 >= 0.0 * 1.5` evaluated # True and reported an all-miss run as a clean decay. if hi.seconds < SCALE_MIN_HI_SECONDS: raise RuntimeError( "harness self-check FAILED: the largest scale point " f"({hi_scale}) loaded in {hi.seconds * 1000:.1f}ms, below the " f"{SCALE_MIN_HI_SECONDS * 1000:.0f}ms floor. At that duration the " "measurement is dominated by process startup jitter, so a ratio " "against the small config would be noise, not a scale curve. " "Raise the scale points rather than lowering this floor." ) # STRICT relative rise: `>`, never `>=`, so two equal costs cannot be read # as a rising curve. if not hi.seconds > lo.seconds * SCALE_MIN_RELATIVE_RISE: raise RuntimeError( "harness self-check FAILED: config-load cost did not rise with " f"scale. At {lo_scale} the load took {lo.seconds * 1000:.1f}ms; " f"at {hi_scale} it took {hi.seconds * 1000:.1f}ms (need strictly " f"> {SCALE_MIN_RELATIVE_RISE}x). This harness exists to price " "per-location and per-entry config work; a flat curve means the " "generated configs never exercised it, so any before/after " "difference reported from this run would be a workload artifact " "rather than a measurement. Check that the generated directives " "are actually reaching the module's merge path." ) def run_workload( arms: list[Arm], workload: str, scales: list[int], rounds: int, dict_file: pathlib.Path | None, ) -> dict[str, dict[int, ConfigSample]]: """Interleave the arms round by round, returning per-arm medians. Interleaving (not A-then-B) so machine drift is shared by both arms; see the module docstring. """ raw: dict[str, dict[int, list[ConfigSample]]] = { arm.label: {scale: [] for scale in scales} for arm in arms } for round_index in range(rounds): for arm in arms: for scale in scales: with tempfile.TemporaryDirectory(prefix="config_bench-") as tmp: root = pathlib.Path(tmp) (root / "logs").mkdir(parents=True, exist_ok=True) conf = generate_config(root, arm, workload, scale, dict_file) sample = measure_config_test(arm, conf, root) raw[arm.label][scale].append(sample) print( f" round {round_index + 1}/{rounds} done ({workload})", file=sys.stderr, ) medians: dict[str, dict[int, ConfigSample]] = {} for label, by_scale in raw.items(): medians[label] = {} for scale, samples in by_scale.items(): ok = all(s.ok for s in samples) medians[label][scale] = ConfigSample( seconds=statistics.median(s.seconds for s in samples), peak_rss_kb=int(statistics.median(s.peak_rss_kb for s in samples)), ok=ok, detail=next((s.detail for s in samples if not s.ok), ""), ) return medians def print_table( workload: str, scales: list[int], medians: dict[str, dict[int, ConfigSample]] ) -> None: print() print(f"== config load: {workload} (median of rounds) ==") print(f"{'arm':<20} {'scale':>8} {'nginx -t':>12} {'peak RSS':>12}") for label, by_scale in medians.items(): for scale in scales: s = by_scale[scale] print( f"{label:<20} {scale:>8} " f"{s.seconds * 1000:>10.1f}ms " f"{s.peak_rss_kb / 1024:>10.1f}MB" ) labels = list(medians) if len(labels) == 2: a, b = labels print() print(f"{'scale':>8} {'time delta':>14} {'RSS delta':>14}") for scale in scales: sa, sb = medians[a][scale], medians[b][scale] t_delta = ( (sb.seconds - sa.seconds) / sa.seconds * 100 if sa.seconds else 0.0 ) r_delta = ( (sb.peak_rss_kb - sa.peak_rss_kb) / sa.peak_rss_kb * 100 if sa.peak_rss_kb else 0.0 ) print(f"{scale:>8} {t_delta:>+13.1f}% {r_delta:>+13.1f}%") print(f"(delta = {b} relative to {a}; negative is faster/smaller)") def _sample(seconds: float, rss_kb: int = 4096, ok: bool = True) -> ConfigSample: return ConfigSample(seconds=seconds, peak_rss_kb=rss_kb, ok=ok) def test_scale_self_check_negative_control() -> None: """Exercise the self-check against input it MUST reject, plus one it must accept. Synthetic samples only -- no nginx, no config generation. Each rejected case below is a way a config-load harness can report a confident number while measuring nothing, which is precisely the defect class that shipped three times in ab_bench.py before its own check held. """ scales = [10, 4000] def must_reject(name: str, results: dict[int, ConfigSample]) -> None: try: evaluate_scale_self_check(results, scales) except RuntimeError: return raise AssertionError( f"self-check accepted {name}, which proves nothing about config scale" ) # A flat curve: the config got 400x bigger and cost the same. The harness # is timing process startup. must_reject("a flat cost curve", {10: _sample(0.20), 4000: _sample(0.20)}) # An inverted curve: bigger config, less time. Noise, not a measurement. must_reject("an inverted curve", {10: _sample(0.30), 4000: _sample(0.10)}) # A rise that is real but below the floor: 1.2x is inside run-to-run # jitter for a process this short. must_reject( "a rise below the relative floor", {10: _sample(0.20), 4000: _sample(0.24)}, ) # Both points tiny. The ratio is a healthy 3x, but at sub-millisecond # durations it is fork+exec jitter -- this is the case a purely relative # check would wave through. must_reject( "a large ratio between two sub-floor timings", {10: _sample(0.0002), 4000: _sample(0.0006)}, ) # Exactly at the relative boundary. `>` not `>=`, so this must fail. must_reject( "a rise exactly at the boundary", {10: _sample(0.10), 4000: _sample(0.10 * SCALE_MIN_RELATIVE_RISE)}, ) # A config nginx rejected. Fast because it never did the work. must_reject( "a rejected config", {10: _sample(0.20), 4000: _sample(0.90, ok=False)}, ) # A missing measurement at one end: no curve exists. must_reject("a missing high-scale sample", {10: _sample(0.20)}) # One scale point cannot show a trend. try: evaluate_scale_self_check({10: _sample(0.20)}, [10]) except RuntimeError: pass else: raise AssertionError("self-check accepted a single scale point") # The genuine article: a clear superlinear rise, well clear of both # floors. This must pass, or the check rejects everything and is useless. evaluate_scale_self_check({10: _sample(0.06), 4000: _sample(1.80)}, scales) def run_self_test() -> int: test_scale_self_check_negative_control() print( "self-test: OK -- scale self-check rejects flat, inverted, " "sub-floor, boundary, rejected-config and missing-sample input, " "and accepts a genuine curve" ) return 0 def main() -> int: parser = argparse.ArgumentParser( description="Config-load A/B benchmark for the zstd nginx module", formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("--arm-a", help="path/to/nginx[:label]") parser.add_argument("--arm-b", help="path/to/nginx[:label] (optional)") parser.add_argument( "--workload", default="locations-same-profile", choices=sorted(WORKLOAD_SCALES), help="which config shape to generate (default: %(default)s)", ) parser.add_argument( "--scales", help="comma-separated scale points, overriding the workload default", ) parser.add_argument( "--rounds", type=int, default=DEFAULT_ROUNDS, help="interleaved rounds per arm (default: %(default)s)", ) parser.add_argument("--json", help="write results to this path as JSON") parser.add_argument( "--self-test", action="store_true", help="exercise the harness's own oracle and exit; runs no nginx", ) args = parser.parse_args() if args.self_test: return run_self_test() if not args.arm_a: parser.error("--arm-a is required (or use --self-test)") arms: list[Arm] = [] for spec in filter(None, [args.arm_a, args.arm_b]): path, label = parse_arm(spec) binary = pathlib.Path(path).resolve() if not binary.exists(): raise RuntimeError(f"harness error: no such nginx binary: {binary}") arms.append(Arm(label=label, binary=binary)) if len({a.label for a in arms}) != len(arms): raise RuntimeError( "harness error: both arms have the same label; give one an " "explicit `path:label` so the table can tell them apart" ) scales = ( [int(x) for x in args.scales.split(",")] if args.scales else WORKLOAD_SCALES[args.workload] ) dict_file: pathlib.Path | None = None tmp_dict: tempfile.TemporaryDirectory | None = None if args.workload == "dcz-dicts": tmp_dict = tempfile.TemporaryDirectory(prefix="config_bench-dict-") dict_file = pathlib.Path(tmp_dict.name) / "dict.bin" dict_file.write_bytes(os.urandom(4096)) try: medians = run_workload(arms, args.workload, scales, args.rounds, dict_file) # The self-check runs against the FIRST arm -- the baseline. If the # baseline shows no scale curve, the workload never exercised the code # path, and comparing a second arm against it is meaningless. evaluate_scale_self_check(medians[arms[0].label], scales) print_table(args.workload, scales, medians) if args.json: payload = { "workload": args.workload, "scales": scales, "rounds": args.rounds, "arms": { label: { str(scale): dataclasses.asdict(sample) for scale, sample in by_scale.items() } for label, by_scale in medians.items() }, } pathlib.Path(args.json).write_text( json.dumps(payload, indent=2), encoding="utf-8" ) print(f"\nwrote {args.json}") finally: if tmp_dict is not None: tmp_dict.cleanup() return 0 if __name__ == "__main__": try: sys.exit(main()) except RuntimeError as exc: print(f"{exc}", file=sys.stderr) sys.exit(2) myguard-labs-nginx-zstd-module-3486b5b/ci/tools/coverage.sh000077500000000000000000000330461525232545500237310ustar00rootroot00000000000000#!/bin/bash # ci/tools/coverage.sh -- publish a coverage REPORT for src/, never a gate. # # Coverage is the cheapest number to inflate: a test that touches a line and # asserts nothing still moves it, so a floor buys the metric and sells the # thing it was standing in for. This script has no --fail-under default and # no exit-nonzero-on-threshold path; COVERAGE_FAIL_UNDER exists only for a # downstream adopter that has decided otherwise (step 26 note in PROMPT.md). # # Usage: ci/tools/coverage.sh [flavor] [version] # Runs ci-build.sh in "coverage" mode (separate .build tree, --coverage # compiled in), runs ci/t/, the assertion-bearing runtime drivers, and all # testkit scenarios against the instrumented binary so .gcda files land next # to the .gcno objects, then reports with gcovr FILTERED TO src/ # -- unfiltered gcovr walks the whole configured nginx tree (order 200k # lines of upstream C) and buries the module's own number in noise. # # Env: # COVERAGE_FAIL_UNDER optional; if set, gcovr --fail-under-line is used. # Unset by default -- see the header note above. # COVERAGE_HARNESS optional, default 1. When 1, the coverage tree is # built with the TEST_HARNESS probe compiled in and the # ci/t/harness-scenarios/* scenarios are run against it # after the Test::Nginx suites, so the libzstd FAULT # paths (which no ci/t/ test can reach -- there is no # way to make libzstd fail from the outside) are counted # instead of being reported as dead lines. Set to 0 to # reproduce the pre-harness number. # # gcovr flag note: --object-directory is used, NOT --gcov-object-directory, # which fails argparse on gcovr below 7.0. --object-directory is accepted by # every gcovr version this script has been run against (verified on 7.2). set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" MODULE_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")" # shellcheck source=ci/linter/lib.sh . "$MODULE_DIR/ci/linter/lib.sh" cd "$MODULE_DIR" collect_module_gcda() { mapfile_checked MODULE_GCDA find "$1" -name 'ngx_http_zstd_*.gcda' } if [ "${1:-}" = "--selftest-work-list" ]; then collect_module_gcda "${2:?usage: coverage.sh --selftest-work-list DIR}" exit 0 fi FLAVOR="${1:-nginx}" VERSION="${2:-}" command -v gcovr >/dev/null || { echo "ERROR: gcovr not found (pip install gcovr / apt-get install gcovr)" >&2 exit 2 } COVERAGE_HARNESS="${COVERAGE_HARNESS:-1}" echo "== coverage: building $FLAVOR ${VERSION:-} in coverage mode ==" # Measure the portable SHA-256 implementation instead of compiling it beside # the OpenSSL EVP path and then leaving nearly all of it untouched. Production # builds and the PR suite still exercise EVP; this report covers code owned by # the module that would otherwise be represented by a misleading 7% file rate. if [ "$COVERAGE_HARNESS" = "1" ]; then # BOTH switches, always together. TEST_HARNESS=1 alone compiles the probe # sources and still yields a probe-free .so -- measured in this repo # (packaged=0 probe symbols, TEST_HARNESS=1 alone=0, both=18). With a # probe-free .so every scenario below would SKIP or fail to arm, and the # coverage number would silently be the pre-harness one while claiming # otherwise. Verified compatible with --coverage: the coverage tree builds # clean with the probe in and emits .gcno for every unit. echo " (with TEST_HARNESS -- fault paths will be exercised)" NGX_ZSTD_NO_LIBCRYPTO=1 TEST_HARNESS=1 \ CFLAGS="${CFLAGS:-} -DNGX_TEST_HARNESS" \ bash "$SCRIPT_DIR/ci-build.sh" "$FLAVOR" "$VERSION" coverage else NGX_ZSTD_NO_LIBCRYPTO=1 \ bash "$SCRIPT_DIR/ci-build.sh" "$FLAVOR" "$VERSION" coverage fi # ci-build.sh resolves an empty VERSION to the current mainline; recover the # resolved value from the build tree it actually created rather than # re-resolving (a second read of the releases feed could race a release and # disagree). BUILD_ROOT="${BUILD_ROOT:-$MODULE_DIR/.build}" if [ -n "$VERSION" ]; then # An explicit version names exactly one tree. Globbing here instead would # silently report on whatever version sorts highest -- ask for 1.30.4 with a # 1.31.2 tree present and you get a coverage figure for the wrong binary, # which reads as a real measurement. SRCDIR="$BUILD_ROOT/${FLAVOR}-${VERSION}-coverage" else SRCDIR="$(find "$BUILD_ROOT" -maxdepth 1 -type d -name "${FLAVOR}-*-coverage" | sort -V | tail -1)" fi if [ -z "$SRCDIR" ] || [ ! -d "$SRCDIR" ]; then echo "ERROR: no coverage build tree found under $BUILD_ROOT" >&2 [ -n "$VERSION" ] && echo " expected: $SRCDIR" >&2 exit 1 fi echo "Coverage build tree: $SRCDIR" BIN="$SRCDIR/objs/$FLAVOR" [ -x "$BIN" ] || { echo "ERROR: $BIN not found or not executable" >&2; exit 1; } REPORT_DIR="$MODULE_DIR/.build/coverage-report" mkdir -p "$REPORT_DIR" # ci-build.sh reuses an existing build directory. A changed compiler/configure # checksum makes libgcov reject old counters instead of replacing them, so every # invocation starts profile collection from a known empty state. find "$SRCDIR/objs/addon" -name '*.gcda' -delete echo "== coverage: exercising the binary via ci/t/ ==" export TEST_NGINX_BINARY="$BIN" export TEST_NGINX_TIMEOUT="${TEST_NGINX_TIMEOUT:-20}" STRICT_SERVROOT="$HOME/.nginx-servroot-coverage-filter-${GITHUB_RUN_ID:-local-$$}-${GITHUB_RUN_ATTEMPT:-1}" export TEST_NGINX_SERVROOT="$STRICT_SERVROOT" trap 'rm -rf -- "$STRICT_SERVROOT"' EXIT rm -rf -- "$TEST_NGINX_SERVROOT" mkdir -p "$TEST_NGINX_SERVROOT" # Both dcz and static tests assert validators derived from these fixtures. # Normalize before either suite so checkout/build time cannot change the ETag. touch -d @1541504307 ci/t/suite/test ci/t/suite/test.zst prove -v ci/t/00-filter.t ci/t/02-conf-warn.t ci/t/03-dcz.t export TEST_NGINX_SERVROOT="$MODULE_DIR/ci/t/servroot-static" mkdir -p "$TEST_NGINX_SERVROOT" prove -v ci/t/01-static.t echo "== coverage: exercising runtime regression drivers ==" # Keep every endpoint inside the coverage workflow's declared 64-port band. # These are assertion-bearing end-to-end drivers already used by the PR suite; # running them against the instrumented binary turns their reachability into # line/branch evidence without inventing coverage-only tests. COVERAGE_RUNTIME_PORT_BASE="${COVERAGE_RUNTIME_PORT_BASE:-${TEST_BASE_PORT:-19330}}" p="$COVERAGE_RUNTIME_PORT_BASE" python3 ci/tools/test_encoding.py --nginx-binary "$BIN" --port "$((p + 0))" python3 ci/tools/test_compression_matrix.py \ --nginx-binary "$BIN" --port "$((p + 1))" --backend-port "$((p + 2))" python3 ci/tools/test_concurrent_cctx_isolation.py \ --nginx-binary "$BIN" --port "$((p + 3))" --backend-port "$((p + 4))" python3 ci/tools/test_cctx_reuse.py --nginx-binary "$BIN" --port "$((p + 5))" python3 ci/tools/test_proxy_unbuffered_truncation.py \ --nginx-binary "$BIN" --port "$((p + 6))" --backend-port "$((p + 7))" python3 ci/tools/test_slow_drain.py \ --nginx-binary "$BIN" --port "$((p + 8))" --backend-port "$((p + 9))" python3 ci/tools/test_sub_filter_cctx_reset.py \ --nginx-binary "$BIN" --port "$((p + 10))" --backend-port "$((p + 11))" python3 ci/tools/test_terminal_frame.py --nginx-binary "$BIN" --port "$((p + 12))" python3 ci/tools/test_reload_under_load.py \ --nginx-binary "$BIN" --port "$((p + 13))" --backend-port "$((p + 14))" python3 ci/tools/test_window_cap.py \ --nginx-binary "$BIN" --port "$((p + 15))" --backend-port "$((p + 16))" python3 ci/tools/test_ldm_budget_config.py --nginx-binary "$BIN" --port "$((p + 17))" python3 ci/tools/test_cctx_advisory_policy.py --nginx-binary "$BIN" --port "$((p + 18))" python3 ci/tools/test_bufs_bound_policy.py --nginx-binary "$BIN" --port "$((p + 19))" python3 ci/tools/test_bypass_vary_config_policy.py \ --nginx-binary "$BIN" --port "$((p + 20))" python3 ci/tools/test_zstd_long_ldm.py \ --nginx-binary "$BIN" --port "$((p + 21))" --backend-port "$((p + 22))" python3 ci/tools/test_dcz.py --nginx-binary "$BIN" \ --port "$((p + 23))" --tls-port "$((p + 24))" --insecure-port "$((p + 25))" python3 ci/tools/test_dcz_cache_partition.py \ --nginx-binary "$BIN" --port "$((p + 26))" --origin-port "$((p + 27))" python3 ci/tools/test_dcz_budget_ring.py \ --nginx-binary "$BIN" --port "$((p + 28))" --backend-port "$((p + 30))" python3 ci/tools/test_dcz_prefix_alloc.py --nginx-binary "$BIN" --port "$((p + 31))" python3 ci/tools/test_var_dynamic_cacheable.py \ --nginx-binary "$BIN" --port "$((p + 32))" # The dictionary path-hardening matrix (the PR gate runs it too): every # strict-path refusal in the opener and the walk, the non-regular-input # refusals, the rejected-reload check. Those arms are otherwise reached # only by the unit fixtures, which this report does not see. The .so # files sit beside the binary in objs/ on the dynamic coverage build; the # one listening fixture takes the next port in the band. DICT_HARDENING_PORT="$((p + 33))" \ bash ci/tools/test_dict_path_hardening.sh "$BIN" "$SRCDIR/objs" # The testkit is the only layer that reaches worker-internal fault/counter # paths. Use the same canonical six-scenario runner as the PR and Memcheck # jobs; a failure stays visible AND fails this script at the end (after the # report is still published) -- a bailed-out scenario, whose TAP plan/oracle # evidence run-testkit-scenarios.sh already verifies per scenario, must never # pass silently just because the coverage PERCENTAGE has no floor. HARNESS_SCENARIOS_FAILED=0 if [ "$COVERAGE_HARNESS" = "1" ]; then echo "== coverage: exercising all module testkit scenarios ==" if [ ! -x "$MODULE_DIR/ci/t/harness/ci/prober/run-scenario.sh" ]; then echo "WARNING: harness submodule not checked out --" \ "fault/counter paths will report as uncovered" >&2 else # Always (re)build: the prober binary can exist but be staler than its # sources, which silently bails every fault/counter scenario while # this script still reports coverage success (audit A30-F7). if ! bash "$MODULE_DIR/ci/t/harness/ci/prober/build.sh"; then echo "ERROR: prober build failed --" \ "fault/counter paths will report as uncovered" >&2 HARNESS_SCENARIOS_FAILED=1 fi scen_version="${SRCDIR##*/}" scen_version="${scen_version%-coverage}" scen_version="${scen_version#"$FLAVOR-"}" if ! "$SCRIPT_DIR/run-testkit-scenarios.sh" \ --root "$MODULE_DIR" \ --build "$SRCDIR" \ --flavor "$FLAVOR" \ --version "$scen_version" \ --log-dir "$REPORT_DIR"; then echo "ERROR: one or more testkit scenarios failed their TAP" \ "plan/oracle; their missing lines remain visible in the" \ "report, but this run will exit non-zero" >&2 HARNESS_SCENARIOS_FAILED=1 fi fi fi echo "== coverage: gcovr over src/ only ==" # gcovr's --filter/--gcov-filter narrow the REPORT and which gcda match a # pattern, but gcovr still WALKS every .gcda under the search path first -- # the module's own .o/.gcno/.gcda land flat in objs/ (as # ngx_http_zstd__module_modules.*), alongside gcda for the WHOLE # upstream core (order 200k lines) this addon-module build also compiles # from scratch, and upstream's gcda resolve to a source layout gcovr can't # find from here -- a hard error, not a filtered-out line, even under # --gcov-filter. Pass the module's own .gcda as explicit search_paths so # gcovr never opens an upstream one at all. # The addon module's .gcda land at objs/addon/src/, mirroring the # --add-dynamic-module path nginx's configure stages the module source into # -- NOT flat in objs/ (that directory holds .gcno for a *_modules.c shim # that is a compile-time registration stub, never itself executed, so it # never gets a matching .gcda). collect_module_gcda "$SRCDIR/objs/addon" if [ "${#MODULE_GCDA[@]}" -eq 0 ]; then echo "ERROR: no .gcda for the zstd module under $SRCDIR/objs -- ci/t/ did not" \ "exercise the coverage-instrumented binary" >&2 exit 1 fi # Even given only the module's OWN .gcda, gcov still resolves an inlined # call target's compilation unit and reports on gcda for upstream objects # (ngx_event.c, ngx_open_file_cache.c, ...) that are not under $MODULE_DIR -- # a real gcov behaviour, not a gcovr bug. Those errors are non-fatal to the # run (gcovr logs and continues) and --filter drops them from the report # regardless; --gcov-ignore-errors=no_working_dir_found silences the noise # without hiding a real parse failure in the module's own files. GCOVR_ARGS=( --root "$MODULE_DIR" --filter "$MODULE_DIR/src/" # gcov records nginx inline-header paths relative to the source root. Using # objs/ here makes it discard the entire filter gcda when it cannot resolve # src/core/ngx_string.h, yielding a plausible report with the main module # silently absent. --object-directory "$SRCDIR" --gcov-ignore-errors=no_working_dir_found --print-summary --xml "$REPORT_DIR/coverage.xml" --html-details "$REPORT_DIR/coverage.html" "${MODULE_GCDA[@]}" ) if [ -n "${COVERAGE_FAIL_UNDER:-}" ]; then GCOVR_ARGS+=(--fail-under-line "$COVERAGE_FAIL_UNDER") fi gcovr "${GCOVR_ARGS[@]}" echo "== coverage: report written to $REPORT_DIR ==" if [ "$HARNESS_SCENARIOS_FAILED" -ne 0 ]; then echo "ERROR: coverage report published above, but one or more testkit" \ "scenarios did not verify -- see the ERROR lines above for which" \ "scenario/TAP plan bailed" >&2 exit 1 fi myguard-labs-nginx-zstd-module-3486b5b/ci/tools/cppcheck-suppressions.txt000066400000000000000000000011241525232545500266630ustar00rootroot00000000000000// cppcheck suppressions baseline for the zstd nginx module. // // Accepted exceptions to the gated cppcheck run in .github/workflows/ // build-test.yml (--enable=warning,performance,portability). Add one entry // per accepted finding as `id:relative/path.c:line` or a bare `id` to // suppress a category project-wide. Keep this list minimal — it exists so the // scanner can FAIL on new findings while documenting the known ones, per the // CI1 audit fix. Empty = nothing suppressed yet. // cppcheck's own "limiting analysis of branches" advisory — not a defect. normalCheckLevelMaxBranches myguard-labs-nginx-zstd-module-3486b5b/ci/tools/dcz_refprefix_cost_bench.c000066400000000000000000000650231525232545500267640ustar00rootroot00000000000000/* * Microbenchmark for A31b-F2: does ZSTD_CCtx_refPrefix() (the dcz raw-prefix * path, ngx_http_zstd_filter_module.c:~3870) pay a per-request dictionary * table build that ZSTD_CCtx_refCDict() (the trained zstd_dict_file path, * :~3911) amortises once at config load? * * This is a PROOF/MEASUREMENT tool only -- it does not touch the module. It * links libzstd directly (ZSTD_STATIC_LINKING_ONLY, for * ZSTD_createCDict_advanced(), exactly as the module itself gates it) and * times two code paths at the module's real parameters: * * PREFIX path (dcz): ZSTD_CCtx_reset(session_and_parameters) * + ZSTD_CCtx_setParameter(level, windowLog) * + ZSTD_CCtx_refPrefix(dict, dictLen) * + ZSTD_compress2(body) * -- mirrors init_cctx()'s per-request sequence exactly: reset first * (module :3694), then refPrefix (module :3870/:3874). * * CDICT path (hypothetical fix direction): CDict built ONCE outside the * timed loop with ZSTD_createCDict_advanced(..., ZSTD_dct_rawContent, ...) * -- rawContent, not dct_auto, to preserve RFC 9842 raw-prefix wire * semantics (dct_auto risks treating a dictionary-shaped buffer as a * trained dictionary structure instead of raw content) -- then per * request only ZSTD_CCtx_reset() + ZSTD_CCtx_refCDict() + compress2(). * * Sweep: dictionary size x {64 KiB, 1 MiB, 4 MiB, 10 MiB (module's * NGX_HTTP_ZSTD_MAX_DICT_SIZE cap)} x body size x {4 KiB (normal dcz shape, * body << dict) , 256 KiB (body larger than dict)}. * * Level fixed at 3 (the module's compiled-in default, module :4143 / * NGX_HTTP_ZSTD_LEVEL_UNSET resolves to 3). windowLog is derived the same * way ngx_http_zstd_dcz_window_log() derives it for dcz: ceil_log2(dict + * body), clamped to [ZSTD_WINDOWLOG_MIN=10, 23] -- see module comment at * :270-303. * * NOISE FLOOR: before trusting any prefix-vs-cdict delta, this program * measures identical-vs-identical (prefix path timed against itself, two * disjoint sets of iterations) at the same sweep points and reports that * delta first. Only a prefix-vs-cdict gap that clearly exceeds the * identical-vs-identical noise floor is reported as a real effect. * * BYTE IDENTITY (A33-F1): a separate, untimed sweep compares the compressed * bytes of the two paths across the level axis (1..19) in two content modes. * * The original version of this check swept only LEVEL 3 and only "disjoint" * content -- dictionary and body seeded from different constants, hence * statistically unrelated. Under those conditions the compressor finds no * cross-matches at all, the dictMatchState search is never exercised, and the * two paths agree trivially. That vacuous PASS was then cited as proof that a * config-time CDict is a drop-in replacement for the per-request refPrefix. * * It is not. With overlapping content -- the case a dcz dictionary exists for * -- the paths CAN diverge: libzstd loads a prefix with ZSTD_dtlm_fast and a * CDict with ZSTD_dtlm_full, and refCDict searches a separate dictMatchState * instead of the merged window, so different matches can be selected. Measured * 2026-09-03 on libzstd 1.5.7, 9 of 56 overlapping points diverged while all * 56 disjoint points agreed. Representative rows (prefix vs cdict bytes): * * level 1, 1 MB dictionary, 256 KB body: 213164 vs 206183 * level 3, 4 MB dictionary, 256 KB body: 182892 vs 181012 * level 3, 10 MB dictionary, 256 KB body: 249102 vs 248040 * level 15, 64 KB dictionary, 256 KB body: 983 vs 1192 * * Three of the nine sit at level 3, this module's default, so the divergence * is reachable in a stock deployment rather than only at operator-raised * levels. It also runs both ways: at level 15 the CDict path emits MORE bytes, * so this is not "one path simply compresses better". * * The 9-of-56 rate characterises THIS corpus, not a general frequency: how * often the paths diverge depends on how much the dictionary matches the body. * The finding is that divergence exists at all, not that it is rare. * * Consequence: hoisting dcz refPrefix to a config-time CDict changes dcz wire * output for some inputs. It is a product decision about output stability, not * a transparent performance refactor. * * EXIT STATUS: an OVERLAP mismatch is the documented finding above and is * reported without failing (exit 0). A DISJOINT mismatch, in either sweep, is * a wire-format regression and exits 1. The two are counted separately on * purpose: sharing one counter would let a regression print under the * "this is expected" banner and exit 0. * * Runtime: bounded, single-process, a few seconds total. No server, no * soak, no fuzzing. * * Build: cc -Wall -Wextra -O2 -DZSTD_STATIC_LINKING_ONLY \ * dcz_refprefix_cost_bench.c -lzstd -o dcz_refprefix_cost_bench * Run: ./dcz_refprefix_cost_bench */ #include #include #include #include #include #define ZSTD_STATIC_LINKING_ONLY #include #define LEVEL 3 #define WLOG_MIN 10 #define WLOG_CAP 23 #define WARMUP_ITERS 10 #define TIMED_ITERS 120 #define NOISE_ITERS 120 static const size_t dict_sizes[] = { 64 * 1024, 1 * 1024 * 1024, 4 * 1024 * 1024, 10 * 1024 * 1024, /* NGX_HTTP_ZSTD_MAX_DICT_SIZE */ }; static const size_t body_sizes[] = { 4 * 1024, /* normal dcz shape: body << dictionary */ 256 * 1024, /* body larger than the smallest dictionaries */ }; /* Levels swept by the byte-identity check (A33-F1). The timing sweep above * stays pinned at LEVEL because its purpose is a per-request cost number at * the module's default; identity is a wire-format contract and must hold at * every level an operator can configure via zstd_comp_level. */ static const int identity_levels[] = { 1, 3, 6, 9, 12, 15, 19 }; static unsigned ceil_log2_clamped(size_t v) { unsigned log = 0; size_t x = 1; while (x < v && log < 63) { x <<= 1; log++; } if (log < WLOG_MIN) { log = WLOG_MIN; } if (log > WLOG_CAP) { log = WLOG_CAP; } return log; } /* Deterministic pseudo-content: reproducible across runs, and NOT trivially * compressible to zero-length (a real dictionary/body has structure). */ static void fill_buf(unsigned char *buf, size_t len, uint32_t seed) { size_t i; uint32_t state = seed; for (i = 0; i < len; i++) { state = state * 1103515245u + 12345u; /* bias toward repeats every 37 bytes so the buffer is compressible, * like real HTTP bodies/dictionaries, rather than incompressible * random noise that would mask table-build cost inside i/o-bound * entropy coding. */ buf[i] = (unsigned char) ((i % 37 == 0) ? (state >> 24) : (state >> 16)); } } static double now_ms(void) { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return (double) ts.tv_sec * 1000.0 + (double) ts.tv_nsec / 1e6; } static void check_zstd(size_t rc, const char *operation) { if (ZSTD_isError(rc)) { fprintf(stderr, "%s failed: %s\n", operation, ZSTD_getErrorName(rc)); exit(1); } } /* Runs the dcz-style refPrefix path `iters` times; returns total ms and * (via *out_csize) the compressed size of the LAST iteration, for the * byte-identity check. *out_dst receives that last iteration's bytes * (caller-owned buffer, at least dstCap). */ static double bench_prefix(ZSTD_CCtx *cctx, const unsigned char *dict, size_t dictLen, const unsigned char *body, size_t bodyLen, unsigned wlog, int iters, unsigned char *dst, size_t dstCap, size_t *out_csize) { int i; double t0, t1; size_t rc = 0; if (iters <= 0) { fprintf(stderr, "bench_prefix: iters must be positive\n"); exit(1); } t0 = now_ms(); for (i = 0; i < iters; i++) { check_zstd(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters), "CCtx_reset(prefix)"); check_zstd(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, LEVEL), "setParameter(prefix level)"); check_zstd(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, (int) wlog), "setParameter(prefix windowLog)"); rc = ZSTD_CCtx_refPrefix(cctx, dict, dictLen); check_zstd(rc, "refPrefix"); rc = ZSTD_compress2(cctx, dst, dstCap, body, bodyLen); check_zstd(rc, "compress2(prefix)"); } t1 = now_ms(); *out_csize = rc; return t1 - t0; } /* Runs the CDict path `iters` times against a CDict built ONCE by the * caller (outside this function, outside the timed region) -- that is the * entire point of the comparison. */ static double bench_cdict(ZSTD_CCtx *cctx, ZSTD_CDict *cdict, const unsigned char *body, size_t bodyLen, int iters, unsigned char *dst, size_t dstCap, size_t *out_csize) { int i; double t0, t1; size_t rc = 0; if (iters <= 0) { fprintf(stderr, "bench_cdict: iters must be positive\n"); exit(1); } t0 = now_ms(); for (i = 0; i < iters; i++) { check_zstd(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters), "CCtx_reset(cdict)"); rc = ZSTD_CCtx_refCDict(cctx, cdict); check_zstd(rc, "refCDict"); rc = ZSTD_compress2(cctx, dst, dstCap, body, bodyLen); check_zstd(rc, "compress2(cdict)"); } t1 = now_ms(); *out_csize = rc; return t1 - t0; } /* Noise floor: prefix path timed against itself, two disjoint iteration * sets. Establishes the threshold a real prefix-vs-cdict effect must clear * before it can be trusted. Returns 0 on success, 1 on allocation failure. */ static int run_noise_floor(ZSTD_CCtx *cctx, unsigned char *scratch, size_t dstCap) { size_t di; printf("== noise floor (identical prefix path, two disjoint iter sets) ==\n"); printf("%-10s %-10s %10s %10s %8s\n", "dict", "body", "run_a_ms", "run_b_ms", "delta%"); for (di = 0; di < sizeof(dict_sizes) / sizeof(dict_sizes[0]); di++) { size_t dictLen = dict_sizes[di]; size_t bodyLen = body_sizes[0]; unsigned wlog = ceil_log2_clamped(dictLen + bodyLen); double ms_a, ms_b, delta_pct; size_t csz; unsigned char *dict, *body; int w; dict = malloc(dictLen); body = malloc(bodyLen); if (dict == NULL || body == NULL) { fprintf(stderr, "alloc failed (dictLen=%zu bodyLen=%zu)\n", dictLen, bodyLen); free(dict); free(body); return 1; } fill_buf(dict, dictLen, 0xD1C7); fill_buf(body, bodyLen, 0xB0D7); for (w = 0; w < WARMUP_ITERS; w++) { check_zstd(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters), "CCtx_reset(noise warmup)"); check_zstd(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, LEVEL), "setParameter(noise level)"); check_zstd(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, (int) wlog), "setParameter(noise windowLog)"); check_zstd(ZSTD_CCtx_refPrefix(cctx, dict, dictLen), "refPrefix(noise warmup)"); check_zstd(ZSTD_compress2(cctx, scratch, dstCap, body, bodyLen), "compress2(noise warmup)"); } ms_a = bench_prefix(cctx, dict, dictLen, body, bodyLen, wlog, NOISE_ITERS, scratch, dstCap, &csz); ms_b = bench_prefix(cctx, dict, dictLen, body, bodyLen, wlog, NOISE_ITERS, scratch, dstCap, &csz); delta_pct = 100.0 * (ms_b - ms_a) / ms_a; printf("%-10zu %-10zu %10.3f %10.3f %+7.1f%%\n", dictLen, bodyLen, ms_a, ms_b, delta_pct); free(dict); free(body); } return 0; } /* One sweep point: measure the prefix path and the cdict path at the same * (dictLen, bodyLen), and check the compressed bytes match. Returns 1 if * the two paths mismatched (byte-identity failure), 0 if they matched; * exits the process on allocation/library failure. */ static int run_sweep_point(ZSTD_CCtx *cctx, size_t dictLen, size_t bodyLen, uint32_t seed_salt, unsigned char *dst_a, unsigned char *dst_b, size_t dstCap) { unsigned wlog = ceil_log2_clamped(dictLen + bodyLen); double ms_prefix, ms_cdict, per_op_prefix, per_op_cdict; size_t csz_prefix, csz_cdict; unsigned char *dict, *body; ZSTD_CDict *cdict; ZSTD_compressionParameters cparams; int identical, w; dict = malloc(dictLen); body = malloc(bodyLen); if (dict == NULL || body == NULL) { fprintf(stderr, "alloc failed (dictLen=%zu bodyLen=%zu)\n", dictLen, bodyLen); exit(1); } fill_buf(dict, dictLen, 0xD1C7 + seed_salt); fill_buf(body, bodyLen, 0xB0D7 + seed_salt); for (w = 0; w < WARMUP_ITERS; w++) { check_zstd(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters), "CCtx_reset(prefix warmup)"); check_zstd(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, LEVEL), "setParameter(prefix warmup level)"); check_zstd(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, (int) wlog), "setParameter(prefix warmup windowLog)"); check_zstd(ZSTD_CCtx_refPrefix(cctx, dict, dictLen), "refPrefix(warmup)"); check_zstd(ZSTD_compress2(cctx, dst_a, dstCap, body, bodyLen), "compress2(prefix warmup)"); } ms_prefix = bench_prefix(cctx, dict, dictLen, body, bodyLen, wlog, TIMED_ITERS, dst_a, dstCap, &csz_prefix); /* Build the CDict ONCE, outside the timed region -- this is the * amortisation the fix direction proposes. dct_rawContent preserves * RFC 9842 raw-prefix wire semantics (module comment near * ZSTD_CCtx_refPrefix's call site). */ cparams = ZSTD_getCParams(LEVEL, (unsigned long long) bodyLen, dictLen); cparams.windowLog = wlog; cdict = ZSTD_createCDict_advanced(dict, dictLen, ZSTD_dlm_byRef, ZSTD_dct_rawContent, cparams, ZSTD_defaultCMem); if (cdict == NULL) { fprintf(stderr, "ZSTD_createCDict_advanced failed\n"); exit(1); } for (w = 0; w < WARMUP_ITERS; w++) { check_zstd(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters), "CCtx_reset(cdict warmup)"); check_zstd(ZSTD_CCtx_refCDict(cctx, cdict), "refCDict(warmup)"); check_zstd(ZSTD_compress2(cctx, dst_b, dstCap, body, bodyLen), "compress2(cdict warmup)"); } ms_cdict = bench_cdict(cctx, cdict, body, bodyLen, TIMED_ITERS, dst_b, dstCap, &csz_cdict); per_op_prefix = ms_prefix / TIMED_ITERS; per_op_cdict = ms_cdict / TIMED_ITERS; identical = (csz_prefix == csz_cdict) && (memcmp(dst_a, dst_b, csz_prefix) == 0); printf("%-10zu %-10zu %5u %12.4f %12.4f %10.4f %10s\n", dictLen, bodyLen, wlog, per_op_prefix, per_op_cdict, per_op_prefix - per_op_cdict, identical ? "yes" : "NO -- MISMATCH"); ZSTD_freeCDict(cdict); free(dict); free(body); return identical ? 0 : 1; } /* Builds a body out of slices of the dictionary, so the dictionary actually * MATCHES the content. The timing sweep's fill_buf() seeds dictionary and body * from two different constants, which makes them statistically unrelated: the * compressor finds no cross-matches, never exercises the dictMatchState search * path, and so the two code paths agree trivially. A real dcz dictionary is * chosen precisely because it resembles the response, so identity has to be * checked against overlapping content or it is checked against nothing. */ static void fill_buf_overlapping(unsigned char *body, size_t bodyLen, const unsigned char *dict, size_t dictLen) { size_t off = 0, chunk, src, step = 0; /* Vary the slice length across iterations. Deriving it from `off` alone * self-stabilises (off advances by exactly the chunk it produced, so the * modulo pins to one value after the first step); stepping a counter * keeps the lengths moving. */ while (off < bodyLen) { chunk = 512 + (step % 1024); step += 397; if (chunk > bodyLen - off) { chunk = bodyLen - off; } /* Every dict_sizes[] entry is >= 64 KB and chunk <= 1535, so the * else-branch is unreachable today; it keeps the copy in bounds if a * smaller dictionary is ever added to the sweep. */ src = (dictLen > chunk) ? (off * 7919) % (dictLen - chunk) : 0; memcpy(body + off, dict + src, chunk); off += chunk; } } /* One byte-identity point: compress the same body twice at the same level and * windowLog, once via the dcz per-request refPrefix path and once via the * hoisted refCDict(dct_rawContent) path the A33-F1 fix direction proposes. * Both paths receive the same fully specified compression parameters, so a * mismatch isolates dictionary-loading behavior rather than parameter * derivation. Returns 1 on mismatch. Untimed: this is a correctness oracle, * not a bench. */ static int run_identity_point(ZSTD_CCtx *cctx, int level, size_t dictLen, size_t bodyLen, int overlap, unsigned char *dst_a, unsigned char *dst_b, size_t dstCap) { unsigned wlog = ceil_log2_clamped(dictLen + bodyLen); size_t csz_a, csz_b; unsigned char *dict, *body; ZSTD_CDict *cdict; ZSTD_compressionParameters cparams; int identical; dict = malloc(dictLen); body = malloc(bodyLen); if (dict == NULL || body == NULL) { fprintf(stderr, "alloc failed (dictLen=%zu bodyLen=%zu)\n", dictLen, bodyLen); exit(1); } fill_buf(dict, dictLen, 0xD1C7); if (overlap) { fill_buf_overlapping(body, bodyLen, dict, dictLen); } else { fill_buf(body, bodyLen, 0xB0D7); } cparams = ZSTD_getCParams(level, (unsigned long long) bodyLen, dictLen); cparams.windowLog = wlog; check_zstd(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters), "CCtx_reset(identity prefix)"); check_zstd(ZSTD_CCtx_setCParams(cctx, cparams), "setCParams(identity prefix)"); check_zstd(ZSTD_CCtx_refPrefix(cctx, dict, dictLen), "refPrefix(identity)"); csz_a = ZSTD_compress2(cctx, dst_a, dstCap, body, bodyLen); check_zstd(csz_a, "compress2(identity prefix)"); cdict = ZSTD_createCDict_advanced(dict, dictLen, ZSTD_dlm_byRef, ZSTD_dct_rawContent, cparams, ZSTD_defaultCMem); if (cdict == NULL) { fprintf(stderr, "ZSTD_createCDict_advanced failed\n"); exit(1); } check_zstd(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters), "CCtx_reset(identity cdict)"); check_zstd(ZSTD_CCtx_refCDict(cctx, cdict), "refCDict(identity)"); csz_b = ZSTD_compress2(cctx, dst_b, dstCap, body, bodyLen); check_zstd(csz_b, "compress2(identity cdict)"); identical = (csz_a == csz_b) && (memcmp(dst_a, dst_b, csz_a) == 0); printf("%-9s %-5d %-10zu %-10zu %5u %10zu %10zu %14s\n", overlap ? "overlap" : "disjoint", level, dictLen, bodyLen, wlog, csz_a, csz_b, identical ? "identical" : "MISMATCH"); ZSTD_freeCDict(cdict); free(dict); free(body); return identical ? 0 : 1; } /* Sweeps byte identity across the level axis in both content modes. * * The two modes are counted SEPARATELY, into *disjoint_out and *overlap_out. * They mean opposite things: disjoint content must agree (a mismatch there is * a wire-format regression), while overlapping content is where divergence is * the documented finding. Folding them into one number lets a regression be * reported as the expected finding and exit 0. */ static void run_identity_sweep(ZSTD_CCtx *cctx, unsigned char *dst_a, unsigned char *dst_b, size_t dstCap, int *disjoint_out, int *overlap_out) { size_t li, di, bi; int overlap, mismatch; *disjoint_out = 0; *overlap_out = 0; printf("\n== byte identity: refPrefix vs refCDict(dct_rawContent), " "level axis, both content modes ==\n"); printf("%-9s %-5s %-10s %-10s %5s %10s %10s %14s\n", "content", "level", "dict", "body", "wlog", "prefix_sz", "cdict_sz", "identical"); for (overlap = 0; overlap < 2; overlap++) { for (li = 0; li < sizeof(identity_levels) / sizeof(identity_levels[0]); li++) { for (di = 0; di < sizeof(dict_sizes) / sizeof(dict_sizes[0]); di++) { for (bi = 0; bi < sizeof(body_sizes) / sizeof(body_sizes[0]); bi++) { mismatch = run_identity_point(cctx, identity_levels[li], dict_sizes[di], body_sizes[bi], overlap, dst_a, dst_b, dstCap); if (overlap) { *overlap_out += mismatch; } else { *disjoint_out += mismatch; } } } } } } int main(void) { size_t di, bi; unsigned char *dst_a, *dst_b, *dst_noise; size_t dstCap; ZSTD_CCtx *cctx; int timing_mismatches = 0; int identity_disjoint_mismatches = 0; int identity_overlap_mismatches = 0; int disjoint_mismatches; printf("A31b-F2 dcz refPrefix per-request cost -- microbenchmark\n"); printf("libzstd %s, level %d, %d warmup + %d timed iters per point\n\n", ZSTD_versionString(), LEVEL, WARMUP_ITERS, TIMED_ITERS); cctx = ZSTD_createCCtx(); if (cctx == NULL) { fprintf(stderr, "ZSTD_createCCtx failed\n"); return 1; } dstCap = ZSTD_compressBound(10 * 1024 * 1024 + 256 * 1024); dst_a = malloc(dstCap); dst_b = malloc(dstCap); dst_noise = malloc(dstCap); if (dst_a == NULL || dst_b == NULL || dst_noise == NULL) { fprintf(stderr, "alloc failed\n"); free(dst_a); free(dst_b); free(dst_noise); ZSTD_freeCCtx(cctx); return 1; } if (run_noise_floor(cctx, dst_noise, dstCap) != 0) { free(dst_a); free(dst_b); free(dst_noise); ZSTD_freeCCtx(cctx); return 1; } printf("\n== prefix vs cdict sweep (per-op ms; refPrefix/refCDict setup only,\n"); printf(" CDict itself built ONCE outside the timed loop) ==\n"); printf("%-10s %-10s %5s %12s %12s %10s %10s\n", "dict", "body", "wlog", "prefix_ms/op", "cdict_ms/op", "delta_ms/op", "identical"); for (di = 0; di < sizeof(dict_sizes) / sizeof(dict_sizes[0]); di++) { for (bi = 0; bi < sizeof(body_sizes) / sizeof(body_sizes[0]); bi++) { timing_mismatches += run_sweep_point(cctx, dict_sizes[di], body_sizes[bi], (uint32_t) (di * 8U + bi), dst_a, dst_b, dstCap); } } run_identity_sweep(cctx, dst_a, dst_b, dstCap, &identity_disjoint_mismatches, &identity_overlap_mismatches); /* Every disjoint-content point, in either sweep, must agree. */ disjoint_mismatches = timing_mismatches + identity_disjoint_mismatches; ZSTD_freeCCtx(cctx); free(dst_a); free(dst_b); free(dst_noise); printf("\n"); /* The two sweeps mean opposite things, so they must not share a verdict. * * The timing sweep runs disjoint content only, where both paths are * expected to agree. A mismatch there is a genuine wire-format * regression and still fails the program. * * The identity sweep deliberately includes overlapping content, where * divergence is the documented finding rather than a fault, so a * mismatch there is reported and the program still exits 0. */ if (identity_overlap_mismatches > 0) { printf("BYTE IDENTITY: NOT UNIVERSAL -- %d overlapping-content " "point(s) produced different compressed bytes between " "refPrefix and refCDict(dct_rawContent).\n\n" "This is the documented finding, not a failure. Hoisting the " "dcz per-request refPrefix() to a config-time CDict is a\n" "WIRE-FORMAT CHANGE, not a transparent optimisation: libzstd " "loads a prefix with ZSTD_dtlm_fast and a CDict with\n" "ZSTD_dtlm_full, and refCDict searches a separate " "dictMatchState rather than the merged window, so the two\n" "paths can select different matches whenever the dictionary " "actually matches the body. Such a change needs an explicit\n" "product decision about dcz output stability.\n", identity_overlap_mismatches); } else { printf("BYTE IDENTITY: no mismatch observed in this sweep. That is " "NOT a general guarantee -- identity is sensitive to how much " "the dictionary actually matches the body.\n"); } if (disjoint_mismatches > 0) { printf("\nDISJOINT IDENTITY: FAILED -- %d point(s) diverged on " "unrelated dictionary and body content, where both paths must " "agree (%d in the timing sweep, %d in the identity sweep).\n" "That is a wire-format regression, not the overlap finding " "above.\n", disjoint_mismatches, timing_mismatches, identity_disjoint_mismatches); return 1; } return 0; } myguard-labs-nginx-zstd-module-3486b5b/ci/tools/dcz_refprefix_cost_bench.sh000077500000000000000000000027071525232545500271570ustar00rootroot00000000000000#!/bin/bash # Build-and-run wrapper for dcz_refprefix_cost_bench.c (A31b-F2 cost proof, # A33-F1 byte-identity proof). # # Measures whether the dcz path's per-request ZSTD_CCtx_refPrefix() call # (module :3870/:3874, reached after the unconditional per-request # ZSTD_CCtx_reset() at :3694) pays a dictionary table-build cost the # trained-dictionary CDict path (:3911) avoids by building once at config # load. Read dcz_refprefix_cost_bench.c's header comment for the full # method, sweep, and noise-floor rationale. # # It also runs an untimed byte-identity sweep over the level axis in two # content modes, which is what shows that swapping refPrefix for a config-time # CDict is a wire-format change rather than a transparent optimisation. An # overlap mismatch is reported and exits 0; a disjoint mismatch is a regression # and exits 1. # # No nginx tree needed: links libzstd directly. Bounded runtime (a few # seconds). set -euo pipefail cd "$(dirname "$0")/../.." CC="${CC:-cc}" SRC="ci/tools/dcz_refprefix_cost_bench.c" OUT="$(mktemp -d "${TMPDIR:-/tmp}/zstd-dcz-refprefix-bench.XXXXXX")" trap 'rm -rf "$OUT"' EXIT if ! pkg-config --exists libzstd 2>/dev/null; then echo "SKIP: libzstd not found via pkg-config" >&2 exit 0 fi read -r -a CFLAGS <<<"$(pkg-config --cflags libzstd) -Wall -Wextra -O2" read -r -a LIBS <<<"$(pkg-config --libs libzstd)" "$CC" "${CFLAGS[@]}" -o "$OUT/dcz_refprefix_cost_bench" "$SRC" "${LIBS[@]}" "$OUT/dcz_refprefix_cost_bench" myguard-labs-nginx-zstd-module-3486b5b/ci/tools/fetch-verified-nginx.sh000077500000000000000000000027461525232545500261460ustar00rootroot00000000000000#!/bin/bash # Download an nginx release tarball and verify its detached PGP signature # against the signer keys VENDORED in ci/tools/keys/ (never fetched from # nginx.org at CI time -- see ci-build.sh's file header for why bootstrapping # the keys from the same origin that serves the tarball+signature would let # an origin compromise substitute all three, audit sha e289021 F3). # # This is the same verification ci-build.sh performs before a full test-suite # build, factored out for the CI jobs that only need a source tree to # configure headers from (scan-build, clang-tidy) or to build a debug binary # for a Valgrind soak (memcheck, helgrind) -- neither wants ci-build.sh's # .build/ cache tree, coverage mode or dynamic-module test-suite flags. # # ci/tools/fetch-verified-nginx.sh # # On success, leaves "nginx-.tar.gz" (verified) in the CURRENT # directory and extracts it there. A verification failure removes the # tarball and exits 1 -- callers must not catch and continue past this. set -euo pipefail VERSION="${1:?usage: fetch-verified-nginx.sh }" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DIR="nginx-${VERSION}" TARBALL="${DIR}.tar.gz" URL="https://nginx.org/download/${TARBALL}" if [ ! -f "$TARBALL" ]; then wget -q -O "$TARBALL" "$URL" fi if ! "$SCRIPT_DIR/verify-nginx-tarball.sh" "$TARBALL" "${URL}.asc"; then echo "PGP signature verification failed for ${TARBALL}" >&2 rm -f "$TARBALL" "${TARBALL}.asc" exit 1 fi tar -xzf "$TARBALL" myguard-labs-nginx-zstd-module-3486b5b/ci/tools/filter-codeql-sarif.py000077500000000000000000000160151525232545500260050ustar00rootroot00000000000000#!/usr/bin/env python3 """Drop vendored-nginx results from a CodeQL SARIF file, in place. CodeQL's `paths`/`paths-ignore` config keys apply only to interpreted languages, or to a compiled language analyzed without building it. This project runs a full traced `make` of C/C++, so those keys are inert and nginx-core findings reach the Security tab on every mainline bump. nginx is compiled into the binary via --add-module, so it cannot be kept out of the database either; scoping therefore happens here, after analysis. Results whose location cannot be determined are kept, never dropped: an unplaceable finding is a reason to look, not to hide. """ import json import re import sys # Vendored nginx is extracted into the workspace as nginx-/. # # Every alert this pipeline has actually produced carries a workspace-relative # path ("nginx-1.31.4/src/core/ngx_log.c"), which is what `^nginx-[0-9]` is # anchored for. SARIF nonetheless permits an absolute or `file:`-scheme URI, # and a vendored result that arrived in that shape would slip past a purely # leading-anchored match and land in the Security tab -- the exact leak this # filter exists to stop. So match the vendored directory as a path SEGMENT, # which covers the relative form and the absolute one alike. VENDORED = re.compile(r"(?:^|/)nginx-[0-9][^/]*/") def result_uri(result): for location in result.get("locations", []): uri = ( location.get("physicalLocation", {}).get("artifactLocation", {}).get("uri") ) if uri: return uri return None def filter_sarif(sarif): removed = kept = unlocated = 0 for run in sarif.get("runs", []): keep = [] for result in run.get("results", []): uri = result_uri(result) if uri is None: unlocated += 1 keep.append(result) elif VENDORED.search(uri): removed += 1 else: kept += 1 keep.append(result) run["results"] = keep return removed, kept, unlocated def _result(uri): """A minimal SARIF result; uri=None models a finding with no location.""" if uri is None: return {"ruleId": "unplaceable", "locations": []} return { "ruleId": "placed", "locations": [{"physicalLocation": {"artifactLocation": {"uri": uri}}}], } def selftest(): """Negative controls for the scoping rule this filter exists to enforce. The dangerous failure is not "it dropped too little" -- it is "it dropped something it should have kept", because a dropped result never reaches the Security tab and nothing reports its absence. So every case below asserts a KEPT result too, and the unlocated case is the one that would silently hide a real finding if a future edit made `uri is None` fall into the vendored branch. """ cases = [ # label, uris, expected (removed, kept, unlocated) ("vendored nginx dropped", ["nginx-1.29.4/src/core/ngx_string.c"], (1, 0, 0)), ("module source kept", ["filter/ngx_http_zstd_filter_module.c"], (0, 1, 0)), ("unlocated result kept", [None], (0, 0, 1)), ( "mixed run scoped correctly", [ "nginx-1.29.4/src/http/ngx_http.c", "nginx-1.28.0/src/core/ngx_cycle.c", "filter/ngx_http_zstd_filter_module.c", "static/ngx_http_zstd_static_module.c", None, ], (2, 2, 1), ), # `nginx-` alone must not match: the anchor is `nginx-`, so a # module-owned path that merely starts with the vendor prefix stays. ("nginx-like module path kept", ["nginx-helpers/ngx_zstd_util.c"], (0, 1, 0)), # A vendored path is matched as a path SEGMENT, so the nested, # absolute and file:-scheme forms are dropped too -- SARIF permits # all of them, and a leading-anchored match would have leaked them. ("nested vendored path dropped", ["build/nginx-1.29.4/src/x.c"], (1, 0, 0)), ( "absolute vendored path dropped", ["/home/runner/work/m/m/nginx-1.31.4/src/core/ngx_log.c"], (1, 0, 0), ), ( "file: scheme vendored path dropped", ["file:///home/runner/work/m/m/nginx-1.31.4/src/core/ngx_log.c"], (1, 0, 0), ), # ...while an absolute MODULE path is still kept: the segment match # must not degrade into "contains nginx- anywhere". ( "absolute module path kept", ["/home/runner/work/m/m/filter/ngx_http_zstd_filter_module.c"], (0, 1, 0), ), # A trailing path component is not a directory segment: a file # literally named nginx-1.2.3 is not the vendored tree. ("vendored-looking filename kept", ["src/nginx-1.2.3"], (0, 1, 0)), ("empty run", [], (0, 0, 0)), ] failed = False for label, uris, want in cases: sarif = {"runs": [{"results": [_result(u) for u in uris]}]} got = filter_sarif(sarif) survivors = [r["ruleId"] for r in sarif["runs"][0]["results"]] if got != want: print(f"FAIL {label}: expected {want}, got {got}", file=sys.stderr) failed = True elif len(survivors) != want[1] + want[2]: print(f"FAIL {label}: survivor count {len(survivors)}", file=sys.stderr) failed = True else: print(f"ok {label}") # Multi-run SARIF: each run is filtered independently and none is dropped. sarif = { "runs": [ {"results": [_result("nginx-1.29.4/src/core/ngx_cycle.c")]}, {"results": [_result("filter/ngx_http_zstd_filter_module.c")]}, ] } if filter_sarif(sarif) != (1, 1, 0) or len(sarif["runs"]) != 2: print("FAIL multi-run SARIF not filtered per run", file=sys.stderr) failed = True else: print("ok multi-run SARIF filtered per run") # The `runs` key must survive so upload-sarif still sees a valid document. sarif = {"version": "2.1.0", "runs": [{"results": []}]} filter_sarif(sarif) if "runs" not in sarif or "results" not in sarif["runs"][0]: print("FAIL filter destroyed SARIF structure", file=sys.stderr) failed = True else: print("ok SARIF structure preserved") return int(failed) def main(): if len(sys.argv) == 2 and sys.argv[1] == "--selftest": return selftest() if len(sys.argv) != 2: print( "usage: filter-codeql-sarif.py | --selftest", file=sys.stderr, ) return 2 path = sys.argv[1] with open(path, encoding="utf-8") as fh: sarif = json.load(fh) removed, kept, unlocated = filter_sarif(sarif) with open(path, "w", encoding="utf-8") as fh: json.dump(sarif, fh) print(f"nginx-core results removed: {removed}") print(f"module results kept: {kept}") print(f"unlocated results kept: {unlocated}") return 0 if __name__ == "__main__": sys.exit(main()) myguard-labs-nginx-zstd-module-3486b5b/ci/tools/install-test-nginx-socket.sh000077500000000000000000000027041525232545500271650ustar00rootroot00000000000000#!/usr/bin/env bash set -euo pipefail prefix="${1:-$HOME/perl5}" version="${TEST_NGINX_SOCKET_VERSION:?TEST_NGINX_SOCKET_VERSION is required}" dist="${TEST_NGINX_SOCKET_DIST:-AGENT/Test-Nginx-${version}.tar.gz}" mirror="${CPAN_MIRROR:-https://cpan.metacpan.org}" lock="${TEST_NGINX_SOCKET_LOCK:-ci/tools/test-nginx-socket.cpan.lock}" expected="Test::Nginx::Socket@${version}" author="${dist%%/*}" author_path="${author:0:1}/${author:0:2}/${dist}" tmp="$(mktemp -d)" trap 'rm -rf "$tmp"' EXIT cpanm --info "$expected" 2>/dev/null | awk '/^[A-Z0-9]+\/.*\.tar\.gz$/ {print; exit}' >"$tmp/dist" if ! grep -qx "$dist" "$tmp/dist"; then echo "::error::${expected} resolved to $(cat "$tmp/dist"), expected ${dist}" >&2 exit 1 fi cpanm --showdeps "$expected" 2>/dev/null | awk '/^[A-Za-z0-9_:]+(~[^[:space:]]+)?$/ {print}' | sort >"$tmp/deps" grep -Ev '^\s*(#|$)' "$lock" | sort >"$tmp/lock" if ! diff -u "$tmp/lock" "$tmp/deps"; then echo "::error::${expected} CPAN dependency graph differs from ${lock}" >&2 exit 1 fi cpanm -l "$prefix" --notest "${mirror}/authors/id/${author_path}" echo "PERL5LIB=${prefix}/lib/perl5:${PERL5LIB:-}" >>"${GITHUB_ENV:-/dev/null}" installed="$(perl -I "${prefix}/lib/perl5" -MTest::Nginx::Socket -e 'print $Test::Nginx::Socket::VERSION')" echo "Test::Nginx::Socket installed version: $installed" if [ "$installed" != "$version" ]; then echo "::error::Test::Nginx::Socket version mismatch: pinned $version, got $installed" >&2 exit 1 fi myguard-labs-nginx-zstd-module-3486b5b/ci/tools/keys/000077500000000000000000000000001525232545500225445ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/tools/keys/arut.key000066400000000000000000000160411525232545500242330ustar00rootroot00000000000000-----BEGIN PGP PUBLIC KEY BLOCK----- mQINBGYXyiQBEAC4jm1y+ODV4+YDGj9vp2BgHB4FJeQdgrBiVX+Mb2qCrEqJgeKV fVwKjkVYqnb76TTybdOKqCP5wdQrncKAKlXsMq6sdsiwPSrdRcjkeiE29WWrtbB4 i+VObnoWklMblMxFQ1XQIkjs2wviidKjJw2VV3i4XnLSrHhWaWqviTLZCMQymoPs F+Tfu1WX9OUfOquekZ5KjkyBxB4ep6+NPeuIkPnW0SiTUhU8tbi8v0aBZEHSZLqE mq8KLROVuYSPvtU+NtaXAM09BHEVCfb409aDps9p6AFT+IN8yoOegGdEZjp6hJvS HxbhuwqNEtg4dTEV515YUCgKabqU1QaqI/Y0+Pdkpep1KRFc9YUYttDkCw7Ybu2u fwTGzwAbD+ThAIOdzmMDodzZaEMf+9fQG4bnO1PdNbXzyP7Kv9qzGa65+9oGCPOS qTpISR8pvzoI8w/Z/vG71ob/nQ6Xm0L986ksErdGhu16ZI7lW2eDYqy2IoFfbeSz HHxk484/pEibrlCRbP2Id+zULfxo1HGOGg+PAY9Q2uNzABsGDMnOhIvXHS+hP7oB sO9A4Prqu6K6cMp3QI219tmmOUegJpmGGPzoNgxR7H30wNcjZPv4PWr/c0fP70Ny ilgbdcEMDSHks30AmiuIvcUxo3A21p2nnpxsKAKYx42UJkyEK0HILMzcqwARAQAB tCZSb21hbiBBcnV0eXVueWFuIDxyLmFydXR5dW55YW5AZjUuY29tPokCTgQTAQgA OBYhBEM4eCXdsbuX7Da6XQB8jXwV2HNpBQJmF8pXAhsDBQsJCAcCBhUKCQgLAgQW AgMBAh4BAheAAAoJEAB8jXwV2HNppvQP/AjzdPKkGRzJkb1ioto/IEP1YhA/Eayk hvejJ0vyWVHXXH7FLW9fIZoApcsD1J8/7zIANm+62IfT3QNbL2R44IyhJB3AY22l t0ToLxodfugegF3NPYYyFOSRUoPD4g2T/dMCPOBX4MNEAnAlCmxAMaJNmQUO76IY GwELa3CH3Aqf7bthKy8P36G11hu7NgH6V9mVIRIpfnfpXFQIztj+vsWtswu4M5t7 BNJwx4a2KTCVQpTdff5/0dO/5drQDxLbIg681WZk3Oe8Eu6nSc0Ud02NIkg1TQH/ MryAp7o/ua3LRem+W/cktnT60p4uXPVZ3Rvg3zOmJSNJ+eIXY2+sDeZEPaROKldA IbnBacTsZjdswIlrbzinY8ZVRosaFlvHg/ESTBRItALHWCRdzOR1Wv1qy/PQfEEL qftDsCTQhssP1MHJWlejeqPlND3iT2vBDeOxqd6WhKuAc+L04iyBB6p867pwrgDF ecg82DPehsAnO2XBAFuIE/SLewkYm0B9HK7/J4LZqPwTAksPf/dnbMAmHWoBDqsu 4U4U4SsJKsZ87R9ao8qO7IWCzHrXavHFmnbqweFfHToeKF/L4PB+tYoW3YmUOged CglpJv13bNWmRwL7+x8b7BwpVwClxHBHteDX4RIN5iPH9h20J4jIpzRa1kNJsTu1 v4ZkqLWJlkiiiQEzBBABCAAdFiEEcziXMGntP0Q/TTffpk/VsXrbOagFAmYdpjsA CgkQpk/VsXrbOahISgf/U7ZO0yK0PsOcAFTB0TQBCNsAhxtJAEJoVoweuYiLk8jR 0OeDRCy0BC//qWDLFT7NKuP50SM2u0Csbg+n6b0bdy+vXbbGVzIAYzG09rPYe2Q5 qwqyAx+MMzyICXul9lGNU2qN2qjUXMb0mCWUhxwMvzRUeS7shT1CBhGrnpoYkY56 NhWj7iG1BbLwYVQzDZC/Rp6rvwJQgZo7+DjaMjryGAEI0ujpUp8ywrPaJpwIuXDI D5BhcyUaEd3XOondHQNedlgERXHT4pN+oNMPWwN3+DeQYLS3FHiqyz05ZvoeWnao A2/fWNA+BqIdjilp/TDDI4Ef7c9hp13weaZggYB3M4kBMwQQAQgAHRYhBFc7/Ws9 j7xkEHmmq6v1vYJ72b9iBQJmHabkAAoJEKv1vYJ72b9iDgoIAP1QJjl4ynLAV9Bo Ol4AAzxZ3x/2NEgLSnjLfhb/OduDxQlL9oPulWoLDG41xiZJkepEnQWmSsIYF6Xe RsAB+eREU2uCxqCvBXpyIs5npXvVDV2/PQuVEop7HByx6Hjr9XK8hugihnEi1p+9 Ecbu+89fi93m3C/5uIIil46cHByjRZ+5Yy1UFUB/wsYud1qMcYmvDaqEo5AqWNcM gWUFhUfgGTtBbyvIWTeX0NHnrbzHP7lhmPfWsfOjAtO8PpM8Gz5RdNRq44DdRKdG uWVby/kni868H+8/tHalDR0I9/Mmg2Uax0eggTVpECv/4+xBduqSB2iPwgRnSzhZ 6SVKJvKJAjMEEAEIAB0WIQT5TVS8DF1qZBfIzz/oLBEYr5TfbgUCZh5KVgAKCRDo LBEYr5TfbitgD/wMamMFfFZnPS7JS1NWEMb5fbhHob1EkmedIpbpRDXUtj0ksehW ZAEpmVF9btqS4B+B9tSK1VS2sy4XwEGodNVSGxdtF9W8+iAHAb6Hq1Z7ifWyb991 Kt/pVk/8adxlU4G8h1fq0idhpnI8KvkAlPJR7+PoJOEN1+VdHS6tkE5LMTf6dF9F iVxKQczOS1b/GmfL3kYfu6UvI07ZuaP+90mOt/TZTwkzsWjRY2vofCIPSDY94rLj m6PmVFoU3PHLKW7yDz1YXkVE6SgQYGZ2bqB6OHJZnDXUTSHncHTbDVzZQekIs1lP V6e5N8Xo/VOpv28feKAsBqQ8ML53djmGUL0azjEz1g2kgPmTuZdKzZ5kcUsULdQV aRKcfyYD1oRpwwlw9GJAxliJHck1IdGGaCslrHtzkh3RMULlloAYitzD9jtKsrOj R19s+JK/tIfFZZ5gR5qhzgOL8WgkSrIaq2o9R4sigBz1IxnXXC573RDA2F5FAeE/ K6EmAO+BqVkImZcmP1JsLtr+OM+jihXIILACEJwhOKPtZth9zrLYkXWB1nCaDxHp XEUpp6UPCQNgNX8NCghnJr5gis/SmYppgFlO9R9yZ7/LtP0tUX0CmhOeqGMnHt4R F8n8D7EBwMWvWjlUbsDkMKX4JORgojguHJZciWQC1gVRwJ0iTH/ImtzDnbQhUm9t YW4gQXJ1dHl1bnlhbiA8YXJ1dEBuZ2lueC5jb20+iQJOBBMBCAA4FiEEQzh4Jd2x u5fsNrpdAHyNfBXYc2kFAmYXyiQCGwMFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AA CgkQAHyNfBXYc2kRFw//VFuCnW3EwoLCWWgWCikgI9kbVDr0/Qiyf2Gb9sfOyzBN q/+ZGjTs7EqTHbYUiCTgjy8t0SNKizoCXjSWLToTAXhOeTY3wDuHkdc3C2OPMPgm HPGmdnfplmsZjj689sy0MTnlLmU/87texR/f3REAKtchVjo5AojuZxXJi+ryBvoz KXi82M1JaYlIr15T+OiRtfZ3cgfTkb5CRa0YRV7QQ1zhOiF0AFKVVikFwRuquphT y2cSLILLzOpwG/CjMJzO4VOASmGJmdicIfYSsZSzz37RrcfeYwR6quJ55Y9QF9IU fg5AHWufpXaf6FbMsW1U1mOq0tMvwvdcO+u5I5SBj6IkqO4zavmW/i5zkxaq96wF Qn6+oRkqHnNNn0hl/B4MWdEjDJsaDXfkQ3Snn4Bfl1JPT6cH2NDVYQn1siIOim/W G5lhGLNB1TOAVLHblQ2xILadK0T33y6lfRUV3BOW01BDoF0ndyd7LjG5Di/cjfSo 1hvhTkW7QJGfzVV4IAAxEyHKlmgONfggZoplqukuPsq7eNNRPhvlZq632QXIqt6Y xE43Nk0O41rX/tWtB7eNcPvfNOc+sGljnCSwpRWyx9xO7plELVD9KdtcyHrIgora Flh7KsSbppSQ/iUKRNP+lfCQsMa1yrnQyxazss8OGlB7YpUJL4trQW35f/jXFD+J ATMEEAEIAB0WIQRzOJcwae0/RD9NN9+mT9Wxets5qAUCZh2mQQAKCRCmT9Wxets5 qPBjB/0SDkET7h/Vw2PJKxuYujsL+tn3SKXshgyCM2u00njJM9TqpZbZV681unKM l8uHtj9b0Z4U0nHoNEC37wI5FJlxy1hLBw5f2fd/yi8LsD1KP2htjMUW+I2xjcdo FusQsIF0s8SyW1DZ3vvN2WcZpKHwub1sY9ZFBfxRc6w+33N4dJwXVXP57kj3Ci8j LDLfkaKyiuYgMtFYZiKKX0tfvaM5pXxLvLOzma9vwfjIMIllooZHDSI65jrbmMv0 rfDKOX9Ws5Xi8n85jq6Oyq28QPLZUsmymCbhvBwq4FcdiyTl9sxCY4HLq0MzmJJ5 DMhlFd2Ds3BopFTWCB2fvYyVoXRaiQEzBBABCAAdFiEEVzv9az2PvGQQeaarq/W9 gnvZv2IFAmYdpugACgkQq/W9gnvZv2Jk4Qf+N0P/7FIHowlO01XmBB5KaztBmVb2 Tj+jtYgPDHRf86O0kW40Rjx++zMlIRNWK4Ue5PKAi82Yue5uvZcVlpWpx/sMvL+N C4Xds3Q3qnkxkoemoIMqUKGvePjBpyUWArBkBQ3FrvZtywnzyFWNrvOpeM+5HIuz WBri/SHBHzQm1/Jl2r5pHcbUdSxB2o1v3f+SaS2vGxwigIf8v44pRfyeWgkoxYgN +2zR0Ing6URZCYkAbwILsmmWGxJIuq+N9Xs1CQ1WZd5S78p/JBMDQ1prUDLCLFMc AvlZpQ0HvzEbKGiIVNa1LEQRF4ZWjQOHaPJhg/D3r/Q7VaFlgsOqrwtQaYkCMwQQ AQgAHRYhBPlNVLwMXWpkF8jPP+gsERivlN9uBQJmHkpZAAoJEOgsERivlN9u8fYQ AK0s0CvQNTXrg/Oe92Ajj+CpFIGhEUgXsufpg3OF+4doXOoRrVcv6y/0dGC+u899 Qiz5rzP8JkgT3Bvs/oFbQnESX7zob/GuBiRAnaanQQGjQsc8tXUcIgIB8vZI6Hxr BZYyjXMrc1fAp1zy6F3YfVtjntp6Zt740zlcFSHPL6pKeNC8lCas7f7EPGm9ERlf XvPOsMyKVDRTrtYVrQ17pgmWzMFl9eYzAV81X/cK7O9BmTvLb9HB9THl9QM6iKWd UPNNhMseMA55i1y1trvv2rQSP2tm7xAijlffNu/LHyVjOJA+63rk9JqpQi2O/sI6 naCZ5kLky3+OisbzJLtsIv3KWGF4jnpZJwPI97UbRAxrBCPd8BDXW06qQ0xfF9GA sW46IDnf5uNV5Fj9T1IhZUUCU6XwwhcTENwcaJ2hubPzW19gvxieRpxdvnXhjUxR UgqgFjtlpyBSABYr2REiaBTHkR1qVMa8tThpSyzfmfBNe9chBGQBdDMzTTUDf4dU cw4UGGPXqrBEapleoZBszXLrZxQxCNmLGFBW3vcJDfRRTvg/OMCIwD72kfd8KY1t SRRi5vQ3CvV8E0EEXshjxVk0fwS+5muM1thWZM4xCSgyH6Ka/5biMeUv1VNcKJne J51xs9jfS/JltrT/ahWG4J9msJFtmYyrLh/nMxccXK75uQINBGYXyiQBEAC5tT5O uysy75BcwAg8jIK+Cw6hNy+riOoCIzsMen8ps4tyDFLmRdpJmVOpmtvESaix2MHf Hc/t9hOsQ8LmF3kDG/JisDXcB/v28EOiDpp5Ug/5UOFBnbu4DkxbakJF8KF/rQ9t i29lt03saGCf2XbqzTLI6FvZ2TT8hDwAZF5aOtDEHV3ChBPn6gplnJADiZ9DioMZ ji1HnL8Zu4IYHMNOgpxULi6TMhBH/MkHbyycOdt/EsQFamnLGeV8KR2fubYjrpbH pLZzSRepQyvKIhHAFj6DUeDyEt2XAitxI8YI40IVO75Zu8ZZq0qYGML8Am+t6ZjJ 3ZR8/DWjxRUYeo+YVEe5f+oRl5GRNkLtGvTAD38Nb2/7SUYdSXA3y3Ocfo/bySwa qggeFpDqK5eHXmrO4hvRqYoEyNyW4VQlGyvYq4s2cLeCF/S2w6dV8OFsksIoq8uq R1/IQ8Bonsf7iAYpsMAZZOGKiJzr01W3GA4Ka3B/MmZP5CysUhFlFxMsDr3/TWfg p3CHd5yGAnuWWWkjqVQzx0tcub3gyDsHCPuws8P2OKJ2lzNPqpp08MjYMMRZb4Y6 9REXkKw7kXU8zM5+1IpW2U+z83NU86QR08PTpjATz05ltdGqF82Z+Ygl2nav8oqV RqNd/k+WE60e1eJmgykjmz6nPbm0S2jt1C7QLQARAQABiQI2BBgBCAAgFiEEQzh4 Jd2xu5fsNrpdAHyNfBXYc2kFAmYXyiQCGwwACgkQAHyNfBXYc2mTihAAqB+sv9lw kRorE6iXwvvj2Dt2iIy7jc1AhZQOH/j7B4GHpV3Ej/ptdUwuzj/aX5EnEeDPZ2JU sSKy2q0RpKGKdKOvgy5yVfd8xqujkawXv26QU53mgyfgQCZLhFFhq0MIAqnxPb8h SCQeol18Wqs++LjeDMwkgMrHJeNhW2U2llqTS37YfRMOo0Vr022ZHlMlkyMz1sQH +C2/nzmmtkI4+vlPeccoN+3239YzndW1+XM8S3dXNcsGTyLAbkCowfpuqQdIP0MY lBwx/Xj9fxBNAuqGVCjrjGMg7mozMkeCDzrAoZiaD3Kud8zSs9VpAyAymrPQJSSS 96b+vr2mDKbV11QJeJZv/d02n4JMjK7Ai//3j/TqkJF4UoYH45g5hvGSrym1UKrf n8TqHdtTFjcxAMXLbWICHdDk7/0ole8Bl8csiSHyKy/sGJ0b/7zcB88CS8OfsR3C OanK13emeD6rHOp8wEWA1/PA1JoAC5suS/uIgPWa5ujLaViJ9pW6ohfzMqOtLABF BB/FgD/qgPF+uTPPLQZw3XO8Q61kFq6x0RJGNgBEOpseounx+T6FCxZqrvjWm/WK VQUiRBtJIvD7Z8UCP+NUzdj3hwLAXpXrPz0gkcbI+hdlTJHCC6i61Qf5OIWnhtw6 kZv2zEcTtzlAYNEumy8KrJzICmPLS7BEC8w= =ilJ3 -----END PGP PUBLIC KEY BLOCK----- myguard-labs-nginx-zstd-module-3486b5b/ci/tools/keys/maxim.key000066400000000000000000000117201525232545500243720ustar00rootroot00000000000000-----BEGIN PGP PUBLIC KEY BLOCK----- mQINBF4TqFoBEADNbls05thIAYVVKdMDRdtzGk7HXGqx60u/kh4BL9HskUpyYFTp N07RJ1TyyusfD7I3skuGHvtQhqdTwHPDEPL5qrAnHps9XWUQrtU7hflcIKt43iDe TvfVVhN0nPir2++C4qvNnrC/UCisyz00H/I9mobl2qzyKyLT8BnUBVuXDfOTlUCY oF4z5BieOMvg1DZNKFDnK67ZuO4JXgtMlu4Q3tFd7qSWCWGuCuAGgn6eWFYMzCbB rPyBYwb7xyycQzqmJiD7Qm9OeVHmZj5rG5hGM14MyTSUVJle0U+CJCF9lmfVuR/c ySy7WmQgIg327x5Y5xa3pKZAvIAycnDabAk/08p59BG7UdAi2S7+2SicAH89/81V g4BI4mZp+IuxaP+S+ckaRf1CUvRAJuLTqUeBSuOzjag+ibD6rqusuZ1MZqLxnXyu gAztNDcmEFa/pqp5bgWbrlTF6zKt4cQf+a/JqFGatsfSzmrIyIZ6GEqgb8oXDDIt Z1AqsTfp6ZBC1vITE9+b0zBw6qq/nGD0Iq47Vp1VxmlxmnoeR4ir8z/oSukPulLU K3IqkmRNGEilINrtBt5jFbBlx8kwdCYvxEF6ymibBBqvwwv65jrrKheBQm+HrrVS aMQmo4Qzj/h/ZLL9KENHibNwUypJnvwEvw0YkAyjICvoNzDUsM+92+B/ewARAQAB tCFNYXhpbSBLb25vdmFsb3YgPG1heGltQG5naW54LmNvbT6JAlcEEwEKAEECGwMF CwkIBwMFFQoJCAsFFgIDAQACHgECF4ACGQEWIQRB25JxPTv0v/PukQacXn+i9Ul3 1AUCY88PtQUJDSXXWwAKCRCcXn+i9Ul31EA9D/9RvgNAn+StYCRAJATj1MhmPN/a DP7DgqBvxjxAQVJtEbVypN0gGNi/pE7AjCG53wtCQaABAaSGikViOUFJu39VSj+a NyW1JQ0XOhZNJ7pMq1S7mgq1ibrbfb7Vlys25xoUsps3vVXt9ciVGW8GQ57sDcpY 8Yp4X4GFbrcgu67bebJVcfzG61JkFxrBIZ45jaMRS6xpGiwvQw6DcpO1SAI7TGPt 3uhRfj3yU9v++/ULzHg7Zq5d74BCcu6o1O59/juPKcS4Bym+d7nlp/AXFiEPAI7v RfXvlzNDncA/s4msEqaT1fPDAG8E9m01Hl3uoCn87H5BBM88rk47PslixNamX8sX pftEPOD4QMFNY0GsVS6/9OIAetaq08R5XjhJoXS2Z7VJWqJYmu0800LpIzNljsEf 9Vt0z/ROKtykSB58OQKIV6EqcA1Dts7TlVFFE5tERMszghBJP++VZ52W6pUAZtXH 26GQzzqcMSfjjw6RpsuyofsKMX2xAG3+z2ZFHzaRLi7kXNKcomOZO1DgsREwkzYL lLeuwcOuZaMauHy/Rds1Wo99eg8+qcFd67sefZMDkgqGZ+xE3+4qwZNBhZLGg7Rq hLQI0d1Qmr7PgJfzvEnm8/+5sN4jsatv6glVfYYUTwB7PQDDqT+y8rsvKJYnNjqI mBmazculVduYimFZ74hdBBARCgAdFiEEZVBsAu/CUPG3o9aU7PDpCywXIIMFAl4T qXUACgkQ7PDpCywXIIN5CQCgyNFrUBGlUvH9QlDSE/umzoyXW/UAn0ve2/HzpMVN uPMAAgnHYE2R0eiEtCNNYXhpbSBLb25vdmFsb3YgPG1heGltQEZyZWVCU0Qub3Jn PokCVAQTAQoAPgIbAwULCQgHAwUVCgkICwUWAgMBAAIeAQIXgBYhBEHbknE9O/S/ 8+6RBpxef6L1SXfUBQJjzw/CBQkNJddbAAoJEJxef6L1SXfUZM0P/j8qRgh1x/d4 IjdlsArTyL5vLJUFzK2s6DEBhDiLJ5ehU77CY9eakITFkzsUbq+BJ7nG6/s763Mq HQ/bVcVQwl0iOq7QICL1N3ficKKd3WAZuuzBxDO3/MT128MDWMKOUu2jZiStctuM OaLaw9JtdTwPTvHFkOb1Ji6YY0XhYyJoPd5LufooHRT7WF7TYGk05JIrxYaYWi8/ Rd+GWnU8qKAmw3NQ305ia6xEaPi5rkl+zxS4+bfs+H9rJ1dRM+nCwXABr5Db1amL t3J6JmXRJLdhGq1ZuAIuO1Irdxe+e0cGSWkUaC+9s9OQcBAxfZ0QSEDrI3/NF362 yKYsmolQwxhvnhlS6maL7ySUSdd5AVzucAX4q3aQ5qzruV+VI9gDZHvUAEwNn3qL H3zhs05LQ3lCM5lWOs78N8yVRyyfzwxK/f01DQfvXm9nLD1mXOz91B0lkaosflfF 8bjqwS930P0aOiGzBJ29qiVIukmrKWpV8L89yUePdFp2Qt59/cLZub1K8S4VuIl+ AaQpaT4TXE9jz9sprChmuE8vrusI0VYNF+M6ZM5OhVb6QfBX911k2esBiVWLbF9A 1E04CIMiYUNc9q+uqqIotJmw1F8QrCZBMm4H3dZHmvYL2tXuiBfaGoc7NljGSJx4 R9bo2E8xd+T+K8PE6jILbnHdtlZiZ3EjiF0EEBEKAB0WIQRlUGwC78JQ8bej1pTs 8OkLLBcggwUCXhOpbwAKCRDs8OkLLBcgg/jfAKCO7DIiB2DGBfLCFftmyuZJN2A6 ZgCfV/cclX++mLyiyYqr2BXnrQk4NVGJAk4EEwEKADgWIQRB25JxPTv0v/PukQac Xn+i9Ul31AUCXhOoqAIbAwULCQgHAwUVCgkICwUWAgMBAAIeAQIXgAAKCRCcXn+i 9Ul31E1LD/4yIPvNAf0x57jvphENedjXzA0DQOnzGc5MVLVsyLw4zYMGJxP2CXYN dUIGBKssoWiUDnEEVF4BNnrkrVO7y9XKWrXhDKI0Q23rau+2TyQ5VLT7Bgsf2QL5 d1H/gdQvToIIQUEITipW81TmFzZlQzjvuivDmcLnYGJQzrEY/4UdC8lzH2noC3PR mC9O98DQgKn72RV1YMAhSfmLFnLFjmZQTHKAbWnVNuPvt4/Hmd/kFem4TOznbtG1 Fa3Hvlb04crLRKns7pf4QQCd/pAp4WjMVyWf4LX44xYziqV3xKVUObv4lKWNO3/P VUb2ka3hKwyIKXiVpqheOd48+c/Kcm/L8mdJdA6+SVrCKBzWRMYPBEfzAVqeaj40 qsNMfbuUgUc+m5f/45UiwWCoQrK5q5lKf9sIQ6RU3Yln43XJ7hP2urgvlW5ZMkW6 qGJ6G6ANVVT2jouqQaYIa2nvMTjEOXh2xzjSw132FHxfr417MeLhQS4JZurMVViA u2SyyLN5oTU2i3+EzhR3DluHMNDz1K9C6SkJJQ+vWAtBaCer/02uxqxqOEnbKxEg RWgKVrMRya/5vDp1Lz/PvR8tc3ne4Yd/OZdqSHFLXIofJStsIlKPnELdIuOg8R7X 594W4XX94V3n12hiiCU+CPFBRL/4PZV8CtbLY5rNFCdpiiN+EP35rbkCDQReE6ha ARAA6aSKum1vKZRHYk/0Os97toQttTDhtwzHj+dCjVDIrGMOFR2BS6Lg6KrGaEyN PYiWAW9xiwx4uJgZr8R0GR1irgO0TBz77Z7JsVADXPoeuT6zF13yd43hrNRrmypW 9TB6dDtQw9yVJAloTWdUYJkopXI2wrfpEmsTQl92YTXMpkVaZMJNn8P+FMKr6GMy kJX5VYVrPeHhL5exmIGAsHj7ObIePJToT55wQu8AehfAxsA+IXPOazzbgVzUEoeO dzwEHN9RV0gEaHE5U0cNKPejE+yIGe69KVAyUbHRg0vD+1N/D4IU6i3mj8oir2y9 Snxb2Z2PqQpqoEs3GmMduFDK9VIrs7m5LrZ6YweRHCC7W2Y/3x3dsaVdGf5L7YwK 7Kp0VorjQHq+eQ+demXmCATDwmgfXVqE14nm1fz8mlf/RpCZqm3sIrByAQslqn5C nyL8wVd5/sUr+cTSUUgv3miX262F39t5fp3P8QTDrWULXs6l3fLQ34PL54nTBtQt 2AqtuVnQ/Gy6Yt4gIYuzv8873/edAclGOYVpUbXS3lsv/vuC9DiXsdy4tLUYYeH5 EzoSwW66Koj5oVwMXJEhOWKDRBGrjL8LqeqFNYTJiuIiLwOronFeKySULVa9IiA/ CrX+ayatuIpwlkU+g20xW053MvedozkhY5rb3RHnjUaOQRUAEQEAAYkCPAQYAQoA JgIbDBYhBEHbknE9O/S/8+6RBpxef6L1SXfUBQJjzwtWBQkNJdL8AAoJEJxef6L1 SXfU/YEP/AnuEZX3l/5bIFauUvA496j9REgSUXvMZhHi7pTd3sjgVZvax/WJ+ZNQ N0sFHFSO0wjHJPpJw3LVBqq1LzuLz4jdnWwtD5XWEBybVbN8TblRMA5NhT/KzZW6 4LF2r64UjMtm03ry8RGOvWlX56/qT7eD6QM8Xo4zL6g1W3+XID0+hQ7zwxSHZqUm It8LE90eTwcGAMVvZ4BJ58bWpujbMsFcWX9oIbDaaJD+hhI209a4znFmF4iOHdEd nLYwe0PiIn3+nyNzJ5EQ/Qvor65BagGYeVQE8BhaPi8vkr9PBY+tIY9pzj+jWB8f MuScfSyGlja1VhrlwMpB4VxcVBgFp31LN5NPn8XqIoOQBDqYA6JDfQtMznZtpb8H u2Tx7bmLwit6MrCu9q66yzYZZ5jg8onbG5QthLER3Qi+m8My7dSlAvruvXS+xoOH kLjtJ+wx2joVAnvf+VPLqlLJGeuNQP8Ji2EpMw73FIeWLakZKAZRHzQgiV3nGDFX 98Wo6hmHbP0eU4NT8kFKte/PIsKUo5LBbZrb5S7MPWvqXDo2iKthaAqA+rxUY0Zu Rv+Ij5IqzTZyStejJGngIQlABi4jtSpf6LdRPW3GpNZ5pUi1DNmfmxVl4xmV5enE 53YQML90r5jsKt/JloNWYEUd7nM+801WiCqlr7lltunhIYyMTA7C =5smL -----END PGP PUBLIC KEY BLOCK----- myguard-labs-nginx-zstd-module-3486b5b/ci/tools/keys/nginx_signing.key000066400000000000000000000270411525232545500261230ustar00rootroot00000000000000-----BEGIN PGP PUBLIC KEY BLOCK----- mQINBGZXLBYBEACxv3nUIdUtFCpH1G4hBB+eVSsWwnHVTDtSYfINHmN8dQfyGy22 XcX2DR6ZW9/I5e06McAz4e3hTuhD5+sF7zv4Dd/xEqxpra08liVvB3QlJ6kawBJa Bn29s/N/A06yUrOVC1ZjhpDLshaHeyHjWDVLUX9ibLx1N3BQoeoH/5lgTmfF4JPk LfnTMwHWQ5phT52MVE+B/XExldIPAn27m2ZfXHXnSUMKCRybQNypBiIp6OBfirwa pyjaRO1AajwalSkbSV9o/fL3liluv1HimQ11/5y0rxMdi+aaeca9oA4Gvfdh/biO MYcTeiZx72BKqDwMfJVXSjQ8XOYbfCjWp8dNkS5Yd4bmX+ITXRkZHqQxgmoKWr7B 9/i+asColt/qqsQ6PROa2y86TbQSfn/HM8L6c85BkJrI41abJ2QHShVzpk0e/464 hqxvnAZCrmdM+GBSuYfDDqHHHgxhIzHnKnyRX/MtfhZA/CUFUOe+m6j214KKtkMQ 6EpZzgH52FFD6Vi1NkQvfYx5pqEdmJfRKR9ABf8fYI8U8ryNgIq7f13bwoX4haZy ql/fC4lTG6OEppgdQe7afyAmdi7G/w1pMcbz5Wwp91R+1372XifynBdeTrUsbK25 P42TH3OADC2Id+MaaGh1AjY1bFifOGRf48rnrcMn0Q4Lw3l56wgjou4MUQARAQAB tCtuZ2lueCBzaWduaW5nIGtleSA8c2lnbmluZy1rZXktMkBuZ2lueC5jb20+iQIi BBMBCgAWBQJmVywWCRAv0hMQtJ9rRgIbAwIZAQAAq08P/jeIVEj9/cJFzdOeBqjg F9DNZljkR+2z5UAkQSHfkzWgHRbdAnjT1bc/ltLi6w/z/97kOZhaiSx6TLRg2mX/ 5nuC4KijhT9rNc/d5j/BHS4U7lFK8c5ED5wxGvJZcF0VCSfeaiuxoO3QiNYX1iiD qEyJ1XL/XHd7LjJ4gKxsohKL1rRLSuvtOkK799YArNit5ueATDWW6EUSZaxOiMNz MaQFMEkjoiPVlj7jNwZN7KHNXkaJjiER0kmJ9XWDtkgSHOZrUNX2PHJpxxCtQj7d YpOFM/DHvNUZ9dHXm3Ioo3R/MUcC4mbZpAvs4YwZ/yRqov/MX4WEUtvcCY36EL5t hUDK09huMMBLBdM0jgVLsJnXn5ksMdVkpgFyeR/SKEaUTmQrgkCIwqvRxDegAkNN lmAiNhxdKD+CrWws+EzQYOeWVRUO9aHKC5ttwhhQuxyvmNgoAMhd8x8Tcm7grC/m ZOqYWzpEWd1DEyi9jaTkhrSWMd5jc5lvCwOHDRzVi1HmIJy+cybPbQpkbFY6vj/7 shx2Aa+QKRJs+33Ztg0drc3j+mDk9NJQy0KPIbqee0gy0pmaKNiJOxdIWI6ra3cM 3lh5OG+CGakga1X9YiCWv4/OgDYY/6cFTqEN0wXruFLNZ7P4iowJgPU1KZauvDZl gfsgBoKJ35Nf6p9PdjcjcyW5iQEzBBABCAAdFiEEcziXMGntP0Q/TTffpk/VsXrb OagFAmZXLlcACgkQpk/VsXrbOaiWowgAvU9HwLkK74VGjosmPpcjurRowUp+/KOA HmIro2wQ6JVlUrSL2Rz+RIBJ1BKTgGnVZznkXywXHWK2LI4nL3aDoAuyyrzQk1pj hO1ZJGJBvh9Zq/kGRgEdlTe2sXVX2G7fr4fhd6BcYYvUBQ5OWR6Hh6uS+G1QVw0y Lu5Gp+7kyolyH6iYlgvxseche+EIqBPyHe5fyb1t8Zcu1uHoQHj9O90FvJSbq4dR d0tTlqK1tDklT+Aod2UobBCurn45udjiAKtzH6Bg2dvF/oY4udSC9/HgNPbm7JuY clEaLukWMdFOCEj9Xr6krHtUh7zTiU6pHvUL2SYMPhsJj6AKZRg52IkBMwQQAQgA HRYhBFc7/Ws9j7xkEHmmq6v1vYJ72b9iBQJmVz0rAAoJEKv1vYJ72b9iVTwH/Awq vgnXbJ5mCGbLdQgrDoUYe+1nw/qWbl7Hpn/px55BEIW5S0itI50c9sOS2QFQMdRh YVqZ+YH4aH5pDNW2kFik4Y+CFoJI9QkrEUx66PYIMu3RVBEE7/HQEwND/IbEAeMg PpGQdEfEDD8kevlinJTyDXJ3dfBa6HEDpK0wDYrBx3mbHP7ouACsZcxqSdx4kOyv U2Xvlc5pVRsdvJ7AsVRhRaRdSO8YlqU1Ue/OM/Ejj+GZ1Qo8EDge5887HiY8gcjy J4FS1n2+3839n990s5xDCFSB1G8KmwgkfbkS6gEpA5wf9nk3tiSPS+HMfjMb50GJ SayUVrAyUupv/Sxvyo+JAjMEEAEIAB0WIQTWeGzjA9mpAimY3GzIRk1UmvdcCgUC ZldKbQAKCRDIRk1UmvdcCn6EEACUhtMnJGtrunotTwywt/jfkqexA+lhQ+S9V5eF IIK6Tlq1asFy0s+twYJBQzTXt+hmL8GrBgeQp26CA8wrbxmnUOrXO1K9ksaXXjj0 SRo9Xr/flCmeFKFRSSVy18UZVwf1vftFwF2lQspU+xZmj7vgr+2vKa3Z+81J8tHw 3/Sc5pt3EGB8GeCiEThe3zr49KpANejy/7feASSS+BBBUbNqnCFImfwLJ2V99mGx GdejudbTYEXsn6jyVWTeKBcaLM4ArS20O0DJkqBcVC1Ymq+K3AGmKnrLJXDSwaV/ +yv5pyqApf6Lu9tx7wy6upBop8KroB9xiTN5UIiYhwtHBlpOLkmXB7K549CYX34y aOHJjez8Txn1bDhbCOe8WOnPEDI8V4RQBr0/xePru6lfwSmSriquVuBGZSir6qxA 1folqrEuoF5aEuxFper6yC/zfVP85znqBOh8OaYTGBeb622UswzLTbW4y2M3E9Ws KhaXzTqXgIn3INCJLCv4CHiGQQB6zN6meGdOkEV0IaZvq3O4iZOAVFmKbN3GZcKT Kjxq295LNO15c0WCauik3FRjSppyvcAqoCEbr+LVAX3/ZV3oELhQPnkZCuAFQUB+ LKxTcTEIdjFKrPEvDgXLL9CNe747ANcLCV02SRRGYnfQ1aoxJNQlzbFw0unHjyDk vKcD44kBswQQAQgAHRYhBBPIKmO2A1dhVuMKTqDqmBtmsNlnBQJmV1HlAAoJEKDq mBtmsNlni3gMALfZSqIL7v66dMyjLQR81G4o6rEAixTuFc3B8xDmWDHKIjmdRMTN mm2KGz0CG7VjdHSe3oOBYok4fDVS0o636EOxndOHszuB9cfhMMXNDFi4T1xcZCLm UTdXCH88cagwTf6REsbfuXF8WiFemNNiPzMzLmnTlUe7Va2t+gKD/Q9vSlDLKz66 IZBMdDoAHDKHZTtvwlAKswnpO0cDIeZjO0C1+YFLLSJ1nYQbh6mH+hJvNLimWPKR ZQCPAa5w0Gutz91cE9nv03yg3FMcjlEgklQ77g/nGGFJnQHAeMhfgUUfPLx1rI9/ 5NON5w7Wf3PXOlTYWO25ieUVKESu8dUCFktKRMnzauej2vjnQlMFG0upzw8dhytn E83WanvRzVynanK38PCNYQ3INsydN3wvJNetHpBdpyPfOa61dOUtu1TBvV80qcBR wIe6vbWZx0WB59b3KV8Sc68j8OJxF6i3E0IRby4f0hcoqogBkry0NPK/rtL2HHnN vcV0wl+DODz9hw== =oWlI -----END PGP PUBLIC KEY BLOCK----- -----BEGIN PGP PUBLIC KEY BLOCK----- mQENBE5OMmIBCAD+FPYKGriGGf7NqwKfWC83cBV01gabgVWQmZbMcFzeW+hMsgxH W6iimD0RsfZ9oEbfJCPG0CRSZ7ppq5pKamYs2+EJ8Q2ysOFHHwpGrA2C8zyNAs4I QxnZZIbETgcSwFtDun0XiqPwPZgyuXVm9PAbLZRbfBzm8wR/3SWygqZBBLdQk5TE fDR+Eny/M1RVR4xClECONF9UBB2ejFdI1LD45APbP2hsN/piFByU1t7yK2gpFyRt 97WzGHn9MV5/TL7AmRPM4pcr3JacmtCnxXeCZ8nLqedoSuHFuhwyDnlAbu8I16O5 XRrfzhrHRJFM1JnIiGmzZi6zBvH0ItfyX6ttABEBAAG0KW5naW54IHNpZ25pbmcg a2V5IDxzaWduaW5nLWtleUBuZ2lueC5jb20+iQE+BBMBAgAoBQJOTjJiAhsDBQkJ ZgGABgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRCr9b2Ce9m/YpvjB/98uV4t 94d0oEh5XlqEZzVMrcTgPQ3BZt05N5xVuYaglv7OQtdlErMXmRWaFZEqDaMHdniC sF63jWMd29vC4xpzIfmsLK3ce9oYo4t9o4WWqBUdf0Ff1LMz1dfLG2HDtKPfYg3C 8NESud09zuP5NohaE8Qzj/4p6rWDiRpuZ++4fnL3Dt3N6jXILwr/TM/Ma7jvaXGP DO3kzm4dNKp5b5bn2nT2QWLPnEKxvOg5Zoej8l9+KFsUnXoWoYCkMQ2QTpZQFNwF xwJGoAz8K3PwVPUrIL6b1lsiNovDgcgP0eDgzvwLynWKBPkRRjtgmWLoeaS9FAZV ccXJMmANXJFuCf26iQFVBBMBCAA/AhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIX gBYhBFc7/Ws9j7xkEHmmq6v1vYJ72b9iBQJmULK1BQkdphrTAAoJEKv1vYJ72b9i 2+AH/RSX5voZXtSAl0fxVc9GDrGesOsykkSELnailOkWiFEHZS842U1EQst9Omki OC14xk9fY36gK8bxXnLwww4hnnh/fpj7vJkJpVCi2uO3RKizyN6rp+7xbZ2lCKfp 5tsDg5U4iaaziTNtb4ISq79gLmLY/gqBwGksRozmChsl2QOVgg0KDTI5TP+41IwW AFuO+XzHZ7OEegxwHta65KeVNipYjCarTRcRhGxA0rpLdBynkZ/OaI5+J6UZVfna 2eyDgHPlMo+v12+g/wOFOwShVWo4PwIsZw1jzBCLhspgezn7IolQFMHtVxCJAkgw XhLgogChbe885HzTB6GlMowXclGJATMEEAEIAB0WIQRzOJcwae0/RD9NN9+mT9Wx ets5qAUCZlcuRQAKCRCmT9Wxets5qD1GB/4/NIcvCRj3LvFbrtmtbExBoBP6Hv/8 U4wUpuJbAAxImJ9uNKKaH+cmvoshkWTSUBXTvNjAQW3SM9oW+V3G7wicUtH+7cnd xExuqf5e6f6IGqKCgrV25g0WWvJZG6ynMDDkgnyu3fTE7GkVKwoWQ6qV6Akar8oV 29P+xe2U7AWPvw+O+SBghl32x8DA/nUjIyLbvBQuXb6BjHOxrTw3WOJDfwHwOyMd P7NHe7RE70cSj/TNabuNw9c31H0+PAj+UWfvgs5diPVJ9Fd/PK4pWQoh/4poMEbc /1Ol0G7SItUKO6v4aHn89g00xnqUxrfwbCWCEF9EjnfFtlsDbGSWIdz8iQE+BBMB AgAoAhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAUCV2K1+AUJGB4fQQAKCRCr 9b2Ce9m/YloaB/9XGrolkocm7l/tsVjaBQCteXKuwsm4XhCuAQ6YAwA1L1UheGOG /aa2xJvrXE8X32tgcTjrKoYoXWcdxaFjlXGTt6jV85qRguUzvMOxxSEM2Dn115et N9piPl0Zz+4rkx8+2vJGF+eMlruPXg/zd88NvyLq5gGHEsFRBMVufYmHtNfcp4ok C1klWiRIRSdp4QY1wdrN1O+/oCTl8Bzy6hcHjLIq3aoumcLxMjtBoclc/5OTioLD wSDfVx7rWyfRhcBzVbwDoe/PD08AoAA6fxXvWjSxy+dGhEaXoTHjkCbz/l6NxrK3 JFyauDgU4K4MytsZ1HDiMgMW8hZXxszoICTTiQEcBBABAgAGBQJOTkelAAoJEKZP 1bF62zmo79oH/1XDb29SYtWp+MTJTPFEwlWRiyRuDXy3wBd/BpwBRIWfWzMs1gnC jNjk0EVBVGa2grvy9JtxJKMd6l/PWXVucSt+U/+GO8rBkw14SdhqxaS2l14v6gyM eUrSbY3XfToGfwHC4sa/Thn8X4jFaQ2XN5dAIzJGU1s5JA0tjEzUwCnmrKmyMlXZ aoQVrmORGjCuH0I0aAFkRS0UtnB9HPpxhGVbs24xXZQnZDNbUQeulFxS4uP3OLDB AeCHl+v4t/uotIad8v6JSO93vc1evIje6lguE81HHmJn9noxPItvOvSMb2yPsE8m H4cJHRTFNSEhPW6ghmlfWa9ZwiVX5igxcvaIRgQQEQIABgUCTk5b0gAKCRDs8OkL LBcgg1G+AKCnacLb/+W6cflirUIExgZdUJqoogCeNPVwXiHEIVqithAM1pdY/gca QZmIRgQQEQIABgUCTk5fYQAKCRCpN2E5pSTFPnNWAJ9gUozyiS+9jf2rJvqmJSeW uCgVRwCcCUFhXRCpQO2YVa3l3WuB+rgKjsSJAjMEEAEIAB0WIQTWeGzjA9mpAimY 3GzIRk1UmvdcCgUCZldKdQAKCRDIRk1UmvdcCj1hEACv1XfhwpsBPVNzcfzMIpfY xAQF28m/VFLwD8FYKoVgb4rF2wLBtt9kaoPZxphEvV/FWHhpa3Tyr3L320r6sVk2 5Ou6G/AH6kNF6vYn98chEmbCc7DE2B03G1HFFuRSOmp0ZwafJ6MYUhjpDrf6fFDL fmdkr/hjLwCYvFQsHXYiIWDFBPZ6RvVC6ozbdFr4eWj+CIPZM4jcGTgSI/u67tC6 8tOdX4a8/ujdkLDjyf2xgbWT8ZxY3o0fvfLFEQVpNMUsYtiW/kTPBsq48Gq2BWow /2Ld86KjgBOyElnVy9kMLCB4d/DPnSdBkjHzWWDx2c/PDGWIGnES6O7NYvRQ9Sr0 bQwtr70nvai2OkpYVszVwOqyr4vDeTIt0GFKOMRDRrscVGmlGr2mpExiCEgGyAjR Z/aZDCzEnsswfJ+6IARYzE5nB3+pbJnzQNvj9r/YL8T9HkWID4sWJnnNmaFoWEMF m+yvI8vyVMGPSqfVtN9pEpx/pzV/Q525nFYuUlEsqGgaDydnwe6AV9gZsRyA+YjE H3gI1gxGwRyupldmstzoYzTktb4o1KL/vGj/onUIk8mFKx8p1X9VPWW0+8LqnAYf Ui3jDoXE/9avsF6ipS7y1k8ga81z01NOvuhai3c9pvMAIYrNTvoQVz8vTIOtJac1 PEoU6jdm8blCt2UjGp8A4okBswQQAQgAHRYhBBPIKmO2A1dhVuMKTqDqmBtmsNln BQJmV1HrAAoJEKDqmBtmsNlntoEMANBPdskGMrU4ZxHMlOTd1JX74ucp5jez0Y2o bwlxOiWroraYVBnWT9v150kNf1Tb5mDxi820qebiSPZxhlI1Kj7NrPFNxQkhhNzN 7Xr/M9OGpkwxosEpcMAiWfofyAdrnwos+MA/edu/EoyVRs6zpo75nP9GKUZwVcjH KtvPMojkZYpxjxsio0aK8LW8VwDtsbwPIXDIHzE7sxUvThrMdXumrh7gKqaC6gep HZB2lL5ES0kVE3/yjZR1khmcmF1zELeC0IddJjX2R9HMcSLixdJ2V8/VFsWMb2KQ pGtDzCuRyyxbugzBIxiGV2Xb7XwOByaikc1duqFv3gtk7Vk8wgQN3YwLkZ6pztlK vCbqy2b2wlPviGjApQ2GVd6EEmlCk2gKPkjrn2lxS2BXWorM+ANSswJT+eILi9yW Q5zzmYK2vFTzL7FAMeqS/671jNhZQ8O7jvbY/mRhl66k2MY7/JgI+coP0cY+HHr2 ozw9yNdOZmnk2Prj7+mBuchbT3BJOQ== =AgHy -----END PGP PUBLIC KEY BLOCK----- -----BEGIN PGP PUBLIC KEY BLOCK----- mQINBGZXO1wBEADEm061e/MGo2f7rpSqokI59in/egWbeQE26vwxB7vPu4e7j+cU Vg3AezwCbf3nVRAE9DpJ+yuB0KVkM/0QszjOEEBuehZYJrUiwMyiY6jAk8xtqjpV PsOMyZrypoJhwzg/sYNadUPw4UoHJ/xq4wNA2ZG9Xf0l8M3shYJPmKWLz/eefa5V Ef/toQ7a55l0aJ7XyACTU6dv4bkHHqomDImK2C94s+KyCxaFyz6NgFz25V/j66Am gB1m6UGGsvP4qYXW+KTsLz9XDvJeLLHWNcqQoyUO5Vs5C3hGozL7kEkyK/1qHcou XXkeGN365z93ZeK+VdBZKJtsCswPk2wdDBByU9lAUNHYcLHf6S8fwCACeIqJ6LaY MKmZUN2gR/boTyMERHEA8XnWXTDp7EsSNIc+LkU5AT8yesANcczH5k/XOI4hltJC piEsSgg9V7FvO4eA2iQWGv/Y4nlUfw3lbRuRFvd7oqVQKlX4iIs++kVCCegBvtNA 1naxPbvTqrC4THvBSSZpOW/y/6XibAr/scCNNW1mEhwm5SPBHq9Sv35p6xKDTcgQ 8o3KLM8tKKt6kokAqlrXk9Nq6LYrZKwg5a9crFF7nCL2xgxZy1OJQVcPuhhZy5WT WReE5RJdlF5VGRT9nMJ3B4Vlp5luQnMUFYXTAKQd6Cogbb99J4MjDttAlwARAQAB tCtuZ2lueCBzaWduaW5nIGtleSA8c2lnbmluZy1rZXktM0BuZ2lueC5jb20+iQIi BBMBCgAWBQJmVztcCRC83NijjYiiswIbAwIZAQAA9FMQAJ/e8F1egZGbRIV6qU/Q bJD3EsKZZlitQSVXbBpxqDlkD+uzSFATGjiLGvJoTzfpJpJjI7FwrtO74lRkjCl9 wQUNJ+wm2Kod6rEEQc6lWkDsgxpjqAAGVS0lmMf+VPBGQ+kc8S3ZdCOWEeq7nThZ /xWR+UuQQcz1vCKmEgwTrr5MJVcqDg4wiH1Z4lRVfjTezf9IWk+xeE3mV8h7Ltbr N5ZvOkiw88JLrbQsurxx+lYEaGIZyIk3huiDE/KpsMdw9KXUfoDcBqWc7oDjqKL+ QEaq7TW6VetKyJaakP6Do+Opx0BtS3eH86PEZqtULEw9WifC86GtRr50iTXWBTfI MFZo4AwigHXvZ5WrJvLfldY+scoU1rPMouYlZJ9W+6YHLjf/jpr4W1w6LKKXX3ah h4VLtlOmrOLA21E7RQ0PwoE6nT7DAm1DsMFCXy7lyp3u5IXGahnJddWCb0Px3RTm PZgOt+YAGJDsP46ngl5LxhilMK5f5R8v5n1lJ/XzFcXCEN4i/d8A1jx9DQx4CJN1 wp/WZzJ6GjnCqMCdOBlQ2eNmhR+q1bAI79kSv86ahaM/aS1FvHMz8ppzwkRhv5jY eR9aRlAwaCPOjbWhYJt/xveOWmxCdg5ta+Pj5g+41wHZyNf9aqR314aKwsxo2AYH uUe+PgpsHbe1sQTkb/W1OfSCiQEzBBABCAAdFiEEcziXMGntP0Q/TTffpk/VsXrb OagFAmZXO+kACgkQpk/VsXrbOajGgwf8CAXJwSIhGOWFSgV6vpvZPChTsgteZxhT 8NrJJLxL8X34Rw5YctSli4akkchTonm5RRp/SlvI2fPe0o6q2ymF4BASPJ/oSI3p Gs/jwctHz8hwaVN0xQ4SBXgquIFWrLRNOjCxEV/vMRJRzuF9jrrdv3vxZEugETI+ rnoEZu2Z2ZlMj7PPeiScf8dFXax67+Xi5S2KJCaXm1QGAJvttHrwsbBAIE9CVUg4 UmXwADQ6HkOKjY+QS5AP8Ak1dg8/oadgyMqB4GrcE44KUpo4YafP37XnwXfQNKpk Rb0bO9Qm9lM/LhPulBY8WIPkmrFCVhGTE6K5ZvI59R4nECHHx24/LYkBMwQQAQgA HRYhBFc7/Ws9j7xkEHmmq6v1vYJ72b9iBQJmVzzzAAoJEKv1vYJ72b9iPPIIAJ5k hTz2d7CaJefHzoraogKSIeBnA3OR+nDgdDl9Mp8i2WLGu9YYhIrPU0iSVw8jqa8t GIjCw4/bS9HN8oub2Ip802xDLugCz1Yz6CXjCXN2rlNPsdBV8IIKNHOv93qMvnZS DwyBUAvAs4XzF7zbYgfZ30B0gRI0g0+Nt44oDOn3PfO/kNUJyBVPT9m7l3JUHuZT FPOD8a0oJPvW+iYlSkmPELBvgehsX7MVLoeQ5qtS1KkuWr+y1wqD5kxqabMPcfdU jAr4ssXs/pSsYJVyS4CuUWkY4FiCJm4KtU+XPDs1RCTzMkW6HHgSebocTZzLETYw XsDx80qd21UAdGc116qJAjMEEAEIAB0WIQTWeGzjA9mpAimY3GzIRk1UmvdcCgUC ZldKYgAKCRDIRk1UmvdcCoG/D/9qLmHYOGnsmedUbgtLmuBJOuA6oqnaWxYI45eV +vaAaI2+QfRoJTrjklTXv29Pi4LTzN5YBySSIkv/z9ry5Xsz5yroNY9Xb6JdrqOt fLa/U0wddNuJbmIom4gUPXGInhHUBbP6mNz+s6e2ukBEWvb2XIsGe5v291QXMohQ /PT8zTIwNYaw2zVF6Sa/0spA9/9XA5BdUcrtl7xPgYL7pLVmKYGJlCf5TOaWfLDJ mIMeeUznVK9vK+vT+YqUPfFyIqO7dvio/+MRFjePoD6csT4UBT009ugy8vrYg2YR K9uaRxP3laz9b6xdUM648ycUQLoI4fLhyKAHwPU9/Q+4rOFdrL72ZGVKzv1XOB0H VXf0/E4JmJBydM7AyXHNxIPDtNFydosGn6VZsEvSPZdQSCsCeBs9UuBWgwFb1XBB 61XiHGnheb3U3ZRkajS1ZNdxfohHrBzHnd8tbDkv5Rq+XoUmDauoeM0VcN15hl4a M/JzkeOrHuJicn3mg+HRHxQSCl3D37bVQT7O36n7cff22GykT7XQUBBxMlhKzygD SgdQUtSEt0eu7AXIvr6yl0kobgZQS3wzUIaY0JEuv2ahtEXXjoPzCVWB2OHIpPbu D58cpyyEVqr+ZecaI4HlaO9lVShf+K0rf/6DC12rC2gNzzv/fCIinDiqiMsPTfEM fduRSYkBswQQAQgAHRYhBBPIKmO2A1dhVuMKTqDqmBtmsNlnBQJmV1HlAAoJEKDq mBtmsNlnhI4L/0MHtfCZ2nuKTF/BkxJ7oB3Uule0tWiFj5SU97GjcVj1LgawGY7Y +zoyEd6Twpl6H/+QkZBB55Bf8+cTzRbDzH1Og0fSORu0pGC0uxWdYu1sTLeTnn93 mesXAvevHFNbsPchIWwsVJopTdzMWuAQS5hMMMtNb/14ZfnBadzhjvaJeH3DlZVK 0cGFp0qfbMfjr9yRJzQ1IkiXsS4G4uKg9T+KRsPr4+JalurWJgLnBXZGetNNjjUa UCV1KZY/iWCAlZjkZ5z7yBRj5nUWLb5AVouEQPEDbn+i/0uEjukC+G6EMq2mgbrh m0bFHbHAYBaf9EH0eP799HpoAx2aziDB5igAC516i3BnqxINI9mXHh92tU/H797I oYZvpBsAHDWDHj6O74jwk5lXF5Qwri8gjA8aTudmuQX3uX4h0/FyGGQJW4/wWecH /1fMuvHHyRtOSsJsheDwcSjrw5WlsyNjvSIbBPV2fIx60W2haVMUVX6CrxAeq44F UYda9m8fOnaIew== =TEOn -----END PGP PUBLIC KEY BLOCK----- myguard-labs-nginx-zstd-module-3486b5b/ci/tools/keys/pluknet.key000066400000000000000000000076301525232545500247460ustar00rootroot00000000000000-----BEGIN PGP PUBLIC KEY BLOCK----- mQINBGKE4psBEADpHSM/IxFD1nXBmnODYXzcl2A+6b6m9m1m2Y4Dlr0ed+y5Lxne QidE9I74A2KSm6+eHW2yh4i1ZwZbmwpmQqM+j5BMt7axoXOdKSyN+fYtUakzNbBN EDRKT79q/zIzkgTJradHkCQkwF1W3go+qPXjR2ZEnLma9dZED9VNI6PmOpeYaASo IkEfbKbwa/vPrvnDSSYY6Y02RXSRk5U1NvQgVUTJP9WGK7NlPUcTBDELLQv6fFPU kjBOel6MecsQ+v8iq4RJF2cbVF0hNjbAiNldjLV74Xd7yWVRlCbdb2agyvQjMNrD jHSvbEMiNB3R8yBHVW2Zldv8q0XjcwoDfdiZYFJe3lRUYmv6I2p+/DptD4r/3ILI peGZtSeOdQEw+vvODL/Ehq03anTrzcpZ6sDLfLrYJhYcrltj0/LMUnLDAjciwRUq XI46EfxwqsdLeqoZFQeO3LOFsh0kJKR2xOrUHIVy84NJ4Gmro6WmUkb1NfdjyHzF z8Lfbo46NKoTcwFsFF0q74jVVIVNUyIS91DusiMqLCsP8jqDOz/kyP4bOJQ+aUXf BANn4Ll1TFWsJ417moxz+Pi5sTaI0na8z2XB1N9WPsSml3FS75hJPJshN2T3VIea zB7GFWqk33ynSDt+cAisG5nsK9fFdcH+t5wm59oobyFbFhKxwX6ROuxlZwARAQAB tCRTZXJnZXkgS2FuZGF1cm92IDxwbHVrbmV0QG5naW54LmNvbT6JAk4EEwEKADgW IQTWeGzjA9mpAimY3GzIRk1UmvdcCgUCYoTimwIbAwULCQgHAwUVCgkICwUWAwIB AAIeAQIXgAAKCRDIRk1UmvdcCqbOD/9Htgk3mWvUFmrApkWQTIDNmLACZ1Sw1PXj Uqte8StYB0bYY+nmAXs7O5eC2h1ViParl7En1joEEMQQmH0qSnw4X1CM/hA8TAYW mBPITTNWo/R52WoyWeWGFnFNIperQmuIZc+pXm0VEFVPiX/2DXbCIu+jaXySvlCN LekmOD4VC7dJS8/ohoaXOR2T8ufS+1CsyPXomEb+COhqRZ3EVBa+k7pnElkFft3Y a1fR0AgatZFQpy+ukePhK7s/M5RGhDJWHgSAZFkf+X2jVV4NRJ+XsY80gU5DD2ZX QT6Je6Knxqk7FnWNSxkhReH6Ss5flZSoGDCmJ2AsPtGeUhus2fGqeN+waGKTZC35 die2V4/cro1SWswSI6Y5GFDZT1olIUztPmSXU/A3oyizJI7XZybwUbpk5kK83VXm el3U/7Qr/VErlDWFefZWeUvT1RILZ8IRoNj4dv158RnKHt9G508A5qz4hUPKoSeq SiXhYwfkc31WPzIJ4ev+X5Ka2sG/CKbEMJ7qwc0Kadiu+ePPfqqbXjpTWRyrbcRM hRNcLNUi1SLWMBClOQG+5GNG1dPPHkbj4dO1OZuaUMwQdu8R8NlsGoVWS40bmVv5 pXstzYCl7k/UnC/Ytlq61GeAoq8ILa6jGj0EWqlhvi0ZNMN+fROhzrRlTzIr/+WE Xf8EiVNFSbQlU2VyZ2V5IEthbmRhdXJvdiA8cy5rYW5kYXVyb3ZAZjUuY29tPokC TgQTAQoAOBYhBNZ4bOMD2akCKZjcbMhGTVSa91wKBQJihO2zAhsDBQsJCAcDBRUK CQgLBRYDAgEAAh4BAheAAAoJEMhGTVSa91wKgLQQANaf4UMndkWoefDQPkJ5qR4K fuV0WRz59riZEApTkVpPXzl8Y1i8Rgt9pa1v1i12vPyIXKav1rJXQcuDEzqrhQ2G yvuAE2U/t2mYaMUmwxWO2d8JA3slvBSgOkiYpbLooDizAdKMT5UQWGyw31Wm51iz HjoztebsyXeXgq9VDjv3D8LUBr/OY3Hguj6HV+zRtC95qgXYadW2FiCtvBK6RTDb iShTuseLSheGh9dZIUSnzaOiJpDA61ZDYtFZxSpe67vEzhSfHVsF+ZdCjoWhhVv+ +2wR4E0VQQtOM9uX1PMlZ5Ymr02/gidsXCM0ZjYXx4cDDhnq+nKomN64VloXWY9t PIi86XmzcSWlGUd+Ac6LyW7/f64bUWs4Ih0Idl0PF0sAr/6axKUsIs1nbn5MEtXk ZPAjcDLqLb9IIQaXRurm/il8v+bLXVBOJq33YUuGRuz8pu4vPA5Q97zglqhlIgbu prHMJ9hl5q39JwS3As2rK0o6Q9VVKr29rqSEfk4wEttvk0QMMU5zEvVl8MtqPj42 qURqpHOadFbYMTwhUmRBUszRZPa5/pWqq0gWOtpyCWFVAsHFWQGJM1Eo6gGEyHZM YgBp+d29p2p409r1+06U67GBnXvUy0RyIpkLQtU+lyOJ6vvrBmmsDs/gc69GnlSC tZmCt0pLesJ7ZJzGdDkduQINBGKE4psBEADQr/enuDeVT11v6ejuYrg7aaZaGFUe 3i28bQ4pRUKNfxs7zVYDDHi2i2bhS5j2yQnbsQtGcgoenw6lapmdQRzr4vjQAz9o kT6l4qpqvFFQM0wZTnigVDmmO9vTHR8Uk3iCKTd2ax3oko/xPWWYJautJ6ex8cOA coHSDeOjuIWSxCKq0BDFp6LoxkM8nuyLAX2cbhI3LncaZhVveMeN+Fmcsv+WpkKs yhX92umZuGwlraSyFy23FiRWSZPu9qVIxMMHvVrQJIgfhyWaHFzoF4M4qDoSKx92 uWfUWgFwPOxOJ6/YcPsX4T8qTl9htmwPN0BibPTlcWaIFXtiU5bE1MivUPeACrI/ gwUfCR3Mg+GYc13C6jzepREUhI7PLi3+A203PlMZd/aaSZkP6j+h4cwdapH5P4uF 7T1EQ0MSdx3neAvu5p0IM6JpriwxfT3HsG+Y952T6MIeXcjNRebsBrygJhJ0/vyr wV5t8jL0yQty4CiE/QFnBs42l+rngi7K7Y1AZRBGK7JA09XaoLrfLmS+PrbYPsaJ flkM8GzUB7BBCLozxDHPzmPkf/A1w3XHZnYuZmS+pvjWCIoKpLQHI99oSUGho/TR gMRO4v7EAzluqCiepMl0xwFfHB115ND/mATazc4Pt6FxUsqffzfZrN01e1UVPrp5 4x6YLO80JnOY6QARAQABiQI2BBgBCgAgFiEE1nhs4wPZqQIpmNxsyEZNVJr3XAoF AmKE4psCGwwACgkQyEZNVJr3XAp9ghAAgCgErxQYn/Lh/mzsxYXPnisggcBpceks mGw7knj1EGkXqq9CHn3EjCw8dB5N857UFlUr++DHwpFL5O36PRQo33RIUFbmBypG 8C/xX1jWGu3xcaqS3P1ncsSSl6ckdvy9pjMxThm/RkXO0eJCn7FcanwPJXEB3Pbb mm0wLI2OXl/m7l5QAr7kErnPvGNzcbX6G35Q/MY8mumBWQ9H53R5ZPpi+OS40Wfn pZNKdh/Acwa7+2RokPqoOcJfxVdBOUigXTzb45qZgqEsSR7bkZAy2E80A/sJKPqs OGjp9cog3rBYyNBn5dasfR9KeBtluKnjUbzutXsQoKUSECY00YGrtneSXMku5hoE Dguk68w/L63ZApYHO/JTgJAYvqPOErAVUegPIw2CT1/2qi5vpClBcKkNS7RXrssA X+lElE0zbzX3bNG+lQuXby7jNUFYltkEiz6vTtc4HuHy8u40DHMswzkoDr0T8IE0 7ZRAWXwV1nlA/dI337cHCsWMJyqem5wZZO13iqe07qaCg1uvBPeqDo81hOCn1us7 l5SYRUTlt7KSFEHZ+Sx4bmVneAuRi5okaQdmrepy/ss/vVpRwWuQxsPkvT8boS7s mqOVsZFcNOuUJPUyOz1dHUL6FMYpk1dw+9n41gO4fLBzJekFTB/fxL6SRbYFWWn7 x0VGHDmuaYQ= =HmVo -----END PGP PUBLIC KEY BLOCK----- myguard-labs-nginx-zstd-module-3486b5b/ci/tools/keys/sb.key000066400000000000000000000045541525232545500236720ustar00rootroot00000000000000-----BEGIN PGP PUBLIC KEY BLOCK----- Version: GnuPG v1.4.11 (FreeBSD) mQENBE5E4vkBCADPkWWzk7W5cXOqeZ1ULNSj8nt5azbYjfQ8OyR2AaDW8J7oazYH reIHKid5uZVJxwr1uLoMloGiYTdy4XYIF2WcOfDnjNGumrAT0Nd4Kdax/pHr5Pdp jFsO4BkHyWk/5/zDCijyoGYLBR6I8hqn+WDuLG/sTtVuTWkUeOlfxb2eZdLyZ3oP 5T5FXtWTpKvr2y7RGshmS6EJnjiVvvErdbNItFXghqvBBaFOJaS2PRBEO9RfKpti i+eS/cmlrm+Tjv44EPfQyLtAmCQ8uqfL50uIKEp6/dsC/OVJ6JlJOYl4j90DX7vB TJaOyUm4s+BLF2BK+Ow8+s+B6jQ5noa/o16NABEBAAG0IFNlcmdleSBCdWRuZXZp dGNoIDxzYkBuZ2lueC5jb20+iQE+BBMBAgAoBQJOROQ6AhsDBQkJZgGABgsJCAcD AgYVCAIJCgsEFgIDAQIeAQIXgAAKCRCmT9Wxets5qEQgB/43Mxmiy7DjXEbxIYkC 9xPC4kf1X+bHkJ9BtAgaYDQewjtQ7vS98TKJBibm3l4egmBjFWjCpL8845n966+u XDqrDWJtOPUXvSEQNXGlijDGSxxpdK2dxDOKIOC8nIlZq/Xz/Uqjb2ZrszmYK2LD IHI1mN9HdI6aTt41QbtG0nkaPPgv3MEvxSMVCzVddroyPXvf/ErT4OSYU+dqJhH+ SBIezuF0suzH/siCksbSBZHIst5rggpjsZvijP5YFH/hpEsR+tKXo9EFk49xn9Ou WdmpOEs7CKDbTApkh9XN/Pk5nJQ/HIDuW8pkgzf2wxNWlMSYw6xnozDkeIqpJcDD 4niqiEYEEBECAAYFAk5OYocACgkQ7PDpCywXIIMKtQCfaAl2rvbEImu6MnDR32KG HTDH2TEAoNeWrSlavyFzbSQka53E9Gs6gF63tCBTZXJnZXkgQnVkbmV2aXRjaCA8 c2JAd2FlbWUubmV0PokBQQQTAQIAKwIbAwUJCWYBgAYLCQgHAwIGFQgCCQoLBBYC AwECHgECF4AFAk5OR38CGQEACgkQpk/VsXrbOagPmAf/QmIEDkkiovc1MgQ81lh4 eeHfvtptb+U4GVCu07DQUR9kEtN6Jqi65gKb95fEztI14PpX+euiWrc/RlnsxWc0 jYF0UmyacWLN6oHPoxlCK5+7zyoz5UTNrYGkTfWfcNtTU509CEZRClBNjMZOTZjP QhdR+Ce6tngRcQvMGNaLjJkKuY7vPh6FjT5oqxpnEIRTsWq6bUaeCXm7j9x0as1Z w1E5D5it3Ug3VlAe58jFJmRgatOsWznKuNoLRjQ2Chp2ce+dLgXriuJMrvEsn5S4 dImUGL5DVYWDVZNG+r85XnOhMfKG308pZby1uzFvD+j3P6yMj1tpaCAAi5lUkHh6 bIhGBBARAgAGBQJOTmJ/AAoJEOzw6QssFyCDH50AoMyJPvPDTYXK5KHOlPYPZQ5M OuCAAJ9zQ/3hKedm3xCLGl4Y6hjxJNlUTbkBDQROROL5AQgAuGIfx9aVOOXVdj8b XvjBQt+UkBURYGACHFQ69w71Aupsg9pZ7FgwgVKxnoNlmRag8sInjQbs3M/lS0sB dg75zZ7Ph7aPev8RAqdtX5+xxvujv1cmkFBExFuC5Wp/Yfzk/lPWZR4vXZrTpRiF PLMlRu0CEJFqoqPPygGFar02Q7rO+da35pxAuYrOWGM7MNr8H/vk13+GiqniBQCa uSoWwZQzaEdG5VGgm/vAwPzO+Cbam3r+Hs7OieykAy8fv+B+qhHn8Vc/520iGvdO IAKpxl6oZrkbNL/wozOOLZni7iWl30C43ujxPiGRlg/YotHmhlnMic85QKyakXCS WXI/JQARAQABiQElBBgBAgAPBQJOROL5AhsMBQkJZgGAAAoJEKZP1bF62zmoGCwH /2a6zlu4Jwmv21vuroaAzECV8gp1luBeagn23EgMMukYhkbwLtL/0twAHmZlkpzl atfq/EH2PgOasl2biJixqp7o9V7Uw6PS5JoY+1IrLEurG+FU2TN/Ysp12al4Z0Hh p4yBRSEikISO9gkeUThixDPX1PjCpx8G/ZYqk+8jRCcDgWsUc/WV3VGPht68oDd7 56/hfQYc/V3eJmm5WYLVGV7Q69tGtp6D09SpoeqCD2K77auEBRVJ4jaT4B2/EfSb x6y7Dy4Oxm8TBOQ2EZw2vEixKxtEt86/oBtLUkqVockPq/Ek9AL+KzT6VR1xU+Cm CoHAyoqJeb/xLBwuKWg0/4U= =iFlP -----END PGP PUBLIC KEY BLOCK----- myguard-labs-nginx-zstd-module-3486b5b/ci/tools/keys/thresh.key000066400000000000000000000222261525232545500245570ustar00rootroot00000000000000-----BEGIN PGP PUBLIC KEY BLOCK----- mQGNBFrwMiUBDADo56OlDknN+ReCMP+8CN1biK5izmGd755TxktHLI9nAP8ociIq Hjrps22pBtAIQ6eZpwCFBys2mR/441rOgZW+O6uqBYrttbxTMvE43EmKYGuFCmuR u0JGMPuqnzF3Y+6uoKzqMzazSrZIBWsBKAkNYTw8+yPlxGgffhBp1ueME7Lskglh EV9gmrEM0QlWod7wSQvyruExPm5INx3MG63Xfvc0bPiWUOGKyMb7kXA5VgnWuzmS BCMm17+A32vMyxhYcvSEgUayQjGghI1uPDSqBQBMEFTgSK2wWzvAXf/M45nxKBgQ IEDmvoC8RM9JTtUr7RE/E1mjsuefF2vYYYsWBstRFGAlUV1/lPNNibu3NqbCug6b 1IWJuV1DX9T9/f81GZJrsPgYYKC6Ai8C1B0NGWjos7/GzgEFENQgf5duOhFPadQz QbRxBoId4Fe/Uwe2HxI8ESCQMwsq8bowcCn6XRA2EYkAt17Kab6LH6tTP54XG9TL bV7bAhyrvZAk1lUAEQEAAbQjS29uc3RhbnRpbiBQYXZsb3YgPGsucGF2bG92QGY1 LmNvbT6JAdcEEwEIAEECGwMFCwkIBwICIgIGFQoJCAsCBBYCAwECHgcCF4AWIQQT yCpjtgNXYVbjCk6g6pgbZrDZZwUCYoTfvAUJEPqvFwAKCRCg6pgbZrDZZxFYDADK R02XgC+AoyrqMwBNXC8Y6aiilEsyppsgj+KwZcGKDYN488gEmff+/KIEdtglw3I3 tCMbo+FzFjHveeVCb0qrIMerWJg+o4YrxxqlQ9Q1InpduKLrIuGae0J1ybITS8+v iYAmwzy1Wb2CDDuCnhCR/QDfOE1CvRILVqIKezC0tRrBTEvRO84m6YMBtJ1DP75Z 2cTNyjPos9+uxi4JcMKrMUBwZKya+z5i+Uxd66wuPj9KmggNG1x+bqMWmpTrSKUn gbLabFUth+uWumpj3/7HBT8Ov7rPgzY/vn3Fn5mKdLQm+kRwSX9/FbtHAE3Qsm+f 6WW8CZ4XzL9ONfhQYwO2Jrq4HzgYloZkL+1Zs61X+zeEyr4o/mzt5DHbQRsD1UzQ gnh7t3YdSAy6gBqevjPWkQlq9e8eoFRydN/htwjS7dleikOsYktSnTIKlRXAWGCm jkRpQyZYuuPcWcGRt/0MVewRJmLemH6O+NviqhgGRePO9QR0R+yfdCwewPJEDk6J AjMEEAEKAB0WIQTWeGzjA9mpAimY3GzIRk1UmvdcCgUCYoeH1wAKCRDIRk1Umvdc Cqa9EAC8Li+w/sRwiu39vNUBogWiAKj3mlfS9lEdmPWx/MSzWtik+IlI931flFWI GL3OWC0ZXVV9G3WXQmVUqMtW2Eachy1DOSwAh4nRn03udfeMG79DUJBvMpAKTSua cVr2tRCFXQcx+6hmkZaANGjalzVu8tEcWfOiT19LS1QM+PH36adQCtRD+wwLgvVq qVowo6yO6jdhCATakRWO9uqeQXvdhJ7n5A3/Hg4QKtbb5vbz6QTPOs1+prICBdfF rVEdLx9BeZGVVoWeJNzbv9ZciC+8YYo/HOTbkccJSJ+G/FeHvshYL9Saxrsl1nUX yNCHBdrUyxPfZMgPWD2k431uplUVCwV5MOaQR4KU8AO3lcKVs02viw4smo0mWa6O pnMIHQ/cWgNxB5/66ch3r7YqosBi8KWHMVBejD+tOv/Y1Ey7v0mF7nBdIclbQz8t 6PlKN8cOggqWjczPo1BtwPxiAkI8Y4VyhOk4ncZnluY1CtM2rQipLfcVFC/z3UGh ZuZ9WIi31ns8Va+msHyIaQx51PB0hSmL+AkDjUuB5APO9zFE2tGV9elbmant6f5c k4F65i19kDcfPe397FjqgyCdIduEDDtoaSS+a6oUgffHgXMXhtP2hI9zQ6c8Bnnd f10HDxakJEcNEz7m8i7VZ0xb+UsOej2rSgdyTIW+an9t8NF9eIkBMwQQAQgAHRYh BHM4lzBp7T9EP00336ZP1bF62zmoBQJii0M3AAoJEKZP1bF62zmoEZYIAIK8SaCJ KT/0NtCyzmFdjX6v+H+EYjEUJCx1QPsHt35Qglco24L/X9hnPJF9P6MY3S3PDLyd 9JsmD+mujgsShqYFME/GzSScYy5Mzm5FM0xXs9UJ51YL+frKknenN5eIr7WVjXnh g0fKn2ZqXlZ/MozHKjKQhhzl9SN6b8eDbi1SFHS/FC7C4Tymnrkhi2KAvpEtUyvg mRSCU5Hrqh6wvi1bCpZ4+vXzQG20CT2cxa1YmgJIDhBqKiWGLyEY2hMCoRKsx5CI UVllc83Hrpk182DDOoVVhxFpStYD/4CNCP46oSeOtjv6EPLIIug25rsjBHPHPfMf p64DcAoKkk6cuFWJAjMEEAEKAB0WIQRB25JxPTv0v/PukQacXn+i9Ul31AUCYoeM ZQAKCRCcXn+i9Ul31EVUD/kB3lxEMDKFg/lFpSBxm1nxplmOCp5Nq9F8Rs9KDsbR Rc4zKL+2PLkgfxh/Nk5+9zjclUjFMBzYS0vEEml7f1R6ceG1a9r7HrdkO581Mvwe x90qVkMMKsShqIcuLzOK0LpvTobBlQpZCBImsNaEVHnmMR3hCz5OmUsGjxNgym87 +ovRJKCZRbbJ36w+COf/jVEkczm+7OrG5BeTTPwWjoIkqs6dajYikfZI79J7FZ2C pWpWeIgJA5emc3sAZWi0KTxlPZ9K4ff3iuV+Xf2PyuRC3iZlOuO66RJ/sl441ebN ckn1Ngu3s48PyMjgD3VG8WDh4RCqBtLpMQJc60wboq9gPMhyyd5eyTYMI90HAEg9 pYGsw6Wk8NpUmBzbSzqSOOdN/SvAXkJmQVGKEzgvDLEsmTeddsjE6U+KUS+8Y69k Dc3sRIR3p5cKoPgZuK2mgbiXvF+TyVGODsyUUCygCGBNN8vsDDw4gpTuOhUm1nMP 3jagHWz2NnMRo00x2nayjffjpMHCKSoNy+UTBKhVLffeZ8df6fCD9SAK+UavPVFW kMKhd+gofhrIbnca9ZL4K+CdyD1d0sxWNtoiDGi9HSnTwXhyGujv2QnNpBxCUZTD nvOEUSNFP/9N+tkAAGiAvk5L5ZuwHRppvnv6t6JEbM7ryRBwWHwgWHConwiFWImN XYkCMwQQAQoAHRYhBC6ZFqS4exJw9J8ez+sX9nTHmkCiBQJii1dOAAoJEOsX9nTH mkCiKu4P/0+je/GsBE69YVAwEFBrrfhEJtVUY8GSYM8WeFoq20SX8SqwltGLFB5R kbZGgPLe0lJrgXzL01GqjU1tnXPbtI7LEq1FKiTkcKVdne140oX1XJuxmFWBcldG 1IetinhJt5EkaYc6nyk9iWgCz9n5YDq9Lr/9jLhFQAgawuicwAfuB13MGbJZYm/Z 5eSdxnivXbrGAYR2TI6/kcf0JLGR03fKbrEM8uBnfZNkKZELyYrBCj4FYODT++Sx pDyrNr2/FlierISJrs272JT7ICg7Knjh6X7BSzsgK7JxyG2UtJKK7qJXYEqMtYhH U1tdh4Ru6zSd4DklgrFHwuUNlTm8f1gPQ4I46p2RCQy2HMnA9WhJ8kwE2JOAj83y 87f9hDwjmn8Pf/iksXGRFQcfDqkOIUf2EnyBvxrzS57Dfvk6WCaH+OLKn1jMyxL8 BekCyk7L7wrMJI4yH51jyJySScGBg1CM0fYqLFWU/I+jw9bHROdCOK2LBajkAYgx /eLG9WtS4etlNmpsxhSOi48wxa6kIOnD2rJGvQMALxhWJlVBEOMumv96qNCQCzHd 6NRLBWBva4qlKM5RlZreeVyArFtTiUmnp6RST4FrMpVgmhoeyos6P6GIG6QVPS2b 4dSRbeKmJFb15kZN8eYP4/BW7DMBzkFwtkRFDV5f/4W6CU6UIGzViQEcBBABCAAG BQJii68XAAoJEFIKmZOhwFL4HY0IAKejouSXBCQWJmpdsA9TV2WVdMspUZHDGRAH epQetm0+eX5Jh62ktuAZG+KCZ0bMdd8FJd6+RRpftUGhDibu9IFfyIK1v8jrChTU /EwK8cPgLn4KveTgC58UrKt4NMpqcETUCrXHVwZzYK/sGZxxKVHhmnQJtfsvg7FV 7Ia9ohiUy1/rz9UlwLPUGmrDnSemSR9w1B3XeNN8SmTHQ5gpZt/rvsII0wMhvS7p TXDpK5YNAqItC+7ZDaU1T21xeZx9OGSt/T2ETXb0rjIJAhKiSShqbiRonZHrxOcg p0vSM1IAsgfnRihHu9YZ3Vj5ntegHh4fWdcTSZUx0n/YggArsyG0JEtvbnN0YW50 aW4gUGF2bG92IDx0aHJlc2hAbmdpbnguY29tPokB1AQTAQgAPgIbAwULCQgHAwUV CgkICwUWAwIBAAIeAQIXgBYhBBPIKmO2A1dhVuMKTqDqmBtmsNlnBQJihN+8BQkQ +q8XAAoJEKDqmBtmsNlncQ0L/0Yk1QejO06gWwV1J2eK9LmjbMofy2ujZBgW1IGt /goo5R4PzC8lBBcsBtsKyN0Rsh7QdLrtKKLQrE/gpwMTMdKhJTdP/c5tUY3EwgId BMYVaxArZQiWlPgSnoKuKydnn6Rb+Qtrhvb9pjn5XlGd/VSbAXZe8YTj6B8qjUa2 YY+IreyB6wkPN/ytV5vcocbS7mzXaibGPVT35e0Pl1Be+xbJkbTmJTSJCSPwyHm9 t2Vuq4e/c3fMwhOUbBjfssspR103vo91XO5sY+v2aQJOctNrv4ZpHMrwBH7MeqDI SCWg9PICUv0ewHzAEGB+K0v342rVAzVNEctwM3Jic7fEJYsItdw+Zk4r8NYqACoR CdSUEHqhP0DbYoWdthpUwD1J5ryWyKTCpTL4wNhKEMcNaiHH3qorSssyMHMFRPoX Kw9Pcay+Uo8NXc2KKxhEHTbQts0jYUNcq0yuWHoNQ4vhKkf9CHBrb/vS22vfEJyd 6FX6ZRYK56A3EFAV8hK0BvZAw4kCMwQQAQoAHRYhBNZ4bOMD2akCKZjcbMhGTVSa 91wKBQJih4fSAAoJEMhGTVSa91wKipoQAI3wkWd8HLQ0w4IFA6W3/igrZTut9sV+ K5Veb61zCbJn6I2aO3ldSClMWpJfvG1OPKyaA6o4QfWt7KV9of8tu68k1rTrKKYe qXe/0KNp9nzEwVmLASG2U6onwaCehGocvhWc9tE6MF2Gi+l+OufqsMzmx7gkdwE+ 4d/VpY/i+eZzqNi1WWNUR45mrItvw84enGW2u4JOaFdSOE2PAbSTUOlcLxfC9yCo lxAkCsy+CsXM8WKlIDH8GpWh/mWyqjoAhZhrlGhdABjygqFAOrDhIaecc8eSOcD3 6MQvhj/y1kh0Fe0rMCSdxUWtSjv+Sw5g1IG6GxhsqFxunxfGDpdbaLnyTQWahDfi 5OsOFl6JbPFiTaF9Xqz+8r0hiwusT4AJvM5M+q18f5dNCeqVKmuAn3BVBw4RdG62 WXt4q6uE5rDI513dR8t84dTgOr9+tHKh5TJqw46aI+kMe36z7FPXBgDsGSkNtM4J BYdZzxSoJCfsGCjlfapkLHrvI+S7AP2952WfYy36uuxBiuTp3vCghvKkXZUeN2kh P++0Zo4OjZGOllhab1X5xZGO8AjWeei4pq66Ys94Veidw5VRi/eWyvB3OhfCq9fb qZIKUfbgTu0y7vOEWWY9wQml12gpxQfkcI72NTiNMCH268WZoXYQJp0+NZtxjsHQ PdhNxQOaJPqziQEzBBABCAAdFiEEcziXMGntP0Q/TTffpk/VsXrbOagFAmKLQzAA CgkQpk/VsXrbOairRggArvsikhDrA1d/x1BXnzOxE2sznq/d84QCKMSQpavrzXHF LQF/qIB+ePA4bmzwvTxQup7yTLK3mQDl0rejXEQMnXHvgfH73c6l6TdAwsoLmrpt oGNzfzJsbiKD2hJT9jJVnipuqqOA7hPT73TA5KM4GzPupFTadB57lDxzzcRfALXi t5Qa6A83tLelQXLOWP6IdyPjraa/kva5jYsMavZU0xWTx9nPeGCwqAnqdEN4Hp8K WKYn9EzkBOL6pPB7GyG/G20ocTCv/ZCJMkamAxjprUovu9BUEg5fCcHrSBtsgGE0 doPfqyOb4tCofZ8aXZYIu3+BEcNO0e5la+eW0YYYPIkCMwQQAQoAHRYhBEHbknE9 O/S/8+6RBpxef6L1SXfUBQJih4xhAAoJEJxef6L1SXfUb8AQAML5vwKOTw6Bn0tA 1ypo6DmlJUWalGgEkFheUC02s+BT+bL/fMsiXd6dBHHl/93bVBQBL/AjVBVv7viQ kfQLLk7iQmEQ/mljvImGkA/W+vyHKDue6n79Ccjfx/ECQB4Y8mmFhOqhDjEC6oR6 ny77QbqmzvjkhfncD26cJq+qRGnE7EwuQI49bR1deQGxr5apqx5XRbf+GPnXlPTc nKxctRsw6PLOjFoyGhBnvC/rEzBUx+wE7jK+bY1TSdW8x91LA/SseWqsmEFzbZRt KKaHE9wD2DB9UvdBAjXdBZvKQ35zSJRWQByODztI9ZcaOWopK3UtIhG/eNIaJGcD 9h3SaeVE8PcUkvZqhLtQf49KlUBc8/g6Nj1wqcBbHDXjbwzt9Qoh6uFyjMkbG3NP BXn7cT8888fJ9Oi53XjjZEVKA88AdcqWpUZtyElNwGtj8IvJ0R9SMKR/7KIYPFWm R04Uok+oj0wQABHkcLmYMUd8psw6aQWG7oybfgPokRChExigLWrCJbYd00banL18 W6RxOQzceiKeZ5sZ5Y+yjQIrKxXKSLl42s8zol05TPScnBn+SAWigG4eEEJhT2by 2WqbhCG9snN9/YMlY8MffOFnD05ps40CSdSCsRgcmaqxgjy75h/z5LYO4HnHwPdY p2ysNzlruScewHvijYJhEKxo17lBiQIzBBABCgAdFiEELpkWpLh7EnD0nx7P6xf2 dMeaQKIFAmKLV00ACgkQ6xf2dMeaQKLLQg//etbDTflbm+HbxI/YyNQhyQfk7icE ytLL+wT9zDW9iq3AMdaPZwT690CsJhr7yzqjk0AGoMyuPfntvcvYb1mPTObXHMzh Rh7+tViPixkJd3hnjSrPBEOkpAghk6xWMx1wldZ9x5XyJ0yC+toBkSaB/KIQeRG2 8/jHtxIQKvPGL28gUjdzW+jopSA4x6gSZAgQLyfsjoUHcMrRJXrwWcmSe8faD8qX XD4z4hN3wQg6olSuaxLM7OoNgbiEjKaL1LaX/xzvC0lGs9o2JBfNFDrng9Y/fZ4o 9aGqx7AZey+4wTKjXqbdEqfDiHfzHxkLBunPxSjJAploOcuvhNOQAY7tv19/mYY1 UoILY9ninCrXthe9ZqhaXxhRhqYhzrE8svF+R01I/U+N4985AnDKRkJ944pZfeh1 wYzEZOPXWvvTsiBLbgi9LuAzoFjA4WJsJBp4AP/U7DtsuhMTmxyBJa+zg8PHj1Ew jBYYuE++ulsilS+76sQawT5KbszpYmEDJiQUuEJkujPQ+hGzuuocoqHrM/IcoAoy i5I/JMAYRqCQfGMFjirmVj3c01jgsOYl7ZgchtCBJfG8V6rlYdTq2FTdaLYdleZC kS7N4jtm+6/KEsf6ukeGNEMbsxTSPHq4RL13eSitRd9Ms+ukSZFFgE0rEiztcdxQ h1PeaEVaxHaSSWiJARwEEAEIAAYFAmKLrxcACgkQUgqZk6HAUvihvAgAk1ETByL3 FZtIlk8scREfwzyqyXuSYWdJ5ED61fKnpcfwGKsOkd+4MwHOSgvxPdnLhBEsMkNq sV82EqX7lTIGoFBLTeW8ZGAxmt/88j3z6mnm33lSTreeVwsQ+B9ZKVAv4E/liDVm 6iq9aYJni4FUoFjFhtgsvJUNs3oX0gaEXdaCqzIDysU2m01vOPx0HTeI95+HdlJW Iwwh/cp+YuclHppI+b0OQKJwLQDVyudzX0JYTWvgE/NCS6/rP8fjaqtFMWwL0tZl 3JJAoLSAuhPyc+V2LkRVoETQGF9nRil2zSyy77Stfm2fRGstnQGOrNTud06el68/ hYfWcCqooHNiMrkBjQRa8DInAQwA2Rk7UdUgpCWl+BMz9B9eKj0XtsNEciXHHKnS FYaSNCWNwib/FsiMfcPFh7xwUTof7e7HBFkvv0QEMCEp7R1MVNBfMiGtG1ICFIt9 nByznPsRk4VvbY/prK4DZy2AmlwhNcT2pQO3AascgsCWdf6G+wcwnHg9tWCp0Xs9 BNXuppmcRrpP4M1PPRIVeG1jeVXvuSHO2HjqPSXP5DhGgSGN7uLOhiLTnPINd186 vf6tqRdqYw3g0W1ImEjGXHeNQfnieIWdU3X4C8KTEPsV3lvtmSAQCoge0CyKfz4c ORi4j8Edp8JpDQlbAThe529+R3eKUw7I/3ESxJBdqzLE/ItWvAcbGEserLDFrg9J 1ojiKhsw3TVcDk+HIDzVakMz6HTd4ExSijMqTehzgKSVHDL+l2jc0f4VSecI+xwC 3/kNsNTBpiPoUYtXBbJllHgQAakREkSKQBas02eqRu8SlQ3yEn87zTtNW8L7xpe7 ZVtxwUgp40PUrsb8uMDJG7ZP5rhLABEBAAGJAbwEGAEIACYCGwwWIQQTyCpjtgNX YVbjCk6g6pgbZrDZZwUCYoTfwQUJEPqvGgAKCRCg6pgbZrDZZ3oEDAC1J3BVwlkX +eoo8VsXAYxMXm8kIaTqOn/tHMOYepK+cWUdHaeCH3N8LigwN4Ve2LtzLBqN3WRA xFNy0DIzdBfA7QdcAoDLnB2FNrWTmwvC9nXkCogFfSCq7c+1oFHdn7M/VZNU4o0n hVOnqM8NLGcgzX3K3hr+WLYUgNQ9G6x0N9VU43tqVwJhvNv4pyiRpRdLlmhOEf35 a/sWE1dttSKdrBhyzTbptw4dXr4lUpvlswWs+dLpSPPhWAuifORv/amWh3bxIxYE qE4o5NI/PQLJvJJLsJvMIIjpKlAGBJg5h3WCiIAkl7H+BesOUIIg8ava5ZUyjlFd szBMaBosZvRgFAlfnYhSGqzhip6PvXfK1YokNv7kqw43c0f1SmtSXZR43SRv/4vp XG7IqtTuqgSwn1qDJgr4yfs8QQykO/jG+cz7X+5OKSAulWi9OoqLyDWlsm3WccPI cJfbm71P+I/ha7ESVQfOxC92fQ7HQAboj7NhecJ4RLqjzrWSHmPGClI= =t1B0 -----END PGP PUBLIC KEY BLOCK----- myguard-labs-nginx-zstd-module-3486b5b/ci/tools/lsan_log_verdict.sh000077500000000000000000000064231525232545500254530ustar00rootroot00000000000000#!/usr/bin/env bash # Copyright (C) 2026 Thijs Eilander # SPDX-License-Identifier: BSD-2-Clause # # Grade verbose ASan/LSan log_path output without treating LLVM's ptrace # preflight heuristic as proof that leak detection failed. Usage: # # ci/tools/lsan_log_verdict.sh '/tmp/lsan-smoke.*' # # The caller must enable verbosity=1 so every process that reaches the LSan # exit hook writes "LeakSanitizer: checking for leaks". This script also runs # the shared deliberate-leak control. It returns 0 only when every log reached # the exit hook, no real detector failure/finding exists, and the control was # detected. Return 1 means a sanitizer finding; return 2 is indeterminate. set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=ci/linter/lib.sh . "$SCRIPT_DIR/../linter/lib.sh" # shellcheck source=ci/tools/lsan_positive_control.sh . "$SCRIPT_DIR/lsan_positive_control.sh" LSAN_FINDING_RE='ERROR: (Address|Leak|UndefinedBehavior)Sanitizer|SUMMARY: (Address|Leak|UndefinedBehavior)Sanitizer|runtime error:' LSAN_FAILURE_RE='Failed suspending threads|Failed spawning a tracer thread|Waiting on the tracer thread failed|LeakSanitizer has encountered a fatal error' LSAN_PREFLIGHT_RE='ptrace appears to be blocked|LeakSanitizer may hang|Child exited with signal' lsan_log_verdict() { local control_rc="$1" shift local file missing=0 if [ "$#" -eq 0 ]; then echo "LSan INDETERMINATE: no verbose sanitizer logs were produced" >&2 return 2 fi if grep -qE "$LSAN_FINDING_RE" "$@" 2>/dev/null then echo "Sanitizer finding:" >&2 grep -nE -m 20 "$LSAN_FINDING_RE" "$@" >&2 || true return 1 fi if grep -qE "$LSAN_FAILURE_RE" "$@" 2>/dev/null then echo "LSan INDETERMINATE: the real stop-the-world operation failed" >&2 grep -nE -m 20 "$LSAN_FAILURE_RE" "$@" >&2 || true return 2 fi for file in "$@"; do if ! grep -q 'LeakSanitizer: checking for leaks' "$file" 2>/dev/null; then echo "LSan INDETERMINATE: exit hook marker missing from $file" >&2 missing=1 fi done [ "$missing" -eq 0 ] || return 2 case "$control_rc" in 0) ;; 1) echo "LSan INDETERMINATE: deliberate leak was not detected" >&2 return 2 ;; *) echo "LSan INDETERMINATE: deliberate-leak control could not run" >&2 return 2 ;; esac if grep -qE "$LSAN_PREFLIGHT_RE" "$@" 2>/dev/null then echo "LSan preflight warning ignored: real exit hooks and deliberate-leak control passed" fi echo "LSan verified: no sanitizer findings across $# process log(s)" } lsan_check_log_glob() { local pattern="$1" control_rc local -a logs=() # The quoted program executes in the child. # shellcheck disable=SC2016 mapfile_checked logs env LC_ALL=C bash -c \ 'compgen -G "$1" || [ $? -eq 1 ]' _ "$pattern" || return $? if lsan_positive_control; then control_rc=0 else control_rc=$? fi lsan_log_verdict "$control_rc" "${logs[@]}" } if [ "${BASH_SOURCE[0]}" = "$0" ]; then if [ "$#" -ne 1 ]; then echo "usage: $0 ''" >&2 exit 2 fi set -e lsan_check_log_glob "$1" fi myguard-labs-nginx-zstd-module-3486b5b/ci/tools/lsan_positive_control.sh000077500000000000000000000116571525232545500265610ustar00rootroot00000000000000#!/usr/bin/env bash # Copyright (C) 2026 Thijs Eilander # SPDX-License-Identifier: BSD-2-Clause # # lsan_positive_control() -- prove LeakSanitizer's exit-time check can # actually detect a DELIBERATE leak in the current environment, before any # caller trusts a "no leak" verdict from it. # # Why this exists: this runner pool intermittently blocks ptrace (LXC # seccomp/yama on builder02), which is what LSan's exit-time tracer needs. # When blocked, LSan emits "ptrace appears to be blocked (is seccomp # enabled?)" and detects NOTHING -- the process just exits, and a caller # that only greps its log for that one signature string is trusting an # implementation detail of the warning message, not proving the detector # ran. Observed directly in build-test.yml run 33084048220 ("Tests # ASAN+UBSAN" job): the "Run smoke tests under ASAN+UBSAN" step printed # "All smoke tests clean under ASAN+UBSAN" (neither a finding nor the # ptrace signature in its log), and SECONDS later in the SAME job the # "zstd_dict_file reload leak check" step (ci/tools/test_reload_leak.sh) # hit the ptrace-blocked signature 4 times in a row. Same job, same # runner, same environment -- so the smoke step's "clean" almost # certainly meant "LSan's exit-time check never ran here either", not "no # leak found". A clean log and a never-ran log are bitwise # indistinguishable without an independent proof the detector fired. # # This function is that proof: it compiles a tiny known-leaking canary # (a bare `malloc` with no free) with the SAME sanitizer the caller is # using, runs it with detect_leaks=1 and a distinct exitcode, and requires # that exact exitcode back. Only then is "clean" from the caller's real # log trustworthy; anything else means the detector could not run here, # which is INDETERMINATE, not a pass. # # Originally inline in ci/tools/test_reload_leak.sh; extracted so the # smoke-tests step in .github/workflows/build-test.yml can share the # exact same proof instead of a second, divergent copy. # # Verdict (return code) -- THREE distinct codes, not two, because "the # control could not even be attempted" (no toolchain here) is a different # fact than "the control ran and LSan failed it" (ptrace is blocked here): # collapsing them would make a caller skip its real leak test on any # runner that simply lacks an ASan-capable `cc`, silently losing coverage # that runner always had before this function existed. # 0 -- positive control PASSED: LSan caught the deliberate leak here. # The caller's own verdict for this run can be trusted. # 1 -- positive control ATTEMPTED and FAILED: the canary compiled and # ran but its deliberate leak was not caught. LSan's exit-time # check is provably not working in this environment right now # (ptrace blocked is the known cause on this runner pool). Any # "clean" reading from the caller's real log is INDETERMINATE, # not a pass -- the caller should skip trusting its own leak # check and report INDETERMINATE instead. # 2 -- control could NOT be attempted at all (no `cc` on PATH, or the # ASan compile itself failed -- e.g. no ASan runtime installed). # This says nothing about whether LSan's exit-time check works; # it means this function has no evidence either way. A caller # MUST NOT treat this as "indeterminate" and skip its real leak # check -- fall back to running that check exactly as if this # function did not exist (its historical behavior). # # Nothing is printed on success; failure paths are silent too -- the # caller decides how loudly to report each code for its own context (a # GitHub ::warning::, a distinct exit code, whatever fits the step). # # Usage (sourced): # if lsan_positive_control; then trust_clean # elif [ $? -eq 2 ]; then run_real_check_unverified # could not attempt # else report_indeterminate; fi # attempted, failed # Usage (direct): ci/tools/lsan_positive_control.sh; echo $? lsan_positive_control() { local work canary_c canary_bin crc command -v cc >/dev/null 2>&1 || return 2 work="$(mktemp -d "${TMPDIR:-/tmp}/lsan-positive-control.XXXXXX")" || return 2 canary_c="$work/canary.c" canary_bin="$work/canary" cat >"$canary_c" <<'EOF' #include int main(void) { malloc(1234); return 0; } EOF if ! cc -fsanitize=address -o "$canary_bin" "$canary_c" 2>/dev/null; then rm -rf "$work" return 2 fi crc=0 ASAN_OPTIONS="detect_leaks=1:exitcode=23" \ timeout 30 "$canary_bin" >/dev/null 2>&1 || crc=$? rm -rf "$work" [ "$crc" -eq 23 ] && return 0 return 1 } # Allow direct execution for ad hoc probing of the current environment: # ci/tools/lsan_positive_control.sh; echo $? # Sourcing (as test_reload_leak.sh and the smoke-tests step do) must not # run this -- BASH_SOURCE differs from $0 only when sourced. if [ "${BASH_SOURCE[0]}" = "$0" ]; then lsan_positive_control exit $? fi myguard-labs-nginx-zstd-module-3486b5b/ci/tools/max-port.sh000077500000000000000000000126561525232545500237110ustar00rootroot00000000000000#!/usr/bin/env bash # Copyright (C) 2026 Thijs Eilander # SPDX-License-Identifier: BSD-2-Clause # # ci/tools/max-port.sh -- prove a job's test port band is safe to bind before # anything tries to bind it. # # ci/tools/max-port.sh [base] [width] # base : first port of the band (default $TEST_BASE_PORT, else 18880) # width : ports in the band (default $TEST_PORT_WIDTH, else 64) # # Prints the highest port in the band on stdout. Exits non-zero, with the # reason, if the band is unusable. # # WHY THIS EXISTS. Two ways a test port band goes wrong, both of which present # as an unreproducible flake rather than as a configuration error: # # 1. THE BAND REACHES THE EPHEMERAL RANGE. Above # /proc/sys/net/ipv4/ip_local_port_range's floor (32768 by default) the # kernel hands out ports for OUTBOUND connections. A test that binds there # races every other process on the box: the kernel wins at random, the # listener fails with "Address already in use", and the run dies before a # single assertion executes. The victim is arbitrary and a rerun "fixes" # it, which is exactly how this masquerades as a fixture flake -- it cost a # sibling repo weeks. # 2. THE BAND IS ALREADY OCCUPIED. On a persistent self-hosted runner a # previous run's server can outlive its job. Binding on top of it does not # fail loudly; the old process keeps answering and the new suite tests the # OLD BINARY. That is worse than a crash -- see # [[feedback-stale-redis-squats-test-port-fakes-outage]] for the same shape # in another repo. # # Both checks are cheap and run before the fixture starts, so the failure names # the cause instead of leaving a timeout to be interpreted. # # Usage in a workflow, once per runtime-bearing job: # - run: ci/tools/max-port.sh "$TEST_BASE_PORT" # ci/linter/lint-ci-ports.sh fails the build if a job starts the runtime suite # without declaring TEST_BASE_PORT at all. # # Extend: a new fixture that binds outside the band is the thing to fix, not # this script -- the band is the contract. set -euo pipefail BASE="${1:-${TEST_BASE_PORT:-18880}}" WIDTH="${2:-${TEST_PORT_WIDTH:-64}}" case "$BASE" in ''|*[!0-9]*) echo "ERROR: base port must be numeric, got '$BASE'" >&2; exit 2 ;; esac case "$WIDTH" in ''|*[!0-9]*) echo "ERROR: width must be numeric, got '$WIDTH'" >&2; exit 2 ;; esac # Force base 10 before ANY arithmetic. `$(( ))` reads a leading zero as octal, # so TEST_BASE_PORT=040000 evaluated to 16384 -- a band this script would have # cleared as safely below the ephemeral floor, while the python driver parses # --port with int() and binds 40000, inside it. The two disagreeing is worse # than either being wrong: the check passes and the bind races the kernel. BASE=$((10#$BASE)) WIDTH=$((10#$WIDTH)) # Port 0 is not a port: bind(0) makes the kernel pick an ephemeral one, so a # band based at 0 would sail through this preflight while reserving nothing. if [ "$BASE" -lt 1 ]; then echo "ERROR: base port must be at least 1, got $BASE" >&2 exit 2 fi if [ "$WIDTH" -lt 1 ]; then echo "ERROR: width must be at least 1, got $WIDTH" >&2 exit 2 fi MAX=$((BASE + WIDTH - 1)) # The floor is READ, never assumed: this box and a GitHub-hosted runner do not # have to agree, and a hardcoded 32768 would pass here and fail there. EPHEMERAL_FLOOR=32768 RANGE_FILE=/proc/sys/net/ipv4/ip_local_port_range if [ -r "$RANGE_FILE" ]; then EPHEMERAL_FLOOR="$(awk '{print $1}' "$RANGE_FILE")" fi if [ "$MAX" -ge "$EPHEMERAL_FLOOR" ]; then echo "ERROR: port band $BASE..$MAX reaches the kernel ephemeral range" >&2 echo " (floor $EPHEMERAL_FLOOR from $RANGE_FILE)." >&2 echo " The kernel allocates outbound ports there, so a listener in" >&2 echo " this band races it and fails at random. Move the band down." >&2 exit 1 fi if [ "$MAX" -gt 65535 ]; then echo "ERROR: port band $BASE..$MAX runs past 65535" >&2 exit 1 fi # Occupancy check. ss is in iproute2, present on every target runner; if it is # somehow absent, say so rather than silently skipping -- a check that # disappears when its tool is missing reports green while checking nothing. if ! command -v ss >/dev/null 2>&1; then echo "ERROR: ss(8) not found; cannot verify the band is free." >&2 echo " Install iproute2 rather than skipping this check." >&2 exit 2 fi # ss runs on its own first. Under `set -euo pipefail` a non-zero ss inside the # pipeline below would fail the command substitution and kill the script with no # message at all -- the unexplained exit this file exists to replace with a # named cause. if ! listeners="$(ss -Hltn 2>&1)"; then echo "ERROR: ss(8) failed, so the band could not be checked:" >&2 printf ' %s\n' "$listeners" >&2 echo " Not treating an unverifiable band as free." >&2 exit 2 fi busy="$(printf '%s\n' "$listeners" \ | awk -v lo="$BASE" -v hi="$MAX" ' { # Local address is field 4, "addr:port" -- take the last colon so # an IPv6 address does not confuse the split. n = split($4, a, ":") p = a[n] + 0 if (p >= lo && p <= hi) print p }' | sort -un | paste -sd, -)" if [ -n "$busy" ]; then echo "ERROR: ports already listening inside band $BASE..$MAX: $busy" >&2 echo " A leftover server from a previous run would keep answering," >&2 echo " so this suite would test the OLD binary and pass." >&2 exit 1 fi echo "$MAX" myguard-labs-nginx-zstd-module-3486b5b/ci/tools/nginx-releases.sh000077500000000000000000000147261525232545500250660ustar00rootroot00000000000000#!/usr/bin/env bash # # The newest nginx stable and mainline releases, from the GitHub releases # feed -- the one place this repo answers "which nginx is current": # # ci/tools/nginx-releases.sh stable -> 1.30.4 # ci/tools/nginx-releases.sh mainline -> 1.31.5 # ci/tools/nginx-releases.sh both -> 1.30.4 1.31.5 # # The CI workflows resolve the mainline they build with it (a hardcoded # version 404s once nginx.org drops the tarball), ci-build.sh does the same # for a versionless nginx build, and ci/tools/bump-versions.sh sources this # file for the functions below (gh_api_json serves its other feeds too). # A line the feed does not show is a refusal, never a guess. # # GH_TOKEN, when set, authenticates the API calls: the shared runner egress # IP routinely exhausts the unauthenticated allowance. In a workflow, pass # `GH_TOKEN: ${{ github.token }}` on the step. Needs curl and python3. # GET an api.github.com path. The runners share an egress IP, so the # unauthenticated API allowance (60/hr) is routinely exhausted and the call # 403s; GH_TOKEN, when present, buys the far larger authenticated quota. The # header goes in through stdin so the token never appears in curl's process # arguments on a shared runner. gh_api_json() { # path local json if [ -n "${GH_TOKEN:-}" ]; then json="$(printf 'header = "Authorization: Bearer %s"\n' "$GH_TOKEN" \ | curl --config - -fsSL "https://api.github.com/$1")" || json="" else json="$(curl -fsSL "https://api.github.com/$1")" || json="" fi if [ -z "$json" ]; then echo "error: could not query https://api.github.com/$1 (rate limit? set GH_TOKEN)" >&2 return 1 fi printf '%s\n' "$json" } # nginx publishes every release on GitHub as a Release (not just a tag): # `release-X.Y.Z`, non-draft, carrying the same signed tarball nginx.org # serves (byte-identical, same .asc, verified by ci/tools/keys). That is a # structured feed; nginx.org/en/download.html is HTML whose section # headings a parser has to know by name, and a renamed heading is a # refusal at best. The stable and mainline lines follow nginx's own # versioning rule -- even minor is stable, odd minor is mainline -- so # each line's newest release is the newest of its parity. Tarballs and # signatures still come from nginx.org (the pinned URLs do not change); # only "what is newest" moves here. # # The feed is paged and ordered by creation time, not by version: a # security release of an older line (1.28.x) can be created months after # the current stable's last release, so the first even-minor entry seen # is not necessarily the newest stable, and a long run of mainline and # legacy releases can push the real one to a later page. Every page is # read until a short page ends the list, and the newest of each line is # the highest version across all of them. The cap bounds a broken feed, # not a real one (the feed holds every release since nginx moved to # GitHub, 29 as of 2026-09 at roughly fifteen a year, so five pages of # thirty is about a decade away); reaching it with a full page is a # refusal, never a pick from what was read. NGINX_FEED_PAGE=30 NGINX_FEED_PAGES=5 NGINX_LINE_PY=$(cat <<'PYEOF' import json, re, sys # argv: current best even, current best odd ("-" when none yet). # stdout: " ", "-" when none. best = {"even": sys.argv[1], "odd": sys.argv[2]} try: releases = json.load(sys.stdin) except ValueError: sys.exit("FATAL: the nginx release feed is not JSON -- refusing to guess") if not isinstance(releases, list): sys.exit("FATAL: the nginx release feed is not a release list -- refusing to guess") def parse(v): m = re.fullmatch(r"(\d+)\.(\d+)\.(\d+)", v) return tuple(int(x) for x in m.groups()) if m else None for r in releases: if (not isinstance(r, dict) or r.get("draft") is not False or r.get("prerelease") is not False): continue m = re.fullmatch(r"release-(\d+\.\d+\.\d+)", str(r.get("tag_name", ""))) if not m: continue v = parse(m.group(1)) line = "even" if v[1] % 2 == 0 else "odd" cur = parse(best[line]) if best[line] != "-" else None if cur is None or v > cur: best[line] = "%d.%d.%d" % v print(best["even"], best["odd"], len(releases)) PYEOF ) # Prints " " with "-" for a line the feed did not show, # which the blank-version check below reports by name. nginx_lines() { local page even="-" odd="-" json count for ((page = 1; page <= NGINX_FEED_PAGES; page++)); do json="$(gh_api_json "repos/nginx/nginx/releases?per_page=${NGINX_FEED_PAGE}&page=${page}")" \ || return 1 read -r even odd count < <(printf '%s' "$json" | python3 -c "$NGINX_LINE_PY" "$even" "$odd") if [ "$count" -lt "$NGINX_FEED_PAGE" ]; then break # a short page is the end of the list fi done # The cap is not an end-of-list marker: a full last page means pages # beyond it were never read and may hold a newer release of either # line, so the answer is unknown, not "the best so far". if [ "$count" -ge "$NGINX_FEED_PAGE" ]; then echo "FATAL: the nginx release feed did not end within ${NGINX_FEED_PAGES} pages" \ "of ${NGINX_FEED_PAGE} -- refusing to pick from an incomplete read" >&2 return 1 fi printf '%s %s\n' "$even" "$odd" } # nginx_release stable|mainline|both -- prints the version(s); a line the # feed did not show is fatal by name. nginx_release() { local stable mainline # The selector is checked before the feed is asked, so a bad name is # usage (status 2) whether or not the API is reachable. case "$1" in stable|mainline|both) ;; *) echo "usage: nginx-releases.sh stable|mainline|both" >&2 return 2 ;; esac read -r stable mainline < <(nginx_lines) || return 1 if [ "$1" != mainline ] && [ "$stable" = "-" ]; then echo "FATAL: the nginx release feed shows no stable (even-minor) release" >&2 return 1 fi if [ "$1" != stable ] && [ "$mainline" = "-" ]; then echo "FATAL: the nginx release feed shows no mainline (odd-minor) release" >&2 return 1 fi case "$1" in stable) echo "$stable" ;; mainline) echo "$mainline" ;; both) echo "$stable $mainline" ;; esac } # Executed rather than sourced: resolve and print. if [ "${BASH_SOURCE[0]}" = "$0" ]; then set -euo pipefail shopt -s inherit_errexit nginx_release "${1:-mainline}" fi myguard-labs-nginx-zstd-module-3486b5b/ci/tools/patches/000077500000000000000000000000001525232545500232205ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/tools/patches/headers-more-nginx-module/000077500000000000000000000000001525232545500301775ustar00rootroot000000000000000001-msvc-precompiled-header-include-order.patch000066400000000000000000000051571525232545500407600ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/tools/patches/headers-more-nginx-moduleFix MSVC compilation of headers-more-nginx-module. nginx's win32 build compiles with a precompiled header (-Yungx_config.h): cl discards everything in a translation unit before the #include line. headers-more includes ngx_config.h first inside ddebug.h, so each .c file's DDEBUG define (and everything else preceding the include chain) was silently thrown away, breaking the build. Move ngx_config.h after the DDEBUG define in each .c file and last among the nginx includes in ddebug.h. Based on the technique from https://github.com/ewwink/headers-more-nginx-module-for-windows (which reorders the includes inside the .c files; this variant adds ngx_config.h before ddebug.h and drops the stray later include — a cleaner equivalent). headers-more-nginx-module is BSD-2-Clause. git apply ignores everything above the first "diff --git" line, so this prose travels with the patch. diff --git a/src/ddebug.h b/src/ddebug.h index 13879af..6be37a7 100644 --- a/src/ddebug.h +++ b/src/ddebug.h @@ -2,10 +2,10 @@ #define DDEBUG_H -#include #include #include #include +#include #if defined(DDEBUG) && (DDEBUG) diff --git a/src/ngx_http_headers_more_filter_module.c b/src/ngx_http_headers_more_filter_module.c index 0bb6fec..26556b7 100644 --- a/src/ngx_http_headers_more_filter_module.c +++ b/src/ngx_http_headers_more_filter_module.c @@ -7,6 +7,7 @@ #ifndef DDEBUG #define DDEBUG 0 #endif +#include #include "ddebug.h" @@ -14,7 +15,6 @@ #include "ngx_http_headers_more_headers_out.h" #include "ngx_http_headers_more_headers_in.h" #include "ngx_http_headers_more_util.h" -#include /* config handlers */ diff --git a/src/ngx_http_headers_more_headers_in.c b/src/ngx_http_headers_more_headers_in.c index 983be5b..27d4f4b 100644 --- a/src/ngx_http_headers_more_headers_in.c +++ b/src/ngx_http_headers_more_headers_in.c @@ -7,6 +7,7 @@ #ifndef DDEBUG #define DDEBUG 0 #endif +#include #include "ddebug.h" diff --git a/src/ngx_http_headers_more_headers_out.c b/src/ngx_http_headers_more_headers_out.c index db412a2..f6511e0 100644 --- a/src/ngx_http_headers_more_headers_out.c +++ b/src/ngx_http_headers_more_headers_out.c @@ -7,6 +7,7 @@ #ifndef DDEBUG #define DDEBUG 0 #endif +#include #include "ddebug.h" diff --git a/src/ngx_http_headers_more_util.c b/src/ngx_http_headers_more_util.c index e1f3636..a309e79 100644 --- a/src/ngx_http_headers_more_util.c +++ b/src/ngx_http_headers_more_util.c @@ -7,6 +7,7 @@ #ifndef DDEBUG #define DDEBUG 0 #endif +#include #include "ddebug.h" myguard-labs-nginx-zstd-module-3486b5b/ci/tools/perf_stat_recipe.py000077500000000000000000000751361525232545500255000ustar00rootroot00000000000000#!/usr/bin/env python3 """CPU cache counter-delta recipe for the data-layout repack optimization row. HYBRID CPU PINNING STRATEGY ----------------------------- This host is a hybrid Intel 12th-gen (P-core + E-core) CPU: perf stat splits every event across two PMUs (cpu_core/* and cpu_atom/*), and the non-pinned core type returns with 0.00% enable time. Naive perf stat then yields half-empty, non-comparable numbers. Solution: pin the workload to ONE core type and select explicit PMU-qualified event names. Hybrid CPUs declare their cores via /sys/devices/cpu_core/cpus and /sys/devices/cpu_atom/cpus; this script detects both and pins to P-cores (the first range) by default. Pass --core-type=E to pin to E-cores instead. Performance core (P) CPUs 0-15, Efficiency core (E) CPUs 16-31 on this host. PURPOSE ------- Measure the cache-miss and instruction-count overhead of dictionary negotiation on the request-hot path. The optimization row proposes splitting a 104-byte struct-of-arrays lookup into two arrays (a 40-byte index + pointer indirection) to reduce cache misses on negotiation. This recipe establishes: 1. A baseline of cache-misses, cache-references, instructions, and cycles for the current code across 1, 4, and 16 configured dictionaries, with a hit and full-scan miss at each size 2. An A/B delta normalized by successful measured requests. perf attaches only to the pinned nginx worker after preflight and warmup, so harness, backend, wrk, startup, fixture, and cleanup work cannot contaminate the counters The repack row is retained ONLY if the counter delta supports the "third-cacheline" claim (fewer cache misses). Struct size reduction alone (pahole) does not retain the row -- smaller struct != fewer misses. USAGE ----- # Baseline: measure current code python3 ci/tools/perf_stat_recipe.py --nginx /path/to/nginx --json baseline.json # After optimization: measure again python3 ci/tools/perf_stat_recipe.py --nginx /path/to/nginx --json optimized.json # Compare python3 ci/tools/perf_stat_recipe.py --compare baseline.json optimized.json # Preserve the original plain-zstd single measurement when needed python3 ci/tools/perf_stat_recipe.py --nginx /path/to/nginx \ --workload zstd --json zstd.json Environment: requires - perf (installed at /usr/bin/perf) - /proc/sys/kernel/perf_event_paranoid <= 1 (this host has 1) - ab_bench.py in the same ci/tools directory - A working nginx binary - wrk load-testing tool Exit status ----------- 0 on success (measurements recorded or comparison printed) 1 on harness error (missing tools, bad args, workload failed) """ from __future__ import annotations import argparse import dataclasses import json import os import pathlib import re import signal import subprocess import sys import tempfile import time TOOLS_DIR = pathlib.Path(__file__).resolve().parent DCZ_DICTIONARY_COUNTS = (1, 4, 16) DCZ_CASES = ("hit", "miss") def cpu_identity() -> str: """Return stable CPU/PMU identity fields for cross-run compatibility.""" wanted = ("vendor_id", "cpu family", "model", "stepping", "model name") fields: dict[str, str] = {} for line in pathlib.Path("/proc/cpuinfo").read_text(encoding="utf-8").splitlines(): if not line.strip(): break key, separator, value = line.partition(":") if separator and key.strip() in wanted: fields[key.strip()] = value.strip() if not all(field in fields for field in wanted): raise RuntimeError("could not determine CPU identity from /proc/cpuinfo") return "; ".join(f"{field}={fields[field]}" for field in wanted) @dataclasses.dataclass class PerfCounters: """CPU cache and execution counters from perf stat.""" cache_misses: int cache_references: int instructions: int cycles: int def miss_rate(self) -> float: """Cache miss rate (misses / references).""" if self.cache_references == 0: return 0.0 return self.cache_misses / self.cache_references def ipc(self) -> float: """Instructions per cycle.""" if self.cycles == 0: return 0.0 return self.instructions / self.cycles def to_dict(self) -> dict: """Serialize to JSON.""" return { "cache_misses": self.cache_misses, "cache_references": self.cache_references, "instructions": self.instructions, "cycles": self.cycles, "miss_rate": self.miss_rate(), "ipc": self.ipc(), } @staticmethod def from_dict(d: dict) -> PerfCounters: """Deserialize from JSON.""" return PerfCounters( cache_misses=d["cache_misses"], cache_references=d["cache_references"], instructions=d["instructions"], cycles=d["cycles"], ) def normalize_counters( counters: PerfCounters, successful_requests: int ) -> dict[str, float]: """Return raw hardware counts per successful measured request.""" if successful_requests <= 0: raise RuntimeError("cannot normalize perf counters without successful requests") return { "cache_misses": counters.cache_misses / successful_requests, "cache_references": counters.cache_references / successful_requests, "instructions": counters.instructions / successful_requests, "cycles": counters.cycles / successful_requests, } def successful_requests_from_bench(result: dict) -> int: """Read successful requests from the measured release-pass window.""" meta = result.get("meta", {}) meta_total = meta.get("successful_requests") if isinstance(meta_total, int): return meta_total total = 0 for arm in result.get("release", {}).values(): for concurrency in arm.values(): for samples in concurrency.values(): for sample in samples: total += max(sample["requests"] - sample["errors"], 0) return total def dcz_scenarios() -> tuple[tuple[int, str], ...]: """Fixed lookup-depth matrix used by the repack measurement.""" return tuple( (dictionary_count, case) for dictionary_count in DCZ_DICTIONARY_COUNTS for case in DCZ_CASES ) def build_dcz_bench_command( nginx_path: pathlib.Path, dictionary_count: int, case: str, result_path: pathlib.Path, perf_path: pathlib.Path, perf_events: str, ) -> list[str]: """Build one deterministic dcz hit/miss ab_bench invocation.""" return [ "python3", str(TOOLS_DIR / "ab_bench.py"), "--arm-a", f"{nginx_path}:baseline", "--rounds", "1", "--workers", "1", "--duration", "5s", "--workload", "dcz", "--dcz-dictionaries", str(dictionary_count), "--dcz-case", case, "--json", str(result_path), "--perf-stat-output", str(perf_path), "--perf-events", perf_events, ] def detect_hybrid_cores() -> tuple[str, str, int]: """Detect P-core and E-core CPU ranges on hybrid x86 CPU. Returns ------- (pcore_range: str, ecore_range: str, pcore_id: int) pcore_range: space-separated CPU ids (e.g. "0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15") ecore_range: space-separated CPU ids (e.g. "16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31") pcore_id: first P-core id (0 on this host) Raises ------ RuntimeError if hybrid CPU detection fails """ try: with open("/sys/devices/cpu_core/cpus", encoding="utf-8") as f: pcore_str = f.read().strip() with open("/sys/devices/cpu_atom/cpus", encoding="utf-8") as f: ecore_str = f.read().strip() except FileNotFoundError as exc: raise RuntimeError( f"Hybrid CPU detection failed: {exc}. " "This recipe requires /sys/devices/cpu_core/cpus and " "/sys/devices/cpu_atom/cpus. Non-hybrid CPUs are not supported." ) from exc # Parse ranges like "0-15" into space-separated "0 1 2 3 ..." def expand_range(range_str: str) -> str: """Convert '0-15' to '0 1 2 3 ... 15'.""" parts: list[str] = [] for part in range_str.split(","): part = part.strip() if "-" in part: start, end = part.split("-") parts.extend(str(i) for i in range(int(start), int(end) + 1)) else: parts.append(part) return " ".join(parts) pcore_ids = expand_range(pcore_str) ecore_ids = expand_range(ecore_str) pcore_id = int(pcore_str.split("-")[0].split(",")[0]) return pcore_ids, ecore_ids, pcore_id def parse_perf_output( perf_output: str, core_type: str, ) -> dict[str, int]: """Parse perf stat output for cache and execution counters. Parameters ---------- perf_output : str perf stat stderr output core_type : str P or E, for error messaging Returns ------- dict[str, int] Counters: cache_misses, cache_references, instructions, cycles Raises ------ RuntimeError if output is invalid or """ counters = {} lines = perf_output.split("\n") for line in lines: line = line.strip() if not line or line.startswith("#") or "time elapsed" in line: continue # Extract number and event name match = re.match( r"^([\d,]+|)\s+(cpu_\w+/[\w\-]+/)", line, ) if not match: continue count_str, event = match.groups() # Skip entries if count_str == "": raise RuntimeError( f"perf stat returned for {event} on {core_type}" f"-cores. Workload pinned to wrong core type. " f"Output:\n{perf_output}" ) # Convert "7,614" to 7614 count = int(count_str.replace(",", "")) # Map event name to counter key if "cache-misses" in event: counters["cache_misses"] = count elif "cache-references" in event: counters["cache_references"] = count elif "instructions" in event: counters["instructions"] = count elif "cycles" in event: counters["cycles"] = count # Validate we got all four counters required = {"cache_misses", "cache_references", "instructions", "cycles"} if not required.issubset(counters.keys()): missing = required - counters.keys() raise RuntimeError( f"perf stat output missing counters: {missing}.\nOutput:\n{perf_output}" ) return counters def run_perf_stat( command: list[str], cpu_mask: str, core_type: str, ) -> PerfCounters: """Run perf stat on a command pinned to one core type. Parameters ---------- command : list[str] Command to run cpu_mask : str Space-separated CPU ids core_type : str P or E, for log messaging Returns ------- PerfCounters Parsed cache and execution counters Raises ------ RuntimeError if perf stat fails or returns no counts """ # Use taskset to pin to the CPU mask full_cmd = ["taskset", "-c", cpu_mask.replace(" ", ",")] + command # Run perf stat with explicit cpu_core/* or cpu_atom/* PMU events. pmu = "cpu_core" if core_type == "P" else "cpu_atom" perf_events = [ f"{pmu}/cache-misses/", f"{pmu}/cache-references/", f"{pmu}/instructions/", f"{pmu}/cycles/", ] perf_cmd = ["/usr/bin/perf", "stat", "-e", ",".join(perf_events)] + full_cmd print(f"[{core_type}-core] Running: {' '.join(perf_cmd)}", file=sys.stderr) try: result = subprocess.run( perf_cmd, capture_output=True, text=True, timeout=600, check=False, ) except subprocess.TimeoutExpired as exc: raise RuntimeError( f"perf stat timed out after 600s. Command: {' '.join(perf_cmd)}" ) from exc except FileNotFoundError as exc: raise RuntimeError( "/usr/bin/perf not found. Install linux-tools or perf package." ) from exc if result.returncode != 0: raise RuntimeError( f"perf stat failed with exit code {result.returncode}.\n" f"stderr: {result.stderr}\n" f"stdout: {result.stdout}" ) counters_dict = parse_perf_output(result.stderr, core_type) print( f"[{core_type}-core] counters: cache_misses={counters_dict['cache_misses']}, " f"cache_references={counters_dict['cache_references']}, " f"instructions={counters_dict['instructions']}, " f"cycles={counters_dict['cycles']}" ) return PerfCounters(**counters_dict) def qualified_perf_events(core_type: str) -> str: """Return the four PMU-qualified events for one hybrid core type.""" pmu = "cpu_core" if core_type == "P" else "cpu_atom" return ",".join( f"{pmu}/{event}/" for event in ("cache-misses", "cache-references", "instructions", "cycles") ) def kill_process_group(process: subprocess.Popen) -> None: """Kill and reap a detached benchmark process group.""" try: os.killpg(process.pid, signal.SIGKILL) except ProcessLookupError: pass process.wait(timeout=10) def wait_process_without_reaping(process: subprocess.Popen, timeout: float) -> bool: """Wait for exit while retaining the leader PID for group cleanup.""" deadline = time.monotonic() + timeout while True: status = os.waitid( os.P_PID, process.pid, os.WEXITED | os.WNOHANG | os.WNOWAIT, ) if status is not None: return status.si_code == os.CLD_EXITED and status.si_status == 0 if time.monotonic() >= deadline: raise subprocess.TimeoutExpired(process.args, timeout) time.sleep(0.05) def run_pinned_bench(command: list[str], cpu_mask: str) -> None: """Run a benchmark pinned to one core set; perf attaches inside it.""" full_command = ["taskset", "-c", cpu_mask.replace(" ", ","), *command] with ( tempfile.TemporaryFile(mode="w+", encoding="utf-8") as stdout_file, tempfile.TemporaryFile(mode="w+", encoding="utf-8") as stderr_file, ): process = subprocess.Popen( full_command, stdout=stdout_file, stderr=stderr_file, text=True, start_new_session=True, ) try: succeeded = wait_process_without_reaping(process, 600) except subprocess.TimeoutExpired as exc: kill_process_group(process) raise RuntimeError( "benchmark timed out after 600s; process group killed" ) from exc except BaseException: kill_process_group(process) raise if not succeeded: kill_process_group(process) stdout_file.seek(0) stderr_file.seek(0) raise RuntimeError( f"benchmark failed with exit code {process.returncode}.\n" f"stderr: {stderr_file.read()}\nstdout: {stdout_file.read()}" ) process.wait(timeout=10) def measure_baseline( nginx_path: str | pathlib.Path, output_json: str | None = None, core_type: str = "P", ) -> PerfCounters: """Measure cache counters for ab_bench.py release pass. Parameters ---------- nginx_path : str or Path Path to nginx binary output_json : str | None If provided, save result to this JSON file core_type : str 'P' (P-cores, default) or 'E' (E-cores) Returns ------- PerfCounters Measured counters """ nginx_path_obj = pathlib.Path(nginx_path).resolve() if not nginx_path_obj.exists(): raise RuntimeError(f"nginx binary not found: {nginx_path_obj}") # Run ab_bench.py release pass under perf stat # Use 1 round and 5s duration for a quick baseline (faster than 3x10s default) bench_cmd = [ "python3", str(TOOLS_DIR / "ab_bench.py"), "--arm-a", f"{nginx_path_obj}:baseline", "--rounds", "1", "--duration", "5s", # Short measurement window ] # Detect hybrid CPU ranges pcore_ids, ecore_ids, _ = detect_hybrid_cores() cpu_mask = pcore_ids if core_type == "P" else ecore_ids # Run the benchmark under perf stat counters = run_perf_stat(bench_cmd, cpu_mask, core_type) # Save to JSON if requested if output_json: result = { "core_type": core_type, "cpu_mask": cpu_mask, "command": " ".join(bench_cmd), "counters": counters.to_dict(), } with open(output_json, "w", encoding="utf-8") as f: json.dump(result, f, indent=2) print(f"Saved to {output_json}") return counters def measure_dcz_matrix( nginx_path: str | pathlib.Path, output_json: str | None = None, core_type: str = "P", ) -> dict[str, dict]: """Measure the fixed 1/4/16-dictionary hit/miss matrix.""" nginx_path_obj = pathlib.Path(nginx_path).resolve() if not nginx_path_obj.exists(): raise RuntimeError(f"nginx binary not found: {nginx_path_obj}") pcore_ids, ecore_ids, _ = detect_hybrid_cores() cpu_mask = pcore_ids if core_type == "P" else ecore_ids identity = cpu_identity() scenarios: dict[str, dict] = {} with tempfile.TemporaryDirectory(prefix="zstd-perf-stat-") as temp_dir: temp_root = pathlib.Path(temp_dir) for dictionary_count, case in dcz_scenarios(): name = f"dicts-{dictionary_count}-{case}" bench_json = temp_root / f"{name}.json" perf_output = temp_root / f"{name}.perf" command = build_dcz_bench_command( nginx_path_obj, dictionary_count, case, bench_json, perf_output, qualified_perf_events(core_type), ) run_pinned_bench(command, cpu_mask) try: bench_result = json.loads(bench_json.read_text(encoding="utf-8")) except (FileNotFoundError, json.JSONDecodeError) as exc: raise RuntimeError( f"ab_bench produced no usable JSON for {name}: {exc}" ) from exc try: counters = PerfCounters( **parse_perf_output( perf_output.read_text(encoding="utf-8"), core_type ) ) except FileNotFoundError as exc: raise RuntimeError(f"perf produced no output for {name}") from exc if bench_result.get("meta", {}).get("perf_scope") != ( "nginx-workers-measured-release" ): raise RuntimeError( f"ab_bench did not prove worker-only perf for {name}" ) successful_requests = successful_requests_from_bench(bench_result) per_request = normalize_counters(counters, successful_requests) scenarios[name] = { "dictionary_count": dictionary_count, "case": case, "command": command, "measured_successful_requests": successful_requests, "engagement": bench_result.get("meta", {}).get("engagement"), "counters": counters.to_dict(), "per_successful_request": per_request, } result = { "schema_version": 2, "workload": "dcz", "core_type": core_type, "cpu_mask": cpu_mask, "cpu_identity": identity, "normalization": "measured_successful_requests", "scenarios": scenarios, } if output_json: pathlib.Path(output_json).write_text( json.dumps(result, indent=2), encoding="utf-8" ) print(f"Saved to {output_json}") return scenarios def print_counters( label: str, counters: PerfCounters, ) -> None: """Print a set of counter values.""" print(f"\n{label}:") print(f" cache_misses: {counters.cache_misses:>12,}") print(f" cache_references: {counters.cache_references:>12,}") print(f" instructions: {counters.instructions:>12,}") print(f" cycles: {counters.cycles:>12,}") print(f" miss_rate: {counters.miss_rate():>12.2%}") print(f" IPC: {counters.ipc():>12.2f}") @dataclasses.dataclass class CounterDelta: """Delta between two measurements.""" cache_misses: int cache_references: int instructions: int cycles: int miss_rate: float ipc: float @staticmethod def compute(baseline: PerfCounters, optimized: PerfCounters) -> CounterDelta: """Compute delta between baseline and optimized.""" return CounterDelta( cache_misses=optimized.cache_misses - baseline.cache_misses, cache_references=optimized.cache_references - baseline.cache_references, instructions=optimized.instructions - baseline.instructions, cycles=optimized.cycles - baseline.cycles, miss_rate=optimized.miss_rate() - baseline.miss_rate(), ipc=optimized.ipc() - baseline.ipc(), ) def _percent_delta(baseline: float, optimized: float) -> float: if baseline == 0: raise RuntimeError("cannot compare against a zero normalized counter") return (optimized - baseline) / baseline def compare_dcz_results(baseline_result: dict, optimized_result: dict) -> None: """Compare dcz matrix cells using counters per successful request.""" identity_fields = ( "schema_version", "workload", "core_type", "cpu_mask", "cpu_identity", "normalization", ) expected_identity = { "schema_version": 2, "workload": "dcz", "normalization": "measured_successful_requests", } for field, value in expected_identity.items(): if baseline_result.get(field) != value or optimized_result.get(field) != value: raise RuntimeError(f"dcz comparison requires {field}={value!r}") for field in identity_fields: if baseline_result.get(field) != optimized_result.get(field): raise RuntimeError(f"dcz comparison requires matching {field}") for result in (baseline_result, optimized_result): if result.get("core_type") not in ("P", "E"): raise RuntimeError("dcz comparison requires core_type P or E") if not isinstance(result.get("cpu_mask"), str) or not result["cpu_mask"]: raise RuntimeError("dcz comparison requires a non-empty cpu_mask") if ( not isinstance(result.get("cpu_identity"), str) or not result["cpu_identity"] ): raise RuntimeError("dcz comparison requires a non-empty cpu_identity") baseline_cells = baseline_result.get("scenarios", {}) optimized_cells = optimized_result.get("scenarios", {}) expected = {f"dicts-{count}-{case}" for count, case in dcz_scenarios()} if set(baseline_cells) != expected or set(optimized_cells) != expected: raise RuntimeError( "dcz comparison requires matching 1/4/16 dictionary hit/miss matrices" ) for dictionary_count, case in dcz_scenarios(): name = f"dicts-{dictionary_count}-{case}" for cell in (baseline_cells[name], optimized_cells[name]): if ( cell.get("dictionary_count") != dictionary_count or cell.get("case") != case ): raise RuntimeError(f"dcz scenario metadata does not match {name}") expected_encoding = "dcz" if case == "hit" else "zstd" engagement = cell.get("engagement") if not isinstance(engagement, dict) or set(engagement.values()) != { expected_encoding }: raise RuntimeError( f"dcz scenario {name} requires {expected_encoding} engagement" ) print("\nDCZ COUNTERS PER SUCCESSFUL REQUEST") header = ( f"{'scenario':<16}{'base req':>11}{'opt req':>11}" f"{'miss delta':>13}{'ref delta':>12}{'insn delta':>13}" f"{'cycle delta':>14}" ) print(header) print("-" * len(header)) for dictionary_count, case in dcz_scenarios(): name = f"dicts-{dictionary_count}-{case}" baseline = baseline_cells[name] optimized = optimized_cells[name] baseline_norm = normalize_counters( PerfCounters.from_dict(baseline["counters"]), baseline["measured_successful_requests"], ) optimized_norm = normalize_counters( PerfCounters.from_dict(optimized["counters"]), optimized["measured_successful_requests"], ) miss_delta = _percent_delta( baseline_norm["cache_misses"], optimized_norm["cache_misses"] ) reference_delta = _percent_delta( baseline_norm["cache_references"], optimized_norm["cache_references"], ) instruction_delta = _percent_delta( baseline_norm["instructions"], optimized_norm["instructions"] ) cycle_delta = _percent_delta(baseline_norm["cycles"], optimized_norm["cycles"]) print( f"{name:<16}{baseline['measured_successful_requests']:>11}" f"{optimized['measured_successful_requests']:>11}{miss_delta:>+12.1%}" f"{reference_delta:>+12.1%}{instruction_delta:>+13.1%}" f"{cycle_delta:>+14.1%}" ) def compare_results(baseline_json: str, optimized_json: str) -> None: """Compare two measurement runs. Parameters ---------- baseline_json : str Path to baseline measurement result optimized_json : str Path to optimized measurement result """ with open(baseline_json, encoding="utf-8") as f: baseline_result = json.load(f) with open(optimized_json, encoding="utf-8") as f: optimized_result = json.load(f) if "scenarios" in baseline_result or "scenarios" in optimized_result: compare_dcz_results(baseline_result, optimized_result) return baseline = PerfCounters.from_dict(baseline_result["counters"]) optimized = PerfCounters.from_dict(optimized_result["counters"]) print("\n" + "=" * 70) print("COUNTER DELTA ANALYSIS") print("=" * 70) print_counters("BASELINE (current code)", baseline) print_counters("OPTIMIZED (with repack)", optimized) delta = CounterDelta.compute(baseline, optimized) print("\nDELTA:") miss_pct = delta.cache_misses / baseline.cache_misses refs_pct = delta.cache_references / baseline.cache_references insns_pct = delta.instructions / baseline.instructions cycles_pct = delta.cycles / baseline.cycles print(f" cache_misses: {delta.cache_misses:>+12,} ({miss_pct:>+6.1%})") print(f" cache_references: {delta.cache_references:>+12,} ({refs_pct:>+6.1%})") print(f" instructions: {delta.instructions:>+12,} ({insns_pct:>+6.1%})") print(f" cycles: {delta.cycles:>+12,} ({cycles_pct:>+6.1%})") print(f" miss_rate: {delta.miss_rate:>+12.2%}") print(f" IPC: {delta.ipc:>+12.3f}") # Verdict print("\n" + "=" * 70) if delta.cache_misses < 0: claim_pct = delta.cache_misses / baseline.cache_misses print( f"✓ SUPPORTS CLAIM: {abs(delta.cache_misses):,} fewer " f"cache misses ({claim_pct:.1%})" ) print(" Repack row retained.") else: support_pct = delta.cache_misses / baseline.cache_misses print( f"✗ DOES NOT SUPPORT: {delta.cache_misses:+,} cache misses " f"({support_pct:+.1%})" ) print(" Repack row should be REFUTED.") print("=" * 70 + "\n") def normalize_cli_argv(argv: list[str]) -> list[str]: """Map documented flag-first spellings onto the argparse subcommands.""" if argv[:1] in (["-h"], ["--help"]): return argv if argv and argv[0] == "--compare": return ["compare", *argv[1:]] if argv and argv[0].startswith("-"): return ["measure", *argv] return argv def main() -> int: """Entry point.""" parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, ) subparsers = parser.add_subparsers(dest="command") # "measure" subcommand measure_parser = subparsers.add_parser( "measure", help="Measure baseline or optimized code", aliases=["baseline"], ) measure_parser.add_argument( "--nginx", required=True, help="Path to nginx binary", ) measure_parser.add_argument( "--json", help="Save result to JSON file", ) measure_parser.add_argument( "--core-type", choices=["P", "E"], default="P", help="Pin to P-cores (default) or E-cores", ) measure_parser.add_argument( "--workload", choices=("dcz", "zstd"), default="dcz", help="measure the dcz lookup matrix (default) or legacy zstd workload", ) # "compare" subcommand compare_parser = subparsers.add_parser( "compare", help="Compare baseline and optimized measurements", ) compare_parser.add_argument( "baseline", help="Baseline measurement JSON file", ) compare_parser.add_argument( "optimized", help="Optimized measurement JSON file", ) argv = normalize_cli_argv(sys.argv[1:]) args = parser.parse_args(argv) if args.command in ("measure", "baseline"): if not args.nginx: parser.print_help() return 1 try: if args.workload == "dcz": measure_dcz_matrix( args.nginx, output_json=args.json, core_type=args.core_type, ) else: measure_baseline( args.nginx, output_json=args.json, core_type=args.core_type, ) return 0 except (RuntimeError, FileNotFoundError, subprocess.TimeoutExpired) as e: print(f"ERROR: {e}", file=sys.stderr) return 1 elif args.command == "compare": try: compare_results(args.baseline, args.optimized) return 0 except (FileNotFoundError, json.JSONDecodeError, KeyError, RuntimeError) as e: print(f"ERROR: {e}", file=sys.stderr) return 1 else: parser.print_help() return 1 if __name__ == "__main__": sys.exit(main()) myguard-labs-nginx-zstd-module-3486b5b/ci/tools/run-testkit-scenarios.sh000077500000000000000000000171771525232545500264220ustar00rootroot00000000000000#!/usr/bin/env bash # Run and verify this module's nginx-module-testkit consumer scenarios. # # Usage: # ci/tools/run-testkit-scenarios.sh --build DIR --version VERSION [options] # [scenario ...] # # Required: # --build DIR nginx/Angie build tree containing objs/ and the # harness-enabled zstd module. # --version VERSION concrete server version used by the testkit build lookup. # # Options: # --root DIR module root (default: repository root). # --flavor NAME nginx or angie (default: nginx). # --log-dir DIR TAP output directory (default: repository root). # --runner CMD testkit runner prefix, for example valgrind options. # --timeout-scale N scale testkit timeouts for slow tooling. # -h, --help show this help. # # With no scenario arguments, all committed consumer scenarios run. The script # preserves every scenario's narrow expected-log allowance, re-derives the TAP # verdict, rejects all-skip/empty runs, and checks property-specific witness # lines. It keeps running after a failure so one deep run reports every broken # scenario, then exits non-zero. Logs are the only side effect. Add a scenario # to SCENARIOS and its witness checks below; callers intentionally share this # list so PR, coverage, and Memcheck surfaces cannot drift. set -euo pipefail usage() { sed -n '2,29p' "$0" | sed 's/^# \{0,1\}//' } ROOT="" BUILD="" FLAVOR="nginx" VERSION="" LOG_DIR="" RUNNER="" TIMEOUT_SCALE="" SCENARIOS=( fault-arms alloc-neutral fault-palloc codec-call-count setparam-call-count setparam-call-count-nodict ) while [ "$#" -gt 0 ]; do case "$1" in --build) BUILD="${2:?--build requires a directory}"; shift 2 ;; --root) ROOT="${2:?--root requires a directory}"; shift 2 ;; --flavor) FLAVOR="${2:?--flavor requires a name}"; shift 2 ;; --version) VERSION="${2:?--version requires a value}"; shift 2 ;; --log-dir) LOG_DIR="${2:?--log-dir requires a directory}"; shift 2 ;; --runner) RUNNER="${2:?--runner requires a command}"; shift 2 ;; --timeout-scale) TIMEOUT_SCALE="${2:?--timeout-scale requires a value}"; shift 2 ;; -h|--help) usage; exit 0 ;; --) shift; SCENARIOS=("$@"); break ;; -*) echo "ERROR: unknown option: $1" >&2; usage >&2; exit 2 ;; *) SCENARIOS=("$@"); break ;; esac done [ -n "$ROOT" ] || ROOT="$(git rev-parse --show-toplevel)" [ -n "$LOG_DIR" ] || LOG_DIR="$ROOT" [ -n "$BUILD" ] || { echo "ERROR: --build is required" >&2; exit 2; } [ -n "$VERSION" ] || { echo "ERROR: --version is required" >&2; exit 2; } [ "${#SCENARIOS[@]}" -gt 0 ] || { echo "ERROR: no scenarios selected" >&2; exit 2; } [ -d "$ROOT" ] || { echo "ERROR: --root is not a directory: $ROOT" >&2; exit 2; } [ -d "$BUILD" ] || { echo "ERROR: --build is not a directory: $BUILD" >&2; exit 2; } ROOT="$(realpath "$ROOT")" BUILD="$(realpath "$BUILD")" mkdir -p "$LOG_DIR" LOG_DIR="$(realpath "$LOG_DIR")" PROBER_DIR="$ROOT/ci/t/harness/ci/prober" RUN_SCENARIO="${PROBER_RUN_SCENARIO:-$PROBER_DIR/run-scenario.sh}" [ -x "$RUN_SCENARIO" ] || { echo "ERROR: testkit runner is not executable: $RUN_SCENARIO" >&2 exit 2 } export PROBER_ROOT="$ROOT" export PROBER_BUILD="$BUILD" export PROBER_MODULE="${PROBER_MODULE:-ngx_http_zstd_filter_module.so}" export PROBER_DIRECTIVE="${PROBER_DIRECTIVE:-zstd_probe}" export PROBER_PROBE="${PROBER_PROBE:-zstd_probe;}" [ -z "$RUNNER" ] || export PROBER_VALGRIND="$RUNNER" [ -z "$TIMEOUT_SCALE" ] || export PROBER_TIMEOUT_SCALE="$TIMEOUT_SCALE" fail() { printf 'ERROR: %s\n' "$*" >&2 status=1 } require_line() { local log="$1" pattern="$2" message="$3" grep -qE "$pattern" "$log" || fail "$message" } verify_tap() { local scenario="$1" log="$2" total skipped n plan plans_text local -a plans if grep -q '^Bail out!' "$log"; then fail "$scenario reported a TAP bailout" fi if grep -q '^not ok' "$log"; then fail "$scenario reported at least one 'not ok' line" fi if ! plans_text="$(sed -nE 's/^1\.\.([0-9]+)([[:space:]]*#.*)?$/\1/p' "$log")"; then fail "$scenario could not parse its TAP plan" plans=() elif [ -n "$plans_text" ]; then mapfile -t plans <<<"$plans_text" else plans=() fi if [ "${#plans[@]}" -ne 1 ]; then fail "$scenario reported ${#plans[@]} TAP plans; expected exactly one" plan=-1 else plan="${plans[0]}" fi total="$(grep -c '^ok ' "$log" || true)" skipped="$(grep -c '^ok .*# SKIP' "$log" || true)" if [ "$total" -eq 0 ]; then fail "$scenario reported no passing TAP oracle" elif [ "$total" -eq "$skipped" ]; then fail "$scenario skipped every oracle" fi if [ "$plan" -ge 0 ] && ! awk -v plan="$plan" ' /^(ok|not ok) [0-9]+/ { n = ($1 == "not" ? $3 : $2) + 0 if (n < 1 || n > plan || seen[n]++) exit 1 count++ } END { if (count != plan) exit 1 } ' "$log"; then fail "$scenario TAP results do not exactly satisfy plan 1..$plan" fi case "$scenario" in alloc-neutral) skipped="$(grep -cE '^ok [345] .*# SKIP' "$log" || true)" [ "$skipped" -ne 3 ] || fail \ "alloc-neutral skipped all cycle/worker allocation oracles 3-5" ;; fault-palloc) require_line "$log" '^ok 2 - PALLOC site accepted the arm[^#]*$' \ "fault-palloc did not prove the PALLOC site was armed" ;; codec-call-count) for n in 2 4 6 8 10; do require_line "$log" "^ok $n - [^#]*total codec calls is [^#]*$" \ "codec-call-count oracle $n did not assert the call count" done ;; setparam-call-count) for n in 3 7; do require_line "$log" "^ok $n - [^#]*setParameter call count is [^#]*$" \ "setparam-call-count oracle $n did not assert the call count" done require_line "$log" '^ok 4 - [^#]*frame window size is [^#]*$' \ "setparam-call-count did not assert the CDict frame window" ;; setparam-call-count-nodict) require_line "$log" '^ok 3 - [^#]*setParameter call count is [^#]*$' \ "setparam-call-count-nodict did not assert the reference count" require_line "$log" '^ok 4 - [^#]*frame window size is [^#]*$' \ "setparam-call-count-nodict did not assert the reference window" ;; esac } status=0 for scenario in "${SCENARIOS[@]}"; do scenario_dir="$ROOT/ci/t/harness-scenarios/$scenario" log="$LOG_DIR/$scenario-tap.log" [ -d "$scenario_dir" ] || { fail "scenario directory does not exist: $scenario_dir" continue } allow_log="" case "$scenario" in fault-arms) allow_log='zstd: ZSTD_(compressStream2|CCtx_refPrefix)\(\) failed' ;; fault-palloc) allow_log='zstd: |ngx_(palloc|pcalloc|pnalloc)' ;; esac printf '%s\n' "== testkit scenario: $scenario ==" set +e if [ -n "$allow_log" ]; then ( cd "$PROBER_DIR" PROBER_ALLOW_LOG="$allow_log" \ "$RUN_SCENARIO" "$scenario_dir" "$FLAVOR" "$VERSION" ) | tee "$log" else ( cd "$PROBER_DIR" unset PROBER_ALLOW_LOG "$RUN_SCENARIO" "$scenario_dir" "$FLAVOR" "$VERSION" ) | tee "$log" fi run_status="${PIPESTATUS[0]}" set -e [ "$run_status" -eq 0 ] || fail "$scenario runner exited $run_status" verify_tap "$scenario" "$log" done exit "$status" myguard-labs-nginx-zstd-module-3486b5b/ci/tools/sanitizer_suite_verdict.py000077500000000000000000000107141525232545500271120ustar00rootroot00000000000000#!/usr/bin/env python3 """Fail closed on incomplete TAP suites and module-attributable sanitizer reports.""" from __future__ import annotations import argparse import glob import re import sys from pathlib import Path FINDING = re.compile( r"(?:(?:Address|Leak|UndefinedBehavior)Sanitizer:|runtime error:)", re.IGNORECASE, ) MODULE = re.compile(r"(?:src/)?ngx_http_zstd(?:_[A-Za-z0-9_]+)?", re.IGNORECASE) PLAN = re.compile(r"^1\.\.([0-9]+)(?:\s*#.*)?$") RESULT = re.compile(r"^(?:not )?ok\s+([0-9]+)(?:\s|$)") LOG_MARKER = re.compile( r"^(?:Bail out!|(?:not )?ok\s|1\.\.[0-9]+|" r"[0-9]{4}/[0-9]{2}/[0-9]{2} [0-9:]+ \[[a-z]+\])" ) def check_suite(spec: str) -> bool: try: label, path_text, status_text = spec.split(":", 2) status = int(status_text) except ValueError: print( f"invalid --suite value: {spec!r}; expected LABEL:PATH:STATUS", file=sys.stderr, ) return False path = Path(path_text) try: lines = path.read_text(encoding="utf-8", errors="replace").splitlines() except OSError as exc: print(f"{label}: cannot read TAP log {path}: {exc}", file=sys.stderr) return False ok = True if status != 0: print(f"{label}: suite exited {status}, expected 0", file=sys.stderr) ok = False if any(line.startswith("Bail out!") for line in lines): print(f"{label}: TAP bailout reported", file=sys.stderr) ok = False if any(line.startswith("not ok ") for line in lines): print(f"{label}: failing TAP result reported", file=sys.stderr) ok = False plans = [int(match.group(1)) for line in lines if (match := PLAN.match(line))] results = [int(match.group(1)) for line in lines if (match := RESULT.match(line))] if len(plans) != 1: print( f"{label}: expected exactly one TAP plan, found {len(plans)}", file=sys.stderr, ) return False plan = plans[0] if plan <= 0 or len(results) != plan or sorted(results) != list(range(1, plan + 1)): print( f"{label}: TAP results do not exactly satisfy plan 1..{plan} " f"({len(results)} result lines)", file=sys.stderr, ) ok = False if ok: print(f"{label}: suite exit 0 and all {plan} TAP results reported") return ok def finding_blocks(text: str) -> list[str]: lines = text.splitlines() starts = [index for index, line in enumerate(lines) if FINDING.search(line)] blocks = [] for pos, start in enumerate(starts): end = starts[pos + 1] if pos + 1 < len(starts) else len(lines) for index in range(start + 1, end): if LOG_MARKER.match(lines[index]): end = index break blocks.append("\n".join(lines[start:end])) return blocks def check_sanitizers(patterns: list[str]) -> bool: paths = sorted({path for pattern in patterns for path in glob.glob(pattern)}) if not paths: print("no sanitizer report/log files were produced", file=sys.stderr) return False attributable: list[tuple[str, str]] = [] findings = 0 for path_text in paths: text = Path(path_text).read_text(encoding="utf-8", errors="replace") for block in finding_blocks(text): findings += 1 if MODULE.search(block): attributable.append((path_text, block)) if attributable: print("sanitizer finding attributable to the zstd module:", file=sys.stderr) for path_text, block in attributable: print(f"--- {path_text} ---", file=sys.stderr) print(block, file=sys.stderr) return False print( f"no module-attributable sanitizer finding in {len(paths)} log(s); " f"parsed {findings} report(s)" ) return True def main() -> int: parser = argparse.ArgumentParser() parser.add_argument( "--suite", action="append", default=[], metavar="LABEL:PATH:STATUS" ) parser.add_argument("--sanitizer-log", action="append", default=[], metavar="GLOB") args = parser.parse_args() if not args.suite or not args.sanitizer_log: parser.error("at least one --suite and --sanitizer-log are required") suite_results = [check_suite(spec) for spec in args.suite] suite_ok = all(suite_results) sanitizer_ok = check_sanitizers(args.sanitizer_log) return 0 if suite_ok and sanitizer_ok else 1 if __name__ == "__main__": raise SystemExit(main()) myguard-labs-nginx-zstd-module-3486b5b/ci/tools/sha256-unit/000077500000000000000000000000001525232545500235565ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/tools/sha256-unit/ngx_config.h000066400000000000000000000016271525232545500260560ustar00rootroot00000000000000/* * Minimal ngx type surface for compiling ngx_http_zstd_sha256.h outside * nginx (tools/test_sha256_unit.c). The header's only ngx dependencies * are these types and memory helpers. Keeping the shim this small means * a NEW dependency taken by the header shows up as a compile error * here; it does not guard against compatible surface drift (the fake * EVP already declares `void *impl` where real OpenSSL has `ENGINE *`, * and both compile) — real-header compatibility is what the production * builds verify. */ #ifndef NGX_SHIM_CONFIG_H #define NGX_SHIM_CONFIG_H #include #include #include typedef unsigned char u_char; typedef uintptr_t ngx_uint_t; typedef intptr_t ngx_int_t; #define ngx_inline inline #define ngx_memcpy(dst, src, n) memcpy(dst, src, n) #define ngx_memzero(buf, n) memset(buf, 0, n) #endif /* NGX_SHIM_CONFIG_H */ myguard-labs-nginx-zstd-module-3486b5b/ci/tools/sha256-unit/ngx_core.h000066400000000000000000000002511525232545500255310ustar00rootroot00000000000000/* Shim companion to ngx_config.h — everything lives there. */ #ifndef NGX_SHIM_CORE_H #define NGX_SHIM_CORE_H #include #endif /* NGX_SHIM_CORE_H */ myguard-labs-nginx-zstd-module-3486b5b/ci/tools/sha256-unit/openssl/000077500000000000000000000000001525232545500252415ustar00rootroot00000000000000myguard-labs-nginx-zstd-module-3486b5b/ci/tools/sha256-unit/openssl/evp.h000066400000000000000000000017431525232545500262110ustar00rootroot00000000000000/* * FAKE OpenSSL EVP header for the unit fixture: declarations only, no * OpenSSL required. tools/test_sha256_unit.c supplies implementations * whose failure modes are scripted (partial digest write + failure * return, wrong output length, scripted success), so the EVP-to- * portable fallback in ngx_http_zstd_sha256.h can be exercised * deterministically. This directory is passed with -I ahead of the * system paths, so this header shadows the real for * the fixture build only. */ #ifndef NGX_SHIM_FAKE_EVP_H #define NGX_SHIM_FAKE_EVP_H #include typedef struct fake_evp_md_st EVP_MD; typedef struct evp_md_ctx_st EVP_MD_CTX; typedef struct engine_st ENGINE; const EVP_MD *EVP_sha256(void); int EVP_DigestInit_ex(EVP_MD_CTX *ctx, const EVP_MD *type, ENGINE *impl); int EVP_DigestUpdate(EVP_MD_CTX *ctx, const void *data, size_t count); int EVP_DigestFinal_ex(EVP_MD_CTX *ctx, unsigned char *md, unsigned int *size); #endif /* NGX_SHIM_FAKE_EVP_H */ myguard-labs-nginx-zstd-module-3486b5b/ci/tools/soak.sh000077500000000000000000000343741525232545500231000ustar00rootroot00000000000000#!/usr/bin/env bash # # Sustained mixed-load soak for the zstd filter. Drives a real nginx # (ideally an ASAN/UBSAN build, optionally under valgrind) with # concurrent, varied requests for a fixed duration, then asserts the # worker survived cleanly: no sanitizer report, no crash, no leak, no # error-log [alert]/[emerg]. This is the "survives production-shaped # load" check that synthetic single-shot tests do not give. # # Usage: # tools/soak.sh [duration_seconds] [concurrency] # USE_VALGRIND=1 tools/soak.sh 120 8 # # Exit non-zero on ANY of: sanitizer error, valgrind error, nginx # crash/non-clean exit, error-log alert/emerg, or a corrupted response. set -euo pipefail NGINX="${1:?usage: soak.sh [duration] [concurrency]}" DURATION="${2:-60}" CONC="${3:-8}" SOAK_SEED="${SOAK_SEED:-$(( $(date +%s) & 0x7fffffff ))}" if ! [[ "$SOAK_SEED" =~ ^[0-9]+$ ]]; then echo "FAIL: SOAK_SEED must be an integer from 0 through 2147483647" >&2 exit 2 fi SOAK_SEED="$(printf '%s' "$SOAK_SEED" | sed 's/^0*//')" SOAK_SEED="${SOAK_SEED:-0}" if [ "${#SOAK_SEED}" -gt 10 ] \ || { [ "${#SOAK_SEED}" -eq 10 ] && [ "$SOAK_SEED" -gt 2147483647 ]; } then echo "FAIL: SOAK_SEED must be an integer from 0 through 2147483647" >&2 exit 2 fi WORK="$(mktemp -d)" trap 'rm -rf "$WORK"' EXIT mkdir -p "$WORK/conf" "$WORK/logs" "$WORK/html" # Mixed payload sizes: tiny (sub-min_length), medium, large (multi-buffer), # and a chunked/no-Content-Length upstream path. head -c 50 /dev/urandom | base64 >"$WORK/html/tiny" head -c 200000 /dev/urandom | base64 >"$WORK/html/medium" head -c 4000000 /dev/urandom | base64 >"$WORK/html/large" printf 'AAAAAAAAAA%.0s' $(seq 1 20000) >"$WORK/html/compressible" # RFC 9842 dcz dictionary. The soak previously sent only zstd/gzip, so the # whole dcz path -- negotiation, ZSTD_CCtx_refPrefix, the 40-byte frame # header, dictionary lifetime across requests -- ran under NEITHER sanitizer # lane despite being the newest and largest block of code in the module. # The dictionary content overlaps the response bodies below so refPrefix has # real matches to find rather than compressing a miss. printf 'shared-boilerplate compute render %.0s' $(seq 1 200) \ >"$WORK/html/dcz-dict" DICT_B64="$(openssl dgst -sha256 -binary "$WORK/html/dcz-dict" | base64)" printf 'shared-boilerplate compute render %.0s' $(seq 1 400) \ >"$WORK/html/dczbody" cat >"$WORK/conf/nginx.conf" </dev/null; then ready=1 break fi sleep 0.1 done if [ "$ready" -ne 1 ]; then echo "FAIL: nginx never became ready (no successful /tiny request in 10s)" >&2 kill "$NGINX_PID" 2>/dev/null || true tail -40 "$WORK/logs/error.log" || true exit 1 fi echo "soak: ${DURATION}s, concurrency ${CONC}$( [ "${USE_VALGRIND:-0}" = 1 ] && echo ' (valgrind)' [ "${USE_HELGRIND:-0}" = 1 ] && echo ' (helgrind)' )" echo "soak: seed $SOAK_SEED (replay with SOAK_SEED=$SOAK_SEED)" END=$(($(date +%s) + DURATION)) fail=0 worker() { local wid="$1" local paths=(/tiny /medium /large /compressible "/bypass" "/bypass?nozstd=1" /dcz) local prefix_paths=(/tiny /tiny /medium /medium /large /large /compressible /bypass "/bypass?nozstd=1" /dcz /dcz) local prefix_aes=(gzip zstd gzip zstd gzip zstd zstd zstd zstd zstd dcz) local prefix_count=${#prefix_paths[@]} local i=0 ok=0 bad=0 dcz_seen=0 rng=$(((10#$SOAK_SEED + wid) & 0x7fffffff)) local -a counts for ((cell = 0; cell < prefix_count; cell++)); do counts[cell]=0; done while [ "$i" -lt "$prefix_count" ] || [ "$(date +%s)" -lt "$END" ]; do if [ "$i" -lt "$prefix_count" ]; then p=${prefix_paths[$i]} ae=${prefix_aes[$i]} cell=$i else rng=$(((1103515245 * rng + 12345) & 0x7fffffff)) p=${paths[$((rng % ${#paths[@]}))]} rng=$(((1103515245 * rng + 12345) & 0x7fffffff)) ae=$([ $((rng % 4)) -eq 0 ] && echo "gzip" || echo "zstd") cell=-1 fi i=$((i + 1)) body="$WORK/r.${wid}.${i}" # The prefix has separate /dcz plain-zstd and negotiated-dcz cells, # so both the fallback and refPrefix paths always run. dcz_seen below # additionally proves that negotiation produced a real dcz frame. dcz_want=0 bad_before=$bad hdrs=(-H "Accept-Encoding: $ae") if [ "$ae" = "dcz" ]; then p=/dcz dcz_want=1 ae=zstd hdrs=(-H "Accept-Encoding: zstd, dcz" -H "Available-Dictionary: :${DICT_B64}:" -H "Sec-Fetch-Site: same-origin") fi identity_want=0 if [ "$ae" != "zstd" ] || [ "$p" = "/tiny" ] \ || [ "$p" = "/bypass?nozstd=1" ] then identity_want=1 fi if curl -fsS "${hdrs[@]}" \ "http://127.0.0.1:18222$p" -o "$body" 2>/dev/null; then # Every served response variant must be enumerated explicitly. # Unrecognized magics are a hard failure, not a silent pass. # # Legitimate response variants: # 1. Plain zstd: magic 28b52ffd (all paths except /bypass?nozstd=1) # 2. DCZ skippable frame: magic 5e2a4d18 (from /dcz with negotiation) # 3. Identity/uncompressed: no magic (only /bypass?nozstd=1, which # has zstd_bypass active, so the response is not compressed) # # A dcz response starts with 5e2a4d18, NOT 28b52ffd, so testing # only for the zstd magic would let every dcz response fall through # and count as ok without ever being decoded. Decode dcz against # the same dictionary the request advertised: that verifies the # frame header, the refPrefix output and the dictionary content. magic="$(head -c4 "$body" | od -An -tx1 | tr -d ' ')" if [ "$identity_want" -eq 1 ]; then if [ "$magic" = "28b52ffd" ] || [ "$magic" = "5e2a4d18" ]; then echo "BAD compressed identity-only cell $p (Accept-Encoding: $ae, magic: $magic)" bad=$((bad + 1)) else ok=$((ok + 1)) fi elif [ "$magic" = "28b52ffd" ]; then if zstd -dq -c "$body" >/dev/null 2>&1; then ok=$((ok + 1)) else echo "BAD zstd $p" bad=$((bad + 1)) fi elif [ "$magic" = "5e2a4d18" ]; then if zstd -dq -D "$WORK/html/dcz-dict" -c "$body" \ >/dev/null 2>&1; then ok=$((ok + 1)) dcz_seen=$((dcz_seen + 1)) else echo "BAD dcz $p" bad=$((bad + 1)) fi elif [ "$dcz_want" -eq 1 ]; then # Asked for dcz with a matching dictionary and a # same-origin fetch, and got something that is neither a # dcz frame nor a zstd frame. echo "BAD dcz $p: negotiated request returned magic $magic" bad=$((bad + 1)) else # Every other combination (ae advertised zstd, and the path # is not sub-min_length or bypassed) must have compressed. # Identity here is a real "we stopped compressing" # regression, not a benign variant. echo "BAD unknown magic $magic $p (Accept-Encoding: $ae)" bad=$((bad + 1)) fi # A negotiated request that came back as a plain zstd frame is # a dcz regression, not a pass: the gate silently declined and # the 28b52ffd branch above would otherwise have accepted it. if [ "$dcz_want" -eq 1 ] && [ "$magic" = "28b52ffd" ]; then echo "BAD dcz $p: negotiated request fell back to plain zstd" bad=$((bad + 1)) fi if [ "$cell" -ge 0 ] && [ "$bad" -eq "$bad_before" ]; then counts[cell]=$((counts[cell] + 1)) fi else echo "FAIL request $p (curl exit $?)" bad=$((bad + 1)) fi rm -f "$body" done echo "worker $wid: $ok ok, $bad bad/failed, $dcz_seen dcz" for ((cell = 0; cell < prefix_count; cell++)); do echo "worker $wid cell ${prefix_paths[$cell]}+${prefix_aes[$cell]}: ${counts[$cell]}" if [ "${counts[$cell]}" -eq 0 ]; then echo "FAIL worker $wid: deterministic cell ${prefix_paths[$cell]}+${prefix_aes[$cell]} had no verified response" bad=$((bad + 1)) fi done if [ "$dcz_seen" -eq 0 ]; then echo "FAIL worker $wid: no dcz response was ever verified" return 1 fi if [ "$bad" -ne 0 ] || [ "$ok" -eq 0 ]; then return 1 fi return 0 } pids=() for w in $(seq 1 "$CONC"); do worker "$w" & pids+=($!) done for pid in "${pids[@]}"; do wait "$pid" || fail=1; done # Clean shutdown so all pool cleanups (incl. CCtx/CDict) run. kill -QUIT "$NGINX_PID" 2>/dev/null || true wait "$NGINX_PID" 2>/dev/null rc=$? # shellcheck source=ci/tools/lsan_log_verdict.sh . "$(cd "$(dirname "$0")" && pwd)/lsan_log_verdict.sh" problems=0 indeterminate=0 if compgen -G "$WORK/logs/asan*" >/dev/null; then echo "ASAN/UBSAN log:" cat "$WORK"/logs/asan* set +e lsan_check_log_glob "$WORK/logs/asan*" asan_rc=$? set -e case "$asan_rc" in 0) : ;; 2) echo "::warning::LeakSanitizer could not run its exit-time check" \ "(ptrace blocked on this runner — LXC seccomp / yama). Soak" \ "leak check INDETERMINATE, not clean; fix the runner profile" \ "to restore coverage." indeterminate=1 ;; *) echo "FAIL: ASAN/UBSAN report (see log above)" problems=1 ;; esac else echo '::warning::LeakSanitizer produced no verbose exit-hook log; soak leak check INDETERMINATE' indeterminate=1 fi if ls "$WORK"/logs/valgrind.* "$WORK"/logs/helgrind.* >/dev/null 2>&1; then if grep -qE 'ERROR SUMMARY: [1-9]|definitely lost: [1-9]' \ "$WORK"/logs/valgrind.* "$WORK"/logs/helgrind.* 2>/dev/null; then echo "FAIL: valgrind/helgrind errors:" grep -E 'ERROR SUMMARY|definitely lost' \ "$WORK"/logs/valgrind.* "$WORK"/logs/helgrind.* 2>/dev/null # Dump every log holding errors in full: the WORK dir is wiped on # exit, so this is the only place the stacks (and the exact # suppression blocks from --gen-suppressions=all) survive, e.g. # in a CI job log. for _vglog in "$WORK"/logs/valgrind.* "$WORK"/logs/helgrind.*; do [ -f "$_vglog" ] || continue grep -qE 'ERROR SUMMARY: [1-9]|definitely lost: [1-9]' "$_vglog" || continue echo "---- $_vglog ----" cat "$_vglog" done problems=1 fi fi if grep -nE '\[alert\]|\[emerg\]' "$WORK/logs/error.log" 2>/dev/null; then echo "FAIL: alert/emerg in error.log" problems=1 fi if [ "$fail" -ne 0 ]; then echo "FAIL: a worker reported a corrupted response" problems=1 fi # QUIT is a clean exit; valgrind uses 99, ASAN 42 on error. if [ "$rc" -ne 0 ] && [ "$rc" -ne 130 ]; then echo "FAIL: nginx exited $rc" tail -40 "$WORK/logs/error.log" || true problems=1 fi [ "$problems" -ne 0 ] && exit 1 if [ "$indeterminate" -ne 0 ]; then echo "⚠ soak passed: ${DURATION}s @ ${CONC} concurrent, no crash/sanitizer" \ "finding, but the leak lane is INDETERMINATE this run (ptrace" \ "blocked -- see warning above)" else echo "✓ soak clean: ${DURATION}s @ ${CONC} concurrent, no sanitizer/leak/crash" fi myguard-labs-nginx-zstd-module-3486b5b/ci/tools/soak_asan_verdict.sh000077500000000000000000000130121525232545500256040ustar00rootroot00000000000000#!/usr/bin/env bash # Copyright (C) 2026 Thijs Eilander # SPDX-License-Identifier: BSD-2-Clause # # soak_asan_verdict() -- triage an ASAN_OPTIONS log_path glob's content and # decide whether the soak passes, fails, or is INDETERMINATE. # # A27-F9: ci/tools/soak.sh used to gate on whether ANY "$WORK/logs/asan*" # file existed at all, never reading it. log_path (see soak.sh's # ASAN_OPTIONS) makes the sanitizer runtime write EVERY file it wants to # emit into that glob -- including pure diagnostics that are not findings, # such as LeakSanitizer's own "ptrace appears to be blocked (is seccomp # enabled?)" startup warning. On this runner pool (LXC seccomp/yama on # builder02) that warning fires routinely: the same signature shows up in # ci/tools/test_reload_leak.sh and the "Run smoke tests under ASAN+UBSAN" # step in .github/workflows/build-test.yml, both of which already read # content instead of existence for exactly this reason. This file extracts # that same triage into one function so soak.sh and this unit fixture share # one source of truth instead of two copies drifting apart. # # Verdict (mirrors the valgrind/helgrind branch a few lines below this # call site in soak.sh, and the two existing ptrace-aware call sites # above): # 0 (pass, clean) -- no log, or a log with no recognized # sanitizer finding and no ptrace-blocked # signature (diagnostics of some other, # unrecognized kind still count as clean # here -- this function only fails on an # actual finding). # 1 (fail) -- a real ASan/LSan/UBSan finding. # 2 (indeterminate) -- LeakSanitizer's exit-time check could not # run (ptrace blocked). NOT a pass and NOT # a fail: the leak lane checked nothing # this run, so a plain 0 would misreport # "no leaks" when nothing was actually # verified. # # The log content (if any) is always echoed by the caller-visible path # (soak.sh cats it), independent of this function's return code, so the # condition stays visible in the job log instead of going silent -- this # function only classifies, it does not print. # # Usage: soak_asan_verdict [ ...] # A caller like `soak_asan_verdict "$WORK"/logs/asan*` that had no # matching files (glob left unexpanded, nullglob off) is handled: this # function treats a sole non-existent path as "no log" (verdict 0), # the same as the historical `ls ... >/dev/null 2>&1` gate finding # nothing. soak_asan_verdict() { local existing=() local f for f in "$@"; do [ -f "$f" ] && existing+=("$f") done # No log at all: nothing to triage, and nothing was written, so this # is a clean pass (the historical "ls found nothing" case). [ "${#existing[@]}" -eq 0 ] && return 0 # Real sanitizer findings fail unconditionally, checked FIRST: a run # can produce a ptrace warning from one worker and a real finding # from another (log_path writes one file per process), and checking # ptrace first would let the warning short-circuit the verdict and # mask the finding -- the same ordering test_reload_leak.sh and the # build-test.yml smoke-tests step already use, for the same reason. # # Patterns, and why these specifically -- this repo builds with # -fsanitize=address,undefined -fno-sanitize-recover=undefined, and # these are what each sanitizer's real output actually looks like, # not a guess from memory: # ERROR: (Address|Leak|UndefinedBehavior)Sanitizer -- the banner # every ASan/LSan finding opens with (heap-buffer-overflow, # use-after-free, detected memory leaks, ...). # SUMMARY: (Address|Leak|UndefinedBehavior)Sanitizer -- the summary # line every finding also emits; belt and suspenders in case a # report is truncated before its ERROR banner but the summary # line still lands. # runtime error: -- UBSan's OWN finding format. It is not an # "ERROR:"/"SUMMARY:" banner at all -- that shape belongs to # ASan/LSan. -fno-sanitize-recover=undefined means the process # aborts on the first one, so a single line is sufficient. if grep -qE 'ERROR: (Address|Leak|UndefinedBehavior)Sanitizer|SUMMARY: (Address|Leak|UndefinedBehavior)Sanitizer|runtime error:' \ "${existing[@]}" 2>/dev/null; then return 1 fi # The known lockdown signature (see the file header): LSan's # exit-time tracer was blocked/killed before it could inspect # anything. No verdict exists in either direction for the leak lane. if grep -qE 'ptrace appears to be blocked|LeakSanitizer may hang|Child exited with signal' \ "${existing[@]}" 2>/dev/null; then return 2 fi # A log existed but held neither a recognized finding nor the # ptrace-blocked signature: pass. (Nothing currently writes # log_path files outside those two shapes, but this function's job # is to fail on findings, not on "log exists" -- the exact defect # A27-F9 exists to fix.) return 0 } # Allow direct execution for ad hoc triage of an already-captured log: # ci/tools/soak_asan_verdict.sh /path/to/asan.* # Sourcing (as soak.sh and the unit fixture do) must not run this -- # BASH_SOURCE differs from $0 only when sourced. if [ "${BASH_SOURCE[0]}" = "$0" ]; then soak_asan_verdict "$@" exit $? fi myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test-nginx-socket.cpan.lock000066400000000000000000000007601525232545500267540ustar00rootroot00000000000000# Locked resolver graph for Test::Nginx::Socket@0.32. # Regenerate with: # cpanm --showdeps Test::Nginx::Socket@0.32 2>/dev/null \ # | awk '/^[A-Za-z0-9_:]+(~[^[:space:]]+)?$/ {print}' | sort Encode ExtUtils::MakeMaker ExtUtils::MakeMaker~6.59 Fcntl File::Find File::Path File::Temp Filter::Util::Call HTTP::Response IO::Select IO::Socket IO::Socket::INET IPC::Run LWP::UserAgent List::MoreUtils List::Util POSIX Test::Base Test::LongString Text::Diff Time::HiRes URI::Escape perl~5.010001 myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_baseline_loaded.py000077500000000000000000000224171525232545500263050ustar00rootroot00000000000000#!/usr/bin/env python3 """Baseline: the zstd filter module is actually loaded and blocking. Every other live-server driver in ci/tools/ (test_reload_under_load.py, test_concurrent_cctx_isolation.py, ...) asserts a *property* of zstd compression under some stressor -- reload, concurrency, a timing race. None of them prove the module ran at all versus, say, a config that silently no-ops zstd and serves identity responses that happen to also pass every other assertion in isolation. ci/t/00-filter.t covers this in the Test::Nginx layer already, but per PR2 step 22 the driver keeps its own loaded-and-blocking control too: without it, a module that failed to load reads as a run of green concurrency/reload cases, because none of them individually depend on compression actually having happened -- they mostly assert response *correctness*, which an unmodified passthrough of a small enough body can also satisfy by accident (Content-Length equal, bytes identical, decode step skipped by a permissive comparison). Two configs, same backend, same request: * WITH `zstd on;` (module present, static --add-module build) -> the filter module is expected to compress: Content-Encoding: zstd and zstd-magic body bytes. * WITHOUT any zstd directive at all (module compiled in, but no directive turns it on) -> the response must NOT be zstd-encoded: this is the compiled-in-default case (see grind worker-contract's warning on default-flip reachability -- an opt-in module must default OFF). The interesting failure mode this catches is not "module missing" (that fails to configure/build at all) but "module present, and something makes it decide not to act" -- a decision-flip bug in the accept/ok predicates step 21 unit-tests, exercised here end-to-end through a real request instead of directly. Requires an nginx binary built --add-module (static) with the zstd filter module, plus the `zstd` CLI for the magic-byte / decode check. """ from __future__ import annotations import argparse import http.server import pathlib import socket import socketserver import subprocess import sys import tempfile import threading import time import urllib.request BODY = b"BASELINE." * 4096 # 36KB, well above zstd_min_length, compressible ZSTD_MAGIC = b"\x28\xb5\x2f\xfd" def parse_args() -> argparse.Namespace: p = argparse.ArgumentParser( description="Baseline: module loaded and blocking (PR2 step 22)." ) p.add_argument("--nginx-binary", required=True) p.add_argument("--zstd-bin", default="zstd") p.add_argument("--port", type=int, default=18110) p.add_argument("--off-port", type=int, default=18112) p.add_argument("--backend-port", type=int, default=18111) return p.parse_args() class _Handler(http.server.BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" def log_message(self, *a): pass def do_GET(self): self.send_response(200) # text/html: the module's compiled-in default `zstd_types` value # (src/ngx_http_zstd_filter_module.c ~line 235). Using anything # else would make the "no directive" case decline on the # content-type gate regardless of the `enable` default, which # would mask -- not exercise -- the enable-default mutation this # baseline exists to catch. self.send_header("Content-Type", "text/html") self.send_header("Content-Length", str(len(BODY))) self.end_headers() self.wfile.write(BODY) class _Srv(socketserver.ThreadingMixIn, socketserver.TCPServer): allow_reuse_address = True daemon_threads = True def wait_port(port: int, timeout: float = 10.0) -> None: end = time.time() + timeout while time.time() < end: try: with socket.create_connection(("127.0.0.1", port), 0.5): return except OSError: time.sleep(0.1) raise RuntimeError(f"nothing listening on 127.0.0.1:{port}") def run_one( nginx: pathlib.Path, port: int, backend_port: int, zstd_directive: str ) -> tuple[bool, bool, bytes]: """Start nginx with (or without) the zstd directive, fetch once, return (advertised_zstd, payload_is_zstd, raw_body). The header and the payload are reported SEPARATELY on purpose. Collapsing them into one boolean makes the default-off case unable to distinguish "no Content-Encoding" from "advertised zstd but sent something else" -- the second is a real bug and would read as a pass. """ with tempfile.TemporaryDirectory(prefix="zstd-baseline-") as td: root = pathlib.Path(td) logs = root / "logs" logs.mkdir() pid_file = root / "nginx.pid" conf = root / "nginx.conf" conf.write_text( f"""master_process off; error_log {logs}/error.log warn; pid {pid_file}; events {{ worker_connections 32; }} http {{ access_log off; default_type application/octet-stream; {zstd_directive} server {{ listen 127.0.0.1:{port}; location / {{ proxy_pass http://127.0.0.1:{backend_port}/; }} }} }} """, encoding="utf-8", ) proc = subprocess.Popen( [str(nginx), "-p", str(root), "-c", str(conf), "-g", "daemon off;"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, ) try: wait_port(port) req = urllib.request.Request( f"http://127.0.0.1:{port}/", headers={"Accept-Encoding": "zstd"} ) with urllib.request.urlopen(req, timeout=10) as resp: ce = (resp.headers.get("Content-Encoding") or "").lower() blob = resp.read() advertised = ce == "zstd" payload_is_zstd = blob[:4] == ZSTD_MAGIC return advertised, payload_is_zstd, blob finally: proc.terminate() try: proc.wait(timeout=10) except subprocess.TimeoutExpired: proc.kill() proc.wait(timeout=10) def main() -> int: args = parse_args() nginx = pathlib.Path(args.nginx_binary) if not nginx.exists(): raise FileNotFoundError(nginx) backend = _Srv(("127.0.0.1", args.backend_port), _Handler) threading.Thread(target=backend.serve_forever, daemon=True).start() try: wait_port(args.backend_port) # Case 1: zstd explicitly enabled -> must compress. on_adv, on_payload, on_body = run_one( nginx, args.port, args.backend_port, "zstd on; zstd_comp_level 3; zstd_min_length 1;", ) if not (on_adv and on_payload): print( "FAIL baseline: 'zstd on;' did not produce a " "zstd-encoded response -- the module is not blocking " "as expected", file=sys.stderr, ) return 1 r = subprocess.run( [args.zstd_bin, "-dq", "-c"], input=on_body, capture_output=True, check=False, ) if r.returncode != 0 or r.stdout != BODY: print( "FAIL baseline: 'zstd on;' response does not decode " "byte-exact to the origin body", file=sys.stderr, ) return 1 print("ok 'zstd on;' compresses and decodes byte-exact") # Case 2: no zstd directive at all -> compiled-in default is OFF, # response must NOT be zstd-encoded (see grind worker-contract's # reachability warning: an opt-in module defaulting ON would make # this pass vacuously). # Deliberately its OWN port, distinct from both --port and # --backend-port: colliding with the backend's listening socket # here silently routes the client straight to the backend, # bypassing nginx (and the module) entirely -- the request would # still get a non-zstd response, but for the wrong reason, and # the check would pass even if `enable`'s default were flipped. # (Caught exactly this way during the mutation pass, see # ci/adoption-findings.md.) off_adv, off_payload, off_body = run_one( nginx, args.off_port, args.backend_port, "" ) # Either half is a failure on its own. Advertising zstd without a zstd # payload is not "safely off", it is a lie to the client. if off_adv or off_payload: print( "FAIL baseline: response was zstd-encoded with NO " "zstd directive present -- compiled-in default is not " f"OFF (Content-Encoding: zstd={off_adv}, " f"zstd magic in body={off_payload})", file=sys.stderr, ) return 1 if off_body != BODY: print( "FAIL baseline: undirected response body does not " "match origin (proxy path itself broken, unrelated to " "zstd)", file=sys.stderr, ) return 1 print( "ok no zstd directive -> compiled-in default is OFF, " "origin body served unmodified" ) print("\n2/2 checks passed") return 0 finally: backend.shutdown() if __name__ == "__main__": sys.exit(main()) myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_bufs_bound_policy.py000077500000000000000000000436311525232545500267210ustar00rootroot00000000000000#!/usr/bin/env python3 """Config-load policy: bound the aggregate ``zstd_buffers number size``. ``zstd_buffers`` delegates to nginx core's ``ngx_conf_set_bufs_slot()``, which rejects only a parse error or either argument being zero -- it never looks at the PRODUCT. ``ngx_conf_merge_bufs_value()`` (invoked from ``ngx_http_zstd_merge_loc_conf()``) then either keeps an explicit value, inherits the parent's, or applies this module's own default (``2 x ZSTD_CStreamOutSize()``); none of those three paths re-validates the pair either. A typo ("zstd_buffers 100000 100000;" instead of "100 100k;") or a value inherited from an outer block could therefore request an overflowing OR merely enormous per-response output-chain pool that nginx would happily commit per concurrent response. A first version of this policy only warned above 8 MB and refused only an actual ``size_t`` overflow -- which meant ``zstd_buffers 2147483647 1024m`` (~2.3 EB per response) loaded successfully with nothing but a log line, because ``size_t`` overflow needs operands nginx's own integer parsing barely admits. That is not a bound; it is a warning that the config commits 2.3 exabytes. This version adds a real hard cap in between. The policy under test -- four tiers on the representable (non-overflowing) product, ascending severity -------------------------------------------------------------------------- 1. ``<= NGX_HTTP_ZSTD_BUFS_ADVISORY_BYTES`` (8 MB) -- silent. 2. ``> ADVISORY_BYTES``, ``<= NGX_HTTP_ZSTD_BUFS_HARD_CAP_BYTES`` (256 MB) -- ``[warn]``, never fails; a config that loads today keeps loading. 3. ``> HARD_CAP_BYTES``, no ``zstd_buffers_unsafe on;`` -- ``[emerg]``, config REFUSED. 4. ``> HARD_CAP_BYTES``, WITH ``zstd_buffers_unsafe on;`` -- ``[warn]`` (not silent -- the operator still sees what they acknowledged), config PASSES. ``num * size`` overflowing ``size_t`` is refused unconditionally at every tier, including with the acknowledgement flag set -- there is no representable size for which "the operator meant this". The hard-cap tier is deliberately NOT advisory-shaped like the CCtx budget or the 8 MB tier below it: unlike a libzstd memory ESTIMATE (which depends on library internals the operator never directly sees), ``number`` and ``size`` are two integers the operator typed out literally. At the 256 MB scale, refusing by default and requiring an explicit ``zstd_buffers_unsafe on;`` is the correct default reading of "this is almost certainly a mistake, but if you really mean it, say so." The check runs once at merge time (``ngx_http_zstd_merge_loc_conf()``), which is the single point that sees the value however it got there -- explicit, inherited via ``ngx_conf_merge_bufs_value()``, or this module's own default (all three land in the same ``conf->bufs`` the filter reads from, and ``conf->bufs_unsafe`` merges by the same inheritance rule). The parse-time slot (``ngx_http_zstd_set_bufs_slot()``) additionally catches an overflowing EXPLICIT value at the earliest point, but never repeats the advisory/cap tiers -- so a bare explicit large value is reported exactly once, not twice. Why this test and not ``ci/t/02-conf-warn.t``: Test::Nginx's error-log greps can assert a substring appeared, but cannot assert a config-load run PASSED with a specific warning present, is REFUSED with a specific message, and PASSES again once acknowledged -- all as one matrix with an explicit exit-code check per cell. A signal death (e.g. a bad cast crashing "nginx -t") would still satisfy a plain string grep; this harness explicitly separates that case out. Arms ---- 1. default zstd_buffers (module default, 2 x stream_out_size) -> PASS, silent. 2. advisory cap - 1 byte (``zstd_buffers 1 8388607``) -> PASS, silent. 3. advisory cap + 1 byte (``zstd_buffers 1 8388609``) -> PASS, warn. 4. hard cap - 1 byte (``zstd_buffers 1 268435455``) -> PASS, warn (still tier 2 -- the boundary is inclusive of "advisory, not refused"). 5. hard cap exact (``zstd_buffers 1 268435456``) -> PASS, warn (same reason as arm 4 -- the cap itself is not yet a breach). 6. hard cap + 1 byte, NO acknowledgement (``zstd_buffers 1 268435457``) -> REFUSED, naming the cap and the acknowledgement spelling. 7. hard cap + 1 byte, WITH ``zstd_buffers_unsafe on;`` -> PASS, warn (not silent). 8. the reported gap's exact reproduction (``zstd_buffers 2147483647 1024m`` -- ~2.3 EB per response), NO acknowledgement -> REFUSED. This is the realistic-typo-shaped operand pair the first version of this policy let straight through with only a log line; it must now be the hard-refused case, not the overflow case. 9. same reproduction, WITH acknowledgement -> PASS, warn. 10. product overflow (``zstd_buffers ``), acknowledgement flag ALSO set -> REFUSED, naming both operands. Proves the acknowledgement never reaches the overflow tier -- there is no spelling that accepts an unrepresentable product. 11. inherited from an outer (http) block: both ``zstd_buffers`` and ``zstd_buffers_unsafe on;`` written at http level, a bare ``location`` underneath -> PASS, warn (proves the acknowledgement flag inherits the same way the aggregate bound does, not just the location that wrote it). 12. explicit large value in the 8-256 MB band fires exactly ONCE, not twice (parse-time slot must not duplicate the merge-time advisory). Arms 1, 2, 4, 5 are the controls that keep this test from being vacuous: a build that warns or refuses unconditionally on any zstd_buffers value fails them. Arm 8 is the regression control for the exact gap this policy exists to close. """ import argparse import pathlib import re import subprocess import sys import tempfile # NGX_HTTP_ZSTD_BUFS_HARD_CAP_BYTES in src/ngx_http_zstd_filter_module.c. # (NGX_HTTP_ZSTD_BUFS_ADVISORY_BYTES, 8 MB, is exercised by the boundary # arms below but not compared against directly in this file.) HARD_CAP_BYTES = 256 * 1024 * 1024 # Matched only on a line carrying "[warn]" (see WARN_RE usage below) so a # [emerg] hard-cap refusal -- which restates the same "requests N x M # bytes = ~T bytes" figure in its own message -- is never miscounted as a # warning. WARN_RE = re.compile( r'\[warn\].*"zstd_buffers" \((?P[^)]+)\) requests (?P\d+) x ' r"(?P\d+) bytes = ~(?P\d+) bytes" ) OVERFLOW_RE = re.compile( r'"zstd_buffers" \([^)]+\) requests (\d+) buffers of (\d+) bytes each; ' r"that product overflows" ) HARD_CAP_ACK_RE = re.compile(r"acknowledges it") HARD_CAP_REFUSE_RE = re.compile(r"above the \d+ MB hard cap") def parse_args(): p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) p.add_argument("--nginx-binary", required=True) p.add_argument("--filter-module") p.add_argument("--port", type=int, default=18140) return p.parse_args() def detect_module(explicit, nginx: pathlib.Path): if explicit: return pathlib.Path(explicit) sib = nginx.parent / "ngx_http_zstd_filter_module.so" return sib if sib.exists() else None def config_test(nginx, module, port, http_directives, srv_directives): """Run ``nginx -t``; return (returncode, combined output).""" with tempfile.TemporaryDirectory() as td: root = pathlib.Path(td) (root / "conf").mkdir() (root / "logs").mkdir() load = f"load_module {module};\n" if module else "" (root / "conf" / "nginx.conf").write_text( f"{load}" "events {}\n" "http {\n" f" {http_directives}\n" f" server {{\n" f" listen 127.0.0.1:{port};\n" " zstd on;\n" f" {srv_directives}\n" " }\n" "}\n" ) proc = subprocess.run( [str(nginx), "-p", str(root), "-c", "conf/nginx.conf", "-t"], capture_output=True, text=True, timeout=60, check=False, ) return proc.returncode, proc.stdout + proc.stderr def check_not_a_signal(name, rc, out): if rc < 0: return [f"{name}: nginx -t died on signal {-rc}; output: {out.strip()!r}"] if rc > 128: return [ ( f"{name}: nginx -t exited {rc} (128+{rc - 128}), a signal " f"death, not a diagnostic; output: {out.strip()!r}" ) ] return [] def run_arm(nginx, module, port, name, http_directives, srv_directives, spec): """One matrix cell. ``spec`` carries the expectations. Returns (failures, printable summary).""" want_pass = spec["want_pass"] want_warning = spec.get("want_warning", False) want_overflow_error = spec.get("want_overflow_error", False) want_hard_cap_refuse = spec.get("want_hard_cap_refuse", False) want_hard_cap_ack = spec.get("want_hard_cap_ack", False) min_total = spec.get("min_total") max_warn_count = spec.get("max_warn_count", 1) rc, out = config_test(nginx, module, port, http_directives, srv_directives) failures = check_not_a_signal(name, rc, out) if want_pass and rc != 0: failures.append( f"{name}: expected nginx -t to PASS but it exited {rc}. The " f"bufs bound policy must never turn a working config into a " f"config-load failure at this tier; output: {out.strip()!r}" ) if not want_pass and rc == 0: failures.append( f"{name}: expected nginx -t to be REFUSED but it exited 0; " f"output: {out.strip()!r}" ) warns = WARN_RE.findall(out) if want_warning and not warns: failures.append( f"{name}: expected the [warn] naming the aggregate buffer " f"total, but it was not emitted; output: {out.strip()!r}" ) if not want_warning and warns: failures.append( f"{name}: a warning was emitted but must NOT be at this tier. " f"Either a threshold is wrong or a boundary is off-by-one; " f"output: {out.strip()!r}" ) if warns and len(warns) > max_warn_count: failures.append( f"{name}: warning fired {len(warns)} times (want at most " f"{max_warn_count}) -- the parse-time slot and the merge-time " f"check are double-reporting the same explicit value; output: " f"{out.strip()!r}" ) if warns and "[warn]" not in out: failures.append( f"{name}: matched warning text but not at [warn] level; " f"output: {out.strip()!r}" ) total = None if warns: total = int(warns[0][3]) if min_total is not None and total < min_total: failures.append( f"{name}: reports {total} bytes, below the expected " f"{min_total}. Computed from the wrong operands." ) if "per response" not in out.lower(): failures.append( f"{name}: does not state the total is PER RESPONSE; " f"output: {out.strip()!r}" ) if "zstd_max_cctx_memory" not in out: failures.append( f"{name}: does not cross-reference the CCtx memory " f'advisory ("zstd_max_cctx_memory"); output: ' f"{out.strip()!r}" ) if want_overflow_error: m = OVERFLOW_RE.search(out) if m is None: failures.append( f"{name}: expected the overflow diagnostic naming both " f"operands, but it was not found; output: {out.strip()!r}" ) elif "[emerg]" not in out: failures.append( f"{name}: overflow diagnostic present but not at [emerg] " f"level; output: {out.strip()!r}" ) if want_hard_cap_refuse: if HARD_CAP_REFUSE_RE.search(out) is None: failures.append( f"{name}: expected the hard-cap refusal naming the cap, " f"but it was not found; output: {out.strip()!r}" ) elif "[emerg]" not in out: failures.append( f"{name}: hard-cap diagnostic present but not at [emerg] " f"level; output: {out.strip()!r}" ) if "zstd_buffers_unsafe" not in out: failures.append( f"{name}: hard-cap refusal does not name the " f'acknowledgement spelling ("zstd_buffers_unsafe"); ' f"output: {out.strip()!r}" ) if want_hard_cap_ack and HARD_CAP_ACK_RE.search(out) is None: failures.append( f"{name}: expected the hard-cap ACKNOWLEDGED wording " f"(the acknowledgement must still be visible, not " f"silent), but it was not found; output: {out.strip()!r}" ) if ( total is not None and total > HARD_CAP_BYTES and not (want_hard_cap_refuse or want_hard_cap_ack) ): failures.append( f"{name}: total {total} bytes exceeds the {HARD_CAP_BYTES}-byte " f"hard cap but this arm did not mark either the refuse or the " f"acknowledge expectation -- the matrix spec is wrong for this " f"arm." ) summary = ( f" {name:<52} exit {rc} (want {'0' if want_pass else 'non-zero'}), " f"warnings {len(warns)}" + (f", total {total} bytes" if total is not None else "") ) return failures, summary def main() -> int: args = parse_args() nginx = pathlib.Path(args.nginx_binary).resolve() if not nginx.exists(): print(f"FAIL: nginx binary not found: {nginx}", file=sys.stderr) return 1 module = detect_module(args.filter_module, nginx) if module is not None: module = module.resolve() if not module.exists(): print(f"FAIL: filter module not found: {module}", file=sys.stderr) return 1 huge = 9223372036854775807 # ngx_int_t / int64_t max on a 64-bit build. ack = "zstd_buffers_unsafe on;" matrix = [ ( "1 default bufs (silent tier)", "", "", {"want_pass": True, "want_warning": False}, ), ( "2 advisory cap-1 (8388607, silent)", "", "zstd_buffers 1 8388607;", {"want_pass": True, "want_warning": False}, ), ( "3 advisory cap+1 (8388609, warn)", "", "zstd_buffers 1 8388609;", {"want_pass": True, "want_warning": True, "min_total": 8388609}, ), ( "4 hard cap-1 (268435455, still warn)", "", "zstd_buffers 1 268435455;", {"want_pass": True, "want_warning": True, "min_total": 268435455}, ), ( "5 hard cap exact (268435456, still warn)", "", "zstd_buffers 1 268435456;", {"want_pass": True, "want_warning": True, "min_total": 268435456}, ), ( "6 hard cap+1, NO ack -> REFUSED", "", "zstd_buffers 1 268435457;", { "want_pass": False, "want_warning": False, "want_hard_cap_refuse": True, }, ), ( "7 hard cap+1, WITH ack -> PASS+warn", "", f"zstd_buffers 1 268435457;\n {ack}", { "want_pass": True, "want_warning": True, "want_hard_cap_ack": True, "min_total": 268435457, }, ), ( "8 reported gap repro, NO ack -> REFUSED", "", "zstd_buffers 2147483647 1024m;", { "want_pass": False, "want_warning": False, "want_hard_cap_refuse": True, }, ), ( "9 reported gap repro, WITH ack -> PASS+warn", "", f"zstd_buffers 2147483647 1024m;\n {ack}", { "want_pass": True, "want_warning": True, "want_hard_cap_ack": True, "min_total": 2305843008139952128, }, ), ( "10 product overflow, ack ALSO set -> still REFUSED", "", f"zstd_buffers {huge} {huge};\n {ack}", { "want_pass": False, "want_warning": False, "want_overflow_error": True, }, ), ( "11 inherited hard-cap breach + ack from http block", f"{ack}\n zstd_buffers 1 268435457;", "", { "want_pass": True, "want_warning": True, "want_hard_cap_ack": True, "min_total": 268435457, }, ), ( "12 explicit 16 MB value warns once, not twice", "", "zstd_buffers 1 16777217;", { "want_pass": True, "want_warning": True, "min_total": 16777217, "max_warn_count": 1, }, ), ] failures = [] print("zstd_buffers aggregate bound policy matrix:") for i, (name, http_directives, srv_directives, spec) in enumerate(matrix): f, summary = run_arm( nginx, module, args.port + i, name, http_directives, srv_directives, spec ) failures += f print(summary) if failures: print("\nFAIL: zstd_buffers bound policy", file=sys.stderr) for f in failures: print(f" - {f}", file=sys.stderr) return 1 print(f"\nOK: zstd_buffers bound policy ({len(matrix)}/{len(matrix)} arms)") return 0 if __name__ == "__main__": sys.exit(main()) myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_build_config_policy.sh000077500000000000000000000212711525232545500271750ustar00rootroot00000000000000#!/bin/sh set -eu root=$(cd "$(dirname "$0")/../.." && pwd) # The advisory must be a real compiler decision, not parsed -E output. grep -q 'ngx_feature="libzstd 1.5.6 or newer"' "$root/auto/zstd" if grep -q -- '-x c -E -' "$root/auto/zstd"; then exit 1 fi # Static OpenSSL integration is optional even when nginx requested OpenSSL: # headers/functions must be compile-probed before the macro is defined. grep -q 'ngx_feature="OpenSSL EVP SHA-256 headers"' "$root/filter/config" grep -Fq 'ngx_feature_path="$CORE_INCS"' "$root/filter/config" grep -q 'OPENSSL/.openssl/include' "$root/filter/config" grep -Fq '${USE_OPENSSL:-NO}' "$root/filter/config" if grep -Fq '[ "$USE_OPENSSL" = YES -a ' "$root/filter/config"; then exit 1 fi "$root/ci/tools/test_evp_prepared_tree_probe.sh" # Exercise the authoritative reorder helper and its hard-failure control. grep -Fq '. "$_ngx_zstd_root/filter/reorder-static.sh"' "$root/filter/config" # The next= anchor must come from the shared helper, not an inline copy. grep -Fq 'next=`ngx_http_zstd_static_next`' "$root/filter/config" if grep -q 'next=ngx_http_brotli_filter_module' "$root/filter/config"; then echo 'filter/config still inlines the next= matrix; it must use ngx_http_zstd_static_next' >&2 exit 1 fi . "$root/filter/reorder-static.sh" # Consumed by the sourced helper. # shellcheck disable=SC2034 ngx_module_name=ngx_http_zstd_filter_module # shellcheck disable=SC2034 next=ngx_http_gzip_filter_module HTTP_FILTER_MODULES='ngx_http_gzip_filter_module ngx_http_range_header_filter_module' ngx_http_zstd_reorder_static_filter [ "$HTTP_FILTER_MODULES" = 'ngx_http_gzip_filter_module ngx_http_zstd_filter_module ngx_http_range_header_filter_module' ] HTTP_FILTER_MODULES=ngx_http_range_header_filter_module if ngx_http_zstd_reorder_static_filter >/dev/null 2>&1; then echo 'absent reorder anchor unexpectedly succeeded' >&2 exit 1 fi # --------------------------------------------------------------------------- # Cross-mode equivalence: dynamic ngx_module_order vs. static reorder-static.sh # # nginx idiom (verified against src/core/ngx_module.c:ngx_add_module and every # *_filter_module.c postconfiguration): each filter's postconfiguration does # next = ngx_http_top_body_filter; ngx_http_top_body_filter = this_filter; # so modules are chained in cycle->modules[] init order, and the LAST filter # *initialized* is the FIRST filter to *run* at request time. # # For a DYNAMIC module, ngx_add_module() walks its declared ngx_module_order # starting just after the module's own name, and inserts the module in # cycle->modules[] immediately before the first *already-loaded* module named # in the remainder of that list ("before" stays cycle->modules_n, i.e. append # at the end, if none of the remaining names are loaded yet). # # zstd's ngx_module_order (filter/config:111-127) starts with: # ngx_http_zstd_filter_module, ngx_pagespeed, # ngx_http_postpone_filter_module, ... # It never names ngx_http_gzip_filter_module. But nginx's own stock # HTTP_FILTER_MODULES order (auto/modules) always registers gzip (when # HTTP_GZIP=YES) strictly before ngx_http_postpone_filter_module. So zstd's # insertion point search lands on "postpone_filter_module" and inserts zstd # immediately before it -- i.e. zstd is *initialized after* gzip whenever # gzip is present, with no direct comparison ever performed. Combined with # the last-initialized-runs-first idiom above, this means: dynamic mode # initializes zstd after gzip, so zstd RUNS BEFORE gzip at request time. # # The static path achieves the identical *runtime* relationship by a # different mechanism: reorder-static.sh splices zstd immediately AFTER # gzip in HTTP_FILTER_MODULES (the static init-order array), which by the # same last-initialized-runs-first idiom also means zstd RUNS BEFORE gzip. # # Scope limit: the two checks below are PREMISE GUARDS, not a derivation of # the dynamic insertion index. ngx_add_module() inserts at the lowest index # among loaded modules named in order[], so postpone's presence alone does not # by itself fix the position; fully deriving it would mean modelling the module # loader here. What these guards do assert is that the two facts the argument # above rests on -- gzip absent from the list, postpone present as the anchor -- # still hold, so the argument is re-checked rather than assumed on every run. # # Property under test: both paths initialize zstd strictly after gzip in # their respective init-order arrays (dynamic: implicitly, via the shared # postpone anchor; static: explicitly, via reorder-static.sh's splice # target), so both agree that zstd is the codec that wins when a client # accepts both encodings. This test locks that agreement; it does not # claim the ordering is "correct" in any absolute sense. # --- dynamic path: parse ngx_module_order out of filter/config ------------- ngx_dyn_order=$(sed -n '/^ngx_module_order="\$ngx_module_name \\$/,/"$/p' \ "$root/filter/config" | tr -d '\\"' | tr -s ' \t\n' ' ') case " $ngx_dyn_order " in *' ngx_http_gzip_filter_module '*) echo 'FAIL: dynamic ngx_module_order now names gzip directly -- re-derive the cross-mode property, do not assume it still holds' >&2 exit 1 ;; esac case " $ngx_dyn_order " in *' ngx_http_postpone_filter_module '*) ;; *) echo 'FAIL: dynamic ngx_module_order no longer anchors on postpone_filter_module -- the gzip-precedes-postpone argument no longer applies' >&2 exit 1 ;; esac # --- static path: exercise the full next= decision matrix ------------------ # Mirrors filter/config:132-140 verbatim so drift in that matrix is caught. # No local reimplementation of the next= matrix: a test that copies the policy # it checks cannot catch drift in the original. Call the same # ngx_http_zstd_static_next() that filter/config uses in production. zstd_static_next() { # $1 = HTTP_FILTER_MODULES, $2 = HTTP_GZIP HTTP_FILTER_MODULES=$1 # Consumed by the sourced reorder helper. # shellcheck disable=SC2034 HTTP_GZIP=$2 ngx_http_zstd_static_next } assert_static_case() { # $1=label $2=HTTP_FILTER_MODULES $3=HTTP_GZIP $4=expected anchor _label=$1 _hfm=$2 _gz=$3 _expect=$4 _got=$(zstd_static_next "$_hfm" "$_gz") if [ "$_got" != "$_expect" ]; then echo "FAIL: $_label: next= matrix gave '$_got', expected '$_expect'" >&2 exit 1 fi # Consumed by the sourced helper. # shellcheck disable=SC2034 ngx_module_name=ngx_http_zstd_filter_module # shellcheck disable=SC2034 next=$_got HTTP_FILTER_MODULES=$_hfm ngx_http_zstd_reorder_static_filter case " $HTTP_FILTER_MODULES " in *" $_expect ngx_http_zstd_filter_module "*) ;; *) echo "FAIL: $_label: zstd not spliced immediately after $_expect (got: $HTTP_FILTER_MODULES)" >&2 exit 1 ;; esac } # 1. brotli present -> zstd goes after brotli, regardless of HTTP_GZIP. assert_static_case brotli-present \ 'ngx_http_brotli_filter_module ngx_http_gzip_filter_module ngx_http_range_header_filter_module' \ YES ngx_http_brotli_filter_module # 2. no brotli, HTTP_GZIP=YES -> zstd goes after gzip. assert_static_case gzip-yes \ 'ngx_http_gzip_filter_module ngx_http_range_header_filter_module' \ YES ngx_http_gzip_filter_module # 3. no brotli, HTTP_GZIP!=YES, pagespeed_etag present -> zstd goes after it. assert_static_case pagespeed-etag-fallback \ 'ngx_pagespeed_etag_filter ngx_http_range_header_filter_module' \ NO ngx_pagespeed_etag_filter # 4. no brotli, no gzip, no pagespeed -> fallback anchor is range_header. assert_static_case bare-fallback \ 'ngx_http_range_header_filter_module' \ NO ngx_http_range_header_filter_module # --- negative control: prove the assertion above can go red ---------------- # Deliberately assert the WRONG expected anchor for the gzip-yes case (the # stale pre-brotli fallback, range_header, instead of the correct gzip) and # require assert_static_case to fail closed. Run in a subshell so its `exit 1` # does not abort this script; capture stderr to show the real failure text. _neg_status=0 _neg_out=$( assert_static_case gzip-yes-WRONG-on-purpose \ 'ngx_http_gzip_filter_module ngx_http_range_header_filter_module' \ YES ngx_http_range_header_filter_module 2>&1 ) || _neg_status=$? if [ "$_neg_status" -eq 0 ]; then echo 'FAIL: negative control did not go red (deliberately wrong anchor was accepted)' >&2 exit 1 fi case "$_neg_out" in *"next= matrix gave 'ngx_http_gzip_filter_module', expected 'ngx_http_range_header_filter_module'"*) ;; *) echo "FAIL: negative control failed for the wrong reason: $_neg_out" >&2 exit 1 ;; esac echo "negative control observed red: $_neg_out" echo 'build config policy: PASS' myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_build_input_manifest.sh000077500000000000000000000114141525232545500273740ustar00rootroot00000000000000#!/usr/bin/env bash # Regression coverage for the canonical persistent-build input manifest. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" MODULE_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")" HASHER="$SCRIPT_DIR/build-input-hash.sh" WORK="$(mktemp -d)" trap 'rm -rf "$WORK"' EXIT # This list is deliberately independent of the manifest under test. Without a # separate oracle, deleting an input from the manifest also deletes its test. required_inputs=( ci/build-inputs.manifest ci/tools/build-input-hash.sh ci/tools/ci-build.sh config auto/zstd filter/config filter/reorder-static.sh static/config src ) for required in "${required_inputs[@]}"; do if ! grep -Fxq "$required" "$MODULE_DIR/ci/build-inputs.manifest"; then echo "FAIL: required build input missing from manifest: $required" >&2 exit 1 fi done copy_inputs() { local input mkdir -p "$WORK/module" while IFS= read -r input || [ -n "$input" ]; do case "$input" in '' | \#*) continue ;; esac mkdir -p "$WORK/module/$(dirname "$input")" cp -a "$MODULE_DIR/$input" "$WORK/module/$input" done < "$MODULE_DIR/ci/build-inputs.manifest" } assert_mutation_changes_hash() { local input="$1" local file="$WORK/module/$input" local relative="$input" local before after if [ -d "$file" ]; then file="$(find "$file" -type f -print -quit)" relative="${file#"$WORK/module/"}" fi before="$(bash "$HASHER" "$WORK/module")" printf '\n# manifest regression mutation\n' >> "$file" after="$(bash "$HASHER" "$WORK/module")" if [ "$before" = "$after" ]; then echo "FAIL: $input mutation did not change build-input hash" >&2 exit 1 fi cp -a "$MODULE_DIR/$relative" "$WORK/module/$relative" } copy_inputs while IFS= read -r input || [ -n "$input" ]; do case "$input" in '' | \#*) continue ;; esac assert_mutation_changes_hash "$input" done < "$MODULE_DIR/ci/build-inputs.manifest" before="$(bash "$HASHER" "$WORK/module")" printf 'documentation only\n' > "$WORK/module/README.md" after="$(bash "$HASHER" "$WORK/module")" if [ "$before" != "$after" ]; then echo "FAIL: docs-only mutation changed build-input hash" >&2 exit 1 fi for specification in \ '.github/workflows/build-test.yml:1' \ '.github/workflows/ci-deep.yml:2'; do workflow="${specification%:*}" expected="${specification##*:}" actual="$(grep -c 'key: .*steps.build-inputs.outputs.hash' \ "$MODULE_DIR/$workflow" || true)" if [ "$actual" -ne "$expected" ]; then echo "FAIL: $workflow uses the canonical hash in $actual/$expected cache keys" >&2 exit 1 fi done echo "✓ every manifest fragment invalidates build caches; docs-only changes do not" # Exercise ci-build.sh itself with a tiny offline server tarball. The fake # Makefile records a compile/link input in the addon object, making a stale # tree directly observable without downloading and compiling nginx. integration="$WORK/integration" module="$integration/module" root="$integration/build" source="$integration/angie-9.9.9" mkdir -p "$module" "$root" "$source" cp -a "$WORK/module/." "$module/" cat > "$source/configure" <<'EOF' #!/usr/bin/env bash set -euo pipefail for argument in "$@"; do case "$argument" in --add-dynamic-module=*) module="${argument#*=}" ;; esac done : "${module:?missing addon module path}" cat > Makefile < objs/ngx_http_zstd_filter_module.o cp objs/ngx_http_zstd_filter_module.o objs/ngx_http_zstd_filter_module.so touch objs/ngx_http_zstd_static_module.so objs/angie MAKEFILE EOF chmod +x "$source/configure" tar -czf "$root/angie-9.9.9.tar.gz" -C "$integration" angie-9.9.9 rm -rf "$source" digest="$(sha256sum "$root/angie-9.9.9.tar.gz" | cut -d' ' -f1)" sed -i "/declare -A ANGIE_SHA256=(/a\\ [\"9.9.9\"]=\"$digest\"" \ "$module/ci/tools/ci-build.sh" BUILD_ROOT="$root" bash "$module/ci/tools/ci-build.sh" angie 9.9.9 coverage \ > "$integration/first.log" object="$root/angie-9.9.9-coverage/objs/ngx_http_zstd_filter_module.o" first_object="$(cat "$object")" printf '\n# compile/link mutation\n' >> "$module/filter/config" BUILD_ROOT="$root" bash "$module/ci/tools/ci-build.sh" angie 9.9.9 coverage \ > "$integration/second.log" second_object="$(cat "$object")" if [ "$first_object" = "$second_object" ]; then echo "FAIL: compile/link-input mutation reused the stale addon object" >&2 exit 1 fi if ! grep -q 'Build inputs changed.*rebuilding from scratch' \ "$integration/second.log"; then echo "FAIL: compile/link-input mutation did not invalidate the build tree" >&2 exit 1 fi echo "✓ compile/link-input mutation rebuilds the addon object" myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_build_windows_inputs.sh000077500000000000000000000104541525232545500274460ustar00rootroot00000000000000#!/usr/bin/env bash set -euo pipefail root=$(cd "$(dirname "$0")/../.." && pwd) source_script=$root/ci/tools/build-windows.sh tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT HUP INT TERM toggle_block=$(awk ' /^### module toggles/ { copy = 1 } /^# OpenSSL build options/ { exit } copy { print } ' "$source_script") grep -q 'BUILDLIB_ZSTD=' <<<"$toggle_block" pins=$tmp/pins.sh for name in VER_NGINX SHA_NGINX VER_PCRE2 SHA_PCRE2 VER_OPENSSL SHA_OPENSSL \ VER_ZLIB SHA_ZLIB VER_NASM SHA_NASM VER_ZSTD SHA_ZSTD; do printf '%s=x\n' "$name" >>"$pins" done resolve() { env -i PATH="$PATH" PINS_FILE="$pins" "$@" bash -c " set -euo pipefail $toggle_block printf '%s %s %s %s %s %s\\n' \"\$WITH_ZSTD\" \"\$WITH_BROTLI\" \ \"\$WITH_COMPRESSION\" \"\$WITH_HEADERS_MORE\" \ \"\$BUILDLIB_ZSTD\" \"\$BUILDLIB_BROTLI\" " "$source_script" } [ "$(resolve)" = '1 0 0 1 1 0' ] [ "$(resolve WITH_ZSTD=0 WITH_COMPRESSION=1 WITH_HEADERS_MORE=0)" = \ '0 0 1 0 1 1' ] [ "$(resolve WITH_ZSTD=0 WITH_COMPRESSION=1 BUILDLIB_ZSTD=0 BUILDLIB_BROTLI=0)" = \ '0 0 1 1 0 0' ] if resolve WITH_ZSTD=1 BUILDLIB_ZSTD=0 >/dev/null 2>&1; then echo 'standalone zstd accepted BUILDLIB_ZSTD=0' >&2 exit 1 fi if resolve WITH_ZSTD=maybe >/dev/null 2>&1; then echo 'invalid module toggle was accepted' >&2 exit 1 fi if PINS_FILE="$tmp/missing" bash -c "set -euo pipefail; $toggle_block" \ "$source_script" >/dev/null 2>&1; then echo 'missing pins file was accepted' >&2 exit 1 fi function_block=$( sed -n '/^verify() {/,/^}/p' "$source_script" sed -n '/^extract() {/,/^}/p' "$source_script" sed -n '/^extract_complete() {/,/^}/p' "$source_script" ) grep -q '^extract_complete()' <<<"$function_block" eval "$function_block" DIR_DL=$tmp/dl mkdir -p "$DIR_DL" payload=$tmp/payload printf 'verified payload\n' >"$payload" payload_sha=$(sha256sum "$payload" | cut -d' ' -f1) curl_mode=success curl() { local output [ "$1" = -fsSL ] [ "$2" = -o ] output=$3 if [ "$curl_mode" = wrong ]; then printf 'wrong payload\n' >"$output" else cp "$payload" "$output" fi [ "$curl_mode" != fail ] } export -f curl export payload curl_mode verify https://example.invalid/archive.tar.gz "$payload_sha" >/dev/null [ -f "$DIR_DL/archive.tar.gz" ] [ ! -e "$DIR_DL/archive.tar.gz.part" ] # A completed download with the wrong digest remains a partial file. This # catches promotion before verification; the following good retry recovers. unlink "$DIR_DL/archive.tar.gz" curl_mode=wrong export curl_mode set +e ( set -e verify https://example.invalid/archive.tar.gz "$payload_sha" ) \ >/dev/null 2>&1 status=$? set -e if [ "$status" -eq 0 ]; then echo 'wrong payload unexpectedly verified' >&2 exit 1 fi [ -f "$DIR_DL/archive.tar.gz.part" ] [ ! -e "$DIR_DL/archive.tar.gz" ] curl_mode=success export curl_mode verify https://example.invalid/archive.tar.gz "$payload_sha" >/dev/null [ -f "$DIR_DL/archive.tar.gz" ] [ ! -e "$DIR_DL/archive.tar.gz.part" ] # A failed transfer is never promoted; a retry overwrites the partial file. unlink "$DIR_DL/archive.tar.gz" curl_mode=fail export curl_mode set +e ( set -e verify https://example.invalid/archive.tar.gz "$payload_sha" ) \ >/dev/null 2>&1 status=$? set -e if [ "$status" -eq 0 ]; then echo 'failed transfer unexpectedly verified' >&2 exit 1 fi [ -f "$DIR_DL/archive.tar.gz.part" ] [ ! -e "$DIR_DL/archive.tar.gz" ] curl_mode=success export curl_mode verify https://example.invalid/archive.tar.gz "$payload_sha" >/dev/null [ -f "$DIR_DL/archive.tar.gz" ] [ ! -e "$DIR_DL/archive.tar.gz.part" ] # A partial extraction is repaired until the completion marker is present. archive_root=$tmp/archive-root mkdir -p "$archive_root/tree" printf 'complete\n' >"$archive_root/tree/member" tar -C "$archive_root" -czf "$DIR_DL/tree.tar.gz" tree cd "$tmp" mkdir -p tree printf 'partial\n' >tree/member printf 'preserve\n' >tree/nonmember extract_complete tree tree.tar.gz [ "$(cat tree/member)" = complete ] [ "$(cat tree/nonmember)" = preserve ] [ -f tree/.extracted ] printf 'kept\n' >tree/member extract_complete tree tree.tar.gz [ "$(cat tree/member)" = kept ] unlink tree/.extracted extract_complete tree tree.tar.gz [ "$(cat tree/member)" = complete ] [ "$(cat tree/nonmember)" = preserve ] echo 'Windows build inputs/cache/extraction policy: PASS' myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_build_windows_openssl_recovery.sh000077500000000000000000000042121525232545500315200ustar00rootroot00000000000000#!/usr/bin/env bash set -euo pipefail root=$(cd "$(dirname "$0")/../.." && pwd) source_script=$root/ci/tools/build-windows.sh tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT HUP INT TERM block=$(awk ' /^OPENSSL_INSTALL_MARKER=/ { copy = 1 } copy { print } copy && /^fi$/ { exit } ' "$source_script") grep -q 'OPENSSL_INSTALL_MARKER=' <<<"$block" grep -q 'nmake install_sw' <<<"$block" VER_OPENSSL=fixture OPENSSL_OPT=no-shared export VER_OPENSSL OPENSSL_OPT tree=$tmp/objs/lib/openssl-$VER_OPENSSL mkdir -p "$tree" log=$tmp/nmake.log export log perl() { :; } cygpath() { printf '%s\n' "$2"; } nmake() { printf '%s\n' "${1:-build}" >>"$log" if [ "${1:-}" = install_sw ]; then mkdir -p "$PWD/openssl/include/openssl" : >"$PWD/openssl/include/openssl/evp.h" if [ "${FAIL_INSTALL:-0}" = 1 ]; then return 1 fi mkdir -p "$PWD/openssl/lib" : >"$PWD/openssl/lib/libssl.lib" : >"$PWD/openssl/lib/libcrypto.lib" fi } export -f perl cygpath nmake cd "$tmp" FAIL_INSTALL=1 export FAIL_INSTALL set +e ( set -e eval "$block" ) status=$? set -e if [ "$status" -eq 0 ]; then echo 'interrupted OpenSSL install unexpectedly succeeded' >&2 exit 1 fi marker=$tree/openssl/.install-complete [ -f "$tree/openssl/include/openssl/evp.h" ] [ ! -e "$marker" ] [ "$(grep -c '^install_sw$' "$log")" -eq 1 ] FAIL_INSTALL=0 export FAIL_INSTALL eval "$block" [ -f "$marker" ] [ -f "$tree/openssl/lib/libssl.lib" ] [ -f "$tree/openssl/lib/libcrypto.lib" ] [ "$(grep -c '^install_sw$' "$log")" -eq 2 ] # A completed install is restartable and must not invoke nmake again. eval "$block" [ "$(grep -c '^install_sw$' "$log")" -eq 2 ] # Mutation control: the old evp.h guard would accept the partial first # install and fail to create the completion marker on retry. unlink "$marker" mutant=${block/\[ ! -f \"\$OPENSSL_INSTALL_MARKER\" \]/[ ! -f \"objs\/lib\/openssl-\$VER_OPENSSL\/openssl\/include\/openssl\/evp.h\" ]} eval "$mutant" if [ -e "$marker" ]; then echo 'evp.h-guard mutation unexpectedly repaired the partial install' >&2 exit 1 fi echo 'Windows OpenSSL interrupted-install recovery: PASS' myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_build_windows_zstd_default.sh000077500000000000000000000076651525232545500306260ustar00rootroot00000000000000#!/usr/bin/env bash # # Regression test for audit A30-F3: ci/tools/build-windows.sh defaulted # REF_ZSTD_MODULE to a fixed historical commit (~154 commits behind at audit # time) even though README.md presents this script alongside CURRENT # behaviour, so the documented default silently built a stale module. # # The fix resolves REPO_ZSTD_MODULE/REF_ZSTD_MODULE from the running # script's own checkout (HEAD) by default, keeping the old fixed commit only # as a fallback for a standalone copy of the script outside any checkout, # and keeps the REPO_ZSTD_MODULE/REF_ZSTD_MODULE overrides fully working. # # This cannot run the real script end-to-end (no MSVC / Windows toolchain # here), so it isolates the default-resolution block -- the part of # build-windows.sh between "### version + SHA-256 pins" and # "# Download + verify" -- and drives it standalone with git stubs, which is # testable without a Windows runtime. # # Usage: ci/tools/test_build_windows_zstd_default.sh set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" BUILD_WINDOWS_SH="$SCRIPT_DIR/build-windows.sh" fail=0 ok() { printf 'OK: %s\n' "$*"; } bad() { printf 'FAIL: %s\n' "$*" fail=1 } # extract_resolution_block -- pulls the REPO_ZSTD_MODULE/REF_ZSTD_MODULE # default-resolution logic (and its _zstd_module_repo_root helper) out of # build-windows.sh, verbatim, between two anchor comments. If the anchors # ever drift this fails loudly rather than silently testing stale logic. extract_resolution_block() { awk ' /^# Module refs/ { grab=1 } grab { print } /^fi$/ && grab { exit } ' "$BUILD_WINDOWS_SH" } BLOCK="$(extract_resolution_block)" if ! grep -q '_zstd_module_repo_root' <<<"$BLOCK"; then echo "FATAL: could not extract the zstd-module default-resolution block from $BUILD_WINDOWS_SH -- anchors drifted" >&2 exit 2 fi echo "== case: inside a checkout, no override -> defaults to HEAD of that checkout ==" CKROOT="$(mktemp -d)" ( cd "$CKROOT" git init -q git config user.email test@example.com git config user.name test mkdir -p ci/tools cp "$BUILD_WINDOWS_SH" ci/tools/build-windows.sh git add -A git commit -q -m init ) HEAD_SHA="$(git -C "$CKROOT" rev-parse HEAD)" unset REPO_ZSTD_MODULE REF_ZSTD_MODULE OUT="$(bash -c ' set -euo pipefail '"$BLOCK"' echo "REPO=$REPO_ZSTD_MODULE" echo "REF=$REF_ZSTD_MODULE" ' "$CKROOT/ci/tools/build-windows.sh")" if grep -q "REPO=$CKROOT" <<<"$OUT" && grep -q "REF=$HEAD_SHA" <<<"$OUT"; then ok "in-checkout default resolves REPO to the checkout and REF to its HEAD ($HEAD_SHA)" else bad "in-checkout default did not resolve to the checkout's HEAD: $OUT" fi echo "== case: explicit REPO_ZSTD_MODULE/REF_ZSTD_MODULE override wins ==" OUT="$(REPO_ZSTD_MODULE="https://example.invalid/fork.git" REF_ZSTD_MODULE="deadbeef" bash -c ' set -euo pipefail '"$BLOCK"' echo "REPO=$REPO_ZSTD_MODULE" echo "REF=$REF_ZSTD_MODULE" ' "$CKROOT/ci/tools/build-windows.sh")" if grep -q "REPO=https://example.invalid/fork.git" <<<"$OUT" && grep -q "REF=deadbeef" <<<"$OUT"; then ok "explicit override is preserved, default resolution does not clobber it" else bad "override was not preserved: $OUT" fi echo "== case: outside any checkout -> falls back to the documented pin ==" OUTSIDE="$(mktemp -d)" unset REPO_ZSTD_MODULE REF_ZSTD_MODULE OUT="$(bash -c ' set -euo pipefail '"$BLOCK"' echo "REPO=$REPO_ZSTD_MODULE" echo "REF=$REF_ZSTD_MODULE" ' "$OUTSIDE/build-windows.sh")" if grep -q "REPO=https://github.com/myguard-labs/nginx-zstd-module.git" <<<"$OUT" \ && grep -q "REF=37cf9ac6b58284ae2da95620f4905930d1277b54" <<<"$OUT"; then ok "outside a checkout, falls back to the documented fixed pin" else bad "fallback pin did not apply outside a checkout: $OUT" fi rm -rf "$CKROOT" "$OUTSIDE" if [ "$fail" -eq 0 ]; then echo "all build-windows.sh zstd-module default cases pass" else echo "one or more build-windows.sh zstd-module default cases FAILED" fi exit "$fail" myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_bump_versions.sh000077500000000000000000001071201525232545500260630ustar00rootroot00000000000000#!/usr/bin/env bash # # Hermetic regression test for ci/tools/bump-versions.sh. # # Audit A30-F4 first: the script used stale pre-move paths (tools/keys/, # tools/ci-build.sh) that do not exist after the tools/ -> ci/tools/ move, # so a real bump could not import the PGP keyring, and a matrix/pin # replacement that matched nothing failed silently, leaving a partial edit. # # Then the pins the weekly bump did not know about: the harness jobs' # static mainline pin (HARNESS_NGINX_VERSION, in two workflows) and the # Windows build's sources (ci/tools/windows-pins.sh) were bumped by hand or # not at all. The cases below cover every pin the script rewrites, the # forward-only rule for release-API feeds, the fail-loud / no-partial-edit # rule for every mutator, the resolve/apply split (every digest fetched # before any file is touched, so a dead mirror part-way edits nothing), and # the staged apply (mutators edit copies, installed together, so a mutator # failing on the second file leaves every tracked file unchanged). # # This test builds a scratch copy of the repo layout the script actually # needs (ci/tools/ci-build.sh, ci/tools/keys/*.key, ci/tools/windows-pins.sh, # .github/workflows/{ci-deep,harness-fault-arms}.yml) and stubs curl/gpg on # PATH so no network is used. Each case name is asserted individually. # # Usage: ci/tools/test_bump_versions.sh set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" BUMP_SCRIPT="$REPO_ROOT/ci/tools/bump-versions.sh" fail=0 SANDBOXES=() cleanup_sandboxes() { local dir for dir in "${SANDBOXES[@]}"; do rm -rf -- "$dir" done } trap cleanup_sandboxes EXIT say() { printf '%s\n' "$*"; } ok() { printf 'OK: %s\n' "$*"; } bad() { printf 'FAIL: %s\n' "$*" fail=1 } # The sandbox's current pins. Every stubbed feed answers these unless a case # overrides it, so a case changes exactly the pins it names. CUR_STABLE=1.30.4 CUR_MAINLINE=1.31.4 CUR_ANGIE=1.12.1 CUR_PCRE2=10.44 CUR_ZLIB=1.3.1 CUR_OPENSSL=4.0.1 CUR_ZSTD=1.5.7 # make_sandbox VARNAME -- creates a scratch repo layout under a temp dir and # assigns its path to VARNAME. make_sandbox() { # shellcheck disable=SC2034 # nameref out-parameter, assigned for the caller local -n out="$1" local dir dir="$(mktemp -d "${TMPDIR:-/tmp}/http-zstd-bump-test.XXXXXX")" SANDBOXES+=("$dir") mkdir -p "$dir/ci/tools/keys" "$dir/.github/workflows" cat >"$dir/ci/tools/keys/dummy.key" <<'EOF' dummy keyring placeholder EOF cat >"$dir/ci/tools/ci-build.sh" <<'EOF' #!/bin/bash SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" declare -A ANGIE_SHA256=( ["1.12.1"]="aaaa" ) declare -A NGINX_SHA256=( ["1.30.4"]="bbbb" ) KEYRING_DIR="$SCRIPT_DIR/keys" EOF cat >"$dir/ci/tools/windows-pins.sh" <<'EOF' # sandbox copy of ci/tools/windows-pins.sh -- data only # shellcheck shell=sh disable=SC2034 VER_NGINX=1.31.4 SHA_NGINX=cccc1 VER_PCRE2=10.44 SHA_PCRE2=cccc2 VER_OPENSSL=4.0.1 SHA_OPENSSL=cccc3 VER_ZLIB=1.3.1 SHA_ZLIB=cccc4 VER_NASM=3.02 SHA_NASM=cccc5 VER_ZSTD=1.5.7 SHA_ZSTD=cccc6 EOF cat >"$dir/.github/workflows/ci-deep.yml" <<'EOF' env: NGINX_VERSION: "" HARNESS_NGINX_VERSION: "1.31.4" jobs: build-flavors: strategy: matrix: include: - version: "1.31.2" label: mainline - version: "1.30.4" label: stable - version: "1.12.1" label: angie EOF cat >"$dir/.github/workflows/harness-fault-arms.yml" <<'EOF' env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true HARNESS_NGINX_VERSION: "1.31.4" EOF cp "$BUMP_SCRIPT" "$dir/ci/tools/bump-versions.sh" cp "$REPO_ROOT/ci/tools/nginx-releases.sh" "$dir/ci/tools/nginx-releases.sh" # shellcheck disable=SC2034 # nameref out-parameter, read by the caller out="$dir" } # stub_bin DIR -- creates PATH-shadowing curl/gpg stubs and prints the bin # dir to prepend to PATH. Reads NEW_* env for what each feed's "latest" # should resolve to; any not set answers the sandbox's current pin. FAIL_URL, # when set, makes every fetch whose URL contains it fail like a dead mirror. stub_bin() { local dir="$1" bindir="$1/bin" mkdir -p "$bindir" cat >"$bindir/curl" <>"\$CURL_ARGV_LOG" url="" out="" prev="" fail_url="${FAIL_URL:-}" nginx_feed="${NGINX_FEED:-json}" for a in "\$@"; do case "\$a" in http*) url="\$a" ;; esac [ "\$prev" = "-o" ] && out="\$a" [ "\$prev" = "--config" ] && [ "\$a" = "-" ] && cat >>"\${CURL_HEADER_LOG:-/dev/null}" prev="\$a" done if [ -n "\$fail_url" ]; then case "\$url" in *"\$fail_url"*) echo "curl: (22) stub: \$url unreachable" >&2; exit 22 ;; esac fi case "\$url" in *api.github.com/repos/nginx/nginx/releases?per_page=30\&page=*) page="\${url##*page=}" # The feed as GitHub serves it, plus decoys every case must ignore: # a legacy line listed FIRST (order must not matter), a draft and a # prerelease newer than everything, and a tag of another shape. # A full page (30 entries) tells the script there may be more. full_page_of_mainline() { # 30 odd-minor entries older than the mainline pin local i sep="" echo "[" for i in \$(seq 1 30); do echo " \${sep}{\"tag_name\": \"release-1.29.\$i\", \"draft\": false, \"prerelease\": false}" sep="," done echo "]" } case "\$nginx_feed" in html) echo "Mainline version nginx-${NEW_MAINLINE:-$CUR_MAINLINE}.tar.gz" ;; no-even) echo "[{\"tag_name\": \"release-${NEW_MAINLINE:-$CUR_MAINLINE}\", \"draft\": false, \"prerelease\": false}]" ;; malformed-only) echo '[{"tag_name": "release-9.98.1", "prerelease": false},' echo ' {"tag_name": "release-9.98.2", "draft": 0, "prerelease": false},' echo ' {"tag_name": "release-9.98.3", "draft": "false", "prerelease": false},' echo ' {"tag_name": "release-9.98.4", "draft": null, "prerelease": false},' echo ' {"tag_name": "release-9.98.5", "draft": false},' echo ' {"tag_name": "release-9.98.6", "draft": false, "prerelease": 0},' echo ' {"tag_name": "release-9.98.7", "draft": false, "prerelease": "false"},' echo ' {"tag_name": "release-9.98.8", "draft": false, "prerelease": null}]' ;; paged) # Stable only appears on page two, behind a full page of mainline. if [ "\$page" = 1 ]; then echo "[{\"tag_name\": \"release-${NEW_MAINLINE:-$CUR_MAINLINE}\", \"draft\": false, \"prerelease\": false}," for i in \$(seq 1 28); do echo " {\"tag_name\": \"release-1.29.\$i\", \"draft\": false, \"prerelease\": false}," done echo " {\"tag_name\": \"release-1.29.30\", \"draft\": false, \"prerelease\": false}]" else echo "[{\"tag_name\": \"release-1.28.0\", \"draft\": false, \"prerelease\": false}," echo " {\"tag_name\": \"release-${NEW_STABLE:-$CUR_STABLE}\", \"draft\": false, \"prerelease\": false}]" fi ;; legacy-first) # Page one is full and carries a recently created LEGACY even # release beside the mainline; the newer stable line is on # page two. Stopping at "both parities seen" would pin the # legacy release as stable. if [ "\$page" = 1 ]; then echo "[{\"tag_name\": \"release-${NEW_MAINLINE:-$CUR_MAINLINE}\", \"draft\": false, \"prerelease\": false}," echo " {\"tag_name\": \"release-1.28.9\", \"draft\": false, \"prerelease\": false}," for i in \$(seq 1 27); do echo " {\"tag_name\": \"release-1.29.\$i\", \"draft\": false, \"prerelease\": false}," done echo " {\"tag_name\": \"release-1.29.30\", \"draft\": false, \"prerelease\": false}]" else echo "[{\"tag_name\": \"release-${NEW_STABLE:-$CUR_STABLE}\", \"draft\": false, \"prerelease\": false}]" fi ;; endless) # Every page is full and never shows a stable: the cap must end it. full_page_of_mainline ;; capped) # Every page is full and carries BOTH parities, newer than the # pins: a read that stops at the cap must not pick from them. echo "[" for i in \$(seq 1 15); do echo " {\"tag_name\": \"release-1.30.\$((i + 4))\", \"draft\": false, \"prerelease\": false}," echo " {\"tag_name\": \"release-1.31.\$((i + 4))\", \"draft\": false, \"prerelease\": false}\$([ "\$i" -lt 15 ] && echo ,)" done echo "]" ;; *) echo "[{\"tag_name\": \"release-1.28.0\", \"draft\": false, \"prerelease\": false}," echo " {\"tag_name\": \"release-1.33.0\", \"draft\": true, \"prerelease\": false}," echo " {\"tag_name\": \"release-1.33.1\", \"draft\": false, \"prerelease\": true}," echo " {\"tag_name\": \"release-9.98.1\", \"prerelease\": false}," echo " {\"tag_name\": \"release-9.98.2\", \"draft\": 0, \"prerelease\": false}," echo " {\"tag_name\": \"release-9.98.3\", \"draft\": \"false\", \"prerelease\": false}," echo " {\"tag_name\": \"release-9.98.4\", \"draft\": null, \"prerelease\": false}," echo " {\"tag_name\": \"release-9.98.5\", \"draft\": false}," echo " {\"tag_name\": \"release-9.98.6\", \"draft\": false, \"prerelease\": 0}," echo " {\"tag_name\": \"release-9.98.7\", \"draft\": false, \"prerelease\": \"false\"}," echo " {\"tag_name\": \"release-9.98.8\", \"draft\": false, \"prerelease\": null}," echo " {\"tag_name\": \"v9.9.9\", \"draft\": false, \"prerelease\": false}," echo " {\"tag_name\": \"release-${NEW_STABLE:-$CUR_STABLE}\", \"draft\": false, \"prerelease\": false}," echo " {\"tag_name\": \"release-${NEW_MAINLINE:-$CUR_MAINLINE}\", \"draft\": false, \"prerelease\": false}]" ;; esac exit 0 ;; *api.github.com/repos/webserver-llc/angie/releases/latest*) echo "{\"tag_name\": \"${NEW_ANGIE:-$CUR_ANGIE}\"}" exit 0 ;; *api.github.com/repos/PCRE2Project/pcre2/releases/latest*) echo "{\"tag_name\": \"pcre2-${NEW_PCRE2:-$CUR_PCRE2}\"}" exit 0 ;; *api.github.com/repos/madler/zlib/releases/latest*) echo "{\"tag_name\": \"v${NEW_ZLIB:-$CUR_ZLIB}\"}" exit 0 ;; *api.github.com/repos/openssl/openssl/releases/latest*) echo "{\"tag_name\": \"openssl-${NEW_OPENSSL:-$CUR_OPENSSL}\"}" exit 0 ;; *api.github.com/repos/facebook/zstd/releases/latest*) echo "{\"tag_name\": \"v${NEW_ZSTD:-$CUR_ZSTD}\"}" exit 0 ;; esac # tarball / .asc fetch: write a stand-in whose bytes depend only on the # file name, so the same tarball from two hosts (zlib) hashes the same and # a case can predict the digest the script records. [ -n "\$out" ] && echo "dummy \${url##*/}" > "\$out" exit 0 EOF cat >"$bindir/gpg" <<'EOF' #!/bin/bash # Accept any --import / --verify as success (hermetic stub). exit 0 EOF chmod +x "$bindir/curl" "$bindir/gpg" if [ "${FAIL_SORT:-0}" = 1 ]; then cat >"$bindir/sort" <<'EOF' #!/bin/sh exit 42 EOF chmod +x "$bindir/sort" fi echo "$bindir" } # The digest the stub tarball for FILE hashes to. stub_sha() { printf 'dummy %s\n' "$1" | sha256sum | awk '{print $1}'; } # The files the updater may rewrite -- the one list every "edited nothing" # case compares against, so another tracked file is added here once and # every such case keeps proving what it claims. TRACKED_FIXTURES=( .github/workflows/ci-deep.yml .github/workflows/harness-fault-arms.yml ci/tools/ci-build.sh ci/tools/windows-pins.sh ) declare -A TRACKED_BEFORE=() snapshot_tracked() { # remember every tracked file of $SANDBOX local f TRACKED_BEFORE=() for f in "${TRACKED_FIXTURES[@]}"; do TRACKED_BEFORE[$f]="$(sha256sum "$SANDBOX/$f" | awk '{print $1}')" done } changed_tracked() { # prints the tracked files that differ from the snapshot local f out="" for f in "${TRACKED_FIXTURES[@]}"; do [ "${TRACKED_BEFORE[$f]}" = "$(sha256sum "$SANDBOX/$f" | awk '{print $1}')" ] || out="$out $f" done printf '%s' "$out" } # run_case NAME [NEW_X=value ...] -- runs bump-versions.sh in a fresh sandbox # with the stubbed feeds answering the given "latest" values and returns its # exit status; sandbox path is left in SANDBOX for the caller to inspect. run_case() { shift make_sandbox SANDBOX local bindir bindir="$( for kv in "$@"; do export "${kv?}"; done stub_bin "$SANDBOX" )" ( cd "$SANDBOX" PATH="$bindir:$PATH" bash ci/tools/bump-versions.sh >"$SANDBOX/out.log" 2>&1 ) return $? } pins_line() { grep -E "^$1=" "$SANDBOX/ci/tools/windows-pins.sh"; } harness_line() { grep -c 'HARNESS_NGINX_VERSION: "'"$1"'"' "$SANDBOX/.github/workflows/$2"; } say "== case: stable-only bump ==" if run_case stable-only NEW_STABLE=1.30.5; then if grep -q 'version: "1.30.5"' "$SANDBOX/.github/workflows/ci-deep.yml" \ && grep -q '\["1.30.5"\]' "$SANDBOX/ci/tools/ci-build.sh"; then ok "stable-only: matrix and sha256 pin both updated" else bad "stable-only: pin(s) missing after bump" fi else bad "stable-only: script exited non-zero unexpectedly: $(cat "$SANDBOX/out.log")" fi say "== case: angie-only bump ==" if run_case angie-only NEW_ANGIE=1.13.0; then if grep -q 'version: "1.13.0"' "$SANDBOX/.github/workflows/ci-deep.yml" \ && grep -q '\["1.13.0"\]' "$SANDBOX/ci/tools/ci-build.sh"; then ok "angie-only: matrix and sha256 pin both updated" else bad "angie-only: pin(s) missing after bump" fi else bad "angie-only: script exited non-zero unexpectedly: $(cat "$SANDBOX/out.log")" fi say "== case: both bump ==" if run_case both NEW_STABLE=1.30.5 NEW_ANGIE=1.13.0; then if grep -q 'version: "1.30.5"' "$SANDBOX/.github/workflows/ci-deep.yml" \ && grep -q 'version: "1.13.0"' "$SANDBOX/.github/workflows/ci-deep.yml" \ && grep -q '\["1.30.5"\]' "$SANDBOX/ci/tools/ci-build.sh" \ && grep -q '\["1.13.0"\]' "$SANDBOX/ci/tools/ci-build.sh"; then ok "both: matrix and sha256 pins all updated" else bad "both: pin(s) missing after bump" fi else bad "both: script exited non-zero unexpectedly: $(cat "$SANDBOX/out.log")" fi say "== case: no-change ==" if run_case no-change; then if grep -q 'CHANGED=0' "$SANDBOX/out.log"; then ok "no-change: script reports CHANGED=0 and exits 0" else bad "no-change: expected CHANGED=0, got: $(cat "$SANDBOX/out.log")" fi else bad "no-change: script exited non-zero unexpectedly: $(cat "$SANDBOX/out.log")" fi say "== case: mainline bump (harness pin in both workflows + Windows nginx pin; matrix entry untouched) ==" if run_case mainline NEW_MAINLINE=1.31.5; then if [ "$(harness_line 1.31.5 ci-deep.yml)" = 1 ] \ && [ "$(harness_line 1.31.5 harness-fault-arms.yml)" = 1 ] \ && [ "$(pins_line VER_NGINX)" = "VER_NGINX=1.31.5" ] \ && [ "$(pins_line SHA_NGINX)" = "SHA_NGINX=$(stub_sha nginx-1.31.5.tar.gz)" ] \ && grep -q 'version: "1.31.2"' "$SANDBOX/.github/workflows/ci-deep.yml" \ && grep -q 'version: "1.30.4"' "$SANDBOX/.github/workflows/ci-deep.yml" \ && ! grep -q '1.31.5' "$SANDBOX/ci/tools/ci-build.sh"; then ok "mainline: both harness pins and the Windows nginx pin+digest updated, matrix and ci-build.sh untouched" else bad "mainline: unexpected result: $(cat "$SANDBOX/out.log")" fi else bad "mainline: script exited non-zero unexpectedly: $(cat "$SANDBOX/out.log")" fi say "== case: windows library bumps (pcre2, zlib, openssl, zstd) ==" if run_case windows-libs NEW_PCRE2=10.48 NEW_ZLIB=1.3.2 NEW_OPENSSL=4.0.2 NEW_ZSTD=1.5.8; then if [ "$(pins_line VER_PCRE2)" = "VER_PCRE2=10.48" ] \ && [ "$(pins_line SHA_PCRE2)" = "SHA_PCRE2=$(stub_sha pcre2-10.48.tar.gz)" ] \ && [ "$(pins_line VER_ZLIB)" = "VER_ZLIB=1.3.2" ] \ && [ "$(pins_line SHA_ZLIB)" = "SHA_ZLIB=$(stub_sha zlib-1.3.2.tar.gz)" ] \ && [ "$(pins_line VER_OPENSSL)" = "VER_OPENSSL=4.0.2" ] \ && [ "$(pins_line SHA_OPENSSL)" = "SHA_OPENSSL=$(stub_sha openssl-4.0.2.tar.gz)" ] \ && [ "$(pins_line VER_ZSTD)" = "VER_ZSTD=1.5.8" ] \ && [ "$(pins_line SHA_ZSTD)" = "SHA_ZSTD=$(stub_sha zstd-1.5.8.tar.gz)" ] \ && [ "$(pins_line VER_NGINX)" = "VER_NGINX=1.31.4" ] \ && [ "$(pins_line SHA_NASM)" = "SHA_NASM=cccc5" ]; then ok "windows-libs: all four version+digest pairs updated, nginx and nasm untouched" else bad "windows-libs: unexpected pins: $(cat "$SANDBOX/ci/tools/windows-pins.sh") -- $(cat "$SANDBOX/out.log")" fi else bad "windows-libs: script exited non-zero unexpectedly: $(cat "$SANDBOX/out.log")" fi say "== case: release feed behind the pin (must hold, never move backwards) ==" if run_case backwards NEW_OPENSSL=3.5.5; then if grep -q 'CHANGED=0' "$SANDBOX/out.log" \ && grep -q 'not moving a pin backwards' "$SANDBOX/out.log" \ && [ "$(pins_line VER_OPENSSL)" = "VER_OPENSSL=4.0.1" ] \ && [ "$(pins_line SHA_OPENSSL)" = "SHA_OPENSSL=cccc3" ]; then ok "backwards: reported the hold, CHANGED=0, pin untouched" else bad "backwards: unexpected result: $(cat "$SANDBOX/out.log")" fi else bad "backwards: script exited non-zero unexpectedly: $(cat "$SANDBOX/out.log")" fi say "== case: every nginx/Angie feed behind its pin (all must hold) ==" if run_case backwards-webservers NEW_STABLE=1.30.3 NEW_ANGIE=1.11.0 NEW_MAINLINE=1.31.3; then if grep -q 'CHANGED=0' "$SANDBOX/out.log" \ && [ "$(grep -c 'not moving a pin backwards' "$SANDBOX/out.log")" -eq 4 ] \ && grep -q 'version: "1.30.4"' "$SANDBOX/.github/workflows/ci-deep.yml" \ && grep -q 'version: "1.12.1"' "$SANDBOX/.github/workflows/ci-deep.yml" \ && [ "$(pins_line VER_NGINX)" = "VER_NGINX=1.31.4" ] \ && [ "$(harness_line 1.31.4 ci-deep.yml)" = 1 ] \ && [ "$(harness_line 1.31.4 harness-fault-arms.yml)" = 1 ]; then ok "backwards webservers: all four pins held without edits" else bad "backwards webservers: unexpected result: $(cat "$SANDBOX/out.log")" fi else bad "backwards webservers: script exited non-zero: $(cat "$SANDBOX/out.log")" fi say "== case: version comparator failure is fatal, never a backwards hold ==" if run_case comparator-failure NEW_ANGIE=1.11.0 FAIL_SORT=1; then bad "comparator-failure: updater exited 0 after sort -V failed" elif ! grep -q 'FATAL: could not compare versions' "$SANDBOX/out.log"; then bad "comparator-failure: missing fatal diagnostic: $(cat "$SANDBOX/out.log")" else ok "comparator-failure: failed closed with a fatal diagnostic" fi say "== case: transient fetch failure part-way (must edit NOTHING: every digest is fetched before any file is touched) ==" # Stable resolves first and its tarball fetch succeeds; the angie tarball # fetch then fails. Before the resolve/apply split, the stable matrix and # digest edits had already landed by then, leaving a half-bumped tree. if run_case transient NEW_STABLE=1.30.5 NEW_ANGIE=1.13.0 FAIL_URL=angie-1.13.0.tar.gz; then bad "transient: script exited 0 despite a failed tarball fetch" elif grep -q '1.30.5' "$SANDBOX/.github/workflows/ci-deep.yml" || grep -q '1.30.5' "$SANDBOX/ci/tools/ci-build.sh"; then bad "transient: the stable bump was written before the angie fetch failed (partial edit): $(cat "$SANDBOX/out.log")" elif ! grep -q 'bump nginx stable: 1.30.4 -> 1.30.5' "$SANDBOX/out.log"; then bad "transient: the stable bump was not even resolved: $(cat "$SANDBOX/out.log")" elif ! grep -q 'could not fetch' "$SANDBOX/out.log"; then bad "transient: exited non-zero but did not name the failed fetch: $(cat "$SANDBOX/out.log")" else ok "transient: named the failed fetch, exited non-zero, no file edited" fi say "== case: missing nasm pin (not bumped, but required by build-windows.sh -- must FAIL before any edit) ==" make_sandbox SANDBOX sed -i '/^SHA_NASM=/d' "$SANDBOX/ci/tools/windows-pins.sh" snapshot_tracked bindir="$(NEW_STABLE=1.30.5 stub_bin "$SANDBOX")" rc=0 ( cd "$SANDBOX" PATH="$bindir:$PATH" bash ci/tools/bump-versions.sh >"$SANDBOX/out.log" 2>&1 ) || rc=$? if [ "$rc" -eq 0 ]; then bad "missing-nasm: script exited 0 with SHA_NASM missing from windows-pins.sh" elif [ -n "$(changed_tracked)" ]; then bad "missing-nasm: a file was mutated before the missing pin was reported (partial edit)" elif ! grep -q 'does not set SHA_NASM' "$SANDBOX/out.log"; then bad "missing-nasm: exited non-zero but did not name the missing pin: $(cat "$SANDBOX/out.log")" else ok "missing-nasm: exited non-zero, named the missing pin, and edited nothing" fi say "== case: apply-phase failure on the SECOND file (must leave EVERY tracked file byte-for-byte unchanged) ==" make_sandbox SANDBOX # The stable entry is well-formed, the angie entry is drifted: the stable # bump lands (on the staged copy), then the angie mutator fails. Before the # staged apply, ci-deep.yml and ci-build.sh already carried the stable bump. cat >"$SANDBOX/.github/workflows/ci-deep.yml" <<'EOF' env: HARNESS_NGINX_VERSION: "1.31.4" jobs: build-flavors: strategy: matrix: include: - version: "1.31.2" label: mainline - version: "1.30.4" label: stable - label: angie version: "1.12.1" EOF snapshot_tracked bindir="$(NEW_STABLE=1.30.5 NEW_ANGIE=1.13.0 stub_bin "$SANDBOX")" rc=0 ( cd "$SANDBOX" PATH="$bindir:$PATH" bash ci/tools/bump-versions.sh >"$SANDBOX/out.log" 2>&1 ) || rc=$? changed="$(changed_tracked)" if [ "$rc" -eq 0 ]; then bad "apply-failure: script exited 0 despite the drifted angie entry" elif [ -n "$changed" ]; then bad "apply-failure: tracked file(s) changed despite the failed mutator:$changed -- $(cat "$SANDBOX/out.log")" elif ! grep -q 'bump nginx stable: 1.30.4 -> 1.30.5' "$SANDBOX/out.log" || ! grep -q 'no matrix entry matched' "$SANDBOX/out.log"; then bad "apply-failure: unexpected log: $(cat "$SANDBOX/out.log")" elif ! grep -Fq -- '.github/workflows/ci-deep.yml' "$SANDBOX/out.log" || grep -Fq -- 'updates/.github/workflows/ci-deep.yml' "$SANDBOX/out.log"; then bad "apply-failure: diagnostic did not name the tracked repository path: $(cat "$SANDBOX/out.log")" else ok "apply-failure: the stable bump was resolved, the angie mutator failed, every tracked file unchanged" fi say "== case: authenticated release queries keep the token out of curl argv ==" make_sandbox SANDBOX bindir="$(stub_bin "$SANDBOX")" argv_log="$SANDBOX/curl.argv" header_log="$SANDBOX/curl.header" token='test-token-not-for-output' if ( cd "$SANDBOX" GH_TOKEN="$token" CURL_ARGV_LOG="$argv_log" CURL_HEADER_LOG="$header_log" \ PATH="$bindir:$PATH" bash ci/tools/bump-versions.sh >"$SANDBOX/out.log" 2>&1 ); then if grep -Fq "$token" "$argv_log"; then bad "token-argv: the GitHub token appeared in curl arguments" elif ! grep -Fxq -- '--config' "$argv_log" || ! grep -Fxq -- '-' "$argv_log"; then bad "token-argv: authenticated queries did not use stdin config: $(cat "$argv_log")" elif ! grep -Fqx "header = \"Authorization: Bearer $token\"" "$header_log"; then bad "token-argv: the stdin header did not carry the bearer token" else ok "token-argv: authenticated header used stdin, carried the token, and kept argv clean" fi else bad "token-argv: script exited non-zero unexpectedly: $(cat "$SANDBOX/out.log")" fi say "== case: nginx feed is not JSON (HTML came back) fails closed without edits ==" make_sandbox SANDBOX snapshot_tracked bindir="$(NGINX_FEED=html stub_bin "$SANDBOX")" rc=0 ( cd "$SANDBOX" PATH="$bindir:$PATH" bash ci/tools/bump-versions.sh >"$SANDBOX/out.log" 2>&1 ) || rc=$? changed="$(changed_tracked)" if [ "$rc" -eq 0 ]; then bad "feed-not-json: script exited 0 on an HTML answer from the release feed" elif [ -n "$changed" ]; then bad "feed-not-json: tracked file(s) changed despite the feed failure:$changed" elif ! grep -Fq 'the nginx release feed is not JSON' "$SANDBOX/out.log"; then bad "feed-not-json: unexpected log: $(cat "$SANDBOX/out.log")" else ok "feed-not-json: failed closed, named the bad feed, and edited nothing" fi say "== case: nginx feed with no even-minor release cannot invent a stable ==" make_sandbox SANDBOX snapshot_tracked bindir="$(NGINX_FEED=no-even stub_bin "$SANDBOX")" rc=0 ( cd "$SANDBOX" PATH="$bindir:$PATH" bash ci/tools/bump-versions.sh >"$SANDBOX/out.log" 2>&1 ) || rc=$? changed="$(changed_tracked)" if [ "$rc" -eq 0 ]; then bad "feed-no-even: script exited 0 with no stable release in the feed" elif [ -n "$changed" ]; then bad "feed-no-even: tracked file(s) changed despite the missing line:$changed" elif ! grep -Fq 'could not determine NEW_STABLE' "$SANDBOX/out.log"; then bad "feed-no-even: unexpected log: $(cat "$SANDBOX/out.log")" else ok "feed-no-even: failed closed instead of pinning a mainline release as stable" fi say "== case: nginx feed with only schema-invalid release records fails closed without edits ==" make_sandbox SANDBOX snapshot_tracked bindir="$(NGINX_FEED=malformed-only stub_bin "$SANDBOX")" rc=0 ( cd "$SANDBOX" PATH="$bindir:$PATH" bash ci/tools/bump-versions.sh >"$SANDBOX/out.log" 2>&1 ) || rc=$? changed="$(changed_tracked)" if [ "$rc" -eq 0 ]; then bad "feed-malformed: script exited 0 with no schema-valid nginx release" elif [ -n "$changed" ]; then bad "feed-malformed: tracked file(s) changed despite the invalid feed:$changed" elif ! grep -Fq 'could not determine NEW_STABLE' "$SANDBOX/out.log"; then bad "feed-malformed: unexpected log: $(cat "$SANDBOX/out.log")" else ok "feed-malformed: rejected missing and non-boolean eligibility flags and edited nothing" fi say "== case: branch transition -- a new even-minor stable above the mainline line moves stable only ==" if run_case transition NEW_STABLE=1.32.0; then if grep -q 'version: "1.32.0"' "$SANDBOX/.github/workflows/ci-deep.yml" \ && grep -q '\["1.32.0"\]' "$SANDBOX/ci/tools/ci-build.sh" \ && [ "$(harness_line 1.31.4 ci-deep.yml)" = 1 ] \ && [ "$(harness_line 1.31.4 harness-fault-arms.yml)" = 1 ] \ && [ "$(pins_line VER_NGINX)" = "VER_NGINX=1.31.4" ] \ && ! grep -q '1.33' "$SANDBOX/.github/workflows/ci-deep.yml"; then ok "transition: stable moved to 1.32.0; mainline pins stayed on 1.31.4; the draft/prerelease 1.33.x were ignored" else bad "transition: unexpected result: $(cat "$SANDBOX/out.log")" fi else bad "transition: script exited non-zero unexpectedly: $(cat "$SANDBOX/out.log")" fi say "== case: stable only on page two of the feed is still found ==" make_sandbox SANDBOX argv_log="$SANDBOX/curl.argv" bindir="$(NGINX_FEED=paged NEW_STABLE=1.30.5 stub_bin "$SANDBOX")" if ( cd "$SANDBOX" CURL_ARGV_LOG="$argv_log" PATH="$bindir:$PATH" bash ci/tools/bump-versions.sh >"$SANDBOX/out.log" 2>&1 ); then if grep -q 'version: "1.30.5"' "$SANDBOX/.github/workflows/ci-deep.yml" \ && grep -q '\["1.30.5"\]' "$SANDBOX/ci/tools/ci-build.sh" \ && grep -q 'releases?per_page=30&page=2' "$argv_log" \ && ! grep -q 'releases?per_page=30&page=3' "$argv_log"; then ok "feed-page-two: read page two, found the stable there, stopped at the short page" else bad "feed-page-two: unexpected result: $(cat "$SANDBOX/out.log"); pages: $(grep -o 'page=[0-9]*' "$argv_log" | tr '\n' ' ')" fi else bad "feed-page-two: script exited non-zero unexpectedly: $(cat "$SANDBOX/out.log")" fi say "== case: a legacy even-minor release on page one must not hide the newer stable on page two ==" make_sandbox SANDBOX argv_log="$SANDBOX/curl.argv" bindir="$(NGINX_FEED=legacy-first NEW_STABLE=1.30.5 stub_bin "$SANDBOX")" if ( cd "$SANDBOX" CURL_ARGV_LOG="$argv_log" PATH="$bindir:$PATH" bash ci/tools/bump-versions.sh >"$SANDBOX/out.log" 2>&1 ); then if grep -q 'version: "1.30.5"' "$SANDBOX/.github/workflows/ci-deep.yml" \ && grep -q '\["1.30.5"\]' "$SANDBOX/ci/tools/ci-build.sh" \ && ! grep -q '1.28.9' "$SANDBOX/out.log" \ && grep -q 'releases?per_page=30&page=2' "$argv_log"; then ok "feed-legacy-first: read past a page that already had both parities; stable is 1.30.5, not the legacy 1.28.9" else bad "feed-legacy-first: unexpected result: $(cat "$SANDBOX/out.log"); pages: $(grep -o 'page=[0-9]*' "$argv_log" | tr '\n' ' ')" fi else bad "feed-legacy-first: script exited non-zero unexpectedly: $(cat "$SANDBOX/out.log")" fi say "== case: a feed that never shows a stable stops at the page cap and fails closed ==" make_sandbox SANDBOX argv_log="$SANDBOX/curl.argv" snapshot_tracked bindir="$(NGINX_FEED=endless stub_bin "$SANDBOX")" rc=0 ( cd "$SANDBOX" CURL_ARGV_LOG="$argv_log" PATH="$bindir:$PATH" bash ci/tools/bump-versions.sh >"$SANDBOX/out.log" 2>&1 ) || rc=$? changed="$(changed_tracked)" if [ "$rc" -eq 0 ]; then bad "feed-endless: script exited 0 without ever seeing a stable release" elif [ -n "$changed" ]; then bad "feed-endless: tracked file(s) changed:$changed" elif ! grep -Fq 'did not end within 5 pages' "$SANDBOX/out.log"; then bad "feed-endless: unexpected log: $(cat "$SANDBOX/out.log")" elif ! grep -q 'page=5' "$argv_log" || grep -q 'page=6' "$argv_log"; then bad "feed-endless: paging did not stop at the cap: $(grep -o 'page=[0-9]*' "$argv_log" | tr '\n' ' ')" else ok "feed-endless: read five pages, no more, and refused the incomplete read" fi say "== case: every capped page full and carrying newer releases of both lines -- must refuse, not pick ==" make_sandbox SANDBOX argv_log="$SANDBOX/curl.argv" snapshot_tracked bindir="$(NGINX_FEED=capped stub_bin "$SANDBOX")" rc=0 ( cd "$SANDBOX" CURL_ARGV_LOG="$argv_log" PATH="$bindir:$PATH" bash ci/tools/bump-versions.sh >"$SANDBOX/out.log" 2>&1 ) || rc=$? changed="$(changed_tracked)" if [ "$rc" -eq 0 ]; then bad "feed-capped: script exited 0 after reading only the capped pages" elif [ -n "$changed" ]; then bad "feed-capped: tracked file(s) changed from an incomplete read:$changed" elif ! grep -Fq 'did not end within 5 pages' "$SANDBOX/out.log"; then bad "feed-capped: unexpected log: $(cat "$SANDBOX/out.log")" elif grep -q 'bump nginx stable' "$SANDBOX/out.log"; then bad "feed-capped: a bump was resolved from the incomplete read: $(cat "$SANDBOX/out.log")" else ok "feed-capped: refused at the cap with both lines visible on every page, edited nothing" fi say "== case: install failure on the SECOND changed file rolls back the first ==" make_sandbox SANDBOX snapshot_tracked bindir="$(NEW_STABLE=1.30.5 NEW_ANGIE=1.13.0 stub_bin "$SANDBOX")" rc=0 ( cd "$SANDBOX" BUMP_VERSIONS_TEST_FAIL_INSTALL_AT=2 PATH="$bindir:$PATH" bash ci/tools/bump-versions.sh >"$SANDBOX/out.log" 2>&1 ) || rc=$? changed="$(changed_tracked)" if [ "$rc" -eq 0 ]; then bad "install-failure: script exited 0 despite the injected failure" elif [ -n "$changed" ]; then bad "install-failure: tracked file(s) changed despite rollback:$changed" elif ! grep -q 'injected install failure at changed file 2' "$SANDBOX/out.log"; then bad "install-failure: unexpected log: $(cat "$SANDBOX/out.log")" else ok "install-failure: every tracked file remained byte-identical" fi say "== case: interruption after the FIRST replacement rolls it back ==" make_sandbox SANDBOX snapshot_tracked bindir="$(NEW_STABLE=1.30.5 NEW_ANGIE=1.13.0 stub_bin "$SANDBOX")" rc=0 ( cd "$SANDBOX" BUMP_VERSIONS_TEST_INTERRUPT_INSTALL_AT=1 PATH="$bindir:$PATH" bash ci/tools/bump-versions.sh >"$SANDBOX/out.log" 2>&1 ) || rc=$? changed="$(changed_tracked)" if [ "$rc" -eq 0 ]; then bad "install-interrupt: script exited 0 despite the injected TERM" elif [ -n "$changed" ]; then bad "install-interrupt: tracked file(s) changed despite rollback:$changed" elif [ "$rc" -ne 143 ]; then bad "install-interrupt: expected status 143, got $rc: $(cat "$SANDBOX/out.log")" else ok "install-interrupt: the replaced file was restored byte-identically" fi say "== case: format-drift (matrix entry does not match -- must FAIL LOUDLY, no partial edit) ==" make_sandbox SANDBOX # Corrupt the matrix so version/label are no longer adjacent the way the # script's pattern expects -- simulates a future reformat/reorder. cat >"$SANDBOX/.github/workflows/ci-deep.yml" <<'EOF' env: HARNESS_NGINX_VERSION: "1.31.4" jobs: build-flavors: strategy: matrix: include: - label: mainline version: "1.31.2" - label: stable version: "1.30.4" - label: angie version: "1.12.1" EOF snapshot_tracked bindir="$(NEW_STABLE=1.30.5 stub_bin "$SANDBOX")" rc=0 ( cd "$SANDBOX" PATH="$bindir:$PATH" bash ci/tools/bump-versions.sh >"$SANDBOX/out.log" 2>&1 ) || rc=$? if [ "$rc" -eq 0 ]; then bad "format-drift: script exited 0 on a no-op replacement (should FAIL LOUDLY)" elif [ -n "$(changed_tracked)" ]; then bad "format-drift: files were mutated despite the failed match (partial edit)" elif ! grep -qi 'no matrix entry matched' "$SANDBOX/out.log"; then bad "format-drift: exited non-zero but did not report the no-op match: $(cat "$SANDBOX/out.log")" else ok "format-drift: exited non-zero, reported the no-op, and left both files untouched" fi say "== case: pins-file drift (a VER_ line missing -- must FAIL LOUDLY before ANY edit) ==" make_sandbox SANDBOX sed -i '/^VER_PCRE2=/d' "$SANDBOX/ci/tools/windows-pins.sh" snapshot_tracked # A stable bump is pending too: the pins check must run before it lands. bindir="$(NEW_STABLE=1.30.5 NEW_PCRE2=10.48 stub_bin "$SANDBOX")" rc=0 ( cd "$SANDBOX" PATH="$bindir:$PATH" bash ci/tools/bump-versions.sh >"$SANDBOX/out.log" 2>&1 ) || rc=$? if [ "$rc" -eq 0 ]; then bad "pins-drift: script exited 0 with a pin missing from windows-pins.sh" elif [ -n "$(changed_tracked)" ]; then bad "pins-drift: a file was mutated before the missing pin was reported (partial edit)" elif ! grep -q 'does not set VER_PCRE2' "$SANDBOX/out.log"; then bad "pins-drift: exited non-zero but did not name the missing pin: $(cat "$SANDBOX/out.log")" else ok "pins-drift: exited non-zero, named the missing pin, and edited nothing" fi say "== case: harness drift (the two workflows disagree -- must FAIL LOUDLY, no edit) ==" make_sandbox SANDBOX sed -i 's/HARNESS_NGINX_VERSION: "1.31.4"/HARNESS_NGINX_VERSION: "1.31.3"/' "$SANDBOX/.github/workflows/harness-fault-arms.yml" snapshot_tracked bindir="$(NEW_MAINLINE=1.31.5 stub_bin "$SANDBOX")" rc=0 ( cd "$SANDBOX" PATH="$bindir:$PATH" bash ci/tools/bump-versions.sh >"$SANDBOX/out.log" 2>&1 ) || rc=$? if [ "$rc" -eq 0 ]; then bad "harness-drift: script exited 0 with the two harness pins disagreeing" elif [ -n "$(changed_tracked)" ]; then bad "harness-drift: a file was mutated despite the disagreement (partial edit)" elif ! grep -q 'must build the same nginx' "$SANDBOX/out.log"; then bad "harness-drift: exited non-zero but did not report the disagreement: $(cat "$SANDBOX/out.log")" else ok "harness-drift: exited non-zero, reported the disagreement, and edited nothing" fi if [ "$fail" -eq 0 ]; then say "all bump-versions.sh cases pass" else say "one or more bump-versions.sh cases FAILED" fi exit "$fail" myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_bypass_vary_config_policy.py000077500000000000000000000234141525232545500304570ustar00rootroot00000000000000#!/usr/bin/env python3 """Config-load table for ``zstd_bypass_vary`` — grind-g6-nits owed control. ``ngx_http_zstd_set_bypass_vary()`` (src/ngx_http_zstd_filter_module.c) hand-validates its single argument as an RFC 9110 field-name token before storing it: reject ``,``/``;``/``"`` and a bare solitary ``*`` outright, accept any other tchar including a non-solitary ``*``, and let normal nginx merge semantics (http-level value overridden by a location-level one) apply on top. None of that validation is expressible as a Test::Nginx ``--- must_die`` table — ``ci/t/02-conf-warn.t`` covers the *emitted warning* half of this directive, not the *acceptance/rejection* table, and no existing file has a rejection-table for this directive. The policy under test ---------------------- * a valid mixed-case token loads AND is retained VERBATIM (exact spelling, not lower/upper-cased) -- proven via the config-load warning ci/t/02-conf-warn.t already asserts fires only when zstd_bypass_vary is set without zstd_bypass, which echoes nothing; so this test instead asserts acceptance (rc==0) and, using a second location, that the RETAINED value round-trips into the emitted Vary response header exactly as typed; * ``,`` ``;`` ``"`` and a bare solitary ``*`` are each REJECTED, with nginx -t exiting non-zero and naming the emerg reason; * a non-solitary ``*`` inside a token (``X-Foo*Bar``) is ACCEPTED; * an http-level value is overridden by a location-level one (the location's Vary reflects the location value, not the http one). Why config_test on nginx -t alone is not enough for arms 1 and 8: a config-load PASS does not prove the value was stored correctly (a setter that silently truncated or normalized the token would still return NGX_CONF_OK). Those two arms therefore additionally start nginx and curl a real response, asserting the Vary header's exact bytes. """ import argparse import pathlib import socket import subprocess import sys import tempfile import time import urllib.request def parse_args(): p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) p.add_argument("--nginx-binary", required=True) p.add_argument("--filter-module") p.add_argument("--port", type=int, default=18170) return p.parse_args() def detect_module(explicit, nginx: pathlib.Path): if explicit: return pathlib.Path(explicit) sib = nginx.parent / "ngx_http_zstd_filter_module.so" return sib if sib.exists() else None def write_conf(root, module, port, http_directive, loc_directive): load = f"load_module {module};\n" if module else "" (root / "conf" / "nginx.conf").write_text( f"{load}" "events {}\n" "http {\n" f" {http_directive}\n" " server {\n" f" listen 127.0.0.1:{port};\n" " zstd on;\n" " zstd_min_length 1;\n" " zstd_bypass $http_x_no_compression;\n" " default_type text/plain;\n" " location /a {\n" f" {loc_directive}\n" ' return 200 "hello world padding padding padding\\n";\n' " }\n" " }\n" "}\n" ) def config_test(nginx, module, port, http_directive="", loc_directive=""): """Run ``nginx -t``; return (returncode, combined output).""" with tempfile.TemporaryDirectory() as td: root = pathlib.Path(td) (root / "conf").mkdir() (root / "logs").mkdir() write_conf(root, module, port, http_directive, loc_directive) proc = subprocess.run( [str(nginx), "-p", str(root), "-c", "conf/nginx.conf", "-t"], capture_output=True, text=True, timeout=60, check=False, ) return proc.returncode, proc.stdout + proc.stderr def free_port(start): port = start while True: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: if s.connect_ex(("127.0.0.1", port)) != 0: return port port += 1 def get_vary_header(nginx, module, port, http_directive, loc_directive): """Start nginx, GET /a, return the Vary header value (or None).""" with tempfile.TemporaryDirectory() as td: root = pathlib.Path(td) (root / "conf").mkdir() (root / "logs").mkdir() write_conf(root, module, port, http_directive, loc_directive) proc = subprocess.Popen( [str(nginx), "-p", str(root), "-c", "conf/nginx.conf", "-g", "daemon off;"], stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True, ) try: deadline = time.time() + 5 connected = False while time.time() < deadline: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: if s.connect_ex(("127.0.0.1", port)) == 0: connected = True break time.sleep(0.1) if not connected: return None, "nginx did not start listening" req = urllib.request.Request( f"http://127.0.0.1:{port}/a", headers={"Accept-Encoding": "zstd", "X-No-Compression": "1"}, ) with urllib.request.urlopen(req, timeout=5) as resp: return resp.headers.get("Vary"), None except Exception as e: # noqa: BLE001 - report, don't hide return None, str(e) finally: proc.terminate() try: proc.wait(timeout=5) except subprocess.TimeoutExpired: proc.kill() proc.wait(timeout=5) def main() -> int: args = parse_args() nginx = pathlib.Path(args.nginx_binary).resolve() if not nginx.exists(): print(f"FAIL: nginx binary not found: {nginx}", file=sys.stderr) return 1 module = detect_module(args.filter_module, nginx) if module is not None: module = module.resolve() if not module.exists(): print(f"FAIL: filter module not found: {module}", file=sys.stderr) return 1 failures = [] port = args.port # --- Rejection arms: nginx -t must fail, naming the reason. --- reject_cases = [ ("comma", "zstd_bypass_vary X-Foo,Bar;", "comma or semicolon"), # nginx's own tokenizer treats an unquoted ';' as the directive # terminator, so the literal byte can only reach the setter's # argument when quoted in the conf syntax itself. ("semicolon", 'zstd_bypass_vary "X-Foo;Bar";', "comma or semicolon"), # Likewise an unquoted '"' would be nginx's own quoting; escape it # inside a quoted string so a literal '"' byte reaches the setter. ("quote", 'zstd_bypass_vary "X-\\"Foo";', "quoted string"), ("bare solitary wildcard", "zstd_bypass_vary *;", "bare wildcard"), ] for name, directive, want_reason in reject_cases: port = free_port(port + 1) rc, out = config_test(nginx, module, port, loc_directive=directive) if rc == 0: failures.append( f"{name}: expected nginx -t to be REFUSED but it exited 0; " f"output: {out.strip()!r}" ) elif want_reason not in out: failures.append( f"{name}: nginx -t was refused (rc={rc}) but the emerg " f"message did not name the reason ({want_reason!r}); " f"output: {out.strip()!r}" ) print( f" reject/{name:<24} exit {rc} (want non-zero), reason " f"{'yes' if want_reason in out else 'no'}" ) # --- Acceptance arm: non-solitary '*' inside a token. --- port = free_port(port + 1) rc, out = config_test( nginx, module, port, loc_directive="zstd_bypass_vary X-Foo*Bar;" ) if rc != 0: failures.append( f"non-solitary wildcard token: expected nginx -t to PASS but " f"it exited {rc}; output: {out.strip()!r}" ) print(f" accept/non-solitary-wildcard exit {rc} (want 0)") # --- Retention arm: mixed-case token loads and is echoed verbatim. --- port = free_port(port + 1) token = "X-My-Header" vary, err = get_vary_header(nginx, module, port, "", f"zstd_bypass_vary {token};") if err is not None: failures.append(f"retention (mixed-case token): {err}") elif vary != token: failures.append( f"retention (mixed-case token): Vary header is {vary!r}, " f"want exact spelling {token!r} (verifies the stored value " f"is not case-folded, truncated or otherwise normalized)" ) print(f" retain/mixed-case-token Vary={vary!r} (want {token!r})") # --- Override arm: http-level value overridden by location-level. --- port = free_port(port + 1) vary, err = get_vary_header( nginx, module, port, http_directive="zstd_bypass_vary X-Http-Level;", loc_directive="zstd_bypass_vary X-Loc-Level;", ) if err is not None: failures.append(f"override (http overridden by location): {err}") elif vary != "X-Loc-Level": failures.append( f"override (http overridden by location): Vary header is " f"{vary!r}, want the LOCATION value 'X-Loc-Level' (the " f"http-level 'X-Http-Level' must not win)" ) print(f" override/loc-wins-over-http Vary={vary!r} (want 'X-Loc-Level')") if failures: print("\nFAIL: zstd_bypass_vary config-load policy", file=sys.stderr) for f in failures: print(f" - {f}", file=sys.stderr) return 1 print( "\nOK: zstd_bypass_vary config-load policy " f"({len(reject_cases) + 3}/{len(reject_cases) + 3} arms)" ) return 0 if __name__ == "__main__": sys.exit(main()) myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_cctx_advisory_policy.py000077500000000000000000000272131525232545500274520ustar00rootroot00000000000000#!/usr/bin/env python3 """Config-load policy: the implicit per-request CCtx memory advisory. ``zstd_max_cctx_memory`` merges to ``0`` when the operator never sets it, and the config-load estimator used to be skipped entirely in that case. A later ``zstd_comp_level 22`` or ``zstd_long on`` could therefore commit hundreds of MB of compressor working set per concurrent response -- multiplied by the ``NGX_HTTP_ZSTD_CCTX_SLOTS`` (4) contexts each worker retains, and again by ``worker_processes`` -- even though the module already knew the number at config load. The policy under test --------------------- Compression enabled and **no** explicit ``zstd_max_cctx_memory`` anywhere in the inheritance chain: * run libzstd's estimator on the merged profile; * if the estimate exceeds ``NGX_HTTP_ZSTD_CCTX_ADVISORY_BYTES`` (32 MB), emit an ``[warn]`` naming the estimate and the retained worker bound; * **warn, never fail** -- turning ``nginx -t`` from pass to fail on a configuration that works today would break deployments on upgrade. An explicit ``zstd_max_cctx_memory 0`` is the operator's acknowledgement and silences the advisory. (``ngx_conf_set_size_slot()`` parses sizes only, so ``0`` -- not the keyword ``off`` -- is the spelling; the directive is capped at ``INT_MAX``, hence ``1024m`` rather than ``1g`` in arm 5.) An explicit non-zero value keeps the pre-existing hard-failure semantics untouched. Why this test and not ``ci/t/00-filter.t``: Test::Nginx's ``--- must_die`` and error-log greps cannot express "must PASS *and* must have emitted exactly this warning", which is the whole contract here -- a policy that failed the config would satisfy any must_die-shaped assertion while being precisely the breaking change this design rejected. Arms ---- 1. default profile (``zstd on`` only) -> PASS, NO advisory. 2. ``zstd_comp_level 22`` -> PASS, advisory. 3. ``zstd_long on`` -> PASS, advisory. 4. ``zstd_window_log 27`` -> PASS, advisory. 5. ``zstd_comp_level 22`` + explicit 1024m budget -> PASS, NO advisory (the explicit-budget path owns this config; the advisory must not double-report). 6. ``zstd_comp_level 22`` + ``zstd_max_cctx_memory 0`` -> PASS, NO advisory (the acknowledgement is honoured). 7. ``zstd_comp_level 22`` + explicit 64m budget -> REFUSED (the hard check is unchanged; the advisory did not soften it). 8. dictionary + default level -> PASS, NO advisory (a dictionary does not move the working set into advisory range). Arms 1, 5, 6 and 8 are the controls that stop this test going vacuous: each asserts the advisory is **absent**, so a build that warned unconditionally -- the obvious wrong implementation -- fails them. """ import argparse import pathlib import re import subprocess import sys import tempfile # NGX_HTTP_ZSTD_CCTX_ADVISORY_BYTES in # src/ngx_http_zstd_filter_module.c. Kept in sync by arms 1 and 2: # level 3 estimates ~3.5 MB (below) and level 22 ~769 MB (above), so a # threshold moved outside the (28.7 MB, 52.7 MB) gap between levels 11 # and 12 does not silently pass this test. ADVISORY_BYTES = 32 * 1024 * 1024 # NGX_HTTP_ZSTD_CCTX_SLOTS. CCTX_SLOTS = 4 # The advisory is identified by this phrase; matching on it rather than # on "[warn]" keeps the unrelated zstd_bypass_vary warning from # satisfying an arm. ADVISORY_RE = re.compile(r"need ~(\d+) bytes of per-request compressor memory") def parse_args(): p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) p.add_argument("--nginx-binary", required=True) p.add_argument("--filter-module") p.add_argument("--port", type=int, default=18130) return p.parse_args() def detect_module(explicit, nginx: pathlib.Path): if explicit: return pathlib.Path(explicit) sib = nginx.parent / "ngx_http_zstd_filter_module.so" return sib if sib.exists() else None def config_test(nginx, module, port, directives, want_dict=False): """Run ``nginx -t``; return (returncode, combined output).""" with tempfile.TemporaryDirectory() as td: root = pathlib.Path(td) (root / "conf").mkdir() (root / "logs").mkdir() extra = "" if want_dict: # A small raw dictionary is enough: the point of the arm is # that loading one does not push the estimate over the # advisory threshold, not the dictionary's content. dic = root / "dict.bin" dic.write_bytes(b"the quick brown fox jumps over the lazy dog" * 64) # zstd_dict_file additionally requires the operator to # acknowledge the non-negotiated Content-Encoding; that gate # is unrelated to this test but must be satisfied to reach # the memory policy at all. extra = f"zstd_dict_file {dic};\n zstd_dict_file_unsafe on;\n " load = f"load_module {module};\n" if module else "" (root / "conf" / "nginx.conf").write_text( f"{load}" "events {}\n" "http {\n" f" {extra}" f" server {{\n" f" listen 127.0.0.1:{port};\n" " zstd on;\n" f" {directives}\n" " }\n" "}\n" ) proc = subprocess.run( [str(nginx), "-p", str(root), "-c", "conf/nginx.conf", "-t"], capture_output=True, text=True, timeout=60, check=False, ) return proc.returncode, proc.stdout + proc.stderr def check_not_a_signal(name, rc, out): if rc < 0: return [f"{name}: nginx -t died on signal {-rc}; output: {out.strip()!r}"] if rc > 128: return [ ( f"{name}: nginx -t exited {rc} (128+{rc - 128}), a signal " f"death, not a diagnostic; output: {out.strip()!r}" ) ] return [] def run_arm( nginx, module, port, name, directives, want_pass, want_advisory, want_dict=False, min_estimate=None, ): """One matrix cell. Returns (failures, printable summary).""" rc, out = config_test(nginx, module, port, directives, want_dict=want_dict) failures = check_not_a_signal(name, rc, out) if want_pass and rc != 0: failures.append( f"{name}: expected nginx -t to PASS but it exited {rc}. The " f"advisory policy must never turn a working config into a " f"config-load failure; output: {out.strip()!r}" ) if not want_pass and rc == 0: failures.append( f"{name}: expected nginx -t to be REFUSED but it exited 0; " f"output: {out.strip()!r}" ) m = ADVISORY_RE.search(out) saw_warn = m is not None and "[warn]" in out if want_advisory and not saw_warn: failures.append( f"{name}: expected the [warn] advisory naming the estimate, " f"but it was not emitted. The operator gets no signal that " f"this profile commits a large per-request working set; " f"output: {out.strip()!r}" ) if not want_advisory and saw_warn: failures.append( f"{name}: the advisory was emitted but must NOT be. Either " f"the threshold is wrong or an explicit budget / " f"acknowledgement is not being honoured; output: " f"{out.strip()!r}" ) est = None if saw_warn: est = int(m.group(1)) if min_estimate is not None and est < min_estimate: failures.append( f"{name}: advisory reports {est} bytes, below the expected " f"{min_estimate}. The estimate was computed against the " f"wrong parameters." ) if est <= ADVISORY_BYTES: failures.append( f"{name}: advisory fired at {est} bytes, which is <= the " f"{ADVISORY_BYTES}-byte threshold. The condition is not " f"the documented one." ) # The message must give the operator the retained worker bound, # not just the per-context figure -- the per-worker number is # what actually shows up in RSS. if str(est * CCTX_SLOTS) not in out: failures.append( f"{name}: advisory does not state the retained worker " f"bound ({est} x {CCTX_SLOTS} = {est * CCTX_SLOTS} " f"bytes); output: {out.strip()!r}" ) if "zstd_max_cctx_memory 0" not in out: failures.append( f"{name}: advisory does not tell the operator how to " f'acknowledge it ("zstd_max_cctx_memory 0"); output: ' f"{out.strip()!r}" ) summary = ( f" {name:<38} exit {rc} (want {'0' if want_pass else 'non-zero'}), " f"advisory {'yes' if saw_warn else 'no':<3} " f"(want {'yes' if want_advisory else 'no'})" + (f", estimate {est} bytes" if est is not None else "") ) return failures, summary def main() -> int: args = parse_args() nginx = pathlib.Path(args.nginx_binary).resolve() if not nginx.exists(): print(f"FAIL: nginx binary not found: {nginx}", file=sys.stderr) return 1 module = detect_module(args.filter_module, nginx) if module is not None: module = module.resolve() if not module.exists(): print(f"FAIL: filter module not found: {module}", file=sys.stderr) return 1 # (name, directives, want_pass, want_advisory, want_dict, min_estimate) # # min_estimate pins the magnitude where the profile implies one, so a # correctly-firing advisory computed against the wrong parameters is # still caught. LDM and window_log 27 both imply a 128 MB window. matrix = [ ("1 default profile", "", True, False, False, None), ( "2 zstd_comp_level 22", "zstd_comp_level 22;", True, True, False, 512 * 1024 * 1024, ), ("3 zstd_long on", "zstd_long on;", True, True, False, 128 * 1024 * 1024), ( "4 zstd_window_log 27", "zstd_window_log 27;", True, True, False, 128 * 1024 * 1024, ), ( "5 level 22 + budget 1024m", "zstd_comp_level 22; zstd_max_cctx_memory 1024m;", True, False, False, None, ), ( "6 level 22 + budget 0 (ack)", "zstd_comp_level 22; zstd_max_cctx_memory 0;", True, False, False, None, ), ( "7 level 22 + budget 64m", "zstd_comp_level 22; zstd_max_cctx_memory 64m;", False, False, False, None, ), ("8 dictionary, default level", "", True, False, True, None), ] failures = [] print("zstd CCtx memory advisory policy matrix:") for i, (name, directives, wp, wa, wd, mn) in enumerate(matrix): f, summary = run_arm( nginx, module, args.port + i, name, directives, wp, wa, want_dict=wd, min_estimate=mn, ) failures += f print(summary) if failures: print("\nFAIL: CCtx memory advisory policy", file=sys.stderr) for f in failures: print(f" - {f}", file=sys.stderr) return 1 print(f"\nOK: CCtx memory advisory policy ({len(matrix)}/{len(matrix)} arms)") return 0 if __name__ == "__main__": sys.exit(main()) myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_cctx_profile_pack.c000066400000000000000000000162761525232545500264670ustar00rootroot00000000000000/* * Unit oracle for the CCtx profile 64-bit key. * * The packer and unpacker (and their layout #defines) are EXTRACTED VERBATIM * from src/ngx_http_zstd_filter_module.c by test_cctx_profile_pack.sh into * generated_profile_pack.inc -- this file never re-implements the arithmetic, * so it cannot quietly agree with a stale copy of itself. * * What the row demands, and therefore what is asserted here: * * 1. REVERSIBLE -- unpack(pack(t)) == t for every in-domain tuple t. * 2. COLLISION-FREE -- distinct tuples never produce the same key. Proved * here without an O(n^2) compare and without a hash set: because (1) * holds for every tuple in the swept domain, pack is injective on that * domain by construction (a left inverse exists), so a collision is * impossible. The sweep additionally checks the reserved high bits are * clear and that no in-domain key ever equals the INVALID sentinel, * which is the only way a real profile could be confused with a refusal. * 3. FAIL-CLOSED -- out-of-domain inputs return NGX_HTTP_ZSTD_PROFILE_INVALID * rather than being masked into a valid-looking key. Masking is what * would let two different compression settings share one CCtx, which * changes bytes on the wire with no diagnostic. * * Domain swept: the full range the config layer can deliver. * level -131072 .. 22 (ZSTD_minCLevel()..ZSTD_maxCLevel(); * negative values are libzstd's fast levels) * window_log 0 .. 63 (0 = "unset"; the accepted maximum is a * libzstd bound -- 31 on 64-bit today -- so the * field is 6 bits wide and the WHOLE field is * swept, not just today's library limit) * long_mode 0 .. 1 */ #include #include #include /* Minimal nginx-typedef shim: the extracted code uses these two names. */ typedef long ngx_int_t; typedef long ngx_flag_t; #include "generated_profile_pack.inc" #define LEVEL_MIN (-131072) #define LEVEL_MAX (22) #define WLOG_MAX ((ngx_int_t) NGX_HTTP_ZSTD_PROFILE_WLOG_MAX) int main(void) { ngx_int_t level, window_log, long_mode; ngx_int_t u_level, u_window_log; ngx_flag_t u_long; uint64_t key; long long swept = 0; long long revfail = 0; long long sentinel_clash = 0; long long reserved_dirty = 0; int rc = 0; for (level = LEVEL_MIN; level <= LEVEL_MAX; level++) { for (window_log = 0; window_log <= WLOG_MAX; window_log++) { for (long_mode = 0; long_mode <= 1; long_mode++) { key = ngx_http_zstd_profile_pack(level, (ngx_flag_t) long_mode, window_log); swept++; if (key == NGX_HTTP_ZSTD_PROFILE_INVALID) { sentinel_clash++; continue; } /* Nothing above the long_mode bit may ever be set. */ if ((key >> (NGX_HTTP_ZSTD_PROFILE_LONG_SHIFT + 1)) != 0) { reserved_dirty++; } ngx_http_zstd_profile_unpack(key, &u_level, &u_long, &u_window_log); if (u_level != level || u_window_log != window_log || (ngx_int_t) u_long != long_mode) { if (revfail < 5) { fprintf(stderr, "REVERSIBILITY: (%ld,%ld,%ld) -> key %llu -> " "(%ld,%ld,%ld)\n", (long) level, (long) long_mode, (long) window_log, (unsigned long long) key, (long) u_level, (long) u_long, (long) u_window_log); } revfail++; } } } } printf("swept %lld in-domain tuples " "(level %d..%d, window_log 0..%ld, long_mode 0..1)\n", swept, LEVEL_MIN, LEVEL_MAX, (long) WLOG_MAX); printf(" reversibility failures : %lld\n", revfail); printf(" INVALID-sentinel clashes: %lld\n", sentinel_clash); printf(" reserved bits set : %lld\n", reserved_dirty); if (revfail || sentinel_clash || reserved_dirty) { rc = 1; } /* * Injectivity follows from reversibility over the swept domain: unpack is * a left inverse of pack there, so pack cannot map two distinct tuples to * one key. State it rather than implying it. */ if (!rc) { printf(" => pack is injective over the swept domain " "(left inverse exists for every tuple): 0 collisions\n"); } /* --- fail-closed checks: out-of-domain must REFUSE, not mask. --- */ { struct { ngx_int_t lvl, wlog; const char *what; } bad[] = { { LEVEL_MIN - 1, 0, "level below bias" }, { 1 << 20, 0, "level above field" }, { 0, WLOG_MAX + 1, "window_log above field" }, { 0, -1, "window_log negative" }, }; size_t i; uint64_t key1, key2; for (i = 0; i < sizeof(bad) / sizeof(bad[0]); i++) { key = ngx_http_zstd_profile_pack(bad[i].lvl, 0, bad[i].wlog); if (key != NGX_HTTP_ZSTD_PROFILE_INVALID) { fprintf(stderr, "FAIL-CLOSED: %s (level=%ld window_log=%ld) returned " "%llu, expected INVALID\n", bad[i].what, (long) bad[i].lvl, (long) bad[i].wlog, (unsigned long long) key); rc = 1; } } if (!rc) { printf(" => all %zu out-of-domain inputs refused " "(no silent masking)\n", sizeof(bad) / sizeof(bad[0])); } /* * Assert that two distinct out-of-domain profiles collapse to the * SAME sentinel: this is the aliasing the cache guard must detect. * Both should pack to INVALID, proving they compare equal, which is * why acquire_cctx must refuse to borrow or seed on INVALID. */ key1 = ngx_http_zstd_profile_pack(LEVEL_MIN - 1, 0, 0); key2 = ngx_http_zstd_profile_pack(0, 0, WLOG_MAX + 1); if (key1 != NGX_HTTP_ZSTD_PROFILE_INVALID || key2 != NGX_HTTP_ZSTD_PROFILE_INVALID || key1 != key2) { fprintf(stderr, "SENTINEL-ALIASING: two distinct out-of-domain profiles " "did not both collapse to INVALID: " "key1=%llu key2=%llu INVALID=%llu\n", (unsigned long long) key1, (unsigned long long) key2, (unsigned long long) NGX_HTTP_ZSTD_PROFILE_INVALID); rc = 1; } else { printf(" => distinct out-of-domain profiles alias to INVALID " "(cache guard must refuse)\n"); } } if (rc) { printf("FAIL\n"); return 1; } printf("PASS\n"); return 0; } myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_cctx_profile_pack.sh000077500000000000000000000077441525232545500266620ustar00rootroot00000000000000#!/bin/bash # Unit fixture for the CCtx profile 64-bit key (ngx_http_zstd_profile_pack / # ngx_http_zstd_profile_unpack). # # The row this test exists for demands a COLLISION-FREE, REVERSIBLE key -- # explicitly not a hash. That property is only worth asserting against the # code that actually ships, so this extracts the packer, the unpacker and # their layout #defines VERBATIM from ngx_http_zstd_filter_module.c by line # range (the same technique as test_ceil_log2_unit.sh) and compiles them # standalone. A hand-copied duplicate of the arithmetic would pass forever # while production drifted underneath it -- which is the exact failure this # extraction prevents. # # No nginx tree and no libzstd needed: pure C, self-contained. set -euo pipefail # ci/tools/ -> ci/ -> repo root: TWO levels since the ci/ move. cd "$(dirname "$0")/../.." SRC="src/ngx_http_zstd_filter_module.c" CC="${CC:-cc}" OUT="$(mktemp -d "${TMPDIR:-/tmp}/zstd-profile-pack-unit.XXXXXX")" trap 'rm -rf "$OUT"' EXIT mkdir -p "$OUT" # --- the layout #defines ------------------------------------------------- # Extracted, never re-typed: the widths and the bias ARE the contract under # test. Redeclaring them here would let the test agree with itself. DEF_START="$(grep -n '^#define NGX_HTTP_ZSTD_PROFILE_LEVEL_BIAS' "$SRC" \ | head -1 | cut -d: -f1)" if [ -z "$DEF_START" ]; then echo "FAIL: could not locate NGX_HTTP_ZSTD_PROFILE_LEVEL_BIAS in $SRC" >&2 exit 1 fi DEF_END="$(grep -n '^#define NGX_HTTP_ZSTD_PROFILE_INVALID' "$SRC" \ | head -1 | cut -d: -f1)" if [ -z "$DEF_END" ]; then echo "FAIL: could not locate NGX_HTTP_ZSTD_PROFILE_INVALID in $SRC" >&2 exit 1 fi # --- the packer ---------------------------------------------------------- PACK_START="$(grep -n '^ngx_http_zstd_profile_pack(ngx_int_t level' "$SRC" \ | head -1 | cut -d: -f1)" if [ -z "$PACK_START" ]; then echo "FAIL: could not locate ngx_http_zstd_profile_pack() in $SRC" >&2 exit 1 fi PACK_START=$((PACK_START - 1)) # the "static uint64_t" line REL="$(tail -n "+${PACK_START}" "$SRC" | grep -n '^}' | head -1 | cut -d: -f1)" PACK_END=$((PACK_START + REL - 1)) # --- the unpacker -------------------------------------------------------- UNPACK_START="$(grep -n '^ngx_http_zstd_profile_unpack(uint64_t key' "$SRC" \ | head -1 | cut -d: -f1)" if [ -z "$UNPACK_START" ]; then echo "FAIL: could not locate ngx_http_zstd_profile_unpack() in $SRC" >&2 exit 1 fi UNPACK_START=$((UNPACK_START - 1)) # the "static void" line REL="$(tail -n "+${UNPACK_START}" "$SRC" | grep -n '^}' | head -1 | cut -d: -f1)" UNPACK_END=$((UNPACK_START + REL - 1)) { echo "/* Generated by test_cctx_profile_pack.sh -- verbatim extraction" echo " * from $SRC:" echo " * defines lines ${DEF_START}-${DEF_END}" echo " * pack lines ${PACK_START}-${PACK_END}" echo " * unpack lines ${UNPACK_START}-${UNPACK_END}" echo " * Do not edit; re-run the script to regenerate. */" sed -n "${DEF_START},${DEF_END}p" "$SRC" echo sed -n "${PACK_START},${PACK_END}p" "$SRC" echo sed -n "${UNPACK_START},${UNPACK_END}p" "$SRC" } > "$OUT/generated_profile_pack.inc" # Sanity: a source reflow must fail loudly here, not produce a garbage .inc # that still compiles and "passes". for marker in NGX_HTTP_ZSTD_PROFILE_LEVEL_BIAS \ NGX_HTTP_ZSTD_PROFILE_INVALID \ 'ngx_http_zstd_profile_pack(ngx_int_t level' \ 'ngx_http_zstd_profile_unpack(uint64_t key'; do if ! grep -q "$marker" "$OUT/generated_profile_pack.inc"; then echo "FAIL: extracted fragment missing '$marker' -- range wrong" >&2 cat "$OUT/generated_profile_pack.inc" >&2 exit 1 fi done cp ci/tools/test_cctx_profile_pack.c "$OUT/test_cctx_profile_pack.c" "$CC" -Wall -Wextra -Werror -O2 -I"$OUT" \ -o "$OUT/profile_pack_unit" "$OUT/test_cctx_profile_pack.c" "$OUT/profile_pack_unit" echo "OK: CCtx profile pack/unpack unit fixture (extracted from current $SRC)" myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_cctx_reuse.py000077500000000000000000000355351525232545500253640ustar00rootroot00000000000000#!/usr/bin/env python3 """Regression test for the worker-lifetime ZSTD_CCtx cache. ZSTD_createCCtx() plus its first-use workspace allocation is 34-75% of the per-response libzstd CPU at typical body sizes, so the filter lends one worker-lifetime context to one request at a time instead of creating a fresh context per response. Two properties have to hold at once, and a test for either alone is worthless: 1. The cache actually engages. A green correctness suite proves only that nothing broke -- it passes identically if the cache never hits and every request builds its own context, which is the bug this optimisation exists to avoid. Asserted here from the ngx_log_debug witnesses: N sequential requests through one worker must yield exactly ONE "created cctx" and N-1 "reusing worker cctx". 2. Reuse never bleeds state between requests. Asserted by decoding every response and byte-comparing it to its own distinct origin body. This overlaps test_concurrent_cctx_isolation.py on purpose: that tool proves the CONCURRENT case (a second in-flight claimant must not get the loaned context), this one proves the SEQUENTIAL case (the reused context must carry no residue from the previous response). Also pins the PROFILE-partition rule: a location whose memory-affecting profile differs must NOT reuse a context cached for another profile, because the retained workspace never shrinks on reset. The profile is all three of zstd_comp_level, zstd_long and zstd_window_log -- measured on libzstd 1.5.7, streaming at a fixed level 6 with an unknown pledged size, ZSTD_sizeof_CCtx() is 4.2 MB at window_log 20 and 131 MB at window_log 27, and 147 MB with zstd_long on and no explicit window. Keying on the level alone would let a single high-memory location raise the worker's floor for every request of every other location and silently defeat their zstd_max_cctx_memory budget. Needs an nginx built --with-debug: the reuse witnesses are debug log lines. """ import argparse import importlib.util import os import pathlib import re import subprocess import sys import tempfile import time import urllib.request MODULE_PATH = pathlib.Path(__file__).with_name("test_encoding.py") SPEC = importlib.util.spec_from_file_location("test_encoding", MODULE_PATH) test_encoding = importlib.util.module_from_spec(SPEC) assert SPEC.loader is not None SPEC.loader.exec_module(test_encoding) CREATE_WITNESS = "zstd: created cctx" REUSE_WITNESS = "zstd: reusing worker cctx" # Enough requests that a one-off cache miss cannot be mistaken for the # steady state, small enough to stay fast. ROUNDS = 12 # Comfortably over zstd_min_length, and distinct per request so a stale # context serving the previous body is a byte mismatch, not a silent pass. BODY_SIZE = 24 * 1024 # The three locations whose memory profile differs from the default, each # of which must occupy its OWN ring slot. ALT_LOCATIONS = ( ("/alt/b", 999, "zstd_comp_level 9"), ("/alt/long", 998, "zstd_long on"), ("/alt/wlog", 997, "zstd_window_log 20"), ) # Requests per /alt location. Must be >= 2: the first seeds that profile's # slot, the rest are the reuse witnesses that only a per-profile ring can # produce. ALT_ROUNDS = 4 assert ALT_ROUNDS >= 2 # One context per distinct profile, and no more. WANT_CREATED = 1 + len(ALT_LOCATIONS) # ROUNDS-1 reuses on the default profile (the first /b request seeds its # slot), plus ALT_ROUNDS-1 on each differing profile. WANT_REUSED = (ROUNDS - 1) + len(ALT_LOCATIONS) * (ALT_ROUNDS - 1) def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description="Regression test for the worker-lifetime ZSTD_CCtx cache." ) parser.add_argument("--nginx-binary", required=True) parser.add_argument("--filter-module") parser.add_argument("--port", type=int, default=18116) parser.add_argument( "--log-level", choices=("debug", "warn"), default="debug", help="Location error_log level. The cache-engagement witnesses are " "only asserted at debug. Use warn under sanitizer builds: UBSAN " "(-fno-sanitize-recover) fatally traps nginx core's own debug " 'logging -- every "%%V?%%V" line passes r->args={0,NULL} into ' "ngx_sprintf_str's nonnull argument on query-less URIs " "(ngx_string.c:586) -- so a sanitized nginx cannot log at debug at " "all. The byte-exactness oracles still run and still gate; only the " "witness counts are skipped.", ) parser.add_argument( "--zstd-bin", default=test_encoding.shutil.which("zstd") or "zstd", ) return parser.parse_args() def fixture(rid: int) -> bytes: """A distinct, compressible body per request id. Distinct matters: if every request shared one body, a context that replayed the PREVIOUS response would still decode byte-exact and the isolation half of this test would assert nothing. """ seed = f"cctx-reuse-{rid:04d}-".encode() return (seed * (BODY_SIZE // len(seed) + 1))[:BODY_SIZE] def http_get(port: int, path: str) -> bytes: req = urllib.request.Request( f"http://127.0.0.1:{port}{path}", headers={"Accept-Encoding": "zstd"}, ) with urllib.request.urlopen(req, timeout=10) as resp: if resp.headers.get("Content-Encoding") != "zstd": raise RuntimeError( f"{path}: expected Content-Encoding: zstd, got " f"{resp.headers.get('Content-Encoding')!r} -- the response was " f"not compressed, so this run proves nothing about the cache" ) return resp.read() def main() -> int: args = parse_args() nginx = pathlib.Path(args.nginx_binary) if not nginx.exists(): raise FileNotFoundError(nginx) v = subprocess.run([str(nginx), "-V"], capture_output=True, text=True, check=False) if "zstd" not in v.stderr: raise RuntimeError("nginx -V shows no zstd module") if args.log_level == "debug" and "--with-debug" not in v.stderr: raise RuntimeError( "the reuse witnesses are ngx_log_debug lines: this tool needs an " "nginx built --with-debug, or --log-level warn to skip them" ) filter_so = test_encoding.detect_module_path( args.filter_module, nginx, "ngx_http_zstd_filter_module.so" ) load = f"load_module {filter_so};\n" if filter_so else "" def decode(blob: bytes) -> bytes: r = subprocess.run( [args.zstd_bin, "-d", "-q", "-c"], input=blob, capture_output=True, check=False, ) if r.returncode != 0: raise RuntimeError( "zstd decode failed: " + r.stderr.decode("utf-8", "replace").strip() ) return r.stdout os.umask(0o022) with tempfile.TemporaryDirectory(prefix="zstd-cctx-reuse-") as td: os.chmod(td, 0o755) root = pathlib.Path(td) (root / "logs").mkdir() html = root / "html" html.mkdir() alt = root / "html-alt" alt.mkdir() for rid in range(ROUNDS): (html / f"b{rid}").write_bytes(fixture(rid)) # Served by the three locations that must each take the per-request # path: one differing only in zstd_comp_level, one differing only in # zstd_long, one differing only in zstd_window_log. The latter two # sit at the SAME level as the cached default on purpose -- under a # level-only cache key they would wrongly borrow it. (alt / "b").write_bytes(fixture(999)) (alt / "long").write_bytes(fixture(998)) (alt / "wlog").write_bytes(fixture(997)) lvl = args.log_level conf = root / "nginx.conf" conf.write_text( f"""worker_processes 1; {load}error_log {root}/logs/error.log warn; pid {root}/nginx.pid; events {{ worker_connections 64; }} http {{ access_log off; default_type application/octet-stream; zstd on; zstd_min_length 1; zstd_types application/octet-stream; gzip_vary on; server {{ listen 127.0.0.1:{args.port}; location /b {{ error_log {root}/logs/error.log {lvl}; alias {html}/; }} location /alt/b {{ error_log {root}/logs/error.log {lvl}; alias {alt}/b; zstd_comp_level 9; }} # Same compression level as the cached default context; differs # only in a memory-affecting parameter. Each must create its own # context rather than borrowing the cached one. location /alt/long {{ error_log {root}/logs/error.log {lvl}; alias {alt}/long; zstd_long on; }} location /alt/wlog {{ error_log {root}/logs/error.log {lvl}; alias {alt}/wlog; zstd_window_log 20; }} }} }} """, encoding="utf-8", ) nlog_path = root / "logs" / "nginx-stdout.log" nlog = open(nlog_path, "w", encoding="utf-8") # noqa: SIM115 proc = subprocess.Popen( [ str(nginx), "-p", str(root), "-c", str(conf), "-g", "daemon off; master_process off;", ], stdout=nlog, stderr=subprocess.STDOUT, text=True, ) failures: list[str] = [] try: test_encoding.wait_for_port(args.port) if proc.poll() is not None: nlog.flush() raise RuntimeError( "nginx exited during startup; tail:\n" + nlog_path.read_text("utf-8", "replace")[-2000:] ) # --- property 2: no state bleed across reused contexts --- for rid in range(ROUNDS): want = fixture(rid) got = decode(http_get(args.port, f"/b/b{rid}")) if got != want: failures.append( f"request {rid}: decoded {len(got)}B != origin " f"{len(want)}B -- the reused context carried state " f"from a previous response" ) break # Each differing memory profile gets its OWN ring slot: the # first request seeds that slot, and every further request to # the same location must REUSE it. Requesting each location # ALT_ROUNDS times is what makes the per-profile half of the # ring observable at all -- with a single request each, a ring # and the old single-slot cache produce identical witness # counts (one create, no reuse) and this test could not tell # them apart. Under the single slot the default /b profile owns # the one slot, so every one of these requests is a fresh # create and `reused` falls short by ALT_PROFILES * # (ALT_ROUNDS - 1). for path, rid, label in ALT_LOCATIONS: for _ in range(ALT_ROUNDS): if decode(http_get(args.port, path)) != fixture(rid): failures.append(f"{label} location: decoded body mismatch") break # Flush the debug log before reading witnesses. time.sleep(0.3) elog = (root / "logs" / "error.log").read_text("utf-8", "replace") created = len(re.findall(re.escape(CREATE_WITNESS), elog)) reused = len(re.findall(re.escape(REUSE_WITNESS), elog)) if args.log_level != "debug": print( " witnesses skipped (--log-level warn: a sanitized " "nginx cannot log at debug -- byte-exactness still " "gates, cache engagement is NOT proven in this run)" ) # --- property 1: the cache actually engages --- # # Exactly four contexts are ever created: the default-profile one # seeded by the first /b request, plus one each for the three # /alt locations, none of which may borrow it. Fewer than four # means the cache lent across differing memory profiles -- the # bug this partition exists to prevent, and the two same-level # locations are the ones a level-only key gets wrong. More than # four means the cache is not holding across requests. if args.log_level == "debug" and created != WANT_CREATED: failures.append( f"expected exactly {WANT_CREATED} 'created cctx' " f"witnesses (one per distinct memory profile: the " f"default, plus the zstd_comp_level / zstd_long / " f"zstd_window_log locations, each of which must get its " f"OWN ring slot rather than borrowing another " f"profile's), saw {created} -- fewer means a slot was " f"lent across memory profiles, more means slots are not " f"surviving across requests" ) # One create per distinct profile; every other request reuses # that profile's slot. The ALT_ROUNDS-1 reuses per /alt # location are the per-profile-slot witnesses: a single-slot # cache cannot produce them, because the default profile # permanently owns the only slot. if args.log_level == "debug" and reused != WANT_REUSED: failures.append( f"expected {WANT_REUSED} 'reusing worker cctx' " f"witnesses ({ROUNDS - 1} on the default profile plus " f"{ALT_ROUNDS - 1} on each of the {len(ALT_LOCATIONS)} " f"differing profiles, which a single-slot cache cannot " f"produce at all), saw {reused}" ) if not failures: print( f" {ROUNDS} sequential responses decoded byte-exact to " f"their own origins" ) if args.log_level == "debug": print( f" witnesses: {created} cctx created, {reused} " f"reused -- cache engaged and profile-partitioned" ) finally: proc.terminate() try: proc.wait(timeout=10) except subprocess.TimeoutExpired: proc.kill() nlog.close() if failures: for f in failures: print(f"FAIL: {f}", file=sys.stderr) return 1 print( f"OK: worker CCtx cache reused across {ROUNDS} sequential requests " f"with no cross-request state bleed, and a differing " f"zstd_comp_level, zstd_long or zstd_window_log each correctly took " f"the per-request path" ) return 0 if __name__ == "__main__": sys.exit(main()) myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_ceil_log2_unit.c000066400000000000000000000071751525232545500257040ustar00rootroot00000000000000/* * Unit fixture for ngx_http_zstd_ceil_log2() — grind-g6-nits owed control. * * ngx_http_zstd_ceil_log2() is `static` inside * src/ngx_http_zstd_filter_module.c (a dcz window-log helper, not a * header), so it cannot be linked from outside that TU. Rather than keep * a hand-copied duplicate that silently drifts from the shipped code * (the repo has exactly this "extraction" precedent for the Accept-Encoding * parser fuzz harness), tools/test_ceil_log2_unit.sh extracts the function * verbatim by line range and #includes the generated file here — this * test always exercises the CURRENT shipped implementation. * * ngx_http_zstd_ceil_log2() was shipped BROKEN in this PR (inverted * power-of-two test: incremented wlog for exact powers of two instead of * non-powers) and fixed in c264f4b. This fixture asserts equivalence * against a reference LINEAR shift-and-count implementation (independent * of __builtin_clzll) over every value 0..2^24 inclusive — covering every * exact power of two through 2^23 — and asserts the result always lands * in [10, NGX_HTTP_ZSTD_DCZ_MAX_WINDOW_LOG(=23)]. */ #include #include typedef uintptr_t ngx_uint_t; #define ngx_inline inline #include "generated_ceil_log2.inc" #define WLOG_MIN 10 #define WLOG_MAX 23 /* Reference implementation: plain linear shift-and-count, independent of * __builtin_clzll, mirroring the portable (#else) branch's algorithm but * kept as a SEPARATE hand-written oracle rather than a copy of either * branch under test. */ static ngx_uint_t ref_ceil_log2(size_t x) { ngx_uint_t wlog; size_t pow2; if (x <= 1024) { return WLOG_MIN; } if (x > ((size_t) 1 << WLOG_MAX)) { return WLOG_MAX; } wlog = WLOG_MIN; pow2 = 1024; while (pow2 < x) { pow2 <<= 1; wlog++; } return wlog; } int main(void) { size_t x; ngx_uint_t got, want; long mismatches = 0; long checked = 0; int k; /* Full range 0..2^24, including every power of two through 2^23. */ for (x = 0; x <= ((size_t) 1 << 24); x++) { got = ngx_http_zstd_ceil_log2(x); want = ref_ceil_log2(x); checked++; if (got != want) { if (mismatches < 20) { printf("FAIL - x=%zu got=%lu want=%lu\n", x, (unsigned long) got, (unsigned long) want); } mismatches++; } if (got < WLOG_MIN || got > WLOG_MAX) { printf("FAIL - x=%zu out of bounds: got=%lu (want [%d,%d])\n", x, (unsigned long) got, WLOG_MIN, WLOG_MAX); mismatches++; } } /* Explicitly re-check every power of two through 2^23 (already inside * the swept range above, but named individually so a regression in * just the power-of-two branch shows up as a named failure, not a * buried count). */ for (k = 0; k <= 23; k++) { size_t p = (size_t) 1 << k; got = ngx_http_zstd_ceil_log2(p); want = ref_ceil_log2(p); if (got != want) { printf("FAIL - power-of-two 2^%d: got=%lu want=%lu\n", k, (unsigned long) got, (unsigned long) want); mismatches++; } } printf("checked %ld values (0..2^24) + 24 named powers of two, " "%ld mismatch(es)\n", checked, mismatches); if (mismatches) { printf("FAILED: %ld ceil_log2 mismatch(es) against linear reference\n", mismatches); return 1; } printf("OK: ceil_log2 equivalence (0..2^24, bounds [%d,%d])\n", WLOG_MIN, WLOG_MAX); return 0; } myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_ceil_log2_unit.sh000077500000000000000000000065761525232545500261030ustar00rootroot00000000000000#!/bin/bash # Unit fixture for ngx_http_zstd_ceil_log2() (grind-g6-nits owed control). # The function is `static` inside ngx_http_zstd_filter_module.c, so this # extracts it verbatim by line range (same technique already used by the # repo's Accept-Encoding parser fuzz harness) and compiles it standalone # against a linear-reference oracle in test_ceil_log2_unit.c. No nginx # tree, no zstd needed — pure C, self-contained. set -euo pipefail # ci/tools/ -> ci/ -> repo root: TWO levels since the ci/ move. cd "$(dirname "$0")/../.." SRC="src/ngx_http_zstd_filter_module.c" CC="${CC:-cc}" OUT="$(mktemp -d "${TMPDIR:-/tmp}/zstd-ceil-log2-unit.XXXXXX")" trap 'rm -rf "$OUT"' EXIT # Locate the function by its exact signature line, then its opening and # closing braces, so the extraction survives unrelated edits elsewhere in # the file but fails loudly (grep -c check below) if the function itself # is renamed or removed. START_LINE="$(grep -n '^static ngx_inline ngx_uint_t$' "$SRC" | head -1 | cut -d: -f1)" if [ -z "$START_LINE" ]; then echo "FAIL: could not locate ngx_http_zstd_ceil_log2() start in $SRC" >&2 exit 1 fi SIG_LINE=$((START_LINE + 1)) if ! sed -n "${SIG_LINE}p" "$SRC" | grep -q 'ngx_http_zstd_ceil_log2(size_t x)'; then echo "FAIL: line $SIG_LINE in $SRC is not ngx_http_zstd_ceil_log2's signature" >&2 exit 1 fi # First "^}" at column 0 after START_LINE closes the function. REL_END="$(tail -n "+${START_LINE}" "$SRC" | grep -n '^}' | head -1 | cut -d: -f1)" END_LINE=$((START_LINE + REL_END - 1)) mkdir -p "$OUT" { echo "/* Generated by test_ceil_log2_unit.sh — verbatim extraction of" echo " * ngx_http_zstd_ceil_log2() from $SRC lines ${START_LINE}-${END_LINE}." echo " * Do not edit; re-run the script to regenerate. */" sed -n "${START_LINE},${END_LINE}p" "$SRC" } >"$OUT/generated_ceil_log2.inc" # Sanity: the extraction must actually contain the function body markers, # or a source reflow silently produced an empty/garbage .inc. if ! grep -q 'ngx_http_zstd_ceil_log2(size_t x)' "$OUT/generated_ceil_log2.inc"; then echo "FAIL: extracted fragment does not contain the function signature" >&2 cat "$OUT/generated_ceil_log2.inc" >&2 exit 1 fi if ! grep -q '__builtin_clzll' "$OUT/generated_ceil_log2.inc"; then echo "FAIL: extracted fragment missing the GCC/Clang branch — extraction range wrong" >&2 exit 1 fi cp ci/tools/test_ceil_log2_unit.c "$OUT/test_ceil_log2_unit.c" "$CC" -Wall -Wextra -Werror -O2 -I"$OUT" \ -o "$OUT/ceil_log2_unit" "$OUT/test_ceil_log2_unit.c" "$OUT/ceil_log2_unit" # C89 has neither `unsigned long long` nor C99's mixed declarations. This # strict fixture therefore selects the portable fallback, executes its power- # of-two boundaries, and makes either construct leaking into that path a hard # compile failure. cat >"$OUT/test_ceil_log2_c89.c" <<'EOF' #include typedef unsigned long ngx_uint_t; #define ngx_inline __inline #include "generated_ceil_log2.inc" int main(void) { return ngx_http_zstd_ceil_log2(1024) != 10 || ngx_http_zstd_ceil_log2(1025) != 11 || ngx_http_zstd_ceil_log2(4096) != 12 || ngx_http_zstd_ceil_log2(4097) != 13 || ngx_http_zstd_ceil_log2((size_t) 1 << 23) != 23; } EOF "$CC" -std=c89 -pedantic-errors -Wall -Wextra -Werror -O2 -I"$OUT" \ -o "$OUT/ceil_log2_c89" "$OUT/test_ceil_log2_c89.c" "$OUT/ceil_log2_c89" echo "OK: ceil_log2 unit fixture (extracted from current $SRC)" myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_ci_build_provenance.sh000077500000000000000000000163111525232545500271630ustar00rootroot00000000000000#!/usr/bin/env bash # # Regression tests for audit sha e289021 F2 and F3, both in tools/ci-build.sh: # # F2 - a fresh (or NO_CACHE=1) build always failed: the tarball's top-level # dir name and the script's own SRCDIR were the same string, so # extraction already landed at SRCDIR and the subsequent `mv` tried to # move that dir onto itself ("cannot move to a subdirectory of # itself"). This never actually built anything on a clean root. # F3 - archive provenance: a wrong PGP signer, a corrupted/tampered # (cache-hit) tarball, or a wrong pinned sha256 must all be rejected # BEFORE extraction, not silently accepted. # # This test does not hit the network for the "must reject" cases -- it # stages a fake $BUILD_ROOT by hand so it can run offline and fast. It DOES # need network + real nginx.org access for the "F2: clean root actually # reaches configure" case (skipped if offline). # # Usage: tools/test_ci_build_provenance.sh set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" MODULE_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")" CI_BUILD="$SCRIPT_DIR/ci-build.sh" fail=0 WORK="$(mktemp -d)" trap 'rm -rf "$WORK"' EXIT copy_build_fixture() { local destination="$1" local input while IFS= read -r input || [ -n "$input" ]; do case "$input" in '' | \#*) continue ;; esac mkdir -p "$destination/$(dirname "$input")" || return 1 cp -a "$MODULE_DIR/$input" "$destination/$input" || return 1 done < "$MODULE_DIR/ci/build-inputs.manifest" } # --------------------------------------------------------------------- # F3a: wrong sha256 pin must be rejected for a statically-pinned version, # even though the tarball content and PGP signature (if any) are untouched. # We fabricate a tiny "tarball" and inject a bogus pin table entry via a # throwaway copy of the script so the real pin table isn't touched. # --------------------------------------------------------------------- test_wrong_sha256_rejected() { local root="$WORK/f3-sha256" mkdir -p "$root" # angie path is simpler to fake: sha256-only, no PGP. echo "not a real tarball" >"$root/angie-9.9.9.tar.gz" local fixture="$WORK/f3-module" if ! copy_build_fixture "$fixture"; then echo "FAIL: could not stage the hermetic ci-build fixture" return 1 fi local copy="$fixture/ci/tools/ci-build.sh" if [ ! -f "$fixture/ci/tools/build-input-hash.sh" ] || \ [ ! -f "$fixture/ci/build-inputs.manifest" ]; then echo "FAIL: hermetic ci-build fixture is missing canonical hash inputs" return 1 fi # Force a pin that cannot possibly match the fake tarball's digest. sed -i 's/\["1\.11\.5"\]=.*/["9.9.9"]="0000000000000000000000000000000000000000000000000000000000000"/' "$copy" if BUILD_ROOT="$root" bash "$copy" angie 9.9.9 >"$root/out.log" 2>&1; then echo "FAIL: ci-build.sh accepted a tarball with a mismatched pinned sha256" cat "$root/out.log" return 1 fi if ! grep -q "sha256 mismatch" "$root/out.log"; then echo "FAIL: rejection happened but not via the expected sha256-mismatch path:" cat "$root/out.log" return 1 fi echo "✓ F3: wrong pinned sha256 rejected before extraction" } # --------------------------------------------------------------------- # F3b: corrupted CACHED tarball must be re-verified and rejected, not # silently reused just because a file with the right name already exists. # (This was the literal pre-fix gap: verification only ran on fresh # download.) Same fake-tarball approach, but this time the "cache" is # pre-populated before the script runs. # --------------------------------------------------------------------- test_corrupted_cache_rejected() { local root="$WORK/f3-cache" mkdir -p "$root" echo "corrupted cached tarball" >"$root/angie-1.11.5.tar.gz" if BUILD_ROOT="$root" bash "$CI_BUILD" angie 1.11.5 >"$root/out.log" 2>&1; then echo "FAIL: ci-build.sh accepted a corrupted CACHED tarball for a pinned version" cat "$root/out.log" return 1 fi if ! grep -q "sha256 mismatch" "$root/out.log"; then echo "FAIL: cache-hit path did not re-verify sha256:" cat "$root/out.log" return 1 fi echo "✓ F3: corrupted cached tarball re-verified and rejected on cache-hit" } # --------------------------------------------------------------------- # F2: clean-root / NO_CACHE=1 extraction must not fail with "mv onto # itself". We can exercise the mv-guard logic in isolation (no network) # by reproducing exactly what ci-build.sh does: extract a tarball whose # top-level dir name equals $FLAVOR-$VERSION, then run the same guarded # mv line. This proves the guard itself is correct without needing a real # nginx/angie tarball. # --------------------------------------------------------------------- test_mv_guard_clean_root() { local root="$WORK/f2-mv" mkdir -p "$root" local dir="fake-1.0.0" mkdir -p "$root/$dir" touch "$root/$dir/marker" tar -czf "$root/$dir.tar.gz" -C "$root" "$dir" rm -rf "${root:?}/$dir" local srcdir="$root/$dir" # same string ci-build.sh derives: "$ROOT/${FLAVOR}-${VERSION}" # Reproduce ci-build.sh's exact extraction + guarded-mv sequence. if [ ! -d "$srcdir" ]; then tar -xzf "$root/$dir.tar.gz" -C "$root" if [ "$root/$dir" != "$srcdir" ]; then mv "$root/$dir" "$srcdir" fi fi if [ ! -f "$srcdir/marker" ]; then echo "FAIL: F2 mv-guard: extracted tree missing after guarded mv (regressed to pre-fix unconditional mv-onto-self failure mode)" return 1 fi echo "✓ F2: guarded mv is a no-op when extraction already lands at SRCDIR (clean-root case)" } # --------------------------------------------------------------------- # F2 (live): actually run ci-build.sh against angie on a clean root with # NO_CACHE=1 and confirm it reaches configure (this is the case that was # unconditionally broken pre-fix). Needs network; skipped if unreachable. # --------------------------------------------------------------------- test_clean_root_live() { if ! curl -fsS -o /dev/null --max-time 5 https://download.angie.software/ 2>/dev/null; then echo "SKIP: F2 live clean-root build (no network to download.angie.software)" return 0 fi local root="$WORK/f2-live" mkdir -p "$root" if ! NO_CACHE=1 BUILD_ROOT="$root" timeout 300 bash "$CI_BUILD" angie 1.11.5 >"$root/out.log" 2>&1; then echo "FAIL: F2 live: clean-root NO_CACHE=1 angie build did not reach a successful build:" tail -40 "$root/out.log" return 1 fi if ! grep -q "cannot move to a subdirectory of itself" "$root/out.log"; then echo "✓ F2: live clean-root NO_CACHE=1 angie build succeeded, no mv-onto-self failure" else echo "FAIL: F2 regressed -- mv-onto-self error resurfaced in a live clean-root build" return 1 fi } test_wrong_sha256_rejected || fail=1 test_corrupted_cache_rejected || fail=1 test_mv_guard_clean_root || fail=1 test_clean_root_live || fail=1 if [ "$fail" -ne 0 ]; then echo "FAIL: ci-build.sh provenance/clean-root regression" exit 1 fi echo "✓ all ci-build.sh F2/F3 regression cases pass" myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_ci_dependency_bootstrap.sh000077500000000000000000000257051525232545500300660ustar00rootroot00000000000000#!/usr/bin/env bash # ci/tools/test_ci_dependency_bootstrap.sh -- ensure fork fallbacks install # their non-runner dependencies before CI uses them. # # Usage: ci/tools/test_ci_dependency_bootstrap.sh # Inputs: repository .github/workflows/*.yml files. # Side effects: none. Exit non-zero on a missing bootstrap or package. set -euo pipefail root=${CI_DEPENDENCY_ROOT:-$(git rev-parse --show-toplevel)} cd "$root" # The apt packages a step's `run` installs, and the checks built on it: # require_apt_package accepts the package installed by ANY step of the # file; require_bootstrap_package demands it from EVERY step named # "Install bootstrap dependencies"; require_resolver_jobs walks each JOB # that runs the nginx resolver and demands that the same job has a # bootstrap step before it installing curl and python3 -- a file-wide # check would stay green with one job's bootstrap deleted as long as any # other job still carried one. APT_STEP_PY=' import pathlib, shlex, sys, yaml path, wanted, mode = pathlib.Path(sys.argv[1]), sys.argv[2], sys.argv[3] BOOTSTRAP = "Install bootstrap dependencies" RESOLVER = "ci/tools/nginx-releases.sh" NEEDED = ("curl", "python3") def installed(run): found = set() logical = run.replace("\\\n", " ") for line in logical.splitlines(): try: lexer = shlex.shlex(line, posix=True, punctuation_chars=";&|") lexer.whitespace_split = True lexer.commenters = "#" words = list(lexer) except ValueError: continue start = 0 for end in [ *[i for i, word in enumerate(words) if word in {";", "&&", "||", "|"}], len(words), ]: command = words[start:end] start = end + 1 apt = 1 if command[:1] == ["sudo"] else 0 if len(command) > apt and command[apt] == "apt-get" \ and "install" in command[apt + 1:]: install = command.index("install", apt + 1) found |= {w for w in command[install + 1:] if not w.startswith("-")} return found doc = yaml.safe_load(path.read_text(encoding="utf-8")) if mode == "jobs": bad = 0 resolving = 0 for name, job in (doc.get("jobs") or {}).items(): steps = job.get("steps", []) if isinstance(job, dict) else [] steps = [s for s in steps if isinstance(s, dict)] resolver = [i for i, s in enumerate(steps) if isinstance(s.get("run"), str) and RESOLVER in s["run"]] if not resolver: continue resolving += 1 boots = [i for i, s in enumerate(steps) if s.get("name") == BOOTSTRAP and i < resolver[0]] if not boots: print(f"FAIL: {path}: job {name!r} runs the nginx resolver (step " f"{resolver[0]}) with no {BOOTSTRAP!r} step before it in " "that job", file=sys.stderr) bad += 1 continue packages = installed(steps[boots[-1]].get("run") or "") for pkg in NEEDED: if pkg not in packages: print(f"FAIL: {path}: job {name!r}: the {BOOTSTRAP!r} step " f"before the resolver does not install apt package " f"{pkg}", file=sys.stderr) bad += 1 if resolving == 0: print(f"FAIL: {path}: no job runs {RESOLVER}", file=sys.stderr) bad += 1 raise SystemExit(1 if bad else 0) bootstrap_steps = 0 for job in (doc.get("jobs") or {}).values(): for step in job.get("steps", []) if isinstance(job, dict) else []: if not isinstance(step, dict) or not isinstance(step.get("run"), str): continue packages = installed(step["run"]) if mode == "any": if wanted in packages: raise SystemExit(0) elif step.get("name") == BOOTSTRAP: bootstrap_steps += 1 if wanted not in packages: print(f"FAIL: {path}: the {BOOTSTRAP!r} step itself must install " f"apt package {wanted} (a later step installing it does not " "count -- the resolver runs right after the bootstrap)", file=sys.stderr) raise SystemExit(1) if mode == "bootstrap" and bootstrap_steps: raise SystemExit(0) if mode == "bootstrap": print(f"FAIL: {path} has no {BOOTSTRAP!r} step to check", file=sys.stderr) else: print(f"FAIL: {path} must install apt package {wanted}", file=sys.stderr) raise SystemExit(1) ' require_apt_package() { local file=$1 package=$2 local base=${CI_DEPENDENCY_ROOT:-$root} python3 -c "$APT_STEP_PY" "$base/$file" "$package" any } require_bootstrap_package() { local file=$1 package=$2 local base=${CI_DEPENDENCY_ROOT:-$root} python3 -c "$APT_STEP_PY" "$base/$file" "$package" bootstrap } require_resolver_jobs() { local file=$1 local base=${CI_DEPENDENCY_ROOT:-$root} python3 -c "$APT_STEP_PY" "$base/$file" - jobs } require() { local file=$1 needle=$2 if ! grep -Fq -- "$needle" "$file"; then echo "FAIL: $file must declare $needle for its fork fallback" >&2 exit 1 fi } require_before() { local file=$1 first=$2 second=$3 first_line second_line first_line=$(grep -n -m1 -F -- "$first" "$file" | cut -d: -f1 || true) second_line=$(grep -n -m1 -F -- "$second" "$file" | cut -d: -f1 || true) if [ -z "$first_line" ] || [ -z "$second_line" ] || [ "$first_line" -ge "$second_line" ]; then echo "FAIL: $file must install curl and python3 before resolving nginx" >&2 exit 1 fi } require_regex() { local file=$1 pattern=$2 if ! grep -Eq -- "$pattern" "$file"; then echo "FAIL: $file must declare a matching fork fallback dependency" >&2 exit 1 fi } # These workflows resolve nginx with curl before their normal package profile. # A fork has no vars.POOL, so ubuntu-latest must receive curl explicitly first. for file in \ .github/workflows/asan.yml \ .github/workflows/build-test.yml \ .github/workflows/ci-deep.yml \ .github/workflows/codeql.yml \ .github/workflows/valgrind.yml; do require "$file" 'name: Install bootstrap dependencies' done # The resolver reads the GitHub releases feed with curl and parses it with # python3. Every JOB that runs it must carry its own "Install bootstrap # dependencies" step ahead of the resolver step, installing both; a later # job-specific install, or another job's bootstrap, must not satisfy that. # Checked per job rather than per file: ci-deep.yml resolves in two jobs, # and a file-wide ordering check stayed green with one of them stripped. for file in \ .github/workflows/asan.yml \ .github/workflows/build-test.yml \ .github/workflows/ci-deep.yml \ .github/workflows/codeql.yml \ .github/workflows/valgrind.yml; do require_resolver_jobs "$file" done # Detached nginx signatures are verified by these fallback workflows. Do not # let their green runs depend on gpg having happened to be in a runner image. for file in \ .github/workflows/asan.yml \ .github/workflows/build-test.yml \ .github/workflows/bump.yml \ .github/workflows/ci-deep.yml \ .github/workflows/codeql.yml \ .github/workflows/harness-fault-arms.yml \ .github/workflows/security-scanners.yml \ .github/workflows/valgrind.yml; do require_apt_package "$file" gnupg done # Test::Nginx is deliberately installed from its pinned CPAN distribution; # semgrep is deliberately installed through pipx/pip. Keep both paths # visible in the workflow rather than assuming a persistent builder carries # either tool already. require_apt_package .github/workflows/build-test.yml cpanminus # The validation job runs this script through git, while cvary-interop clones # the real comparison module. Check both declarations rather than allowing a # package in one job to mask a missing package in the other. require_regex .github/workflows/build-test.yml '^[[:space:]]+git[[:space:]]+\\$' require .github/workflows/build-test.yml \ 'sudo apt-get install -y build-essential git gnupg libzstd-dev' require_apt_package .github/workflows/ci-deep.yml cpanminus require .github/workflows/security-scanners.yml 'semgrep==1.173.0' # The soak workers construct their dictionary digest with the openssl CLI; # likewise, the deep fuzz failure notification serializes JSON with Python. require_apt_package .github/workflows/asan.yml openssl require_apt_package .github/workflows/valgrind.yml openssl require_apt_package .github/workflows/ci-deep.yml openssl require_apt_package .github/workflows/ci-deep.yml clang require_apt_package .github/workflows/ci-deep.yml curl require_apt_package .github/workflows/ci-deep.yml python3 # CodeQL inspects its database through Python and unpacks src.zip when the # action stores extracted sources as an archive. require_apt_package .github/workflows/codeql.yml python3 require_apt_package .github/workflows/codeql.yml unzip # The dependency parser itself uses yaml.safe_load; keep that import available # in the validation job that executes this script. require_apt_package .github/workflows/build-test.yml python3-yaml # Negative control: comments are not package installations. mutant=$(mktemp -d "${TMPDIR:-/tmp}/dependency-bootstrap.XXXXXX") trap 'rm -rf "$mutant"' EXIT mkdir -p "$mutant/.github/workflows" cp .github/workflows/asan.yml "$mutant/.github/workflows/asan.yml" sed -i 's/gnupg/libgcrypt20-dev/g' "$mutant/.github/workflows/asan.yml" printf '\n# gnupg was removed from the apt install above\n' \ >>"$mutant/.github/workflows/asan.yml" sed -i '/set -euo pipefail/a\ echo install gnupg' \ "$mutant/.github/workflows/asan.yml" sed -i '/set -euo pipefail/a\ echo "apt-get install gnupg"' \ "$mutant/.github/workflows/asan.yml" if CI_DEPENDENCY_ROOT="$mutant" require_apt_package \ .github/workflows/asan.yml gnupg >/dev/null 2>&1; then echo 'FAIL: dependency comment mutant was accepted as an installation' >&2 exit 1 fi echo 'OK: dependency comment mutant rejected' # Negative control: strip ONE resolver job's bootstrap step (ci-deep.yml's # helgrind job) while the memcheck job keeps its own. A file-wide check # accepted this; the per-job check must not. cp .github/workflows/ci-deep.yml "$mutant/.github/workflows/ci-deep.yml" python3 - "$mutant/.github/workflows/ci-deep.yml" <<'PY' import pathlib, sys, yaml path = pathlib.Path(sys.argv[1]) doc = yaml.safe_load(path.read_text(encoding="utf-8")) job = doc["jobs"]["helgrind"] before = len(job["steps"]) job["steps"] = [s for s in job["steps"] if not (isinstance(s, dict) and s.get("name") == "Install bootstrap dependencies")] if len(job["steps"]) != before - 1: raise SystemExit("mutant did not remove exactly one bootstrap step") path.write_text(yaml.safe_dump(doc, sort_keys=False), encoding="utf-8") PY if CI_DEPENDENCY_ROOT="$mutant" require_resolver_jobs \ .github/workflows/ci-deep.yml >/dev/null 2>&1; then echo 'FAIL: a resolver job with its bootstrap step removed was accepted' >&2 exit 1 fi echo 'OK: single-job bootstrap removal mutant rejected' echo 'OK: fork fallback dependencies are explicitly bootstrapped' myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_ci_scanner_fuzz_boundaries.sh000077500000000000000000000067251525232545500305760ustar00rootroot00000000000000#!/usr/bin/env bash # Gate CI C scanner selectors and the production-sized fuzz domain. set -euo pipefail root=$(git rev-parse --show-toplevel) cd "$root" assert_contract() { local base=${1:-.} grep -Fq "'^(src|ci)/.*\.[ch]$'" "$base/ci/linter/lint-c.sh" || return 1 [ "$(grep -Fc "files: '^(src|ci)/.*\.[ch]$'" "$base/.pre-commit-config.yaml")" -ge 3 ] \ || return 1 # Check each owning step independently: both must use checked discovery and # pass the explicit C-file array rather than a directory to the scanner. python3 - "$base/.github/workflows/security-scanners.yml" <<'PY' || return 1 import pathlib, re, sys, yaml doc = yaml.safe_load(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")) steps = doc["jobs"]["scanners"]["steps"] for name, command in (("flawfinder", "flawfinder"), ("semgrep", "semgrep scan")): run = next(step["run"] for step in steps if step.get("name") == name) required = ( 'mapfile_checked files find', '"$GITHUB_WORKSPACE/src" "$GITHUB_WORKSPACE/ci"', "-name '*.c'", "-name '*.h'", '[ "${#files[@]}" -gt 0 ]', ) if not all(token in run for token in required): raise SystemExit(f"{name} does not discover checked src+ci C/H targets") invocation = rf'^\s*{re.escape(command)}\b.*?"\$\{{files\[@\]\}}".*?\|\s*tee' if re.search(invocation, run, re.MULTILINE | re.DOTALL) is None: raise SystemExit(f"{name} command does not consume the C target array") if name == "flawfinder" and "--error-level=4" not in run: raise SystemExit("flawfinder findings are not blocking at level 4") if name == "semgrep" and re.search(r'(?/dev/null 2>&1; then echo 'FAIL: scanner/max_len mutant did not make the contract gate red' >&2 exit 1 fi assert_contract # The exact CI Semgrep profiles must still block a real C finding. Keep the # source under a non-C suffix so the repository's normal widened scan stays # clean; copy it to the temporary C target only for this negative control. if command -v semgrep >/dev/null 2>&1; then cp ci/linter/fixtures/semgrep-blocking.c.fixture "$work/blocking.c" set +e semgrep scan --config p/c --config p/security-audit \ --severity=WARNING --severity=ERROR --error --jobs=1 --metrics=off \ "$work/blocking.c" >"$work/semgrep-negative.log" 2>&1 semgrep_rc=$? set -e if [ "$semgrep_rc" -ne 1 ]; then echo "FAIL: Semgrep negative control returned $semgrep_rc, want finding status 1" >&2 cat "$work/semgrep-negative.log" >&2 exit 1 fi echo 'OK: Semgrep C finding remains blocking' fi echo 'OK: CI C scanner selectors and >4096 fuzz boundaries are enforced' myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_compression_matrix.py000077500000000000000000000376571525232545500271540ustar00rootroot00000000000000#!/usr/bin/env python3 """Broad correctness matrix for the zstd filter. The module's truncation / zero-size-buffer / terminal-frame bug class (a209f96, 2af5889, PR#23/#49, bug A #36, bug B) keeps recurring in *different* circumstances, and each prior regression test pinned only the one slice that had just broken. This test crosses the axes the bug class has actually hit and asserts the right outcome in **every cell**: * transport : proxy (buffering on), proxy (buffering off), static * payload : incompressible, highly-compressible, real-CSS-like * size : 1, 1024, CStreamInSize-1/=/+1, 200000, 1 MiB * Accept-Encoding: "zstd" -> must be zstd, byte-exact "gzip, br, zstd" -> some C-E set, body decodes to origin (zstd OR a co-filter wins; either is correct as long as it round-trips) "gzip" -> must NOT be zstd; body intact "" (identity) -> no C-E; body intact Every cell verifies the client can reconstruct the exact origin bytes (decode per the returned Content-Encoding, then byte-compare) — never "got 200" or "got non-empty". A truncation/abort shows up as a short body, a decode error, or a connection failure. Rig discipline (learned the hard way): * threaded backend (single-thread is OOM/stall-killed under load) * backend self-check before trusting any end-to-end result * distinct bytes per (payload,size) so a wrong-body is detectable * fresh request each cell; HTTP error => failure, not silent reuse Self-contained: stdlib + the ``zstd`` and ``gzip`` CLIs (+ ``brotli`` if present; brotli cells are skipped when it is not). No PHP-FPM. """ from __future__ import annotations import argparse import gzip as _gzip import http.server import itertools import os import pathlib import shutil import socket import socketserver import subprocess import sys import tempfile import threading import time import urllib.request from typing import ClassVar CSTREAM_IN = 131072 # ZSTD_CStreamInSize on libzstd 1.5.x — the # historical boundary for this bug class. SIZES = [1, 1024, CSTREAM_IN - 1, CSTREAM_IN, CSTREAM_IN + 1, 200000, 1024 * 1024] PAYLOADS = ("incompressible", "compressible", "css") def parse_args() -> argparse.Namespace: p = argparse.ArgumentParser(description="Broad zstd filter correctness matrix.") p.add_argument("--nginx-binary", required=True) p.add_argument("--filter-module") p.add_argument("--static-module") p.add_argument("--zstd-bin", default="zstd") p.add_argument("--port", type=int, default=18096) p.add_argument("--backend-port", type=int, default=18097) p.add_argument( "--repeat", type=int, default=2, help="Re-request each cell N times (intermittent bugs surface on repeats).", ) return p.parse_args() def detect(explicit, nginx: pathlib.Path, name: str): if explicit: return pathlib.Path(explicit) sib = nginx.parent / name return sib if sib.exists() else None def make_payload(kind: str, n: int) -> bytes: if kind == "incompressible": buf = bytearray(n) x = (n * 2654435761) & 0xFFFFFFFF for i in range(n): x = (x * 1103515245 + 12345) & 0xFFFFFFFF buf[i] = (x >> 16) & 0xFF return bytes(buf) if kind == "compressible": # Highly compressible: zstd buffers internally and produces # zero output on early/forced-flush calls — the exact bug-B # path. A short unique prefix keeps (kind,size) bodies distinct. head = f"CELL-{kind}-{n}-".encode() return (head + b"A" * max(0, n - len(head)))[:n] # "css": realistic mid-ratio text, distinct per size. unit = ( f"/* cell {n} */.c{n}{{color:#abcdef;margin:0 auto;padding:1px}}\n" ).encode() return (unit * (n // len(unit) + 1))[:n] class _Handler(http.server.BaseHTTPRequestHandler): """GET /p// -> exactly bytes of , **chunked, no Content-Length**, streamed in small records.""" protocol_version = "HTTP/1.1" # ClassVar: deliberately shared by every handler instance -- the server # populates it once before serving, and each request only reads it. fixtures: ClassVar[dict] = {} def log_message(self, *a): pass def do_GET(self): parts = self.path.split("?")[0].strip("/").split("/") if len(parts) != 3 or parts[0] != "p": self.send_response(404) self.send_header("Content-Length", "0") self.end_headers() return key = (parts[1], int(parts[2])) data = self.fixtures.get(key) if data is None: self.send_response(404) self.send_header("Content-Length", "0") self.end_headers() return self.send_response(200) self.send_header("Content-Type", "text/css") self.send_header("Transfer-Encoding", "chunked") self.end_headers() mv = memoryview(data) for i in range(0, len(mv), 16384): c = bytes(mv[i : i + 16384]) self.wfile.write(b"%X\r\n" % len(c) + c + b"\r\n") self.wfile.write(b"0\r\n\r\n") class _Srv(socketserver.ThreadingMixIn, socketserver.TCPServer): allow_reuse_address = True daemon_threads = True def wait_port(port: int, timeout: float = 10.0) -> None: end = time.time() + timeout while time.time() < end: try: with socket.create_connection(("127.0.0.1", port), 0.5): return except OSError: time.sleep(0.1) raise RuntimeError(f"nothing on 127.0.0.1:{port}") def fetch(port: int, path: str, accept_encoding: str): hdrs = {"User-Agent": "zstd-matrix/1.0"} if accept_encoding: hdrs["Accept-Encoding"] = accept_encoding req = urllib.request.Request(f"http://127.0.0.1:{port}{path}", headers=hdrs) with urllib.request.urlopen(req, timeout=25) as resp: return resp.read(), (resp.headers.get("Content-Encoding") or "").lower() def decode(zstd_bin: str, blob: bytes, enc: str) -> bytes: if enc == "zstd": if blob[:4] != b"\x28\xb5\x2f\xfd": raise RuntimeError(f"zstd C-E but no zstd magic (hex={blob[:8].hex()})") r = subprocess.run( [zstd_bin, "-dq", "-c"], input=blob, capture_output=True, check=False ) if r.returncode != 0: raise RuntimeError( "zstd -d failed (premature end): " + r.stderr.decode("utf-8", "replace") ) return r.stdout if enc == "gzip": return _gzip.decompress(blob) if enc == "br": br = shutil.which("brotli") if not br: raise RuntimeError("br C-E but no brotli CLI to verify") r = subprocess.run([br, "-dc"], input=blob, capture_output=True, check=False) if r.returncode != 0: raise RuntimeError("brotli -d failed") return r.stdout if enc in ("", "identity"): return blob raise RuntimeError(f"unexpected Content-Encoding {enc!r}") def main() -> int: args = parse_args() nginx = pathlib.Path(args.nginx_binary) if not nginx.exists(): raise FileNotFoundError(nginx) mods = [ m for m in ( detect(args.filter_module, nginx, "ngx_http_zstd_filter_module.so"), detect(args.static_module, nginx, "ngx_http_zstd_static_module.so"), ) if m ] have_brotli = shutil.which("brotli") is not None # Everything below must stay readable by the workers when run as # root (they drop to the compiled-in nginx user): a restrictive # inherited umask (e.g. 077) would strip group/other bits from # every fixture and subdir created here, 403ing the workers even # with the scratch root itself opened up. os.umask(0o022) with tempfile.TemporaryDirectory(prefix="zstd-matrix-") as td: # mkdtemp gives 0700: run as root, workers drop to the # compiled-in nginx user and cannot enter it -> 403s os.chmod(td, 0o755) root = pathlib.Path(td) sroot = root / "static" logs = root / "logs" sroot.mkdir() logs.mkdir() fixtures: dict = {} for kind, n in itertools.product(PAYLOADS, SIZES): data = make_payload(kind, n) fixtures[(kind, n)] = data (sroot / f"{kind}-{n}").write_bytes(data) _Handler.fixtures = fixtures backend = _Srv(("127.0.0.1", args.backend_port), _Handler) threading.Thread(target=backend.serve_forever, daemon=True).start() try: wait_port(args.backend_port) # backend self-check (rig discipline) for kind, n in ( ("incompressible", 1), ("compressible", CSTREAM_IN), ("css", 1024 * 1024), ): raw = urllib.request.urlopen( f"http://127.0.0.1:{args.backend_port}/p/{kind}/{n}", timeout=10 ).read() if raw != fixtures[(kind, n)]: raise RuntimeError(f"backend self-check failed for {kind}/{n}") load = "".join(f"load_module {m};\n" for m in mods) conf = root / "nginx.conf" conf.write_text( f"""{load}worker_processes 1; error_log {logs}/error.log warn; pid {root}/nginx.pid; events {{ worker_connections 256; }} http {{ access_log off; default_type text/css; zstd on; zstd_comp_level 6; zstd_min_length 1; zstd_types text/css; gzip on; gzip_comp_level 5; gzip_min_length 1; gzip_types text/css; server {{ listen 127.0.0.1:{args.port}; # transport: proxy, buffering ON (default) location /bufon/ {{ proxy_pass http://127.0.0.1:{args.backend_port}/p/; proxy_http_version 1.1; proxy_set_header Connection ""; }} # transport: proxy, buffering OFF (the bug-B production shape) location /bufoff/ {{ proxy_pass http://127.0.0.1:{args.backend_port}/p/; proxy_http_version 1.1; proxy_set_header Connection ""; proxy_buffering off; }} # transport: static file (sendfile path) location /static/ {{ alias {sroot}/; }} }} }} """, encoding="utf-8", ) proc = subprocess.Popen( [ str(nginx), "-p", str(root), "-c", str(conf), "-g", "daemon off; master_process off;", ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, ) try: wait_port(args.port) ae_axis = ["zstd", "gzip", ""] if have_brotli: ae_axis.append("gzip, br, zstd") else: ae_axis.append("gzip, zstd") # zstd should win transports = { "bufon": lambda k, n: f"/bufon/{k}/{n}", "bufoff": lambda k, n: f"/bufoff/{k}/{n}", "static": lambda k, n: f"/static/{k}-{n}", } failures: list[str] = [] cells = 0 for tname, mkpath in transports.items(): for kind in PAYLOADS: for n in SIZES: origin = fixtures[(kind, n)] for ae in ae_axis: for attempt in range(args.repeat): cells += 1 lbl = ( f"{tname} {kind} n={n} AE={ae!r} #{attempt + 1}" ) try: body, enc = fetch( args.port, mkpath(kind, n), ae ) except Exception as e: # noqa failures.append(f"{lbl}: request failed: {e}") continue # Correctness rules: # AE 'zstd' -> must be zstd. # AE 'gzip' -> must NOT be zstd. # any -> body decodes to origin. if ae == "zstd" and enc != "zstd": failures.append( f"{lbl}: expected zstd, got C-E={enc!r}" ) continue if ae == "gzip" and enc == "zstd": failures.append( f"{lbl}: zstd served to gzip-only client" ) continue try: dec = decode(args.zstd_bin, body, enc) except Exception as e: # noqa failures.append( f"{lbl}: decode failed (C-E={enc!r}): {e}" ) continue if dec != origin: failures.append( f"{lbl}: body mismatch " f"(C-E={enc!r} got " f"{len(dec)}B want {n}B)" ) if failures: sys.stderr.write(f"MATRIX FAILED: {len(failures)}/{cells} cells:\n") for f in failures[:30]: sys.stderr.write(f" - {f}\n") if len(failures) > 30: sys.stderr.write(f" ... +{len(failures) - 30} more\n") el = logs / "error.log" if el.exists(): z = [ ln for ln in el.read_text("utf-8", "replace").splitlines() if "zero size buf in writer" in ln ] if z: sys.stderr.write( f" nginx logged {len(z)} 'zero size buf in writer'\n" ) return 1 print( f"OK: {cells} matrix cells " f"({len(transports)} transports x " f"{len(PAYLOADS)} payloads x {len(SIZES)} sizes " f"x {len(ae_axis)} AE x {args.repeat}) " f"all round-tripped byte-exact / correct " f"fallback" + ("" if have_brotli else " [brotli CLI absent: br-axis skipped]") ) return 0 finally: proc.terminate() try: # 30s, not 5: gcov-instrumented nginx flushing # .gcda at exit on a loaded runner can outlive a # tight reap budget; teardown runs after the # verdict, so patience costs nothing when healthy. proc.wait(timeout=30) except subprocess.TimeoutExpired: proc.kill() proc.wait(timeout=30) finally: backend.shutdown() backend.server_close() if __name__ == "__main__": try: raise SystemExit(main()) except Exception as exc: print(f"ERROR: {exc}", file=sys.stderr) raise myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_concurrent_cctx_isolation.py000077500000000000000000000374201525232545500304770ustar00rootroot00000000000000#!/usr/bin/env python3 """Regression test for per-request ZSTD_CCtx isolation. Bug class --------- Commit 774b4a5 ("fix: use per-request zstd contexts") fixed a defect where the compression context was effectively shared across requests handled by one worker. A shared / improperly-reset ``ZSTD_CCtx`` does NOT crash — it silently produces a *wrong but well-formed* zstd stream: request A's worker state bleeds into request B, so B decompresses to a mix of A's and B's bytes (or a frame that decodes cleanly but to the wrong content). None of the other tests can see this: they each fetch a single body, or many *identical* bodies, so cross-request bleed is invisible. This test makes every concurrent request ask for **distinct content** and asserts each response decompresses to *its own* expected bytes, exactly. Cross-request CCtx contamination shows up as a body that decodes to the wrong request's payload (or a decode error / wrong length). Coverage -------- * High concurrency against a single worker (worker_processes 1) so all requests share the one worker's context lifecycle. * Distinct per-request payloads spanning the ZSTD_CStreamInSize buffer boundary (the size where the module's bugs cluster), with a unique per-request marker prefix so a swapped body is unambiguous. * HTTP/1.1 keepalive reuse: many requests per connection (connection reuse is where a not-properly-reset context leaks between requests). * Several rounds; every response decode-and-diffed against its own origin. Rig discipline (same hard rules as the other regression tests): * threaded backend (single-thread is OOM/stall-killed under load) * backend self-check before trusting any end-to-end result * distinct bytes per request id so a wrong-body is detectable * per-request fresh buffer; an HTTP error is a failure, not a retry * overall + per-request timeouts so a livelock is a hard FAIL Self-contained: stdlib + the ``zstd`` CLI only. """ from __future__ import annotations import argparse import concurrent.futures import hashlib import http.server import os import pathlib import re import socket import socketserver import subprocess import sys import tempfile import threading import time import urllib.request CSTREAM_IN = 131072 # ZSTD_CStreamInSize (libzstd 1.5.x) — the size # where this module's bug history clusters. # Distinct sizes straddling the boundary; each request id maps to one. SIZES = [913, 65536, CSTREAM_IN - 1, CSTREAM_IN, CSTREAM_IN + 1, 180003, 524288] def parse_args() -> argparse.Namespace: p = argparse.ArgumentParser( description="Per-request ZSTD_CCtx isolation regression (774b4a5 class)." ) p.add_argument("--nginx-binary", required=True) p.add_argument("--filter-module") p.add_argument("--static-module") p.add_argument("--zstd-bin", default="zstd") p.add_argument("--port", type=int, default=18098) p.add_argument("--backend-port", type=int, default=18099) p.add_argument( "--concurrency", type=int, default=16, help="Simultaneous in-flight requests." ) p.add_argument( "--rounds", type=int, default=6, help="Times to run the full concurrent batch." ) p.add_argument( "--require-multi-slot", action="store_true", help="Also assert, from the debug-log witnesses, that the worker " "CCtx ring actually lent MORE THAN ONE slot during the run. " "Without this the byte-exactness oracle alone is satisfied by a " "build where concurrent claimants always fall through to a " "per-request context -- which is exactly what the single-slot " "cache did, so the test would pass without ever exercising " "concurrent slot borrowing, the risk surface the ring " "introduces. Needs an nginx built --with-debug; raises the " "location error_log to debug.", ) return p.parse_args() def detect(explicit, nginx: pathlib.Path, name: str): if explicit: return pathlib.Path(explicit) sib = nginx.parent / name return sib if sib.exists() else None def payload_for(req_id: int) -> bytes: """Deterministic, unique-per-id, low-compressibility body. A 64-byte ASCII header carries the request id so a swapped body is instantly identifiable; the remainder is an id-seeded PRNG stream (incompressible -> the stream genuinely crosses output buffers, the regime where context state matters).""" size = SIZES[req_id % len(SIZES)] head = f"REQ-{req_id:08d}-CCTX-ISOLATION-MARKER-".encode() head = head + b"=" * (64 - len(head)) buf = bytearray(head) x = (req_id * 2654435761 + 1) & 0xFFFFFFFF while len(buf) < size: x = (x * 1103515245 + 12345) & 0xFFFFFFFF buf.append((x >> 16) & 0xFF) return bytes(buf[:size]) class _Handler(http.server.BaseHTTPRequestHandler): """GET /r/ -> payload_for(id), chunked, no Content-Length.""" protocol_version = "HTTP/1.1" def log_message(self, *a): pass def do_GET(self): seg = self.path.split("?")[0].rsplit("/", 1)[-1] try: rid = int(seg) except ValueError: self.send_response(404) self.send_header("Content-Length", "0") self.end_headers() return data = payload_for(rid) self.send_response(200) self.send_header("Content-Type", "application/octet-stream") self.send_header("Transfer-Encoding", "chunked") self.end_headers() mv = memoryview(data) for i in range(0, len(mv), 16384): c = bytes(mv[i : i + 16384]) self.wfile.write(b"%X\r\n" % len(c) + c + b"\r\n") self.wfile.write(b"0\r\n\r\n") class _Srv(socketserver.ThreadingMixIn, socketserver.TCPServer): allow_reuse_address = True daemon_threads = True def wait_port(port: int, timeout: float = 10.0) -> None: end = time.time() + timeout while time.time() < end: try: with socket.create_connection(("127.0.0.1", port), 0.5): return except OSError: time.sleep(0.1) raise RuntimeError(f"nothing listening on 127.0.0.1:{port}") def fetch_decoded(port: int, rid: int, zstd_bin: str) -> bytes: req = urllib.request.Request( f"http://127.0.0.1:{port}/r/{rid}", headers={"Accept-Encoding": "zstd", "User-Agent": "zstd-cctx-iso/1.0"}, ) with urllib.request.urlopen(req, timeout=20) as resp: if (resp.headers.get("Content-Encoding") or "").lower() != "zstd": raise RuntimeError( f"rid={rid}: not zstd-encoded " f"(C-E={resp.headers.get('Content-Encoding')!r})" ) blob = resp.read() if blob[:4] != b"\x28\xb5\x2f\xfd": raise RuntimeError(f"rid={rid}: no zstd magic (hex={blob[:8].hex()})") r = subprocess.run( [zstd_bin, "-dq", "-c"], input=blob, capture_output=True, check=False ) if r.returncode != 0: raise RuntimeError( f"rid={rid}: zstd -d failed (premature " f"end / corrupt): " + r.stderr.decode("utf-8", "replace") ) return r.stdout def main() -> int: args = parse_args() nginx = pathlib.Path(args.nginx_binary) if not nginx.exists(): raise FileNotFoundError(nginx) if args.require_multi_slot: # The slot witnesses are ngx_log_debug2 lines, compiled out without # --with-debug. Checked here rather than at the assertion: a # non-debug build yields zero witnesses, which the multi-slot check # would otherwise report as "the ring never lent two slots" -- an # accusation against the module for what is a build-flavor problem. v = subprocess.run( [str(nginx), "-V"], capture_output=True, text=True, check=False ) if "--with-debug" not in v.stderr: raise RuntimeError( "--require-multi-slot reads ngx_log_debug witnesses: this " "tool needs an nginx built --with-debug, or drop the flag" ) mods = [ m for m in ( detect(args.filter_module, nginx, "ngx_http_zstd_filter_module.so"), detect(args.static_module, nginx, "ngx_http_zstd_static_module.so"), ) if m ] # Everything below must stay readable by the workers when run as # root (they drop to the compiled-in nginx user): a restrictive # inherited umask (e.g. 077) would strip group/other bits from # every fixture and subdir created here, 403ing the workers even # with the scratch root itself opened up. os.umask(0o022) with tempfile.TemporaryDirectory(prefix="zstd-cctx-") as td: # mkdtemp gives 0700: run as root, workers drop to the # compiled-in nginx user and cannot enter it -> 403s os.chmod(td, 0o755) root = pathlib.Path(td) logs = root / "logs" logs.mkdir() backend = _Srv(("127.0.0.1", args.backend_port), _Handler) threading.Thread(target=backend.serve_forever, daemon=True).start() try: wait_port(args.backend_port) # Backend self-check: distinct ids -> distinct bodies. for rid in (0, 3, 7): raw = urllib.request.urlopen( f"http://127.0.0.1:{args.backend_port}/r/{rid}", timeout=10 ).read() if raw != payload_for(rid): raise RuntimeError(f"backend self-check failed for rid={rid}") if ( payload_for(0) == payload_for(1) or hashlib.sha256(payload_for(2)).digest() == hashlib.sha256(payload_for(3)).digest() ): raise RuntimeError( "payloads not distinct per id (test would be blind to bleed)" ) elvl = "debug" if args.require_multi_slot else "warn" load = "".join(f"load_module {m};\n" for m in mods) conf = root / "nginx.conf" conf.write_text( f"""{load}worker_processes 1; error_log {logs}/error.log {elvl}; pid {root}/nginx.pid; events {{ worker_connections 512; }} http {{ access_log off; default_type application/octet-stream; zstd on; zstd_comp_level 6; zstd_min_length 1; zstd_types application/octet-stream; keepalive_requests 100000; server {{ listen 127.0.0.1:{args.port}; keepalive_timeout 60s; location /r/ {{ proxy_pass http://127.0.0.1:{args.backend_port}/r/; proxy_http_version 1.1; proxy_set_header Connection ""; }} }} }} """, encoding="utf-8", ) proc = subprocess.Popen( [ str(nginx), "-p", str(root), "-c", str(conf), "-g", "daemon off; master_process off;", ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, ) try: wait_port(args.port) failures: list[str] = [] total = 0 base = 0 for rnd in range(args.rounds): ids = list(range(base, base + args.concurrency)) base += args.concurrency with concurrent.futures.ThreadPoolExecutor( max_workers=args.concurrency ) as ex: futs = { ex.submit(fetch_decoded, args.port, rid, args.zstd_bin): rid for rid in ids } for fut in concurrent.futures.as_completed(futs): rid = futs[fut] total += 1 try: got = fut.result() except Exception as e: # noqa: BLE001 failures.append(f"round {rnd} rid={rid}: {e}") continue want = payload_for(rid) if got != want: # Identify if it is *another request's* # body (the CCtx-bleed signature). culprit = "" if got[:32].startswith(b"REQ-"): culprit = " looks like " + got[:28].decode( "ascii", "replace" ) failures.append( f"round {rnd} rid={rid}: body " f"mismatch (got {len(got)}B want " f"{len(want)}B){culprit}" ) slots_seen: set[str] = set() if args.require_multi_slot: # Flush, then count the DISTINCT ring slots that were # actually lent. One slot is the single-slot cache's # behaviour; the concurrent-borrowing path this test # exists to cover is only reached when at least two # distinct slots were on loan during the run. time.sleep(0.3) elog = (logs / "error.log").read_text("utf-8", "replace") slots_seen = set( re.findall( r"zstd: reusing worker cctx \S+ \(slot:(\d+)\)", elog, ) ) if len(slots_seen) < 2: failures.append( f"only {len(slots_seen)} distinct ring slot(s) " f"were ever lent ({sorted(slots_seen)}) across " f"{total} concurrent requests -- this run never " f"exercised concurrent slot borrowing, so its " f"byte-exactness result proves nothing about " f"the ring" ) if failures: sys.stderr.write( f"CCtx-ISOLATION FAILED: {len(failures)}/{total}:\n" ) for f in failures[:25]: sys.stderr.write(f" - {f}\n") if len(failures) > 25: sys.stderr.write(f" ... +{len(failures) - 25} more\n") return 1 print( f"OK: {total} concurrent requests " f"({args.concurrency} in flight x {args.rounds} " f"rounds, distinct per-request bodies, keepalive " f"reuse) each decoded byte-exact to its own " f"origin — no cross-request CCtx bleed" + ( f"; {len(slots_seen)} distinct ring slots lent concurrently" if args.require_multi_slot else "" ) ) return 0 finally: proc.terminate() try: # 30s, not 5: gcov-instrumented nginx flushing # .gcda at exit on a loaded runner can outlive a # tight reap budget; teardown runs after the # verdict, so patience costs nothing when healthy. proc.wait(timeout=30) except subprocess.TimeoutExpired: proc.kill() proc.wait(timeout=30) finally: backend.shutdown() backend.server_close() if __name__ == "__main__": try: raise SystemExit(main()) except Exception as exc: print(f"ERROR: {exc}", file=sys.stderr) raise myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_coverage_prober_rebuild.sh000077500000000000000000000144621525232545500300500ustar00rootroot00000000000000#!/usr/bin/env bash # Regression test for audit A30-F7: coverage.sh must unconditionally # (re)build the harness prober before running testkit scenarios. The # prober binary can exist but be STALER than its sources -- a guard that # only rebuilds when the binary is ABSENT does not catch that case, and a # stale binary silently bails every fault/counter scenario while # coverage.sh still reports coverage success. # # A second-pass audit could not exercise this guard's negative control # directly: the ci/t/harness submodule is not checked out in every # environment, and a full coverage.sh run is a forbidden long-runner here. # This test instead asserts the STRUCTURAL shape of the fix -- the # prober/build.sh invocation is not nested inside a condition that tests # for the binary's absence -- and proves the assertion can actually fail # by mutating a copy of coverage.sh back into the old buggy (guarded # rebuild) shape and confirming the assertion goes red on that copy. set -euo pipefail root=$(git rev-parse --show-toplevel) src="$root/ci/tools/coverage.sh" # Passes iff `bash ".../prober/build.sh"` is invoked unconditionally # (module-level "always rebuild"), not gated behind an absence check such # as `if [ ! -x .../run-scenario.sh ]; then ... build.sh; fi` or # `if [ ! -f .../prober ]; then ... build.sh; fi`. assert_unconditional_rebuild() { local file=$1 grep -Fq 'prober/build.sh' "$file" || return 1 # Extract the block from the harness-submodule presence check to its # closing `fi` at the same indent, i.e. the whole # `if [ ! -x .../run-scenario.sh ]; then ... else ... fi` construct # that guards scenario execution. local block block=$(sed -n '/harness submodule not checked out/,/^ fi$/p' "$file") [ -n "$block" ] || return 1 # The build.sh call must appear directly under the `else` branch (the # submodule-present path), not inside a NESTED absence-only guard # such as `if [ ! -x ... ] || [ ! -f .../prober ]; then build.sh; fi` # sitting inside that else. Isolate everything between `else` and the # matching `fi` and confirm build.sh is not itself wrapped in another # `if [ ! ... ]; then ... fi` around it. local else_branch else_branch=$(printf '%s\n' "$block" | sed -n '/^ else$/,/^ fi$/p') [ -n "$else_branch" ] || return 1 # The rebuild invocation must actually be present INSIDE the else # branch (not merely somewhere else in the file, e.g. a comment or a # different guarded copy) -- otherwise this whole check is satisfied # by a file that mentions build.sh anywhere and never calls it here. # shellcheck disable=SC2016 # literal pattern: $MODULE_DIR must not expand printf '%s\n' "$else_branch" \ | grep -Fq 'bash "$MODULE_DIR/ci/t/harness/ci/prober/build.sh"' \ || return 1 # Old buggy shape: a second, nested absence guard directly wrapping # the build.sh call, e.g. # if [ ! -x ".../prober" ]; then # bash ".../prober/build.sh" # fi # Detect that shape: an `if [ ! ` line followed (within a few lines, # before any `fi`) by the build.sh invocation. if printf '%s\n' "$else_branch" \ | awk ' /if[[:space:]]*\[[[:space:]]*!/ { guard=1; next } /^[[:space:]]*fi[[:space:]]*$/ { guard=0; next } guard && /prober\/build\.sh/ { found=1 } END { exit !found } '; then return 1 fi return 0 } assert_unconditional_rebuild "$src" work=$(mktemp -d "${TMPDIR:-/tmp}/coverage-prober-rebuild.XXXXXX") trap 'rm -rf "$work"' EXIT cp "$src" "$work/coverage.sh" # Mutate the fix back into the pre-A30-F7 buggy shape: gate the rebuild # behind an absence-only check on the prober binary itself, so a STALE # (present but outdated) binary would again silently skip the rebuild. python3 - "$work/coverage.sh" <<'PY' import sys path = sys.argv[1] with open(path) as f: text = f.read() old = ( ' # Always (re)build: the prober binary can exist but be staler than its\n' ' # sources, which silently bails every fault/counter scenario while\n' ' # this script still reports coverage success (audit A30-F7).\n' ' if ! bash "$MODULE_DIR/ci/t/harness/ci/prober/build.sh"; then\n' ) new = ( ' # (mutated) only rebuild when the prober binary is absent --\n' ' # this is the pre-A30-F7 buggy shape the negative control targets.\n' ' if [ ! -x "$MODULE_DIR/ci/t/harness/ci/prober/prober" ] && \\\n' ' ! bash "$MODULE_DIR/ci/t/harness/ci/prober/build.sh"; then\n' ) if old not in text: sys.exit( "FAIL: mutation anchor text not found in coverage.sh -- " "the A30-F7 fix comment/guard moved; update this test together " "with coverage.sh" ) with open(path, "w") as f: f.write(text.replace(old, new, 1)) PY if assert_unconditional_rebuild "$work/coverage.sh" >/dev/null 2>&1; then echo 'FAIL: mutated (absence-guarded) coverage.sh copy still passed the' \ 'unconditional-rebuild assertion -- the assertion cannot detect' \ 'a regression to the pre-A30-F7 stale-prober bug' >&2 exit 1 fi echo 'OK: negative control -- absence-guarded mutant correctly fails the' \ 'unconditional-rebuild assertion' # Second negative control: drop the build.sh invocation from the else # branch entirely (e.g. accidentally deleted, or moved somewhere the # assertion does not look) and confirm the assertion still catches it -- # a file that merely MENTIONS build.sh elsewhere (a comment, a different # guarded copy) must not satisfy this check. cp "$src" "$work/coverage-removed.sh" # shellcheck disable=SC2016 # literal pattern: $MODULE_DIR must not expand sed -i '/bash "\$MODULE_DIR\/ci\/t\/harness\/ci\/prober\/build\.sh"/d' \ "$work/coverage-removed.sh" if assert_unconditional_rebuild "$work/coverage-removed.sh" >/dev/null 2>&1; then echo 'FAIL: coverage.sh copy with the build.sh call removed from the' \ 'else branch still passed the unconditional-rebuild assertion --' \ 'the assertion does not actually require the call inside that' \ 'branch' >&2 exit 1 fi echo 'OK: negative control -- else-branch-missing-build.sh mutant' \ 'correctly fails the unconditional-rebuild assertion' echo 'OK: coverage.sh unconditionally rebuilds the prober (A30-F7) and' \ 'negative controls' myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_dcz.py000077500000000000000000000713511525232545500237740ustar00rootroot00000000000000#!/usr/bin/env python3 """End-to-end RFC 9842 dcz test against a real nginx. Covers what t/03-dcz.t cannot: the bytes on the wire. For every case the test asserts the negotiated Content-Encoding, and for dcz responses it additionally verifies the 40-byte frame header (skippable-frame magic + dictionary SHA-256), decodes the body with `zstd -d -D ` under the RFC's 8 MB client window guarantee (--memory), and byte-compares against the origin fixture. The dictionary is a simulated "previous version" of the resource, so the test also asserts the dictionary actually engaged: the dcz body must be smaller than the plain zstd body. """ import argparse import base64 import hashlib import os import pathlib import shutil import socket import ssl import subprocess import sys import tempfile import time import urllib.request DCZ_MAGIC = bytes([0x5E, 0x2A, 0x4D, 0x18, 0x20, 0x00, 0x00, 0x00]) RFC_CLIENT_WINDOW = 8 * 1024 * 1024 def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description="Run an end-to-end RFC 9842 dcz test against nginx." ) parser.add_argument( "--nginx-binary", required=True, help="Path to the nginx binary to start for the test.", ) parser.add_argument( "--filter-module", help=( "Optional path to ngx_http_zstd_filter_module.so. " "If omitted, a sibling module next to the nginx binary is used." ), ) parser.add_argument( "--static-module", help=( "Optional path to ngx_http_zstd_static_module.so. " "If omitted, a sibling module next to the nginx binary is used." ), ) parser.add_argument( "--port", type=int, default=18106, help="Local TCP port for the temporary nginx instance.", ) parser.add_argument( "--tls-port", type=int, default=None, help=( "Local TCP port for the native-TLS listener " "(default: --port + 1). RFC 9842 section 8 restricts dcz to " "secure contexts, so the happy path is exercised here." ), ) parser.add_argument( "--insecure-port", type=int, default=None, help=( "Local TCP port for a plain-HTTP listener with NO " "zstd_dcz_assume_secure_transport (default: --port + 2). " "Used for the fail-closed secure-context checks." ), ) parser.add_argument( "--zstd-bin", default=shutil.which("zstd") or "zstd", help="Path to the zstd CLI used for decompression.", ) parser.add_argument( "--fixture-lines", type=int, default=600, help="Number of generated source lines shared between dictionary and resource.", ) return parser.parse_args() def detect_module(explicit, nginx: pathlib.Path, name: str): """A dynamic-module build (--add-dynamic-module, e.g. the Coverage CI job) needs load_module lines or `zstd on;` is an unknown directive and nginx dies at config parse; a static build has no .so and needs none. Mirror the sibling tools: an explicit path wins, else use a module sitting next to the nginx binary if present.""" if explicit: return pathlib.Path(explicit) sib = nginx.parent / name return sib if sib.exists() else None def nginx_has_ssl(nginx: pathlib.Path) -> bool: """Whether this binary can parse `listen ... ssl`. Same probe shape the sibling tools use for --with-debug (test_concurrent_cctx_isolation.py:206, test_slow_drain.py:233): ask the binary itself via -V rather than inferring from the environment or the job name. The sanitizer and valgrind nginx builds in this repo's CI are configured WITHOUT --with-http_ssl_module, where an `ssl` listen parameter is a config-time emerg that kills the tool before any assertion runs -- including every plain-HTTP check, which needs no TLS at all. """ v = subprocess.run([str(nginx), "-V"], capture_output=True, text=True, check=False) return "--with-http_ssl_module" in v.stderr def wait_for_port(port: int, timeout: float = 10.0, stderr_file=None) -> None: deadline = time.time() + timeout while time.time() < deadline: try: with socket.create_connection(("127.0.0.1", port), timeout=0.5): return except OSError: time.sleep(0.1) # A config-parse failure would otherwise surface only as this silent # timeout; include what nginx actually said on stderr. detail = "" if stderr_file is not None and stderr_file.exists(): text = stderr_file.read_text(errors="replace").strip() if text: detail = f"; nginx stderr:\n{text}" raise RuntimeError(f"nginx did not start listening on 127.0.0.1:{port}{detail}") def build_fixtures(root: pathlib.Path, lines: int) -> tuple[bytes, bytes]: """Dictionary = version 1; served resource = version 2 sharing most of its content. Returns (dict_bytes, resource_bytes).""" shared = [ f"function widget_{i}() {{ return compute({i}) + render('{'x' * 20}'); }}" for i in range(lines) ] v1 = "\n".join(shared) + "\n" v2_lines = shared[:] for i in range(0, lines, 50): v2_lines.insert(i, f"// added in v2: handler {i}") v2 = "\n".join(v2_lines) + "\n" (root / "dicts").mkdir() (root / "html").mkdir() dict_path = root / "dicts" / "app-v1.js" dict_path.write_text(v1, encoding="utf-8", newline="\n") (root / "html" / "app.js").write_text(v2, encoding="utf-8", newline="\n") return dict_path.read_bytes(), (root / "html" / "app.js").read_bytes() def make_selfsigned(root: pathlib.Path) -> tuple[pathlib.Path, pathlib.Path]: """Self-signed localhost cert for the native-TLS listener. Generated per run rather than committed: a fixture certificate in-tree expires and turns into a rolling CI failure (feedback-fixed-date-test-ages-into-failure).""" cert = root / "conf" / "test.crt" key = root / "conf" / "test.key" if shutil.which("openssl") is None: # ci/tools/soak.sh already depends on this binary in the same # jobs, so a miss here means the image changed, not that the # test is optional. Say so instead of surfacing FileNotFoundError. raise RuntimeError( "the openssl CLI is required to generate the native-TLS " "listener's certificate (RFC 9842 secure-context checks); " "install the openssl package" ) subprocess.run( [ "openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", "-days", "2", "-subj", "/CN=127.0.0.1", "-addext", "subjectAltName=IP:127.0.0.1", "-keyout", str(key), "-out", str(cert), ], check=True, capture_output=True, ) return cert, key def write_config( root: pathlib.Path, port: int, tls_port: int, insecure_port: int, modules, with_ssl: bool, ) -> pathlib.Path: conf_dir = root / "conf" conf_dir.mkdir() (root / "logs").mkdir() load = "".join(f"load_module {m};\n" for m in modules) # Only emitted for an SSL-capable binary: `listen ... ssl` against an # nginx without ngx_http_ssl_module is [emerg] at config parse, so an # unconditional block would take every non-TLS assertion down with it. tls_server = "" if with_ssl: cert, key = make_selfsigned(root) tls_server = f""" # Native TLS: the secure context the RFC actually describes, with no # acknowledgement directive at all. Proves the gate passes on # r->connection->ssl rather than only on the opt-in. server {{ listen 127.0.0.1:{tls_port} ssl; ssl_certificate {cert}; ssl_certificate_key {key}; root html; }} """ conf = conf_dir / "nginx.conf" conf.write_text( f"""{load} worker_processes 1; daemon off; error_log logs/error.log warn; pid logs/nginx.pid; events {{ }} http {{ access_log off; # $zstd_bytes_out / $zstd_ratio are log-phase variables (only valid # once the filter has finished writing the response); this format # exists so the dcz byte-accounting oracle below can read the # module's own count of bytes it queued downstream and cross-check # it against the bytes actually received over the socket. log_format zstd_bytes_fmt "$zstd_bytes_out $zstd_ratio"; types {{ application/javascript js; }} default_type application/octet-stream; gzip_vary on; zstd on; zstd_min_length 64; zstd_dcz_dict_file {root}/dicts/app-v1.js; # Cleartext listener modelling "TLS terminated by a proxy in front": # RFC 9842 section 8 forbids dcz outside a secure context, and this # listener carries the explicit operator acknowledgement that the # client-facing hop was HTTPS. Every pre-existing check in this file # runs here. server {{ listen 127.0.0.1:{port}; zstd_dcz_assume_secure_transport on; root html; access_log logs/zstd_bytes.log zstd_bytes_fmt; }} {tls_server} # Cleartext with NO acknowledgement: the compiled-in default. Nothing # a client can send may negotiate dcz here. server {{ listen 127.0.0.1:{insecure_port}; root html; }} }} """, encoding="utf-8", newline="\n", ) return conf def fetch(port: int, headers: dict, tls: bool = False): """Returns (email.message.Message, body). The Message preserves repeated header lines — the module legitimately emits two Vary lines (Accept-Encoding via gzip_vary, Available-Dictionary its own), and a dict would silently keep only one of them. tls=True talks to the native-TLS listener. The certificate is the per-run self-signed one from make_selfsigned(), so verification is switched off deliberately: this asserts the module's view of r->connection->ssl, not PKI.""" scheme = "https" if tls else "http" request = urllib.request.Request( f"{scheme}://127.0.0.1:{port}/app.js", headers=headers ) context = None if tls: context = ssl.create_default_context() context.check_hostname = False context.verify_mode = ssl.CERT_NONE with urllib.request.urlopen(request, timeout=10, context=context) as response: return response.headers, response.read() def content_encoding(headers) -> str: value = headers.get("Content-Encoding") return value.strip() if value else "identity" def vary_values(headers) -> str: return ",".join(headers.get_all("Vary") or []).lower() def read_last_log_line(path: pathlib.Path, timeout: float = 10.0) -> str: """Poll for the access log line the most recent request just wrote. $zstd_bytes_out / $zstd_ratio are log-phase variables: nginx writes the access_log line only after the response is fully sent, which can race a fast local request. Poll instead of reading once, mirroring test_encoding.py's validate_ratio_log. """ deadline = time.time() + timeout line = "" while time.time() < deadline: if path.exists(): content = path.read_text(encoding="utf-8", errors="replace") lines = [l for l in content.splitlines() if l.strip()] if lines: line = lines[-1] break time.sleep(0.05) return line def decode_dcz( zstd_bin: str, body: bytes, dict_path: pathlib.Path, work: pathlib.Path ) -> bytes: """Decode a full dcz body (skippable header + frame) with the dictionary, capped at the RFC 9842 client window guarantee so a frame demanding a larger window fails here instead of on real clients.""" src = work / "response.dcz" dst = work / "response.out" src.write_bytes(body) subprocess.run( [ zstd_bin, "-d", "-q", "-f", f"--memory={RFC_CLIENT_WINDOW}", "-D", str(dict_path), "-o", str(dst), str(src), ], check=True, capture_output=True, ) return dst.read_bytes() def main() -> int: args = parse_args() failures: list[str] = [] def check(name: str, condition: bool, detail: str = "") -> None: if condition: print(f"ok - {name}") else: print(f"FAIL - {name} {detail}") failures.append(name) nginx_path = pathlib.Path(args.nginx_binary) modules = [ m for m in ( detect_module( args.filter_module, nginx_path, "ngx_http_zstd_filter_module.so" ), detect_module( args.static_module, nginx_path, "ngx_http_zstd_static_module.so" ), ) if m ] # Everything below must stay readable by the workers when run as # root (they drop to the compiled-in nginx user): a restrictive # inherited umask (e.g. 077) would strip group/other bits from # every fixture and subdir created here, 403ing the workers even # with the scratch root itself opened up. os.umask(0o022) with tempfile.TemporaryDirectory(prefix="zstd-dcz-") as tmp: # mkdtemp gives 0700: run as root, workers drop to the # compiled-in nginx user and cannot enter it -> 403s os.chmod(tmp, 0o755) root = pathlib.Path(tmp) dict_bytes, resource = build_fixtures(root, args.fixture_lines) tls_port = args.tls_port if args.tls_port else args.port + 1 insecure_port = args.insecure_port if args.insecure_port else args.port + 2 with_ssl = nginx_has_ssl(nginx_path) conf = write_config(root, args.port, tls_port, insecure_port, modules, with_ssl) dict_hash = hashlib.sha256(dict_bytes).digest() dict_b64 = base64.b64encode(dict_hash).decode() bad_b64 = base64.b64encode(b"\x01" * 32).decode() stderr_file = root / "nginx-stderr.log" with stderr_file.open("wb") as stderr_handle: nginx = subprocess.Popen( [args.nginx_binary, "-p", str(root), "-c", str(conf)], stdout=subprocess.DEVNULL, stderr=stderr_handle, ) try: wait_for_port(args.port, stderr_file=stderr_file) wait_for_port(insecure_port, stderr_file=stderr_file) if with_ssl: wait_for_port(tls_port, stderr_file=stderr_file) # -- RFC 9842 section 8: dcz is a secure-context-only coding. dcz_headers = { "Accept-Encoding": "zstd, dcz", "Available-Dictionary": f":{dict_b64}:", } if with_ssl: tls_headers, tls_body = fetch(tls_port, dict(dcz_headers), tls=True) check( "secure context: native TLS negotiates dcz with no opt-in", content_encoding(tls_headers) == "dcz", f"(got {content_encoding(tls_headers)})", ) check( "secure context: native-TLS dcz body carries the RFC magic", tls_body[:8] == DCZ_MAGIC and tls_body[8:40] == dict_hash, f"(got {tls_body[:8].hex()})", ) else: # Loud and named, not silent: the sanitizer and valgrind # builds have no ngx_http_ssl_module, so the native-TLS # leg cannot run there. Everything below still gates -- # the fail-closed default and the spoof cases are the # security-relevant assertions and need no TLS. print( " native-TLS checks skipped (nginx -V has no " "--with-http_ssl_module: this binary cannot parse " "`listen ... ssl`); cleartext fail-closed and spoof " "checks below still run" ) insecure_headers, insecure_body = fetch(insecure_port, dict(dcz_headers)) check( "secure context: plain HTTP falls back to zstd " "(compiled-in default, no directive)", content_encoding(insecure_headers) == "zstd", f"(got {content_encoding(insecure_headers)})", ) insecure_src = root / "insecure-fallback.zst" insecure_src.write_bytes(insecure_body) # check=False on purpose: if the gate regresses, this body is # a dcz frame and the dictionary-less decode EXITS NON-ZERO. # With check=True that regression surfaces as a traceback # that skips every remaining assertion instead of as a # readable FAIL line. insecure_decode = subprocess.run( [args.zstd_bin, "-d", "-q", "-c", str(insecure_src)], check=False, capture_output=True, ) check( "secure context: the plain-HTTP fallback is still decodable " "without the dictionary", insecure_decode.returncode == 0 and insecure_decode.stdout == resource, f"(rc={insecure_decode.returncode}, " f"{len(insecure_decode.stdout)} bytes)", ) # An untrusted client-supplied scheme signal must not # re-enable dcz: X-Forwarded-Proto is settable by anyone who # can reach the listener, so the module never consults it. for spoof in ( {"X-Forwarded-Proto": "https"}, {"Forwarded": "proto=https"}, {"X-Forwarded-Ssl": "on"}, {"Front-End-Https": "on"}, ): spoof_headers, _ = fetch(insecure_port, {**dcz_headers, **spoof}) name = next(iter(spoof)) check( f"secure context: spoofed {name} does not enable dcz", content_encoding(spoof_headers) == "zstd", f"(got {content_encoding(spoof_headers)})", ) # -- plain zstd client: baseline and cache-key contract headers, plain_body = fetch(args.port, {"Accept-Encoding": "zstd"}) check( "plain client negotiates zstd", content_encoding(headers) == "zstd", f"(got {content_encoding(headers)})", ) check( "plain variant varies on Available-Dictionary", "available-dictionary" in vary_values(headers), f"(vary: {vary_values(headers)!r})", ) check( "plain variant varies on Accept-Encoding", "accept-encoding" in vary_values(headers), f"(vary: {vary_values(headers)!r})", ) check( # Sec-Fetch-Site selects the representation (dcz is # refused cross-site), so a shared cache must key on it # or it serves one origin's variant to the other. "plain variant varies on Sec-Fetch-Site", "sec-fetch-site" in vary_values(headers), f"(vary: {vary_values(headers)!r})", ) # -- the dcz happy path headers, dcz_body = fetch( args.port, { "Accept-Encoding": "zstd, dcz", "Available-Dictionary": f":{dict_b64}:", }, ) check( "dictionary client negotiates dcz", content_encoding(headers) == "dcz", f"(got {content_encoding(headers)})", ) check( "dcz variant varies on Available-Dictionary", "available-dictionary" in vary_values(headers), f"(vary: {vary_values(headers)!r})", ) check( "dcz variant varies on Sec-Fetch-Site", "sec-fetch-site" in vary_values(headers), f"(vary: {vary_values(headers)!r})", ) check( "dcz body starts with the RFC 9842 magic", dcz_body[:8] == DCZ_MAGIC, f"(got {dcz_body[:8].hex()})", ) check( "dcz header embeds the dictionary SHA-256", dcz_body[8:40] == dict_hash, ) decoded = decode_dcz( args.zstd_bin, dcz_body, root / "dicts" / "app-v1.js", root ) check( "dcz body decodes byte-exact with the dictionary " "within the 8 MB client window", decoded == resource, f"(decoded {len(decoded)} bytes, want {len(resource)})", ) check( "dictionary actually engaged (dcz smaller than plain zstd)", len(dcz_body) < len(plain_body), f"(dcz {len(dcz_body)} vs zstd {len(plain_body)})", ) # -- $zstd_bytes_out / $zstd_ratio byte-accounting oracle. # # Regression for the 40-byte dcz prefix being counted twice: # ngx_http_zstd_filter_get_buf() advances out_buf->last past # the 40-byte skippable-frame prefix AND used to separately # add 40 to ctx->bytes_out, while the emit path in # ngx_http_zstd_filter_compress() already counts the whole # buffer (prefix included) via ngx_buf_size(). Ground truth # is len(dcz_body): the exact bytes this request put on the # wire, already captured above. $zstd_bytes_out must equal it # exactly -- not "close", not "within 40" -- and $zstd_ratio # (bytes_in*1000/bytes_out, truncated to 3 decimals) must be # derivable from that same bytes_out. bytes_log_line = read_last_log_line(root / "logs" / "zstd_bytes.log") check( "$zstd_bytes_out / $zstd_ratio access_log line was written", bool(bytes_log_line), f"(zstd_bytes.log: {bytes_log_line!r})", ) if bytes_log_line: logged_bytes_out_str, logged_ratio_str = bytes_log_line.split(" ", 1) logged_bytes_out = int(logged_bytes_out_str) check( "$zstd_bytes_out equals the actual dcz response byte " "length (prefix counted exactly once)", logged_bytes_out == len(dcz_body), f"(zstd_bytes_out={logged_bytes_out}, wire body={len(dcz_body)})", ) # Ground truth is len(dcz_body), NOT logged_bytes_out: the # ratio must match the ACTUAL wire bytes. Deriving the # expectation from logged_bytes_out instead would make # this check vacuous under the exact bug this test guards # against (bytes_out inflated by 40 would inflate the # expectation the same way and the two would always # agree, however wrong bytes_out is). bytes_in = len(resource) scaled = bytes_in * 1000 // len(dcz_body) expected_ratio = f"{scaled // 1000}.{scaled % 1000:03d}" check( "$zstd_ratio matches bytes_in*1000/bytes_out computed " "from the actual wire byte count", logged_ratio_str == expected_ratio, f"(logged {logged_ratio_str!r}, expected {expected_ratio!r})", ) # -- content checksum (defence in depth): the inner zstd frame # starts right after the 40-byte dcz header, so byte 44 is its # Frame_Header_Descriptor; Content_Checksum_flag is bit 2. fhd = dcz_body[44] if len(dcz_body) > 44 else None check( "dcz frame declares a content checksum (FHD bit 2)", fhd is not None and (fhd & 0x04) != 0, f"(FHD {fhd:#04x})" if fhd is not None else "(body too short)", ) # The property the checksum exists for: decoding against a # WRONG dictionary of the same size structurally succeeds and # yields wrong bytes (raw-prefix references stay in range and # nothing else in the frame ties the output to the content). # With the checksum the decoder must reject it loudly instead # of handing wrong content to the caller. dict_path = root / "dicts" / "app-v1.js" wrong_dict = root / "wrong-same-size.js" wrong_dict.write_bytes(b"Z" * dict_path.stat().st_size) wrong_src = root / "response-wrongdict.dcz" wrong_src.write_bytes(dcz_body) wrong_out = root / "response-wrongdict.out" wrong_rc = subprocess.run( [ args.zstd_bin, "-d", "-q", "-f", f"--memory={RFC_CLIENT_WINDOW}", "-D", str(wrong_dict), "-o", str(wrong_out), str(wrong_src), ], capture_output=True, check=False, ) if wrong_rc.returncode == 0: wrong_len = len(wrong_out.read_bytes()) if wrong_out.exists() else 0 wrong_detail = f"(rc=0, {wrong_len} bytes of wrong content accepted)" else: wrong_detail = "" check( "wrong same-size dictionary fails the decode " "(checksum turns silent corruption into a loud error)", wrong_rc.returncode != 0, wrong_detail, ) # -- every gate miss must fall back to zstd, never break fallback_cases = [ ( "unknown dictionary hash", { "Accept-Encoding": "zstd, dcz", "Available-Dictionary": f":{bad_b64}:", }, ), ( "no dcz token in Accept-Encoding", { "Accept-Encoding": "zstd", "Available-Dictionary": f":{dict_b64}:", }, ), ( "explicit dcz;q=0 refusal", { "Accept-Encoding": "zstd, dcz;q=0", "Available-Dictionary": f":{dict_b64}:", }, ), ( "wildcard does not enable dcz", { "Accept-Encoding": "zstd, *", "Available-Dictionary": f":{dict_b64}:", }, ), ( "cross-site request refused", { "Accept-Encoding": "zstd, dcz", "Available-Dictionary": f":{dict_b64}:", "Sec-Fetch-Site": "cross-site", }, ), ( "malformed Available-Dictionary", { "Accept-Encoding": "zstd, dcz", "Available-Dictionary": "junk", }, ), ] for name, case_headers in fallback_cases: headers, _body = fetch(args.port, case_headers) check( f"fallback: {name} -> zstd", content_encoding(headers) == "zstd", f"(got {content_encoding(headers)})", ) # zstd fallbacks must still decode without any dictionary src = root / "plain.zst" src.write_bytes(plain_body) plain_decoded = subprocess.run( [args.zstd_bin, "-d", "-q", "-c", str(src)], check=True, capture_output=True, ).stdout check( "plain zstd variant decodes without the dictionary", plain_decoded == resource, ) finally: nginx.terminate() try: # 30s, not 10: gcov-instrumented nginx flushing .gcda # at exit on a loaded runner can outlive a tight reap # budget; teardown runs after the verdict, so patience # costs nothing when healthy. nginx.wait(timeout=30) except subprocess.TimeoutExpired: nginx.kill() nginx.wait(timeout=30) if failures: print(f"FAILED: {len(failures)} dcz check(s): {', '.join(failures)}") return 1 print("OK: verified RFC 9842 dcz negotiation, framing, and roundtrip") return 0 if __name__ == "__main__": sys.exit(main()) myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_dcz_budget_ring.py000077500000000000000000000413551525232545500263460ustar00rootroot00000000000000#!/usr/bin/env python3 """Couples a dcz (RFC 9842) request to the CCtx memory budget and the ring key. A dcz request does not compress at ``zstd_window_log``: it derives its own window from the negotiated dictionary, up to 2^23. Two consequences had no test at all before this one, and both are asserted here against the bytes on the wire and against the module's own reuse witnesses: 1. RING PARTITION. The worker CCtx ring keys each slot on the memory-affecting profile (level, long mode, window log) because ``ZSTD_sizeof_CCtx()`` never shrinks on reset. Keying a dcz request on the *unset* directive let it borrow a slot vetted for a small window and permanently raise that slot's retained workspace; every later plain request matching the same key then reused a context whose floor was the dcz figure. A dcz request and a plain request on the SAME location must therefore land in DIFFERENT slots. Asserted from the ``zstd: reusing worker cctx ... (slot:N) ... window_log:W`` debug witnesses, which print the slot index and the profile the slot was built for -- the key is reversible precisely so tests can read it back. This is a real oracle rather than a byte comparison: both requests decode correctly either way, so only the slot identity distinguishes the fixed code from the broken code. 2. BUDGET CLAMP. ``zstd_max_cctx_memory`` is vetted at config load against ``zstd_window_log`` only, so the dcz window walked straight through it (measured, libzstd 1.5.7 level 3: 3 663 393 B at the default window vs 9 954 849 B at wlog 23). With a budget set, the declared window in the dcz frame header must not exceed the budget-derived cap. 3. DEFAULT UNCHANGED. The clamp is opt-in. With neither ``zstd_window_log`` nor ``zstd_max_cctx_memory`` set, the dcz window must still be the dictionary-derived one -- the regression guard for that scoping decision, without which the "fix" would quietly change wire bytes for everyone. Needs an nginx built --with-debug: the ring witnesses are debug log lines. """ import argparse import base64 import hashlib import http.server import os import pathlib import re import socketserver import subprocess import sys import tempfile import threading import time import urllib.request DCZ_MAGIC = bytes([0x5E, 0x2A, 0x4D, 0x18, 0x20, 0x00, 0x00, 0x00]) DCZ_HEADER_LEN = 40 # 8-byte skippable magic + 32-byte dict hash ZSTD_MAGIC = b"\x28\xb5\x2f\xfd" # The dcz window is ceil_log2(dict + content). 4 MB dictionary + 4 MB body is # 2^23 exactly, so with no ceiling the derived window saturates at # NGX_HTTP_ZSTD_DCZ_MAX_WINDOW_LOG -- the worst case the budget escape was # measured at, and comfortably above anything a budget can permit. DICT_SIZE = 4 * 1024 * 1024 BODY_SIZE = 4 * 1024 * 1024 COMP_LEVEL = 3 # What the dictionary + body demand with no ceiling configured: the RFC 9842 # cap itself. UNCLAMPED_WLOG = 23 # Measured, libzstd 1.5.7, ZSTD_estimateCStreamSize_usingCCtxParams at level 3: # # window log estimate # (unset) 3 663 393 # 20 2 614 817 # 21 3 663 393 # 22 5 760 545 # 23 9 954 849 # # The budget must clear the DEFAULT-window estimate or the pre-existing # nginx -t gate rejects the configuration before any of this runs -- that gate # vets conf->window_log, and it is not what this test is about. 4 000 000 B # clears 3 663 393 B, so the config loads, and the largest window log that # still fits it is 21 -- two steps below the 23 the dictionary asks for, which # is what makes the assertion able to fail rather than pass by construction. BUDGET = 4_000_000 BUDGET_EXPECTED_WLOG = 21 REUSE_RE = re.compile( r"zstd: reusing worker cctx \S+ \(slot:(\d+)\) " r"level:(-?\d+) long:(\d+) window_log:(\d+)" ) def parse_args() -> argparse.Namespace: p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) p.add_argument("--nginx-binary", required=True) p.add_argument("--filter-module") p.add_argument("--static-module") p.add_argument("--port", type=int, default=18160) p.add_argument("--backend-port", type=int, default=18162) return p.parse_args() def detect(explicit, nginx: pathlib.Path, name: str): if explicit: return pathlib.Path(explicit) sib = nginx.parent / name return sib if sib.exists() else None def make_dict() -> bytes: """A raw dcz dictionary. RFC 9842 type=raw: any bytes are legal, and the module references them with ZSTD_CCtx_refPrefix, so no training needed.""" out = bytearray() i = 0 while len(out) < DICT_SIZE: out += b'{"tok":%08d,"kind":"dcz-budget-ring-fixture"}\n' % i i += 1 return bytes(out[:DICT_SIZE]) def make_body() -> bytes: """Body that shares structure with the dictionary, so dcz genuinely wins and the response is a dcz frame rather than a fallback.""" out = bytearray() i = 0 while len(out) < BODY_SIZE: out += b'{"tok":%08d,"kind":"dcz-budget-ring-fixture"}\n' % (i + 7) i += 1 return bytes(out[:BODY_SIZE]) class _Handler(http.server.BaseHTTPRequestHandler): """Chunked origin. A static file gives nginx a Content-Length, which becomes libzstd's pledged source size, and libzstd then emits a Single_Segment frame with NO Window_Descriptor -- the declared window is unreadable and the whole oracle collapses. Proxying a chunked body is what keeps the window observable, the same reason test_window_cap.py does it. """ protocol_version = "HTTP/1.1" payload = b"" def log_message(self, *a): pass def do_GET(self): self.send_response(200) self.send_header("Content-Type", "application/javascript") self.send_header("Transfer-Encoding", "chunked") self.end_headers() mv = memoryview(self.payload) for i in range(0, len(mv), 16384): c = bytes(mv[i : i + 16384]) self.wfile.write(b"%X\r\n" % len(c) + c + b"\r\n") self.wfile.write(b"0\r\n\r\n") class _Srv(socketserver.ThreadingMixIn, socketserver.TCPServer): allow_reuse_address = True daemon_threads = True def wait_port(port: int, timeout: float = 15.0) -> None: import socket end = time.time() + timeout while time.time() < end: try: with socket.create_connection(("127.0.0.1", port), 0.5): return except OSError: time.sleep(0.1) raise RuntimeError(f"nothing listening on 127.0.0.1:{port}") def frame_window(body: bytes, expect_dcz: bool) -> int: """Declared window size from the zstd frame header (RFC 8878 3.1.1.1).""" if expect_dcz: if body[:8] != DCZ_MAGIC: raise RuntimeError(f"not a dcz body (hex={body[:8].hex()})") blob = body[DCZ_HEADER_LEN:] else: blob = body if blob[:4] != ZSTD_MAGIC: raise RuntimeError(f"no zstd magic (hex={blob[:8].hex()})") fhd = blob[4] if fhd & 0x20: raise RuntimeError( "frame is Single_Segment: it carries no Window_Descriptor, so " "the window cannot be read -- the fixture must stay chunked" ) wd = blob[5] exp, mantissa = wd >> 3, wd & 7 base = 1 << (10 + exp) return base + (base // 8) * mantissa def fetch(port: int, path: str, dcz: bool, dict_bytes: bytes): headers = {"Accept-Encoding": "zstd", "User-Agent": "zstd-dcz-budget/1.0"} if dcz: digest = hashlib.sha256(dict_bytes).digest() b64 = base64.b64encode(digest).decode("ascii") headers["Accept-Encoding"] = "zstd, dcz" headers["Available-Dictionary"] = f":{b64}:" req = urllib.request.Request(f"http://127.0.0.1:{port}{path}", headers=headers) with urllib.request.urlopen(req, timeout=30) as resp: return (resp.headers.get("Content-Encoding") or "identity").strip(), resp.read() def main() -> int: args = parse_args() nginx = pathlib.Path(args.nginx_binary).resolve() mods = [ m for m in ( detect(args.filter_module, nginx, "ngx_http_zstd_filter_module.so"), detect(args.static_module, nginx, "ngx_http_zstd_static_module.so"), ) if m ] failures: list[str] = [] def check(name: str, ok: bool, detail: str = "") -> None: if ok: print(f"ok - {name}") else: print(f"FAIL - {name}: {detail}") failures.append(name) dict_bytes = make_dict() body = make_body() os.umask(0o022) with tempfile.TemporaryDirectory(prefix="zstd-dczbudget-") as td: os.chmod(td, 0o755) root = pathlib.Path(td) (root / "logs").mkdir() (root / "html").mkdir() for d in ("client_body_temp", "proxy_temp"): (root / d).mkdir() dict_path = root / "dcz.dict" dict_path.write_bytes(dict_bytes) # Served from disk; nginx sends it chunked-free but with a # Content-Length. pledged_size is then known, and the derived window # is ceil_log2(dict + content) either way. (root / "html" / "app.js").write_bytes(body) load = "".join(f"load_module {m};\n" for m in mods) conf = root / "nginx.conf" conf.write_text( f"""{load}worker_processes 1; error_log {root}/logs/error.log debug; pid {root}/nginx.pid; events {{ worker_connections 64; }} http {{ access_log off; client_body_temp_path {root}/client_body_temp; proxy_temp_path {root}/proxy_temp; types {{ application/javascript js; }} default_type application/octet-stream; zstd on; zstd_comp_level {COMP_LEVEL}; zstd_min_length 64; zstd_types application/javascript; zstd_dcz_dict_file {dict_path}; zstd_dcz_assume_secure_transport on; # No zstd_window_log and no zstd_max_cctx_memory: the DEFAULT config, # where the dcz window must be exactly what it always was and the only # thing that changes is which ring slot it uses. server {{ listen 127.0.0.1:{args.port}; location / {{ proxy_pass http://127.0.0.1:{args.backend_port}/; proxy_http_version 1.1; proxy_set_header Connection ""; }} }} # Same location shape, plus an explicit memory budget. This is the only # configuration in which the dcz window is allowed to move. server {{ listen 127.0.0.1:{args.port + 1}; zstd_max_cctx_memory {BUDGET}; location / {{ proxy_pass http://127.0.0.1:{args.backend_port}/; proxy_http_version 1.1; proxy_set_header Connection ""; }} }} }} """, encoding="utf-8", newline="\n", ) _Handler.payload = body backend = _Srv(("127.0.0.1", args.backend_port), _Handler) threading.Thread(target=backend.serve_forever, daemon=True).start() proc = subprocess.Popen( [str(nginx), "-p", str(root) + "/", "-c", str(conf), "-g", "daemon off;"], stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, ) try: wait_port(args.port) # --- 3. default config: dcz window unchanged ----------------- ce, dcz_body = fetch(args.port, "/app.js", True, dict_bytes) check("default config negotiates dcz", ce == "dcz", f"C-E={ce!r}") if ce == "dcz": w = frame_window(dcz_body, True) check( "default config: dcz window is the dictionary-derived one", w == 1 << UNCLAMPED_WLOG, f"declared window {w}, want {1 << UNCLAMPED_WLOG} " f"(2^{UNCLAMPED_WLOG}) -- an opt-in clamp leaked into the " "default configuration", ) # A plain request on the SAME location, twice, so the second is a # reuse witness naming the slot the plain profile settled in. for _ in range(2): ce_p, _plain_body = fetch(args.port, "/app.js", False, dict_bytes) check( "same location also serves plain zstd", ce_p == "zstd", f"C-E={ce_p!r}" ) # And the dcz request again, so it too produces a reuse witness. fetch(args.port, "/app.js", True, dict_bytes) # --- 2. budget clamp ---------------------------------------- ce_b, budget_body = fetch(args.port + 1, "/app.js", True, dict_bytes) check("budgeted location negotiates dcz", ce_b == "dcz", f"C-E={ce_b!r}") if ce_b == "dcz": wb = frame_window(budget_body, True) check( "zstd_max_cctx_memory clamps the dcz window", wb == 1 << BUDGET_EXPECTED_WLOG, f"declared window {wb} != budget-derived cap " f"{1 << BUDGET_EXPECTED_WLOG} (2^{BUDGET_EXPECTED_WLOG}) " f"for zstd_max_cctx_memory {BUDGET} -- the dcz path " "either escaped the vetted budget (over-clamp) or " "clamped tighter than the budget allows (under-clamp)", ) finally: proc.terminate() try: proc.wait(timeout=10) except subprocess.TimeoutExpired: proc.kill() proc.wait(timeout=10) # --- 1. ring partition ------------------------------------------ log = (root / "logs" / "error.log").read_text(errors="replace") witnesses = REUSE_RE.findall(log) if not witnesses: check( "ring reuse witnesses present", False, "no 'reusing worker cctx' lines -- nginx is not built " "--with-debug, or the cache never engaged", ) else: check("ring reuse witnesses present", True) # slot -> set of window_logs that slot was keyed for by_slot: dict[str, set[str]] = {} for slot, _lvl, _lng, wlog in witnesses: by_slot.setdefault(slot, set()).add(wlog) wlogs = {w for s in by_slot.values() for w in s} # The STRONG form, and the weak one is not good enough here. # "at least two slots exist and at least two window logs appear" # is an aggregate over the whole log: it cannot tell the fixed # behaviour (slot A keyed only on the dcz window, slot B keyed # only on the plain one) from a regression in which one slot was # keyed on the plain window and LATER re-keyed by a dcz request, # which is exactly the contamination under test. Two servers run # in this one worker, so both aggregate counts are inflated for # reasons unrelated to the property anyway. # # A slot is never re-parameterised, so every witness for a given # slot must report the SAME window log. More than one window log # on a single slot is the contamination itself, visible directly. mixed = {s: sorted(w) for s, w in by_slot.items() if len(w) > 1} check( "no ring slot is keyed on more than one window log", not mixed, f"slot(s) re-keyed mid-run: {mixed} -- a dcz request borrowed " "a slot vetted for another window and raised its retained " "workspace permanently", ) # And the two profiles must actually be present, in slots of # their own: the dcz window (unclamped, on the default server) # and the plain "unset" 0. Without this a run in which dcz never # negotiated would satisfy the check above vacuously. dcz_slots = {s for s, w in by_slot.items() if str(UNCLAMPED_WLOG) in w} plain_slots = {s for s, w in by_slot.items() if "0" in w} check( "dcz and plain requests occupy DIFFERENT ring slots", bool(dcz_slots) and bool(plain_slots) and not (dcz_slots & plain_slots), f"witnesses {witnesses!r}: slots keyed on the dcz window " f"{UNCLAMPED_WLOG} = {sorted(dcz_slots)}, slots keyed on the " f"unset window 0 = {sorted(plain_slots)}; want both non-empty " "and disjoint. A dcz request keyed on the unset " "zstd_window_log shares the plain slot and raises its " "retained workspace permanently", ) check( "the dcz slot is keyed on the dcz window, not the unset one", str(UNCLAMPED_WLOG) in wlogs, f"window_logs seen in the ring: {sorted(wlogs)}; expected one " f"slot keyed on {UNCLAMPED_WLOG}", ) if failures: print(f"\nFAILED {len(failures)} check(s): {', '.join(failures)}") return 1 print("\nOK: dcz budget + ring key") return 0 if __name__ == "__main__": sys.exit(main()) myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_dcz_cache_partition.py000077500000000000000000000272471525232545500272150ustar00rootroot00000000000000#!/usr/bin/env python3 """Shared-cache partitioning test for the dcz cross-origin decision. ``ngx_http_zstd_dcz_negotiate()`` treats ``Sec-Fetch-Site`` as a response-selection input: absent / ``same-origin`` / ``none`` may negotiate dcz, every other value (``cross-site``, ``same-site``) falls back to plain zstd. A header that selects the representation must be listed in ``Vary`` or a shared cache will serve one client's variant to another. This harness builds a REAL two-server fixture inside one nginx: client -> :front (proxy_cache, no zstd) -> :origin (zstd + dcz) and runs the fill-order matrix {same-origin, cross-site, absent Sec-Fetch-Site} x {both fill orders} recording what the cache actually serves in each cell. Without ``Vary: Sec-Fetch-Site`` the second request in each pair is a cache HIT on the first request's variant: * fill same-origin then request cross-site -> the dcz body is served to a cross-site client, bypassing the origin gate entirely; * fill cross-site then request same-origin -> dcz is suppressed for a legitimate same-origin client. Both are asserted here. The test fails on a module that does not push ``Vary: Sec-Fetch-Site``. Notes on the rig ---------------- * dcz requires a secure context (RFC 9842 section 8). The origin server sets ``zstd_dcz_assume_secure_transport on`` — the "TLS terminated upstream" deployment — because Test::Nginx/this harness speak plain HTTP. Without it nginx never negotiates dcz and every cell looks identical (a vacuous green). * The oracle is the response's own ``Content-Encoding`` plus the dcz magic in the body, NOT body equality: a dcz body and a zstd body of the same resource are both "valid", so only the encoding marker discriminates. ``X-Cache-Status`` is asserted too so a cell that was a MISS cannot masquerade as a correctly-partitioned HIT. * Each matrix cell uses a distinct URI so the cache entries of one cell cannot leak into the next. """ from __future__ import annotations import argparse import base64 import hashlib import http.client import os import pathlib import shutil import socket import subprocess import sys import tempfile import time DCZ_MAGIC = b"\x5e\x2a\x4d\x18" # zstd skippable frame 0x184D2A5E, LE # Each cell gets its own URI: a stale entry from a previous cell must # not be able to answer the next one. CELLS = [ # (label, first request's Sec-Fetch-Site, second request's, uri) ("same-origin then cross-site", "same-origin", "cross-site", "/c/a"), ("cross-site then same-origin", "cross-site", "same-origin", "/c/b"), ("same-origin then absent", "same-origin", None, "/c/c"), ("absent then cross-site", None, "cross-site", "/c/d"), ("cross-site then absent", "cross-site", None, "/c/e"), ("absent then same-origin", None, "same-origin", "/c/f"), ] def dcz_expected(sfs: str | None) -> bool: """What the ORIGIN would serve for this Sec-Fetch-Site, uncached.""" return sfs is None or sfs in ("same-origin", "none") def parse_args() -> argparse.Namespace: p = argparse.ArgumentParser( description="dcz shared-cache partitioning (Vary: Sec-Fetch-Site) test." ) p.add_argument("--nginx-binary", required=True) p.add_argument("--filter-module") p.add_argument("--static-module") p.add_argument("--port", type=int, default=18120, help="Caching front-end port.") p.add_argument("--origin-port", type=int, default=18121) p.add_argument( "--dict", default=str(pathlib.Path(__file__).resolve().parents[1] / "t/suite/dcz-dict"), help="Dictionary fixture (default: ci/t/suite/dcz-dict).", ) return p.parse_args() def detect_module(explicit, nginx: pathlib.Path, name: str): if explicit: return pathlib.Path(explicit) sib = nginx.parent / name return sib if sib.exists() else None def wait_port(port: int, timeout: float = 10.0) -> None: deadline = time.time() + timeout while time.time() < deadline: try: with socket.create_connection(("127.0.0.1", port), 0.5): return except OSError: time.sleep(0.1) raise RuntimeError(f"nothing listening on 127.0.0.1:{port}") def request(port: int, uri: str, sfs: str | None, dict_b64: str): headers = { "Host": "cache.test", "Accept-Encoding": "zstd, dcz", "Available-Dictionary": f":{dict_b64}:", "Connection": "close", } if sfs is not None: headers["Sec-Fetch-Site"] = sfs conn = http.client.HTTPConnection("127.0.0.1", port, timeout=15) try: conn.request("GET", uri, headers=headers) resp = conn.getresponse() body = resp.read() return ( resp.status, (resp.getheader("Content-Encoding") or "").lower(), (resp.getheader("X-Cache-Status") or "").upper(), resp.getheader("Vary") or "", body, ) finally: conn.close() def main() -> int: args = parse_args() nginx = pathlib.Path(args.nginx_binary).resolve() if not nginx.exists(): raise FileNotFoundError(f"nginx binary not found: {nginx}") filt = detect_module(args.filter_module, nginx, "ngx_http_zstd_filter_module.so") stat = detect_module(args.static_module, nginx, "ngx_http_zstd_static_module.so") modules = [m.resolve() for m in (filt, stat) if m is not None] for m in modules: if not m.exists(): raise FileNotFoundError(f"module not found: {m}") dict_path = pathlib.Path(args.dict) dict_raw = dict_path.read_bytes() dict_b64 = base64.b64encode(hashlib.sha256(dict_raw).digest()).decode() os.umask(0o022) with tempfile.TemporaryDirectory(prefix="zstd-dczcache-") as td: os.chmod(td, 0o755) root = pathlib.Path(td) for sub in ("html", "logs", "cache", "tmp"): (root / sub).mkdir() shutil.copyfile(dict_path, root / "dcz-dict") # Compressible, dictionary-friendly payload: the dictionary's own # bytes repeated, so dcz and plain zstd both succeed and differ. (root / "html" / "payload").write_bytes(dict_raw * 8) for _, _, _, uri in CELLS: dst = root / "html" / uri.lstrip("/") dst.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(root / "html" / "payload", dst) load = "".join(f"load_module {m};\n" for m in modules) conf = root / "nginx.conf" conf.write_text( f"""{load}worker_processes 1; error_log {root}/logs/error.log warn; pid {root}/nginx.pid; events {{ worker_connections 64; }} http {{ access_log off; default_type application/octet-stream; proxy_temp_path {root}/tmp; proxy_cache_path {root}/cache levels=1:2 keys_zone=dczc:4m inactive=10m; # Front-end: a SHARED cache. No zstd here — it must serve exactly # what the origin gave it, so the cached variant is observable. server {{ listen 127.0.0.1:{args.port}; server_name cache.test; location / {{ proxy_pass http://127.0.0.1:{args.origin_port}; proxy_http_version 1.1; proxy_set_header Connection ""; proxy_cache dczc; proxy_cache_valid 200 10m; # Deliberately NOT proxy_cache_key'd on Sec-Fetch-Site: the # point is whether the ORIGIN declares the dependency via # Vary, which is the only thing an operator-independent # shared cache (CDN, browser cache) can act on. add_header X-Cache-Status $upstream_cache_status always; }} }} # Origin: the module under test. server {{ listen 127.0.0.1:{args.origin_port}; server_name cache.test; root {root}/html; zstd on; zstd_comp_level 3; zstd_min_length 1; zstd_types application/octet-stream; zstd_dcz_assume_secure_transport on; zstd_dcz_dict_file {root}/dcz-dict; location / {{ }} }} }} """, encoding="utf-8", ) proc = subprocess.Popen( [ str(nginx), "-p", str(root), "-c", str(conf), "-g", "daemon off; master_process off;", ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, ) try: try: wait_port(args.origin_port) wait_port(args.port) except RuntimeError: # nginx refused to start: its own diagnostics are the # answer, not the bare "nothing listening". if proc.poll() is not None and proc.stdout is not None: sys.stderr.write(proc.stdout.read()) elog = root / "logs" / "error.log" if elog.exists(): sys.stderr.write(elog.read_text(errors="replace")[-4000:]) raise # Rig self-check: the ORIGIN itself must discriminate, or # every matrix cell is vacuous. for sfs in ("same-origin", "cross-site", None): st, ce, _, _, body = request( args.origin_port, "/payload", sfs, dict_b64 ) want = dcz_expected(sfs) got = ce == "dcz" if st != 200 or got != want: raise RuntimeError( f"origin self-check failed for Sec-Fetch-Site={sfs!r}: " f"status={st} Content-Encoding={ce!r}, expected " f"{'dcz' if want else 'zstd'}" ) if want and not body.startswith(DCZ_MAGIC): raise RuntimeError( "origin self-check: Content-Encoding dcz but body " f"lacks the skippable-frame magic (first 8B " f"{body[:8].hex()})" ) failures: list[str] = [] print( "cell " "fill second served cache ok" ) for label, first, second, uri in CELLS: st1, _ce1, _cs1, vary1, _ = request(args.port, uri, first, dict_b64) st2, ce2, cs2, _vary2, body2 = request(args.port, uri, second, dict_b64) want2 = dcz_expected(second) got2 = ce2 == "dcz" ok = st1 == 200 and st2 == 200 and got2 == want2 if got2 and not body2.startswith(DCZ_MAGIC): ok = False print( f"{label:34s}{first!s:12s}{second!s:12s}" f"{ce2 or '-':10s}{cs2 or '-':7s}{'PASS' if ok else 'FAIL'}" ) if not ok: failures.append( f"{label}: after filling with Sec-Fetch-Site={first!r}, " f"a request with Sec-Fetch-Site={second!r} was served " f"Content-Encoding={ce2!r} (cache {cs2 or '-'}), expected " f"{'dcz' if want2 else 'zstd'}. " f"Vary on the fill was {vary1!r}." ) if failures: sys.stderr.write( f"dcz shared-cache partitioning FAILED ({len(failures)} cells):\n" ) for f in failures: sys.stderr.write(f" - {f}\n") return 1 print(f"OK: dcz cache partitioning holds across {len(CELLS)} cells") return 0 finally: proc.terminate() try: proc.wait(timeout=10) except subprocess.TimeoutExpired: proc.kill() if __name__ == "__main__": sys.exit(main()) myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_dcz_dict_lookup_unit.c000066400000000000000000000237311525232545500272150ustar00rootroot00000000000000/* * Differential fixture for ngx_http_zstd_dcz_dict_lookup() -- A33-P4. * * ngx_http_zstd_dcz_dict_lookup() switches from a linear ngx_memcmp scan * to a binary search over a hash-sorted array once the configured * dictionary count exceeds NGX_HTTP_ZSTD_DCZ_BSEARCH_THRESHOLD. This * fixture asserts the extracted, CURRENT shipped implementation (see * test_dcz_dict_lookup_unit.sh for the extraction) returns identical * results to a naive, always-correct brute-force linear scan, for every * array size in {1, threshold, threshold+1, 64, 256, 737} and for: * * - a HIT on the first, middle and last element (post-sort position); * - a MISS with a key that would sort before the first entry; * - a MISS with a key that would sort after the last entry; * - a MISS with a key that would sort in the middle (no entry matches). * * A deliberately broken comparator is used to prove this fixture is * ARMED -- see main(): it must make at least one above-threshold lookup * diverge (because ngx_http_zstd_dcz_dict_lookup()'s * bsearch branch is sorted with the same comparator used to search, a * broken order produces wrong verdicts for any input that requires * following the correct direction past the first probe). */ #include #include #include #include typedef uintptr_t ngx_uint_t; typedef intptr_t ngx_int_t; typedef unsigned char u_char; typedef struct { void *elts; ngx_uint_t nelts; size_t size; ngx_uint_t nalloc; void *pool; } ngx_array_t; typedef struct { unsigned char *data; size_t len; } ngx_str_t; #define ngx_memcmp(s1, s2, n) memcmp(s1, s2, n) #define ngx_qsort qsort #define ngx_libc_cdecl #include "generated_dcz_dict_lookup.inc" #define HLEN NGX_HTTP_ZSTD_SHA256_DIGEST_LEN static int checks_run = 0; static int checks_failed = 0; static int quiet_failures = 0; #define CHECK(cond, msg) \ do { \ checks_run++; \ if (!(cond)) { \ checks_failed++; \ if (!quiet_failures) { \ fprintf(stderr, "FAIL: %s (%s:%d)\n", msg, __FILE__, \ __LINE__); \ } \ } \ } while (0) /* Reference oracle: brute-force linear scan, independent of the sort * order or the threshold, always correct by construction. */ static ngx_http_zstd_dcz_dict_t * ref_lookup(ngx_http_zstd_dcz_dict_t *dicts, ngx_uint_t n, const unsigned char *key) { ngx_uint_t i; for (i = 0; i < n; i++) { if (memcmp(dicts[i].hash, key, HLEN) == 0) { return &dicts[i]; } } return NULL; } static void fill_unique_sorted(ngx_http_zstd_dcz_dict_t *dicts, ngx_uint_t n, unsigned seed) { ngx_uint_t i; size_t j; /* Fill with deterministic varied bytes, then force strict ordering * by writing the index into the leading bytes big-endian. This * guarantees n distinct, strictly increasing 32-byte keys; the real * merge-time duplicate-hash rejection guarantees the same invariant * on dcz_dicts. */ for (i = 0; i < n; i++) { for (j = 0; j < HLEN; j++) { dicts[i].hash[j] = (unsigned char) ((seed + i * 131u + j * 17u) & 0xffu); } dicts[i].hash[0] = (unsigned char) ((i >> 24) & 0xff); dicts[i].hash[1] = (unsigned char) ((i >> 16) & 0xff); dicts[i].hash[2] = (unsigned char) ((i >> 8) & 0xff); dicts[i].hash[3] = (unsigned char) (i & 0xff); /* leave hash[4] at 0x01 so index 0's key never collides with * the all-zero "before first" miss key below */ dicts[i].hash[4] = 0x01; dicts[i].file.len = 0; dicts[i].file.data = NULL; } } static void reverse_dicts(ngx_http_zstd_dcz_dict_t *dicts, ngx_uint_t n) { ngx_http_zstd_dcz_dict_t tmp; ngx_uint_t i; for (i = 0; i < n / 2; i++) { tmp = dicts[i]; dicts[i] = dicts[n - i - 1]; dicts[n - i - 1] = tmp; } } /* Runs the differential check for one array size n, using `cmp` (the * real comparator, or a deliberately broken one) to sort. Returns 1 if * every sub-case matched the oracle, 0 if any diverged. */ static int run_size(ngx_uint_t n, int (*cmp)(const void *, const void *)) { ngx_http_zstd_dcz_dict_t *dicts; ngx_array_t arr; unsigned char key[HLEN]; ngx_http_zstd_dcz_dict_t *got, *want; int ok; ngx_uint_t before; dicts = calloc(n, sizeof(*dicts)); if (dicts == NULL) { fprintf(stderr, "calloc failed for n=%lu\n", (unsigned long) n); exit(1); } fill_unique_sorted(dicts, n, 0xC0FFEEu + (unsigned) n); arr.elts = dicts; arr.nelts = n; arr.size = sizeof(*dicts); arr.nalloc = n; arr.pool = NULL; if (cmp == ngx_http_zstd_dcz_dict_cmp) { /* Exercise the production wrapper with deliberately wrong input * order. Feeding it fill_unique_sorted()'s original ascending * order would let a no-op sort implementation remain green. */ reverse_dicts(dicts, n); ngx_http_zstd_dcz_dict_sort(&arr); } else { qsort(dicts, n, sizeof(*dicts), cmp); } ok = 1; before = checks_failed; /* HIT: first, middle, last (post-sort positions). */ memcpy(key, dicts[0].hash, HLEN); got = ngx_http_zstd_dcz_dict_lookup(&arr, key); want = ref_lookup(dicts, n, key); CHECK(got == want, "HIT(first) diverges from oracle"); memcpy(key, dicts[n / 2].hash, HLEN); got = ngx_http_zstd_dcz_dict_lookup(&arr, key); want = ref_lookup(dicts, n, key); CHECK(got == want, "HIT(mid) diverges from oracle"); memcpy(key, dicts[n - 1].hash, HLEN); got = ngx_http_zstd_dcz_dict_lookup(&arr, key); want = ref_lookup(dicts, n, key); CHECK(got == want, "HIT(last) diverges from oracle"); /* MISS: sorts before the first entry (all-zero hash; index 0's * hash[4] is forced to 0x01, so index 0 never equals this key). */ memset(key, 0x00, HLEN); got = ngx_http_zstd_dcz_dict_lookup(&arr, key); want = ref_lookup(dicts, n, key); CHECK(got == want && got == NULL, "MISS(last) diverges from oracle or unexpectedly hit"); /* MISS: a key that sorts in the middle of the range but matches * nothing -- take the middle element's key and flip a trailing * byte that fill_unique_sorted() populated (not the forced index * prefix), so it lands near the middle in sort order without * colliding with any real entry. */ if (n > 0) { memcpy(key, dicts[n / 2].hash, HLEN); key[HLEN - 1] ^= 0xff; got = ngx_http_zstd_dcz_dict_lookup(&arr, key); want = ref_lookup(dicts, n, key); CHECK(got == want, "MISS(mid) diverges from oracle"); } if ((ngx_uint_t) checks_failed != before) { ok = 0; } free(dicts); return ok; } static int run_all(int (*cmp)(const void *, const void *)) { static const ngx_uint_t sizes[] = { 1, NGX_HTTP_ZSTD_DCZ_BSEARCH_THRESHOLD, NGX_HTTP_ZSTD_DCZ_BSEARCH_THRESHOLD + 1, 64, 256, 737 }; ngx_uint_t i; int all_ok; all_ok = 1; for (i = 0; i < sizeof(sizes) / sizeof(sizes[0]); i++) { if (!run_size(sizes[i], cmp)) { all_ok = 0; } } return all_ok; } /* Deliberately broken comparator: inverted sign, so qsort() produces * descending order while ngx_http_zstd_dcz_dict_lookup()'s bsearch * branch still assumes ascending order. Used only to prove the fixture * is armed (main() below). */ static int broken_cmp(const void *one, const void *two) { return -ngx_http_zstd_dcz_dict_cmp(one, two); } int main(void) { int real_ok; real_ok = run_all(ngx_http_zstd_dcz_dict_cmp); if (!real_ok) { fprintf(stderr, "%d/%d checks failed against the REAL comparator " "(should be 0)\n", checks_failed, checks_run); return 1; } printf("%d/%d checks passed against the real comparator\n", checks_run - checks_failed, checks_run); /* Armed proof: re-run with a broken comparator and require at * least one divergence above the threshold (bsearch relies on * ascending order; a linear scan below the threshold does not sort * at all, so small n cannot be expected to fail here -- the armed * assertion targets sizes that exercise the bsearch branch). */ { int checks_before, failed_before; checks_before = checks_run; failed_before = checks_failed; quiet_failures = 1; run_size(NGX_HTTP_ZSTD_DCZ_BSEARCH_THRESHOLD + 1, broken_cmp); run_size(256, broken_cmp); run_size(737, broken_cmp); quiet_failures = 0; if (checks_failed == failed_before) { fprintf(stderr, "FAIL: fixture is NOT armed -- broken comparator " "produced zero divergences (checks_run %d->%d)\n", checks_before, checks_run); return 1; } printf("armed: broken comparator produced %d divergent " "check(s) as expected\n", checks_failed - failed_before); } printf("OK: dcz dict lookup fixture\n"); return 0; } myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_dcz_dict_lookup_unit.sh000077500000000000000000000153321525232545500274060ustar00rootroot00000000000000#!/bin/bash # Differential fixture for ngx_http_zstd_dcz_dict_lookup() -- A33-P4. # # ngx_http_zstd_dcz_dict_lookup() and its sort comparator # ngx_http_zstd_dcz_dict_cmp() are `static` inside # src/ngx_http_zstd_filter_module.c, so this extracts them verbatim by # line range (same technique already used by # ci/tools/test_ceil_log2_unit.sh and the Accept-Encoding parser fuzz # harness) and compiles them standalone against a reference brute-force # linear-scan oracle in test_dcz_dict_lookup_unit.c. No nginx tree, no # zstd needed -- pure C, self-contained. # # This proves the negotiation lookup (which switches from a linear # ngx_memcmp scan to a binary search once the configured dictionary # count exceeds NGX_HTTP_ZSTD_DCZ_BSEARCH_THRESHOLD) returns identical # verdicts to a naive linear scan for every case: sizes straddling the # threshold, a hit on the first/middle/last element, and misses # including keys that sort before the first and after the last entry. set -euo pipefail # ci/tools/ -> ci/ -> repo root: TWO levels since the ci/ move. cd "$(dirname "$0")/../.." SRC="src/ngx_http_zstd_filter_module.c" CC="${CC:-cc}" OUT="$(mktemp -d "${TMPDIR:-/tmp}/zstd-dcz-lookup-unit.XXXXXX")" trap 'rm -rf "$OUT"' EXIT extract_by_signature() { # $1 = signature-line grep pattern (the K&R-style function-name # line, one line below the return-type line); $2 = output file; # $3 = a substring the extraction must contain (extraction-range # sanity check). local sig_pattern="$1" local outfile="$2" local must_contain="$3" local sig_line sig_line="$(grep -n "$sig_pattern" "$SRC" | head -1 | cut -d: -f1 || true)" if [ -z "$sig_line" ]; then echo "FAIL: could not locate '$sig_pattern' in $SRC" >&2 exit 1 fi # The return-type line is directly above the signature line for # every function in this file (K&R style: type on its own line). local start_line=$((sig_line - 1)) # First "^}" at column 0 after start_line closes the function (or # struct, for the type extraction). local rel_end rel_end="$(tail -n "+${start_line}" "$SRC" | grep -n '^}' | head -1 | cut -d: -f1 || true)" if [ -z "$rel_end" ]; then echo "FAIL: could not locate the end of '$sig_pattern' in $SRC" >&2 exit 1 fi local end_line=$((start_line + rel_end - 1)) { echo "/* Generated by test_dcz_dict_lookup_unit.sh -- verbatim" echo " * extraction from $SRC lines ${start_line}-${end_line}." echo " * Do not edit; re-run the script to regenerate. */" sed -n "${start_line},${end_line}p" "$SRC" echo } >>"$outfile" if ! grep -q "$must_contain" "$outfile"; then echo "FAIL: extracted fragment ('$sig_pattern') missing '$must_contain' -- extraction range wrong" >&2 cat "$outfile" >&2 exit 1 fi } mkdir -p "$OUT" INC="$OUT/generated_dcz_dict_lookup.inc" : >"$INC" # 1) The threshold constant (a single #define line, grabbed directly -- # not a function, so the signature/brace extractor doesn't apply). if ! grep -q '^#define NGX_HTTP_ZSTD_DCZ_BSEARCH_THRESHOLD' "$SRC"; then echo "FAIL: NGX_HTTP_ZSTD_DCZ_BSEARCH_THRESHOLD not found in $SRC" >&2 exit 1 fi grep '^#define NGX_HTTP_ZSTD_DCZ_BSEARCH_THRESHOLD' "$SRC" >>"$INC" echo >>"$INC" # 2) NGX_HTTP_ZSTD_SHA256_DIGEST_LEN (needed by the struct/functions # below). Lives in ngx_http_zstd_sha256.h, not $SRC. SHA_HDR="src/ngx_http_zstd_sha256.h" if ! grep -q '^#define NGX_HTTP_ZSTD_SHA256_DIGEST_LEN' "$SHA_HDR"; then echo "FAIL: NGX_HTTP_ZSTD_SHA256_DIGEST_LEN not found in $SHA_HDR" >&2 exit 1 fi grep '^#define NGX_HTTP_ZSTD_SHA256_DIGEST_LEN' "$SHA_HDR" >>"$INC" echo >>"$INC" # NGX_HTTP_ZSTD_DCZ_HEADER_LEN: the struct's frame_header field sizes # against it; grab it too so the verbatim struct extraction compiles. if ! grep -q '^#define NGX_HTTP_ZSTD_DCZ_HEADER_LEN' "$SRC"; then echo "FAIL: NGX_HTTP_ZSTD_DCZ_HEADER_LEN not found in $SRC" >&2 exit 1 fi grep '^#define NGX_HTTP_ZSTD_DCZ_HEADER_LEN' "$SRC" >>"$INC" echo >>"$INC" # 3) The ngx_http_zstd_dcz_dict_t struct (only the fields the lookup # touches are needed by the test harness, but extracting the real # struct verbatim means a field reorder/rename cannot silently drift # the fixture out of sync with the shipped layout). STRUCT_END="$(grep -n '^} ngx_http_zstd_dcz_dict_t;' "$SRC" | head -1 | cut -d: -f1 || true)" STRUCT_START="$(head -n "${STRUCT_END:-0}" "$SRC" | grep -n '^typedef struct {$' | tail -1 | cut -d: -f1 || true)" if [ -z "$STRUCT_START" ] || [ -z "$STRUCT_END" ]; then echo "FAIL: could not locate ngx_http_zstd_dcz_dict_t struct bounds in $SRC" >&2 exit 1 fi { echo "/* Generated: ngx_http_zstd_dcz_dict_t, $SRC lines ${STRUCT_START}-${STRUCT_END}. */" sed -n "${STRUCT_START},${STRUCT_END}p" "$SRC" echo } >>"$INC" if ! grep -q 'ngx_http_zstd_dcz_dict_t;' "$INC"; then echo "FAIL: struct extraction missing its typedef name" >&2 exit 1 fi # 4) ngx_http_zstd_dcz_dict_cmp() -- the sort comparator. extract_by_signature '^ngx_http_zstd_dcz_dict_cmp(const void \*one, const void \*two)$' \ "$INC" 'ngx_http_zstd_dcz_dict_cmp(const void \*one, const void \*two)' # 5) The production sort wrapper. The fixture calls this rather than # sorting its own positive-case array, so removing the merge-time sort # cannot leave this test green. extract_by_signature '^ngx_http_zstd_dcz_dict_sort(ngx_array_t \*dcz_dicts)$' \ "$INC" 'ngx_qsort' # 6) ngx_http_zstd_dcz_dict_lookup() -- the function under test. extract_by_signature '^ngx_http_zstd_dcz_dict_lookup(ngx_array_t \*dcz_dicts,$' \ "$INC" 'ngx_http_zstd_dcz_dict_lookup' if ! grep -q 'NGX_HTTP_ZSTD_DCZ_BSEARCH_THRESHOLD' "$INC"; then echo "FAIL: lookup extraction missing the threshold branch -- extraction range wrong" >&2 exit 1 fi # The wrapper must be wired before pointer inheritance is resolved. This # distinguishes an owned location array from a child sharing its parent's # already-sorted array, and makes deleting the production call fail the gate. SORT_CALL_LINE="$(grep -n 'ngx_http_zstd_dcz_dict_sort(conf->dcz_dicts);' "$SRC" | cut -d: -f1 || true)" MERGE_LINE="$(grep -n 'ngx_conf_merge_ptr_value(conf->dcz_dicts, prev->dcz_dicts, NULL);' "$SRC" | cut -d: -f1 || true)" if [ -z "$SORT_CALL_LINE" ] || [ -z "$MERGE_LINE" ] || [ "$SORT_CALL_LINE" -ge "$MERGE_LINE" ]; then echo "FAIL: owned dcz dictionary array is not sorted before inheritance" >&2 exit 1 fi cp ci/tools/test_dcz_dict_lookup_unit.c "$OUT/test_dcz_dict_lookup_unit.c" "$CC" -std=c89 -pedantic-errors -Wall -Wextra -Werror -O2 -I"$OUT" \ -o "$OUT/dcz_dict_lookup_unit" "$OUT/test_dcz_dict_lookup_unit.c" "$OUT/dcz_dict_lookup_unit" echo "OK: dcz dict lookup differential fixture (extracted from current $SRC)" myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_dcz_prefix_alloc.py000077500000000000000000000214121525232545500265140ustar00rootroot00000000000000#!/usr/bin/env python3 """Regression test for the dcz 40-byte prefix inlining. RFC 9842 dcz responses used to queue the fixed 40-byte skippable-frame prefix (magic + dictionary SHA-256) on its OWN pool buffer and chain link, separate from the compressor's output buffers. ngx_http_zstd_filter_get_buf() now reserves 40 bytes at the front of the FIRST compressor output buffer and writes the prefix there instead, so a dcz response allocates one fewer ngx_buf_t/chain-link pair per response. This is a pure allocation-count and wire-format regression test: * runs a debug build (error_log ... debug) and greps its own log for "zstd get_buf: allocated buffer" (compressor output-buffer pool allocations) and "zstd dcz: 40-byte frame header queued" (the OLD dedicated-buffer code path -- must never fire again); * captures a strace -e trace=writev count for the same request, as a downstream-iovec-reduction proxy; * decodes the dcz body with the negotiated dictionary and byte-compares it against the plain fixture, so a buffer/prefix accounting change can never silently corrupt the wire body while "passing" on allocation counts alone. Exercises three body sizes (empty-compressed-output edge via a 1-byte body, a small body, and a body spanning multiple compressor iterations) so the assertion holds across the END-transition retry path the prefix sits in front of, not only the single-iteration common case. """ import argparse import base64 import hashlib import pathlib import re import shutil import socket import subprocess import sys import tempfile import time DCZ_MAGIC = bytes([0x5E, 0x2A, 0x4D, 0x18, 0x20, 0x00, 0x00, 0x00]) def parse_args() -> argparse.Namespace: p = argparse.ArgumentParser(description=__doc__) p.add_argument("--nginx-binary", required=True) p.add_argument("--filter-module") p.add_argument("--port", type=int, default=18311) return p.parse_args() def detect_module(explicit, nginx: pathlib.Path): if explicit: return pathlib.Path(explicit) sib = nginx.parent / "ngx_http_zstd_filter_module.so" return sib if sib.exists() else None def wait_for_port(port: int, timeout: float = 10.0) -> None: deadline = time.time() + timeout while time.time() < deadline: try: with socket.create_connection(("127.0.0.1", port), timeout=0.5): return except OSError: time.sleep(0.05) raise RuntimeError(f"nginx never opened port {port}") def write_config( root: pathlib.Path, port: int, module, dict_path: pathlib.Path, dict_hash_hex: str ) -> pathlib.Path: (root / "conf").mkdir() (root / "logs").mkdir() load = f"load_module {module};\n" if module else "" conf = root / "conf" / "nginx.conf" conf.write_text( f"""{load} worker_processes 1; daemon off; error_log {root}/logs/error.log debug; pid {root}/logs/nginx.pid; events {{ }} http {{ access_log off; zstd on; zstd_min_length 1; zstd_types *; zstd_dcz_assume_secure_transport on; zstd_dcz_dict_file {dict_path} {dict_hash_hex}; server {{ listen 127.0.0.1:{port}; root {root}/html; }} }} """, encoding="utf-8", newline="\n", ) return conf def fetch_dcz(port: int, dict_digest_b64: str, path: str) -> bytes: req = ( f"GET /{path} HTTP/1.1\r\n" "Host: localhost\r\n" "Accept-Encoding: dcz, zstd\r\n" f"Available-Dictionary: :{dict_digest_b64}:\r\n" "Sec-Fetch-Site: same-origin\r\n" "Connection: close\r\n\r\n" ) with socket.create_connection(("127.0.0.1", port), timeout=5) as s: s.sendall(req.encode()) data = b"" while True: chunk = s.recv(65536) if not chunk: break data += chunk return data def split_response(data: bytes): head, rest = data.split(b"\r\n\r\n", 1) # dechunk (Transfer-Encoding: chunked -- every response here is) body = b"" while rest: size_line, rest = rest.split(b"\r\n", 1) size = int(size_line.split(b";")[0], 16) if size == 0: break body += rest[:size] rest = rest[size + 2 :] return head.decode(errors="replace"), body def decode_dcz( zstd_bin: str, body: bytes, dict_path: pathlib.Path, work: pathlib.Path ) -> bytes: src = work / "response.dcz" dst = work / "response.out" src.write_bytes(body) subprocess.run( [ zstd_bin, "-d", "-q", "-f", "--memory=8388608", "-D", str(dict_path), "-o", str(dst), str(src), ], check=True, capture_output=True, ) return dst.read_bytes() def main() -> int: args = parse_args() failures = [] def check(name: str, cond: bool, detail: str = "") -> None: if cond: print(f"ok - {name}") else: print(f"FAIL - {name} {detail}") failures.append(name) nginx_path = pathlib.Path(args.nginx_binary) module = detect_module(args.filter_module, nginx_path) zstd_bin = shutil.which("zstd") if zstd_bin is None: print("SKIP: no zstd CLI on PATH", file=sys.stderr) return 0 with tempfile.TemporaryDirectory(prefix="zstd-dcz-alloc-") as tmp: root = pathlib.Path(tmp) (root / "html").mkdir() dict_bytes = b"The quick brown fox jumps over the lazy dog. " * 40 dict_path = root / "dict.bin" dict_path.write_bytes(dict_bytes) dict_digest = hashlib.sha256(dict_bytes).digest() dict_hash_hex = hashlib.sha256(dict_bytes).hexdigest() dict_digest_b64 = base64.b64encode(dict_digest).decode() conf = write_config(root, args.port, module, dict_path, dict_hash_hex) bodies = { "1byte.txt": b"x", "small.txt": b"hello world tiny body for dcz prefix test", "multi.txt": (b"The quick brown fox jumps over the lazy dog. " * 50), } for name, content in bodies.items(): (root / "html" / name).write_bytes(content) error_log = root / "logs" / "error.log" for name, content in bodies.items(): error_log.write_text("") # truncate between cases nginx = subprocess.Popen( [str(nginx_path), "-p", str(root), "-c", str(conf)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) try: wait_for_port(args.port) data = fetch_dcz(args.port, dict_digest_b64, name) finally: nginx.terminate() try: nginx.wait(timeout=5) except subprocess.TimeoutExpired: nginx.kill() nginx.wait(timeout=5) head, body = split_response(data) check( f"{name}: negotiated dcz", "Content-Encoding: dcz" in head, f"(head={head!r})", ) check( f"{name}: body starts with RFC 9842 magic", body[:8] == DCZ_MAGIC, f"(got {body[:8].hex()})", ) check( f"{name}: header embeds the dictionary SHA-256", body[8:40] == dict_digest, "", ) decoded = decode_dcz(zstd_bin, body, dict_path, root) check( f"{name}: decode round-trip is byte-exact", decoded == content, f"(decoded {len(decoded)}B, expected {len(content)}B)", ) log_text = error_log.read_text(errors="replace") alloc_count = len(re.findall(r"zstd get_buf: allocated buffer", log_text)) old_path_count = len( re.findall(r"zstd dcz: 40-byte frame header queued", log_text) ) check( f"{name}: old dedicated-prefix-buffer path never fires", old_path_count == 0, f"(matched {old_path_count} times -- prefix inlining " "regressed to the pre-fix separate-buffer path)", ) check( f"{name}: exactly one compressor buffer allocated", alloc_count == 1, f"(got {alloc_count} allocations for a body that fits " "one compressor buffer plus the inlined prefix)", ) print() if failures: print(f"FAILED: {len(failures)} check(s) did not pass:") for f in failures: print(f" - {f}") return 1 print( "OK: dcz prefix rides the first compressor buffer, one pool " "allocation per response, wire body unchanged" ) return 0 if __name__ == "__main__": sys.exit(main()) myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_dcz_window_log_unit.c000066400000000000000000000230231525232545500270430ustar00rootroot00000000000000/* * Copyright (C) 2026 Thijs Eilander * SPDX-License-Identifier: BSD-2-Clause * * Unit oracle for ngx_http_zstd_dcz_window_log(). * * The function and its ngx_http_zstd_ceil_log2() dependency are EXTRACTED * VERBATIM from src/ngx_http_zstd_filter_module.c by * test_dcz_window_log_unit.sh into generated_dcz_window_log.inc -- this file * never re-implements the arithmetic, so it cannot quietly agree with a stale * copy of itself. * * What is asserted, and why each one matters: * * 1. DEFAULT UNCHANGED. With neither clamp configured (both ceilings 0), * the result is exactly ceil_log2(dict + content). Both clamps are * opt-in, and an operator who never asked for a memory bound must not * see dcz wire bytes move. This is the regression guard for the scoping * decision; without it the "fix" silently becomes a global behaviour * change. * * 2. EITHER CEILING BINDS, and only downward. zstd_window_log and * zstd_max_cctx_memory are ceilings of the same class, so a configured * one lowers the window and never raises it -- a ceiling that could * raise the window would be a memory ESCAPE, the exact defect this * function was written to close. * * 3. THE LOWER OF THE TWO WINS when both are set. Honouring the larger * would void whichever bound the operator set more tightly. * * 4. RANGE. The result is always in [10, NGX_HTTP_ZSTD_DCZ_MAX_WINDOW_LOG]. * The lower bound keeps an invalid windowLog out of libzstd; the upper * bound is the RFC 9842 client-window guarantee. Both bounds also keep * the value inside ngx_http_zstd_profile_pack()'s 6-bit window_log * field, whose packer REFUSES rather than masks an out-of-domain value * -- so a violation here would turn into a failed CCtx acquisition, * not a silent aliasing. * * The sweep is exhaustive over the interesting domain rather than a handful * of hand-picked cases: every dictionary size across the power-of-two * boundaries, both pledged-size branches (known length and unknown), and * every value of both ceilings including the out-of-range ones. */ #include #include #include #include /* Minimal nginx shim: the extracted code uses these names only. */ typedef long ngx_int_t; typedef unsigned long ngx_uint_t; #define ngx_inline inline #include "generated_dcz_window_log.inc" #define WLOG_MIN ((ngx_int_t) NGX_HTTP_ZSTD_DCZ_MIN_WINDOW_LOG) #define UNKNOWN ((off_t) -1) static long long failures; static void fail(const char *what, size_t dict, off_t pledged, ngx_int_t conf, ngx_int_t budget, ngx_int_t got, ngx_int_t want) { failures++; if (failures <= 20) { fprintf(stderr, "FAIL %s: dict=%zu pledged=%lld conf_wlog=%ld budget_cap=%ld" " -> %ld, want %ld\n", what, dict, (long long) pledged, (long) conf, (long) budget, (long) got, (long) want); } } int main(void) { /* * Dictionary sizes straddling every power-of-two boundary in range, so * the ceil_log2 rounding is exercised at the exact points it can be off * by one, not only in the middle of a decade. */ static const size_t dicts[] = { 0, 1, 1023, 1024, 1025, 4095, 4096, 4097, (1u << 20) - 1, 1u << 20, (1u << 20) + 1, (1u << 22) - 1, 1u << 22, (1u << 22) + 1, (1u << 23) - 1, 1u << 23, (1u << 23) + 1, (size_t) 1 << 25 }; /* * The last three exercise the overflow fix: pledged_size is * upstream-controlled (r->headers_out.content_length_n) and must not be * able to wrap "required" down to something small. All three are large * POSITIVE off_t values -- (off_t) SIZE_MAX itself is negative on a * platform where off_t and size_t are the same width (verified: it * would silently fall into the pledged_size < 0 "unknown length" branch * and test nothing), so the boundary is expressed as OFF_T_MAX (the * largest legal off_t on ANY platform) and fractions of it, which stay * positive everywhere and still exceed size_t on an ILP32 build where * off_t is wider. */ #define OFF_T_MAX ((off_t) (((uintmax_t) 1 << (sizeof(off_t) * 8 - 1)) - 1)) static const off_t pledges[] = { UNKNOWN, 0, 1, 4096, 1u << 20, 1u << 23, OFF_T_MAX / 2, OFF_T_MAX - 1, OFF_T_MAX, }; size_t di, pi; ngx_int_t conf, budget; long long swept = 0; for (di = 0; di < sizeof(dicts) / sizeof(dicts[0]); di++) { for (pi = 0; pi < sizeof(pledges) / sizeof(pledges[0]); pi++) { size_t dict = dicts[di]; off_t pledged = pledges[pi]; ngx_int_t base; /* Unclamped reference: BOTH ceilings unset. */ base = ngx_http_zstd_dcz_window_log(dict, pledged, 0, 0); swept++; /* 4. range */ if (base < WLOG_MIN || base > NGX_HTTP_ZSTD_DCZ_MAX_WINDOW_LOG) { fail("range(unclamped)", dict, pledged, 0, 0, base, -1); } /* * 1. default unchanged -- the unclamped answer is exactly the * sizing rule, independently recomputed here from ceil_log2 so * this is an oracle rather than a tautology. */ { size_t pledged_contribution, required; ngx_int_t want; /* * Independently recomputed with its OWN saturating * arithmetic (not a call into the saturation the fix adds), * so this stays an oracle rather than the tautology of * calling the same helper the production code now calls. * Saturating at ceil_log2's own 2^23 threshold is enough: * anything at or past it maps to the same window log, so * the oracle does not need SIZE_MAX-exact behaviour. */ if (pledged < 0) { pledged_contribution = 1024 * 1024; } else if ((uintmax_t) pledged > (size_t) 1 << 23) { pledged_contribution = (size_t) 1 << 23; } else { pledged_contribution = (size_t) pledged; } if (pledged_contribution > (size_t) 1 << 23 || dict > (size_t) 1 << 23) { required = (size_t) 1 << 23; } else { required = dict + pledged_contribution; } want = (ngx_int_t) ngx_http_zstd_ceil_log2(required); if (base != want) { fail("default-unchanged", dict, pledged, 0, 0, base, want); } } /* -1 and 0 are the "unset" spellings; sweep past the cap too. */ for (conf = -1; conf <= 25; conf++) { for (budget = -1; budget <= 25; budget++) { ngx_int_t got, want; got = ngx_http_zstd_dcz_window_log(dict, pledged, conf, budget); swept++; /* Expected: base, lowered by whichever ceilings are set. */ want = base; if (conf > 0 && conf < want) { want = conf; } if (budget > 0 && budget < want) { want = budget; } if (want < WLOG_MIN) { want = WLOG_MIN; /* the defence-in-depth floor */ } /* 2 + 3: either ceiling binds, the lower one wins. */ if (got != want) { fail("clamp", dict, pledged, conf, budget, got, want); continue; } /* * "A ceiling never raises the window" needs no separate * assertion: `want` is `base` reduced by the set * ceilings, and the floor cannot lift it back above * `base` because `base` is itself >= the floor. So the * got != want comparison above already enforces it, and * a `got > base` check here could never fire. Stating it * as its own assertion would only look like coverage. */ /* * 4: range again, now under clamping. A configured * ceiling below ZSTD_WINDOWLOG_MIN would otherwise push * an invalid windowLog into libzstd -- assert the floor * holds for EVERY combination, not just the unclamped * one. */ if (got > NGX_HTTP_ZSTD_DCZ_MAX_WINDOW_LOG) { fail("range(above-cap)", dict, pledged, conf, budget, got, NGX_HTTP_ZSTD_DCZ_MAX_WINDOW_LOG); } if (got < WLOG_MIN) { fail("range(below-floor)", dict, pledged, conf, budget, got, WLOG_MIN); } } } } } printf("swept %lld combinations\n", swept); if (failures) { printf("FAILED: %lld assertion(s)\n", failures); return 1; } printf("OK: dcz window log (default unchanged, both ceilings bind " "downward, lower wins, result in range)\n"); return 0; } myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_dcz_window_log_unit.sh000077500000000000000000000143051525232545500272410ustar00rootroot00000000000000#!/bin/bash # Copyright (C) 2026 Thijs Eilander # SPDX-License-Identifier: BSD-2-Clause # # Unit fixture for ngx_http_zstd_dcz_window_log() -- the dcz window log and # its two operator clamps. # # This function is the single source of truth for a value computed in TWO # places that must never disagree: ngx_http_zstd_acquire_cctx() packs it into # the CCtx ring key, and ngx_http_zstd_filter_init_cctx() pushes it into # libzstd. If they diverge, a request borrows a slot keyed on one window and # compresses at another -- the contamination the ring key exists to prevent. # A hand-copied duplicate of the arithmetic in a test would pass forever while # production drifted, so the function and its ngx_http_zstd_ceil_log2() # dependency are extracted VERBATIM by line range and compiled standalone. # # No nginx tree and no libzstd needed: pure C, self-contained. # # REQUIRE_M32: unset/"0"/""/false/no/off (case insensitive) keeps the # default local-dev behaviour -- SKIP with exit 0 when no -m32 toolchain is # present. Any other value (1/true/yes/on, case insensitive, or anything # else non-empty) makes a missing -m32 toolchain a hard FAILURE instead. set -euo pipefail # ci/tools/ -> ci/ -> repo root. cd "$(dirname "$0")/../.." SRC="src/ngx_http_zstd_filter_module.c" CC="${CC:-cc}" OUT="$(mktemp -d "${TMPDIR:-/tmp}/zstd-dcz-wlog-unit.XXXXXX")" trap 'rm -rf "$OUT"' EXIT mkdir -p "$OUT" extract() { # $1 = start line, $2 = label; prints "start end" local start="$1" label="$2" rel end if [ -z "$start" ]; then echo "FAIL: could not locate $label in $SRC" >&2 exit 1 fi rel="$(tail -n "+${start}" "$SRC" | grep -n '^}' | head -1 | cut -d: -f1)" end=$((start + rel - 1)) echo "$start $end" } # --- ngx_http_zstd_ceil_log2() (dependency) ------------------------------ CEIL_SIG="$(grep -n '^ngx_http_zstd_ceil_log2(size_t x)' "$SRC" \ | head -1 | cut -d: -f1)" if [ -z "$CEIL_SIG" ]; then echo "FAIL: could not locate ngx_http_zstd_ceil_log2() in $SRC" >&2 exit 1 fi read -r CEIL_START CEIL_END < <(extract "$((CEIL_SIG - 1))" "ceil_log2") # --- the window bound #defines ------------------------------------------- CAP_LINE="$(grep -n '^#define NGX_HTTP_ZSTD_DCZ_MAX_WINDOW_LOG' "$SRC" \ | head -1 | cut -d: -f1)" if [ -z "$CAP_LINE" ]; then echo "FAIL: could not locate NGX_HTTP_ZSTD_DCZ_MAX_WINDOW_LOG in $SRC" >&2 exit 1 fi MIN_LINE="$(grep -n '^#define NGX_HTTP_ZSTD_DCZ_MIN_WINDOW_LOG' "$SRC" \ | head -1 | cut -d: -f1)" if [ -z "$MIN_LINE" ]; then echo "FAIL: could not locate NGX_HTTP_ZSTD_DCZ_MIN_WINDOW_LOG in $SRC" >&2 exit 1 fi GUESS_LINE="$(grep -n '^#define NGX_HTTP_ZSTD_UNKNOWN_SIZE_GUESS' "$SRC" \ | head -1 | cut -d: -f1)" if [ -z "$GUESS_LINE" ]; then echo "FAIL: could not locate NGX_HTTP_ZSTD_UNKNOWN_SIZE_GUESS in $SRC" >&2 exit 1 fi # --- ngx_http_zstd_dcz_window_log() (the function under test) ------------ DCZ_SIG="$(grep -n '^ngx_http_zstd_dcz_window_log(size_t dict_len' "$SRC" \ | head -1 | cut -d: -f1)" if [ -z "$DCZ_SIG" ]; then echo "FAIL: could not locate ngx_http_zstd_dcz_window_log() in $SRC" >&2 exit 1 fi read -r DCZ_START DCZ_END < <(extract "$((DCZ_SIG - 1))" "dcz_window_log") { echo "/* Generated by test_dcz_window_log_unit.sh -- verbatim extraction" echo " * from $SRC:" echo " * ceil_log2 lines ${CEIL_START}-${CEIL_END}" echo " * window cap line ${CAP_LINE}" echo " * size guess line ${GUESS_LINE}" echo " * dcz_window_log lines ${DCZ_START}-${DCZ_END}" echo " * Do not edit; re-run the script to regenerate. */" sed -n "${CAP_LINE}p" "$SRC" sed -n "${MIN_LINE}p" "$SRC" sed -n "${GUESS_LINE}p" "$SRC" echo sed -n "${CEIL_START},${CEIL_END}p" "$SRC" echo sed -n "${DCZ_START},${DCZ_END}p" "$SRC" } > "$OUT/generated_dcz_window_log.inc" # Sanity: a source reflow must fail loudly here, not produce a garbage .inc # that still compiles and "passes". for marker in 'ngx_http_zstd_ceil_log2(size_t x)' \ 'NGX_HTTP_ZSTD_DCZ_MAX_WINDOW_LOG' \ 'NGX_HTTP_ZSTD_DCZ_MIN_WINDOW_LOG' \ 'NGX_HTTP_ZSTD_UNKNOWN_SIZE_GUESS' \ 'ngx_http_zstd_dcz_window_log(size_t dict_len' \ 'budget_window_cap'; do if ! grep -q "$marker" "$OUT/generated_dcz_window_log.inc"; then echo "FAIL: extracted fragment is missing \"$marker\"" >&2 cat "$OUT/generated_dcz_window_log.inc" >&2 exit 1 fi done cp ci/tools/test_dcz_window_log_unit.c "$OUT/test_dcz_window_log_unit.c" "$CC" -Wall -Wextra -Werror -O2 -I"$OUT" \ -o "$OUT/dcz_window_log_unit" "$OUT/test_dcz_window_log_unit.c" "$OUT/dcz_window_log_unit" echo "OK: dcz window-log unit fixture (extracted from current $SRC)" # ILP32-with-LFS pass: sizeof(off_t) > sizeof(size_t) is the one shape where # a huge upstream content-length can truncate on cast or wrap the addition # in ngx_http_zstd_dcz_window_log(). A plain native build on this (64-bit) # host has off_t == size_t width and cannot reach either defect -- -m32 with # _FILE_OFFSET_BITS=64 is what reproduces it, so it is not optional coverage. # Skipped (not failed) when the runner has no 32-bit toolchain at all -- # UNLESS REQUIRE_M32 is set to a truthy value (1/true/yes/on, case # insensitive; anything else non-empty and not "0" also counts as set), in # which case a missing toolchain is a hard failure rather than a silent # skip (set in CI so a runner image that drops multilib loses this lens # loudly, not quietly). require_m32="${REQUIRE_M32:-0}" case "${require_m32,,}" in 0 | "" | false | no | off) require_m32_set=0 ;; *) require_m32_set=1 ;; esac if "$CC" -m32 -x c -o /dev/null - <<<'int main(void){return 0;}' \ >/dev/null 2>&1 then "$CC" -m32 -D_FILE_OFFSET_BITS=64 -Wall -Wextra -Werror -O2 -I"$OUT" \ -o "$OUT/dcz_window_log_unit_ilp32" "$OUT/test_dcz_window_log_unit.c" "$OUT/dcz_window_log_unit_ilp32" echo "OK: dcz window-log unit fixture (ILP32 + LFS, off_t wider than size_t)" elif [ "$require_m32_set" = "1" ]; then echo "FAIL: no -m32 toolchain available and REQUIRE_M32 is set; ILP32/LFS pass cannot run" >&2 exit 1 else echo "SKIP: no -m32 toolchain available; ILP32/LFS pass not run" >&2 fi myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_dict_path_hardening.sh000077500000000000000000000750601525232545500271550ustar00rootroot00000000000000#!/usr/bin/env bash # # Fixture matrix for two G5 hardening rows against zstd_dict_file and # zstd_dcz_dict_file, both of which route through the shared # ngx_http_zstd_open_dict_file() helper: # # 1. "Reject non-regular dictionary inputs without letting FIFOs block # config test/reload" -- neither loader used to test ngx_is_file() # before reading. A FIFO opened O_RDONLY blocks the config-parsing # master until a writer appears; a directory/device reaches a # confusing later error. Every fixture below runs under `timeout` # so a regression HANGS THE SCRIPT, not the CI job, and is reported # as a failure either way. # # 2. "Strict dictionary-path trust policy for privileged reloads" -- # zstd_dict_strict_path on (default off) refuses a symlink and a # group/world-writable target so a less-privileged local writer # cannot steer or mutate what the master snapshots on the next # reload. A rejected reload (SIGHUP against a mutated dictionary) # must leave the OLD cycle serving, not tear the worker down. # # Usage: ci/tools/test_dict_path_hardening.sh # # holds ngx_http_zstd_filter_module.so (and the static # module, unused here but harmless to have loaded). set -euo pipefail NGINX="${1:?usage: test_dict_path_hardening.sh }" MODDIR="${2:?usage: test_dict_path_hardening.sh }" # The one fixture that serves requests (the rejected-reload check) listens # here. Overridable so a caller running inside a declared port band -- the # coverage job -- can keep this endpoint inside it. PORT="${DICT_HARDENING_PORT:-18198}" # Both arguments MUST be absolutized before anything else. Every fixture # below runs nginx with -p "$WORK" (a mktemp dir), and nginx resolves a # relative load_module path against that prefix -- so a caller passing # "nginx-1.31.4/objs" yields load_module /tmp/tmp.XXXX/nginx-1.31.4/objs/... # and dlopen() fails. The damage is not a plain red: every fixture that # EXPECTS a rejection still "passes", because the config was refused for # the wrong reason. Resolve here rather than trusting the caller. NGINX="$(readlink -f -- "$NGINX")" MODDIR="$(readlink -f -- "$MODDIR")" if [ ! -x "$NGINX" ]; then echo "❌ $NGINX is not an executable nginx binary" exit 1 fi FILTER_MOD="$MODDIR/ngx_http_zstd_filter_module.so" if [ ! -f "$FILTER_MOD" ]; then echo "❌ $FILTER_MOD not found" exit 1 fi # Non-vacuity gate for the whole matrix. A "rejected cleanly" fixture # only means something if the module loads at all in the baseline; a # dlopen failure would satisfy every negative fixture for free. Prove # the plain config loads BEFORE asserting anything about dictionaries. PRELUDE_WORK="$(mktemp -d)" mkdir -p "$PRELUDE_WORK/conf" "$PRELUDE_WORK/logs" cat >"$PRELUDE_WORK/conf/nginx.conf" <"$PRELUDE_WORK/out" 2>&1; then echo "❌ baseline config with $FILTER_MOD does not even load --" echo " every rejection fixture below would pass vacuously. Aborting." cat "$PRELUDE_WORK/out" rm -rf "$PRELUDE_WORK" exit 1 fi rm -rf "$PRELUDE_WORK" echo "✓ baseline: module loads, rejection fixtures are non-vacuous" # NOT under /tmp directly: /tmp is mode 1777 (world-writable), and the # strict-path ancestor-directory vetting this file exercises now refuses # EVERY component of a dictionary path, including /tmp itself, when it # is group/world-writable -- correctly so, since a world-writable /tmp # is exactly the attack surface (an unprivileged user renaming a file # into a sibling of $WORK) that check exists to close. A dictionary # fixture placed directly under /tmp would therefore fail every # "strict on, must still load" case for a reason that has nothing to do # with what that case is testing. $HOME is used as the mktemp base # instead so the whole ancestor chain (/, /home, $HOME, $WORK) is # root- or self-owned and non-group/world-writable, matching a real # deployment's dictionary directory tree. # # $HOME is an environment variable, not a trust boundary -- a root-run # job can inherit a non-root $HOME, and any job can have a # group-writable one, either of which would make strict mode correctly # reject every positive fixture through THIS ancestor, for a reason # unrelated to what each fixture tests. Validate $HOME itself against # the same rule the module now enforces before trusting it as the # mktemp base, and fail loudly with a clear precondition error rather # than let every downstream fixture misreport. Resolve the candidate first so # a symlink in HOME itself does not become part of the fixture path, then vet # every component the module will walk. if [ -z "${HOME:-}" ] || [ ! -d "$HOME" ]; then echo "❌ \$HOME is unset or not a directory; cannot pick a safe base" \ "for the strict-mode fixtures below" >&2 exit 1 fi STRICT_BASE="$(readlink -f -- "$HOME")" validate_strict_ancestor() { local candidate="$1" owner mode owner="$(stat -c '%u' "$candidate")" mode="$(stat -c '%a' "$candidate")" if [ "$owner" != "$(id -u)" ] && [ "$owner" != "0" ]; then echo "❌ strict fixture ancestor $candidate is owned by uid $owner," \ "neither this job's uid $(id -u) nor root" >&2 exit 1 fi if [ $((0$mode & 0022)) -ne 0 ]; then echo "❌ strict fixture ancestor $candidate is mode $mode, writable" \ "by group or other" >&2 exit 1 fi } current=/ validate_strict_ancestor "$current" IFS='/' read -r -a strict_parts <<<"${STRICT_BASE#/}" for part in "${strict_parts[@]}"; do [ -n "$part" ] || continue current="${current%/}/$part" validate_strict_ancestor "$current" done if [ ! -w "$STRICT_BASE" ] || [ ! -x "$STRICT_BASE" ]; then echo "❌ strict fixture base $STRICT_BASE is not writable and searchable" \ "by uid $(id -u); mktemp cannot create the fixture directory there" >&2 exit 1 fi WORK="$(mktemp -d --tmpdir="$STRICT_BASE" zstd-dict-hardening.XXXXXX)" cleanup() { if [ -n "${NGINX_PID:-}" ]; then kill -9 "$NGINX_PID" 2>/dev/null || true fi rm -rf "$WORK" } trap cleanup EXIT mkdir -p "$WORK/conf" "$WORK/logs" "$WORK/html" # mkdir honours umask, which on a typical dev/CI account (002) leaves # these group-writable (0775) -- exactly the bit the strict-path # ancestor vetting now refuses on every walked directory. Pin them # explicitly rather than relying on umask, same reasoning as the # chmod 0644 a few lines below for the leaf fixture file. chmod 0755 "$WORK" "$WORK/conf" "$WORK/logs" "$WORK/html" fail=0 # ── nginx -t against one http_config snippet, bounded ──────────────── # Echoes "OK"/"FAIL: " on stdout. The `timeout` is the control # for fixture class 1: a regression (no O_NONBLOCK) blocks in open() on # a FIFO, so this would hang without it rather than merely fail. conf_test() { local snippet="$1" cat >"$WORK/conf/nginx.conf" <"$WORK/logs/t.out" 2>&1; then echo "OK" else if [ "$?" -eq 124 ]; then echo "FAIL: nginx -t TIMED OUT (hung open — the FIFO-block regression)" else echo "FAIL: $(tail -3 "$WORK/logs/t.out" | tr '\n' ' ')" fi fi } check() { local name="$1" want="$2" got="$3" if [ "$want" = "regular" ] && [ "$got" = "OK" ]; then echo "✓ $name: OK (expected)" elif [ "$want" = "reject" ] && [[ "$got" == FAIL:* ]] && [[ "$got" != *TIMED\ OUT* ]]; then echo "✓ $name: rejected cleanly ($got)" else echo "✗ $name: want=$want got=$got" fail=1 fi } # ── Fixture: regular file, must still load ─────────────────────────── # chmod explicit: umask can leave this group-writable (e.g. 0002 -> # 0664), which would spuriously trip the strict-mode fixtures below # that are meant to start from a clean, non-writable baseline. head -c 8192 /dev/urandom | base64 >"$WORK/html/regular.dict" chmod 0644 "$WORK/html/regular.dict" r=$(conf_test " zstd_dict_file_unsafe on; zstd_dict_file $WORK/html/regular.dict;") check "regular file (zstd_dict_file)" regular "$r" r=$(conf_test " zstd_dcz_dict_file $WORK/html/regular.dict;") check "regular file (zstd_dcz_dict_file)" regular "$r" # ── Fixture: FIFO ───────────────────────────────────────────────────── mkfifo "$WORK/html/fifo.dict" r=$(conf_test " zstd_dict_file_unsafe on; zstd_dict_file $WORK/html/fifo.dict;") check "FIFO (zstd_dict_file)" reject "$r" r=$(conf_test " zstd_dcz_dict_file $WORK/html/fifo.dict;") check "FIFO (zstd_dcz_dict_file)" reject "$r" # ── Fixture: directory ──────────────────────────────────────────────── mkdir -p "$WORK/html/dir.dict" r=$(conf_test " zstd_dict_file_unsafe on; zstd_dict_file $WORK/html/dir.dict;") check "directory (zstd_dict_file)" reject "$r" r=$(conf_test " zstd_dcz_dict_file $WORK/html/dir.dict;") check "directory (zstd_dcz_dict_file)" reject "$r" # ── Fixture: UNIX domain socket ──────────────────────────────────────── if command -v python3 >/dev/null 2>&1; then python3 - "$WORK/html/sock.dict" <<'PYEOF' import socket, sys s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) s.bind(sys.argv[1]) PYEOF r=$(conf_test " zstd_dict_file_unsafe on; zstd_dict_file $WORK/html/sock.dict;") check "socket (zstd_dict_file)" reject "$r" r=$(conf_test " zstd_dcz_dict_file $WORK/html/sock.dict;") check "socket (zstd_dcz_dict_file)" reject "$r" else echo "::warning::python3 not found; socket fixture skipped" fi # ── Fixture: character device (always present, harmless to open RDONLY) ─ r=$(conf_test " zstd_dict_file_unsafe on; zstd_dict_file /dev/null;") check "device /dev/null (zstd_dict_file)" reject "$r" r=$(conf_test " zstd_dcz_dict_file /dev/null;") check "device /dev/null (zstd_dcz_dict_file)" reject "$r" # ── Fixture: symlink, non-strict (default) accepts ──────────────────── ln -s "$WORK/html/regular.dict" "$WORK/html/link.dict" r=$(conf_test " zstd_dict_file_unsafe on; zstd_dict_file $WORK/html/link.dict;") check "symlink, strict off / default (zstd_dict_file)" regular "$r" r=$(conf_test " zstd_dcz_dict_file $WORK/html/link.dict;") check "symlink, strict off / default (zstd_dcz_dict_file)" regular "$r" # ── Fixture: symlink, strict on rejects (the release-symlink escape # hatch is validated by the line above: default OFF still follows it) ─ r=$(conf_test " zstd_dict_strict_path on; zstd_dict_file_unsafe on; zstd_dict_file $WORK/html/link.dict;") check "symlink, strict on (zstd_dict_file)" reject "$r" r=$(conf_test " zstd_dict_strict_path on; zstd_dcz_dict_file $WORK/html/link.dict;") check "symlink, strict on (zstd_dcz_dict_file)" reject "$r" # ── M3: INTERMEDIATE symlink component, strict on rejects ──────────── # # The regression this pins: O_NOFOLLOW on the whole path guards only the # LEAF. Build /…/rel-1/inter.dict and point a sibling symlink "current" # at rel-1, then load /…/current/inter.dict -- every component but the # last is a perfectly ordinary directory to the kernel, so a leaf-only # O_NOFOLLOW open follows "current" silently and strict mode loads # whatever the symlink's owner aimed it at. This is the release-symlink # swap the README warning claims strict mode defends against. # # NON-VACUITY: the SAME dictionary reached through its REAL path must # still load under strict on (the fixture directly below). Without that # pair, "rejected" here could just mean the file or the directory layout # is broken, and the assertion would pass for the wrong reason. mkdir -p "$WORK/html/rel-1" cp "$WORK/html/regular.dict" "$WORK/html/rel-1/inter.dict" chmod 0755 "$WORK/html/rel-1" chmod 0644 "$WORK/html/rel-1/inter.dict" ln -s "$WORK/html/rel-1" "$WORK/html/current" r=$(conf_test " zstd_dict_strict_path on; zstd_dict_file_unsafe on; zstd_dict_file $WORK/html/current/inter.dict;") check "intermediate symlink component, strict on (zstd_dict_file)" reject "$r" r=$(conf_test " zstd_dict_strict_path on; zstd_dcz_dict_file $WORK/html/current/inter.dict;") check "intermediate symlink component, strict on (zstd_dcz_dict_file)" reject "$r" # Positive control for the pair above: the identical bytes via the real # directory path load fine under strict on. r=$(conf_test " zstd_dict_strict_path on; zstd_dict_file_unsafe on; zstd_dict_file $WORK/html/rel-1/inter.dict;") check "real (non-symlink) path, strict on still loads (zstd_dict_file)" regular "$r" r=$(conf_test " zstd_dict_strict_path on; zstd_dcz_dict_file $WORK/html/rel-1/inter.dict;") check "real (non-symlink) path, strict on still loads (zstd_dcz_dict_file)" regular "$r" # The intermediate-symlink path must STILL LOAD with strict off, so the # fix cannot be mistaken for a blanket path restriction and the # documented release-symlink deployment keeps working unconfigured. r=$(conf_test " zstd_dict_file_unsafe on; zstd_dict_file $WORK/html/current/inter.dict;") check "intermediate symlink component, strict off / default" regular "$r" # ── M4: OWNER-writable / foreign-owned dictionary, strict on rejects ── # # The regression this pins: strict mode used to test only S_IWGRP | # S_IWOTH, so a dictionary owned by an unprivileged account with an # entirely ordinary mode 0644 passed while a root master read it -- and # that owner can rewrite the file, steering what the next privileged # reload snapshots. Strict mode now additionally requires the file to be # owned by the loading principal (geteuid()) or by root. # # COVERAGE HONESTY -- READ BEFORE TRUSTING THIS FIXTURE. # # Which half runs depends on whether this script has the privilege to # create a file it does not own: # # * as root -- chown to an unprivileged uid gives a genuine # foreign-owned dictionary and the real M4 case runs. # * non-root -- an unprivileged CI user CANNOT chown a file away from # itself, so the foreign-owner half is NOT COVERED here # and is reported as such. What IS asserted instead is # the complementary, fully observable half: the check is # reached and the positive path (a file owned by # geteuid(), mode 0644) still loads. This is deliberately # NOT dressed up as a pass for the uncovered half. if [ "$(id -u)" -eq 0 ]; then foreign_uid="" for cand in nobody daemon bin; do if id -u "$cand" >/dev/null 2>&1; then foreign_uid="$(id -u "$cand")" break fi done if [ -n "$foreign_uid" ]; then cp "$WORK/html/regular.dict" "$WORK/html/foreign.dict" chmod 0644 "$WORK/html/foreign.dict" chown "$foreign_uid" "$WORK/html/foreign.dict" r=$(conf_test " zstd_dict_strict_path on; zstd_dict_file_unsafe on; zstd_dict_file $WORK/html/foreign.dict;") check "foreign-owned 0644 dictionary, strict on (zstd_dict_file)" reject "$r" r=$(conf_test " zstd_dict_strict_path on; zstd_dcz_dict_file $WORK/html/foreign.dict;") check "foreign-owned 0644 dictionary, strict on (zstd_dcz_dict_file)" reject "$r" # Complement: the same foreign-owned file must still load with # strict OFF, so the ownership rule is confined to strict mode # and does not change the default deployment's behaviour. r=$(conf_test " zstd_dict_file_unsafe on; zstd_dict_file $WORK/html/foreign.dict;") check "foreign-owned 0644 dictionary, strict off / default" regular "$r" else echo "::warning::no unprivileged account (nobody/daemon/bin) found;" echo " M4 foreign-owner half NOT COVERED in this run" fi else echo "• M4 foreign-owner half NOT COVERED: running as uid $(id -u), which" echo " cannot chown a file away from itself. The observable complement" echo " (self-owned 0644 still loads under strict on) is asserted below." fi # Observable in every environment, privileged or not: a dictionary owned # by the loading principal with a sane mode is accepted under strict on. # This is what keeps the M4 ownership rule from being a blanket refusal, # and it is the half the unprivileged CI user genuinely covers. cp "$WORK/html/regular.dict" "$WORK/html/selfowned.dict" chmod 0644 "$WORK/html/selfowned.dict" r=$(conf_test " zstd_dict_strict_path on; zstd_dict_file_unsafe on; zstd_dict_file $WORK/html/selfowned.dict;") check "self-owned 0644 dictionary, strict on loads (zstd_dict_file)" regular "$r" r=$(conf_test " zstd_dict_strict_path on; zstd_dcz_dict_file $WORK/html/selfowned.dict;") check "self-owned 0644 dictionary, strict on loads (zstd_dcz_dict_file)" regular "$r" # ── Fixture: world-writable regular file, strict on rejects ─────────── cp "$WORK/html/regular.dict" "$WORK/html/writable.dict" chmod 0666 "$WORK/html/writable.dict" r=$(conf_test " zstd_dict_strict_path on; zstd_dict_file_unsafe on; zstd_dict_file $WORK/html/writable.dict;") check "world-writable, strict on (zstd_dict_file)" reject "$r" r=$(conf_test " zstd_dict_strict_path on; zstd_dcz_dict_file $WORK/html/writable.dict;") check "world-writable, strict on (zstd_dcz_dict_file)" reject "$r" # ── A33-F2: ancestor-DIRECTORY ownership/mode, not just the leaf ────── # # The regression this pins: M3 (intermediate-symlink fixture above) made # the strict-mode walk resolve every path component with # openat(O_NOFOLLOW|O_DIRECTORY), and M4 (self-owned/world-writable # fixtures above) added leaf ownership+mode checks. Neither ever # fstat()-checked the intermediate DIRECTORY fds the walk opens along # the way. A local user who owns, or can write into, an ancestor # directory can rename() a root-owned 0644 file into the leaf position # and pass both leaf checks while still having fully steered which # bytes strict mode loads -- the directive's promise ("a # less-privileged local writer cannot steer the bytes") only holds when # EVERY component is vetted, not just the last one. # # Two arms below, one per unsafe-ancestor cause (mode, then ownership), # each proved against a PARENT directory of the dictionary file, never # the leaf itself -- the existing M4 fixtures already cover the leaf. # ── Arm 1: world-writable PARENT directory ───────────────────────────── mkdir -p "$WORK/html/unsafe-parent-mode" cp "$WORK/html/regular.dict" "$WORK/html/unsafe-parent-mode/leaf.dict" chmod 0644 "$WORK/html/unsafe-parent-mode/leaf.dict" chmod 0777 "$WORK/html/unsafe-parent-mode" r=$(conf_test " zstd_dict_strict_path on; zstd_dict_file_unsafe on; zstd_dict_file $WORK/html/unsafe-parent-mode/leaf.dict;") check "world-writable PARENT directory, strict on (zstd_dict_file)" reject "$r" r=$(conf_test " zstd_dict_strict_path on; zstd_dcz_dict_file $WORK/html/unsafe-parent-mode/leaf.dict;") check "world-writable PARENT directory, strict on (zstd_dcz_dict_file)" reject "$r" # Complement: the identical layout must still load with strict off, so # the ancestor rule is confined to strict mode. r=$(conf_test " zstd_dict_file_unsafe on; zstd_dict_file $WORK/html/unsafe-parent-mode/leaf.dict;") check "world-writable PARENT directory, strict off / default" regular "$r" # NO STICKY-BIT EXEMPTION: a sticky world-writable ancestor (the # /tmp-style layout) still lets an unprivileged user CREATE the next # path component -- it only stops them deleting/renaming someone # else's existing entry, which is not the attack here (the attacker # creates a new file, they don't need to touch an existing one). This # must reject exactly like the plain 0777 case above, not be # special-cased into passing. mkdir -p "$WORK/html/unsafe-parent-sticky" cp "$WORK/html/regular.dict" "$WORK/html/unsafe-parent-sticky/leaf.dict" chmod 0644 "$WORK/html/unsafe-parent-sticky/leaf.dict" chmod 1777 "$WORK/html/unsafe-parent-sticky" r=$(conf_test " zstd_dict_strict_path on; zstd_dict_file_unsafe on; zstd_dict_file $WORK/html/unsafe-parent-sticky/leaf.dict;") check "sticky world-writable PARENT directory, strict on (zstd_dict_file)" reject "$r" r=$(conf_test " zstd_dict_strict_path on; zstd_dcz_dict_file $WORK/html/unsafe-parent-sticky/leaf.dict;") check "sticky world-writable PARENT directory, strict on (zstd_dcz_dict_file)" reject "$r" # ── Arm 2: PARENT directory owned by a foreign, non-root uid ────────── # Same coverage-honesty split as the M4 leaf-ownership fixture above: # only root can chown a directory away from itself, so the genuine # foreign-owner case runs under root and the observable complement # (self-owned parent still loads) runs everywhere else. if [ "$(id -u)" -eq 0 ]; then foreign_uid="" for cand in nobody daemon bin; do if id -u "$cand" >/dev/null 2>&1; then foreign_uid="$(id -u "$cand")" break fi done if [ -n "$foreign_uid" ]; then mkdir -p "$WORK/html/unsafe-parent-owner" cp "$WORK/html/regular.dict" "$WORK/html/unsafe-parent-owner/leaf.dict" chmod 0644 "$WORK/html/unsafe-parent-owner/leaf.dict" chmod 0755 "$WORK/html/unsafe-parent-owner" chown "$foreign_uid" "$WORK/html/unsafe-parent-owner" r=$(conf_test " zstd_dict_strict_path on; zstd_dict_file_unsafe on; zstd_dict_file $WORK/html/unsafe-parent-owner/leaf.dict;") check "foreign-owned PARENT directory, strict on (zstd_dict_file)" reject "$r" r=$(conf_test " zstd_dict_strict_path on; zstd_dcz_dict_file $WORK/html/unsafe-parent-owner/leaf.dict;") check "foreign-owned PARENT directory, strict on (zstd_dcz_dict_file)" reject "$r" r=$(conf_test " zstd_dict_file_unsafe on; zstd_dict_file $WORK/html/unsafe-parent-owner/leaf.dict;") check "foreign-owned PARENT directory, strict off / default" regular "$r" else echo "::warning::no unprivileged account (nobody/daemon/bin) found;" echo " A33-F2 foreign-owner-ancestor half NOT COVERED in this run" fi else echo "• A33-F2 foreign-owner-ancestor half NOT COVERED: running as uid" echo " $(id -u), which cannot chown a directory away from itself. The" echo " observable complement (self-owned parent, sane mode, strict on" echo " still loads) is asserted by every 'strict on ... loads'" echo " fixture above, whose dictionaries all sit under \$WORK/html," echo " itself self-owned mode 0755 (pinned above)." fi # ── Regression: zstd_dict_strict_path AFTER the dcz directive must not # silently skip the check for a dictionary already loaded by that # point. ngx_conf_parse() runs top-to-bottom, so a bare # "zstd_dict_strict_path on" placed after zstd_dcz_dict_file used to # read the flag as still-unset at load time and treat it as off -- # the dictionary loaded successfully with no error, which is the # same as strict mode being silently ignored. init_main_conf() now # rejects this ordering outright (a config-load error naming the # file), which is what "reject" below actually verifies -- this is # NOT the same case as the strict-on-BEFORE fixtures above, and # without this case the ordering hazard has zero coverage. r=$(conf_test " zstd_dcz_dict_file $WORK/html/writable.dict; zstd_dict_strict_path on;") check "world-writable, strict on declared AFTER dcz directive (ordering)" reject "$r" r=$(conf_test " zstd_dcz_dict_file $WORK/html/link.dict; zstd_dict_strict_path on;") check "symlink, strict on declared AFTER dcz directive (ordering)" reject "$r" # ── Fixture: same world-writable file, strict off (default) accepts ─── r=$(conf_test " zstd_dict_file_unsafe on; zstd_dict_file $WORK/html/writable.dict;") check "world-writable, strict off / default (zstd_dict_file)" regular "$r" # ── Behavior: rejected reload leaves the OLD cycle serving ──────────── # Start nginx on a known-good regular dictionary with strict on, then # SIGHUP after swapping the path in for a symlink. The reload must be # refused (error.log line) and the OLD worker must keep answering -- # proving a rejected reload does not tear down the running cycle. cp "$WORK/html/regular.dict" "$WORK/html/live.dict" chmod 0644 "$WORK/html/live.dict" cat >"$WORK/conf/nginx.conf" </dev/null; then ready=1 break fi sleep 0.1 done if [ "$ready" -ne 1 ]; then echo "✗ old-cycle-active fixture: nginx did not start" fail=1 else # Swap the trusted path for a symlink out from under the running # config -- exactly the local-writer scenario the row defends # against -- then ask for a reload. rm "$WORK/html/live.dict" ln -s "$WORK/html/regular.dict" "$WORK/html/live.dict" kill -HUP "$NGINX_PID" sleep 1 # O_NOFOLLOW makes the open() itself fail (ELOOP, "Too many levels # of symbolic links") before ngx_is_link()'s own message can fire; # accept either wording as proof the symlink swap was refused. if curl -fsS -o /dev/null "http://127.0.0.1:$PORT/" 2>/dev/null \ && grep -qE "is a symlink; refused|levels of symbolic links" \ "$WORK/logs/error.log"; then echo "✓ old-cycle-active fixture: reload refused, old cycle still serving" else echo "✗ old-cycle-active fixture: refusal not logged or old cycle stopped answering" tail -10 "$WORK/logs/error.log" || true fail=1 fi kill -QUIT "$NGINX_PID" 2>/dev/null || true wait "$NGINX_PID" 2>/dev/null || true NGINX_PID="" fi # ── Read completeness: both loaders must read a dictionary to the end ── # # Both loaders used to issue ONE ngx_read_fd() and treat any short count # as a fatal "incomplete read". read() on a regular file may legally # return fewer bytes than asked for, and an EINTR-interrupted read # returns early on any filesystem regardless of O_NONBLOCK -- so a valid # dictionary could fail config load for no reason. They now loop via # ngx_http_zstd_read_dict_file(). # # NOTE ON WHAT THESE FIXTURES DO AND DO NOT PROVE. The read loop itself # is NOT covered here and cannot be: read() on a local tmpfs/ext4 regular # file does not return a short count, and a signal cannot be steered into # the master's read window from a shell. Measured -- with the loop # reverted to a single read whose short count is fatal, every fixture in # this file still passed, the 1 MiB dictionary included. The loop's three # exits (resume-on-short, retry-on-EINTR, fail-on-EOF) are covered by # ci/tools/test_read_dict_file_unit.sh, which extracts the helper and # scripts ngx_read_fd; all three mutants are killed there. # # What these fixtures DO pin is the config-level contract around it: a # large dictionary loads, and the two rejections stay rejections. # A dictionary comfortably larger than one page. This is an end-to-end # smoke check that a big dictionary loads through the real loader on the # real filesystem -- it is NOT the short-read regression test (see the # note above; that lives in test_read_dict_file_unit.sh). head -c 1048576 /dev/urandom >"$WORK/html/large.dict" chmod 0644 "$WORK/html/large.dict" r=$(conf_test " zstd_dict_file_unsafe on; zstd_dict_file $WORK/html/large.dict;") check "1 MiB dictionary loads (zstd_dict_file)" regular "$r" r=$(conf_test " zstd_dcz_dict_file $WORK/html/large.dict;") check "1 MiB dictionary loads (zstd_dcz_dict_file)" regular "$r" # Zero-length dictionary: rejected, and rejected the SAME way by both # loaders. ZSTD_createCDict(buf, 0, level) returns a valid do-nothing # CDict and a 0-byte read "succeeds", so without an explicit check the # operator silently gets no dictionary at all. : >"$WORK/html/empty.dict" chmod 0644 "$WORK/html/empty.dict" r=$(conf_test " zstd_dict_file_unsafe on; zstd_dict_file $WORK/html/empty.dict;") check "zero-length dictionary (zstd_dict_file)" reject "$r" r=$(conf_test " zstd_dcz_dict_file $WORK/html/empty.dict;") check "zero-length dictionary (zstd_dcz_dict_file)" reject "$r" # Unreadable regular file: config load must fail rather than proceed # with an unpopulated buffer. This is refused at open() -- it never # reaches the read loop -- but it pins that a permission failure is # fatal at config time and not deferred to first use. Skipped under # root, which bypasses mode 0000 and would make the fixture vacuous. if [ "$(id -u)" -ne 0 ]; then head -c 8192 /dev/urandom >"$WORK/html/noperm.dict" chmod 0000 "$WORK/html/noperm.dict" r=$(conf_test " zstd_dict_file_unsafe on; zstd_dict_file $WORK/html/noperm.dict;") check "unreadable dictionary (zstd_dict_file)" reject "$r" r=$(conf_test " zstd_dcz_dict_file $WORK/html/noperm.dict;") check "unreadable dictionary (zstd_dcz_dict_file)" reject "$r" else echo "• unreadable dictionary: skipped (running as root bypasses mode 0000)" fi if [ "$fail" -ne 0 ]; then echo "❌ dictionary path hardening: one or more fixtures failed" exit 1 fi echo "✓ all dictionary path hardening fixtures passed" myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_dict_walk_unit.c000066400000000000000000000605501525232545500260020ustar00rootroot00000000000000/* * Unit fixture for ngx_http_zstd_dict_file_open_strict() and * ngx_http_zstd_dict_file_check_dir() -- the strict-mode * component-by-component dictionary open in src/ngx_http_zstd_dict_file.h, * THE authoritative copy, included directly. * * WHY A UNIT FIXTURE BESIDE THE CONFIG MATRIX. ci/tools/test_dict_path_hardening.sh * and ci/t/02-conf-warn.t drive the walk through nginx against real * directories, and they stay the witnesses that the diagnostics an * operator sees are unchanged. But several of the walk's exits cannot be * staged from a shell: open("/") failing, fstat() failing on a directory * the walk just opened, a foreign-owned ancestor (needs root to chown), * a component of NGX_MAX_PATH bytes (no filesystem accepts one), and the * errno-before-close ordering that the openat() diagnostic depends on. * Here the four system calls the walk makes are function-like macros * over scripted fakes, so every exit is reached deterministically, the * exact openat() flag words are observable, and fd hygiene -- every fd * the walk opened is closed exactly once, the returned leaf never -- is * asserted on every path. */ #include #include #include #include #include #include #include #include /* --- minimal nginx surface the header needs --------------------------- */ typedef intptr_t ngx_int_t; typedef uintptr_t ngx_uint_t; typedef int ngx_fd_t; typedef unsigned char u_char; typedef struct { size_t len; u_char *data; } ngx_str_t; #define NGX_INVALID_FILE -1 #define NGX_MAX_PATH 4096 #define NGX_EINTR EINTR #define ngx_errno errno #define ngx_memcpy memcpy #define ngx_strcmp(a, b) strcmp((const char *) (a), (const char *) (b)) #define ngx_read_fd(fd, buf, n) read(fd, buf, n) /* --- the fake filesystem ---------------------------------------------- * * A node is one directory entry the walk may openat(): its parent fd, * its name, what openat() should return, and what fstat() on the * resulting fd reports. fds are handed out from 100 upward so a leaked * or double-closed fd is unmistakable. */ typedef struct { int parent; /* the dirfd openat() must be called with */ const char *name; int open_errno; /* 0: opens; otherwise openat() fails so */ int fstat_errno; /* 0: fstat works; otherwise it fails so */ uid_t uid; mode_t mode; } node_t; #define MAX_NODES 16 #define ROOT_FD 100 static node_t nodes[MAX_NODES]; static int nnodes; static int root_open_errno; /* open("/") fails with this if set */ static int root_fstat_errno; static uid_t root_uid; static mode_t root_mode; static uid_t fake_euid = 500; static int root_open_flags; /* what open("/") was asked for */ /* * Declared before the fakes: a violated fixture invariant -- open() of * anything but "/", openat() through a closed dirfd, close() of an fd * that is not open -- must FAIL the run, not just print. The walk * ignores close()'s return and check_hygiene() counts only fds still * open, so nothing else would notice a double close. */ static int failures; /* per-fd bookkeeping: fd -> node index (-1 for the root), open state */ static int fd_node[256]; static int fd_open[256]; static int next_fd; static int opens, closes; /* the exact openat() calls, in order, for sequence assertions */ typedef struct { int dirfd; char name[64]; int flags; } call_t; static call_t calls[32]; static int ncalls; static void fs_reset(void) { memset(nodes, 0, sizeof(nodes)); memset(fd_node, 0, sizeof(fd_node)); memset(fd_open, 0, sizeof(fd_open)); memset(calls, 0, sizeof(calls)); nnodes = 0; ncalls = 0; opens = 0; closes = 0; next_fd = ROOT_FD; root_open_errno = 0; root_fstat_errno = 0; root_open_flags = -1; root_uid = 0; root_mode = S_IFDIR | 0755; } static int fs_add(int parent, const char *name, uid_t uid, mode_t mode) { node_t *n = &nodes[nnodes]; n->parent = parent; n->name = name; n->uid = uid; n->mode = mode; return nnodes++; } static int fake_open(const char *name, int flags) { int fd; root_open_flags = flags; if (strcmp(name, "/") != 0) { fprintf(stderr, "FAIL: open(\"%s\"): the walk opens only \"/\"\n", name); failures++; errno = EINVAL; return -1; } if (root_open_errno) { errno = root_open_errno; return -1; } fd = next_fd++; fd_node[fd] = -1; fd_open[fd] = 1; opens++; return fd; } /* * Nodes are matched on (dirfd, name). A name with no node under that * dirfd is ENOENT, which is also what a component opened from the WRONG * parent gets -- so the sequence is checked by construction, not just * by the recorded calls. */ static int fake_openat(int dirfd, const char *name, int flags) { int i, fd; if (ncalls < 32) { size_t n = strlen(name); /* the record keeps a bounded prefix: the sequence assertions * compare short component names, and the NGX_MAX_PATH-byte * component is asserted through the walk's own report */ if (n > sizeof(calls[ncalls].name) - 1) { n = sizeof(calls[ncalls].name) - 1; } calls[ncalls].dirfd = dirfd; memcpy(calls[ncalls].name, name, n); calls[ncalls].name[n] = '\0'; calls[ncalls].flags = flags; } ncalls++; if (dirfd < 0 || dirfd >= 256 || !fd_open[dirfd]) { fprintf(stderr, "FAIL: openat() on fd %d, which is not open\n", dirfd); failures++; errno = EBADF; return -1; } for (i = 0; i < nnodes; i++) { if (nodes[i].parent == dirfd && strcmp(nodes[i].name, name) == 0) { if (nodes[i].open_errno) { errno = nodes[i].open_errno; return -1; } fd = next_fd++; fd_node[fd] = i; fd_open[fd] = 1; opens++; return fd; } } errno = ENOENT; return -1; } static int fake_fstat(int fd, struct stat *st) { int e; if (fd < 0 || fd >= 256 || !fd_open[fd]) { errno = EBADF; return -1; } memset(st, 0, sizeof(*st)); if (fd_node[fd] == -1) { e = root_fstat_errno; st->st_uid = root_uid; st->st_mode = root_mode; } else { e = nodes[fd_node[fd]].fstat_errno; st->st_uid = nodes[fd_node[fd]].uid; st->st_mode = nodes[fd_node[fd]].mode; } if (e) { errno = e; return -1; } return 0; } static uid_t fake_geteuid(void) { return fake_euid; } /* * close() deliberately CLOBBERS errno: the walk must have captured the * failing call's errno before closing, or the diagnostic names the wrong * error. A double close is a hard failure of the fixture. */ static int fake_close(int fd) { if (fd < 0 || fd >= 256 || !fd_open[fd]) { fprintf(stderr, "FAIL: close(%d) on an fd that is not open\n", fd); failures++; errno = EBADF; return -1; } fd_open[fd] = 0; closes++; errno = EBADF; return 0; } #define open(name, flags) fake_open(name, flags) #define openat(dfd, name, flags) fake_openat(dfd, name, flags) #define fstat(fd, st) fake_fstat(fd, st) #define geteuid() fake_geteuid() #define ngx_close_file(fd) fake_close(fd) /* * The header's members are `static ngx_inline`; this fixture uses the * walk but not the read loop or the hex decoder, so ngx_inline must be a * real `inline` here or -Werror=unused-function fires on those. */ #define ngx_inline inline #include "../../src/ngx_http_zstd_dict_file.h" /* * The module's logging shell, ngx_http_zstd_log_dict_walk(), is extracted * verbatim by test_dict_walk_unit.sh into the generated file below and * compiled against a counting ngx_conf_log_error() stub: every refusal * code must produce exactly one diagnostic, OK none, and the three codes * that carry an errno must hand it to the logger. A case dropped from the * switch is then a counted failure here, not merely a missing label. */ typedef struct { int unused; } ngx_conf_t; #define NGX_LOG_EMERG 2 static int log_calls; static int log_last_err; static void ngx_conf_log_error(int level, ngx_conf_t *cf, int err, const char *fmt, ...) { (void) level; (void) cf; (void) fmt; log_calls++; log_last_err = err; } #include "generated_log_dict_walk.inc" /* * POSIX only: without the *at() family the header defines no walk and the * first call below fails to compile, which is the intended outcome. No * #error spells that out, deliberately: the repository's standalone * cppcheck pass sees no system headers, so AT_FDCWD is undefined there * and a preprocessor error would abort its analysis of this file. */ /* --- harness ---------------------------------------------------------- */ static void check(const char *name, long got, long want) { if (got != want) { printf("✗ %s: got %ld want %ld\n", name, got, want); failures++; return; } printf("✓ %s\n", name); } static void check_str(const char *name, const char *got, const char *want) { if (strcmp(got, want) != 0) { printf("✗ %s: got \"%s\" want \"%s\"\n", name, got, want); failures++; return; } printf("✓ %s\n", name); } static ngx_str_t str(const char *s) { ngx_str_t r; r.len = strlen(s); r.data = (u_char *) s; return r; } #define FILE_FLAGS (O_RDONLY | O_NONBLOCK) #ifdef O_CLOEXEC #define CLOEXEC O_CLOEXEC #else #define CLOEXEC 0 #endif /* * Every fd the walk opened must be closed exactly once, except the leaf * it returned (which must still be open, and must be the last fd it * opened). */ static void check_hygiene(const char *name, ngx_fd_t returned) { int fd, still_open = 0; char label[128]; for (fd = ROOT_FD; fd < next_fd; fd++) { if (fd_open[fd]) { still_open++; } } snprintf(label, sizeof(label), "%s: fds closed except the returned leaf", name); check(label, still_open, returned == NGX_INVALID_FILE ? 0 : 1); if (returned != NGX_INVALID_FILE) { snprintf(label, sizeof(label), "%s: the returned fd is the open one", name); check(label, fd_open[returned] && returned == next_fd - 1, 1); } } /* A three-component happy path: / -> srv -> dicts -> a.dict. */ static int layout_happy(void) { int srv, dicts; fs_reset(); srv = fs_add(ROOT_FD, "srv", 0, S_IFDIR | 0755); (void) srv; /* fds are allocated in walk order: root=100, srv=101, dicts=102 */ dicts = fs_add(101, "dicts", fake_euid, S_IFDIR | 0750); (void) dicts; fs_add(102, "a.dict", fake_euid, S_IFREG | 0644); return 0; } int main(void) { ngx_http_zstd_dict_walk_t walk; ngx_str_t path; ngx_fd_t fd; printf("# happy path\n"); layout_happy(); path = str("/srv/dicts/a.dict"); fd = ngx_http_zstd_dict_file_open_strict(&path, FILE_FLAGS, &walk); check("absolute three-component path opens the leaf", fd, 103); check("rc is OK", walk.rc, NGX_HTTP_ZSTD_DICT_WALK_OK); check("three openat() calls", ncalls, 3); check("first openat() is relative to the root fd", calls[0].dirfd, ROOT_FD); check_str("first component", calls[0].name, "srv"); check("second openat() is relative to srv's fd", calls[1].dirfd, 101); check_str("second component", calls[1].name, "dicts"); check("leaf openat() is relative to dicts' fd", calls[2].dirfd, 102); check_str("leaf component", calls[2].name, "a.dict"); check("root open flags: O_RDONLY|O_DIRECTORY|O_CLOEXEC", root_open_flags, O_RDONLY | O_DIRECTORY | CLOEXEC); check("intermediate flags: O_RDONLY|O_DIRECTORY|O_NOFOLLOW|O_CLOEXEC", calls[0].flags, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | CLOEXEC); check("leaf flags: caller's flags | O_NOFOLLOW | O_CLOEXEC", calls[2].flags, FILE_FLAGS | O_NOFOLLOW | CLOEXEC); check("leaf is not opened O_DIRECTORY", (calls[2].flags & O_DIRECTORY) != 0, 0); check_hygiene("happy path", fd); /* repeated separators name the same components */ layout_happy(); path = str("//srv///dicts//a.dict"); fd = ngx_http_zstd_dict_file_open_strict(&path, FILE_FLAGS, &walk); check("repeated separators: same leaf", fd, 103); check("repeated separators: same three calls", ncalls, 3); check_hygiene("repeated separators", fd); /* * A trailing separator names a directory, as it does for open(2) * (ENOTDIR on a regular file): the last component is never opened * as a leaf. The intermediate directories before it are still * walked and vetted, so the refusal comes after "dicts" is opened. */ layout_happy(); path = str("/srv/dicts/a.dict/"); fd = ngx_http_zstd_dict_file_open_strict(&path, FILE_FLAGS, &walk); check("trailing separator: refused", fd, NGX_INVALID_FILE); check("trailing separator: rc DIRECTORY", walk.rc, NGX_HTTP_ZSTD_DICT_WALK_DIRECTORY); check("trailing separator: the last component is never opened", ncalls, 2); check_hygiene("trailing separator", fd); layout_happy(); path = str("/srv/dicts/a.dict///"); fd = ngx_http_zstd_dict_file_open_strict(&path, FILE_FLAGS, &walk); check("repeated trailing separators: rc DIRECTORY", walk.rc, NGX_HTTP_ZSTD_DICT_WALK_DIRECTORY); check_hygiene("repeated trailing separators", fd); /* a separator run BETWEEN components is still just a separator */ layout_happy(); path = str("/srv//dicts/a.dict"); fd = ngx_http_zstd_dict_file_open_strict(&path, FILE_FLAGS, &walk); check("separator run between components: leaf opens", fd, 103); check_hygiene("separator run between components", fd); printf("# refusals before any file is touched\n"); layout_happy(); path = str("srv/dicts/a.dict"); fd = ngx_http_zstd_dict_file_open_strict(&path, FILE_FLAGS, &walk); check("relative path is refused", fd, NGX_INVALID_FILE); check("relative path: rc RELATIVE", walk.rc, NGX_HTTP_ZSTD_DICT_WALK_RELATIVE); check("relative path: nothing opened", opens, 0); layout_happy(); path = str(""); fd = ngx_http_zstd_dict_file_open_strict(&path, FILE_FLAGS, &walk); check("empty path is refused", fd, NGX_INVALID_FILE); check("empty path is refused as relative", walk.rc, NGX_HTTP_ZSTD_DICT_WALK_RELATIVE); check("empty path: nothing opened", opens, 0); check_hygiene("empty path", fd); printf("# the root directory\n"); layout_happy(); root_open_errno = EACCES; path = str("/srv/dicts/a.dict"); fd = ngx_http_zstd_dict_file_open_strict(&path, FILE_FLAGS, &walk); check("open(\"/\") failure is refused", fd, NGX_INVALID_FILE); check("open(\"/\") failure: rc OPEN_ROOT", walk.rc, NGX_HTTP_ZSTD_DICT_WALK_OPEN_ROOT); check("open(\"/\") failure: errno reported", walk.err, EACCES); check_hygiene("open root failure", fd); layout_happy(); root_fstat_errno = EIO; fd = ngx_http_zstd_dict_file_open_strict(&path, FILE_FLAGS, &walk); check("fstat(\"/\") failure: rc DIR_FSTAT", walk.rc, NGX_HTTP_ZSTD_DICT_WALK_DIR_FSTAT); check("fstat(\"/\") failure: errno captured before close()", walk.err, EIO); check_str("fstat(\"/\") failure: component is \"/\"", (const char *) walk.component, "/"); check("fstat(\"/\") failure: no openat() issued", ncalls, 0); check_hygiene("fstat root failure", fd); layout_happy(); root_uid = 1000; fd = ngx_http_zstd_dict_file_open_strict(&path, FILE_FLAGS, &walk); check("foreign-owned root: rc DIR_OWNER", walk.rc, NGX_HTTP_ZSTD_DICT_WALK_DIR_OWNER); check("foreign-owned root: uid reported", (long) walk.uid, 1000); check_str("foreign-owned root: component is \"/\"", (const char *) walk.component, "/"); check_hygiene("foreign-owned root", fd); layout_happy(); root_mode = S_IFDIR | 0777; fd = ngx_http_zstd_dict_file_open_strict(&path, FILE_FLAGS, &walk); check("world-writable root: rc DIR_WRITABLE", walk.rc, NGX_HTTP_ZSTD_DICT_WALK_DIR_WRITABLE); check_hygiene("world-writable root", fd); layout_happy(); path = str("/"); fd = ngx_http_zstd_dict_file_open_strict(&path, FILE_FLAGS, &walk); check("\"/\" alone names a directory: rc DIRECTORY", walk.rc, NGX_HTTP_ZSTD_DICT_WALK_DIRECTORY); check("\"/\" alone: no openat() issued", ncalls, 0); check_hygiene("root only", fd); layout_happy(); path = str("///"); fd = ngx_http_zstd_dict_file_open_strict(&path, FILE_FLAGS, &walk); check("\"///\" names a directory: rc DIRECTORY", walk.rc, NGX_HTTP_ZSTD_DICT_WALK_DIRECTORY); check_hygiene("root only, repeated", fd); printf("# components\n"); layout_happy(); path = str("/srv/./a.dict"); fd = ngx_http_zstd_dict_file_open_strict(&path, FILE_FLAGS, &walk); check("\".\" component: rc DOT", walk.rc, NGX_HTTP_ZSTD_DICT_WALK_DOT); check_str("\".\" component named", (const char *) walk.component, "."); check("\".\" component: not passed to openat()", ncalls, 1); check_hygiene("dot", fd); layout_happy(); path = str("/srv/../etc/a.dict"); fd = ngx_http_zstd_dict_file_open_strict(&path, FILE_FLAGS, &walk); check("\"..\" component: rc DOT", walk.rc, NGX_HTTP_ZSTD_DICT_WALK_DOT); check_str("\"..\" component named", (const char *) walk.component, ".."); check_hygiene("dotdot", fd); { static char longpath[NGX_MAX_PATH + 16]; size_t i; layout_happy(); longpath[0] = '/'; for (i = 1; i < NGX_MAX_PATH + 1; i++) { longpath[i] = 'x'; } longpath[NGX_MAX_PATH + 1] = '\0'; path = str(longpath); fd = ngx_http_zstd_dict_file_open_strict(&path, FILE_FLAGS, &walk); check("component of NGX_MAX_PATH bytes: rc COMPONENT_LONG", walk.rc, NGX_HTTP_ZSTD_DICT_WALK_COMPONENT_LONG); check("component of NGX_MAX_PATH bytes: not passed to openat()", ncalls, 0); check_hygiene("long component", fd); /* one byte shorter fits the buffer and reaches openat() (ENOENT) */ layout_happy(); longpath[NGX_MAX_PATH] = '\0'; path = str(longpath); fd = ngx_http_zstd_dict_file_open_strict(&path, FILE_FLAGS, &walk); check("component of NGX_MAX_PATH-1 bytes reaches openat()", ncalls, 1); check("component of NGX_MAX_PATH-1 bytes: rc OPENAT (ENOENT)", walk.rc, NGX_HTTP_ZSTD_DICT_WALK_OPENAT); check("component of NGX_MAX_PATH-1 bytes: NUL-terminated copy", (long) strlen((const char *) walk.component), NGX_MAX_PATH - 1); check_hygiene("long component that fits", fd); } printf("# openat() failures\n"); layout_happy(); nodes[1].open_errno = ELOOP; /* "dicts" is a symlink */ path = str("/srv/dicts/a.dict"); fd = ngx_http_zstd_dict_file_open_strict(&path, FILE_FLAGS, &walk); check("symlinked intermediate: rc OPENAT", walk.rc, NGX_HTTP_ZSTD_DICT_WALK_OPENAT); check("symlinked intermediate: ELOOP captured before close()", walk.err, ELOOP); check_str("symlinked intermediate: component named", (const char *) walk.component, "dicts"); check("symlinked intermediate: walk stopped there", ncalls, 2); check_hygiene("symlinked intermediate", fd); layout_happy(); nodes[2].open_errno = ELOOP; /* the leaf is a symlink */ fd = ngx_http_zstd_dict_file_open_strict(&path, FILE_FLAGS, &walk); check("symlinked leaf: rc OPENAT", walk.rc, NGX_HTTP_ZSTD_DICT_WALK_OPENAT); check_str("symlinked leaf: component named", (const char *) walk.component, "a.dict"); check_hygiene("symlinked leaf", fd); layout_happy(); path = str("/srv/dicts/missing.dict"); fd = ngx_http_zstd_dict_file_open_strict(&path, FILE_FLAGS, &walk); check("missing leaf: rc OPENAT", walk.rc, NGX_HTTP_ZSTD_DICT_WALK_OPENAT); check("missing leaf: ENOENT reported", walk.err, ENOENT); check_hygiene("missing leaf", fd); printf("# intermediate directory vetting\n"); layout_happy(); nodes[1].mode = S_IFDIR | 0777; path = str("/srv/dicts/a.dict"); fd = ngx_http_zstd_dict_file_open_strict(&path, FILE_FLAGS, &walk); check("world-writable intermediate: rc DIR_WRITABLE", walk.rc, NGX_HTTP_ZSTD_DICT_WALK_DIR_WRITABLE); check_str("world-writable intermediate: component named", (const char *) walk.component, "dicts"); check("world-writable intermediate: leaf never opened", ncalls, 2); check_hygiene("world-writable intermediate", fd); layout_happy(); nodes[1].mode = S_IFDIR | S_ISVTX | 0777; fd = ngx_http_zstd_dict_file_open_strict(&path, FILE_FLAGS, &walk); check("sticky world-writable intermediate: still DIR_WRITABLE", walk.rc, NGX_HTTP_ZSTD_DICT_WALK_DIR_WRITABLE); check_hygiene("sticky intermediate", fd); layout_happy(); nodes[1].mode = S_IFDIR | 0775; fd = ngx_http_zstd_dict_file_open_strict(&path, FILE_FLAGS, &walk); check("group-writable intermediate: DIR_WRITABLE", walk.rc, NGX_HTTP_ZSTD_DICT_WALK_DIR_WRITABLE); check_hygiene("group-writable intermediate", fd); layout_happy(); nodes[0].uid = 1000; fd = ngx_http_zstd_dict_file_open_strict(&path, FILE_FLAGS, &walk); check("foreign-owned intermediate: rc DIR_OWNER", walk.rc, NGX_HTTP_ZSTD_DICT_WALK_DIR_OWNER); check("foreign-owned intermediate: uid reported", (long) walk.uid, 1000); check_str("foreign-owned intermediate: component named", (const char *) walk.component, "srv"); check("foreign-owned intermediate: walk stopped there", ncalls, 1); check_hygiene("foreign-owned intermediate", fd); layout_happy(); nodes[0].uid = fake_euid; /* owned by the loader: accepted */ fd = ngx_http_zstd_dict_file_open_strict(&path, FILE_FLAGS, &walk); check("loader-owned intermediate is accepted", fd, 103); check_hygiene("loader-owned intermediate", fd); layout_happy(); nodes[1].fstat_errno = EIO; fd = ngx_http_zstd_dict_file_open_strict(&path, FILE_FLAGS, &walk); check("fstat() failure on an intermediate: rc DIR_FSTAT", walk.rc, NGX_HTTP_ZSTD_DICT_WALK_DIR_FSTAT); check("fstat() failure: errno captured before close()", walk.err, EIO); check_str("fstat() failure: component named", (const char *) walk.component, "dicts"); check_hygiene("fstat intermediate failure", fd); /* * The leaf is NOT vetted by the walk: ownership and mode of the file * itself are the caller's fstat()-side checks. A world-writable leaf * therefore opens here. */ layout_happy(); nodes[2].mode = S_IFREG | 0666; fd = ngx_http_zstd_dict_file_open_strict(&path, FILE_FLAGS, &walk); check("leaf mode is the caller's business: the walk opens it", fd, 103); check_hygiene("world-writable leaf", fd); printf("# ngx_http_zstd_log_dict_walk() -- one diagnostic per refusal\n"); { ngx_conf_t cf; int rc; char label[96]; path = str("/srv/dicts/a.dict"); for (rc = NGX_HTTP_ZSTD_DICT_WALK_OK; rc <= NGX_HTTP_ZSTD_DICT_WALK_DIR_WRITABLE; rc++) { memset(&walk, 0, sizeof(walk)); walk.rc = rc; walk.err = EIO; walk.uid = 1000; memcpy(walk.component, "dicts", sizeof("dicts")); log_calls = 0; log_last_err = -1; ngx_http_zstd_log_dict_walk(&cf, &path, &walk); snprintf(label, sizeof(label), "log shell: code %d logs %s", rc, rc == NGX_HTTP_ZSTD_DICT_WALK_OK ? "nothing" : "once"); check(label, log_calls, rc == NGX_HTTP_ZSTD_DICT_WALK_OK ? 0 : 1); if (rc == NGX_HTTP_ZSTD_DICT_WALK_OPEN_ROOT || rc == NGX_HTTP_ZSTD_DICT_WALK_OPENAT || rc == NGX_HTTP_ZSTD_DICT_WALK_DIR_FSTAT) { snprintf(label, sizeof(label), "log shell: code %d hands the call's errno to the " "logger", rc); check(label, log_last_err, EIO); } } } if (failures) { printf("❌ %d dict walk unit assertion(s) failed\n", failures); return 1; } printf("✓ all dict walk unit assertions passed\n"); return 0; } myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_dict_walk_unit.sh000077500000000000000000000102441525232545500261700ustar00rootroot00000000000000#!/bin/bash # Unit fixture for the strict-mode dictionary path walk, # ngx_http_zstd_dict_file_open_strict() and its per-directory check, in # src/ngx_http_zstd_dict_file.h -- THE authoritative copy, included # directly. The four system calls the walk makes are function-like macros # over scripted fakes in test_dict_walk_unit.c, so every exit is reached # deterministically, including the ones no shell fixture can stage. No # nginx tree, no zstd needed -- pure C, POSIX only. set -euo pipefail # ci/tools/ -> ci/ -> repo root. cd "$(dirname "$0")/../.." HDR="src/ngx_http_zstd_dict_file.h" SRC="src/ngx_http_zstd_filter_module.c" CC="${CC:-cc}" OUT="$(mktemp -d "${TMPDIR:-/tmp}/zstd-dict-walk-unit.XXXXXX")" trap 'rm -rf "$OUT"' EXIT for marker in 'ngx_http_zstd_dict_file_open_strict(ngx_str_t \*path, int flags,' \ 'ngx_http_zstd_dict_file_check_dir(int fd, ngx_http_zstd_dict_walk_t \*walk)' \ 'NGX_HTTP_ZSTD_DICT_WALK_DIR_WRITABLE' 'O_NOFOLLOW'; do if ! grep -q "$marker" "$HDR"; then echo "FAIL: $HDR lacks marker: $marker" >&2 exit 1 fi done # Production must route strict mode through the header's walk and must not # open path components on its own: an openat() CALL anywhere in the module # (an identifier as its first argument -- the diagnostics only ever spell # it with a quote or an empty list) is the walk restored inside it. if ! grep -q 'ngx_http_zstd_dict_file_open_strict(path, flags, &walk)' "$SRC"; then echo "FAIL: $SRC no longer routes strict mode through ngx_http_zstd_dict_file_open_strict()" >&2 exit 1 fi if grep -Eq 'openat\([A-Za-z_]' "$SRC"; then echo "FAIL: $SRC calls openat() on its own again; the walk belongs in $HDR" >&2 exit 1 fi # Every refusal the walk can report must have a message in the module's # switch: a code added to the header without a case is a silent refusal. # The extraction itself is checked first: renamed or reformatted enum # markers would yield no codes, and a loop over nothing passes. codes=$(sed -n '/^typedef enum {$/,/^} ngx_http_zstd_dict_walk_rc_t;$/p' "$HDR" \ | grep -o 'NGX_HTTP_ZSTD_DICT_WALK_[A-Z_]*' | sort -u || true) ncodes=$(printf '%s\n' "$codes" | grep -c . || true) if [ "$ncodes" -lt 2 ]; then echo "FAIL: could not extract the walk refusal codes from $HDR (found $ncodes); the enum was renamed or reformatted" >&2 exit 1 fi for code in $codes; do if ! grep -q "case $code:" "$SRC"; then echo "FAIL: $SRC has no diagnostic for $code" >&2 exit 1 fi done # The logging shell itself is extracted verbatim (it is `static` in the # module) and compiled into the fixture, which calls it with every code and # counts the diagnostics -- so a case label that no longer logs, not just a # missing label, fails the run. Locate pipelines end in `|| true` so a # missing pattern reaches its diagnostic under pipefail. SIG='ngx_http_zstd_log_dict_walk(ngx_conf_t \*cf, ngx_str_t \*path,$' SIG_LINE="$(grep -n "^${SIG}" "$SRC" | head -1 | cut -d: -f1 || true)" if [ -z "$SIG_LINE" ]; then echo "FAIL: could not locate ngx_http_zstd_log_dict_walk() in $SRC" >&2 exit 1 fi START_LINE=$((SIG_LINE - 1)) if ! sed -n "${START_LINE}p" "$SRC" | grep -q '^static void$'; then echo "FAIL: line $START_LINE of $SRC is not the log shell's 'static void' line" >&2 exit 1 fi REL_END="$(tail -n "+${START_LINE}" "$SRC" | grep -n '^}' | head -1 | cut -d: -f1 || true)" if [ -z "$REL_END" ]; then echo "FAIL: could not find the closing brace of ngx_http_zstd_log_dict_walk() in $SRC" >&2 exit 1 fi END_LINE=$((START_LINE + REL_END - 1)) { echo "/* Generated by test_dict_walk_unit.sh: verbatim extraction of" echo " * ngx_http_zstd_log_dict_walk() from $SRC lines ${START_LINE}-${END_LINE}." echo " * Do not edit. */" sed -n "${START_LINE},${END_LINE}p" "$SRC" } > "$OUT/generated_log_dict_walk.inc" if ! grep -q 'switch (walk->rc)' "$OUT/generated_log_dict_walk.inc"; then echo "FAIL: the extracted log shell is not the switch over walk->rc" >&2 exit 1 fi "$CC" -std=gnu99 -Wall -Wextra -Werror -O1 -I ci/tools -I "$OUT" \ -o "$OUT/test_dict_walk_unit" ci/tools/test_dict_walk_unit.c "$OUT/test_dict_walk_unit" echo "OK: dict walk unit fixture (against $HDR)" myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_dio_err_ratelimit_unit.c000066400000000000000000000167641525232545500275460ustar00rootroot00000000000000/* * Unit fixture for ngx_http_zstd_static_dio_err_should_log() — A33-F4(b), * PR #314 review: the directio hard-error probe arm in * ngx_http_zstd_static_module.c logs NGX_LOG_ERR on every request once a * sidecar's directio_alignment mismatches the device, 1:1 with request * volume. The fix rate-limits the LOG only; *malformed stays 0 on that * arm (unchanged — see the header comment in the .c file for why the * bad-file ring cannot absorb a config-mismatch failure). * * ngx_http_zstd_static_dio_err_should_log() is `static` inside * src/ngx_http_zstd_static_module.c, so it cannot be linked from outside * that TU. test_dio_err_ratelimit_unit.sh extracts it (plus its window * constant and file-scope counters) verbatim by line range and #includes * the generated file here — this test always exercises the CURRENT * shipped implementation, never a hand-copied duplicate. * * The extracted function calls ngx_time() and ngx_log_debug1(); both are * stubbed below — ngx_time() as a test-controlled fake clock, including a * backward-step arm matching the production helper's defensive handling, * ngx_log_debug1() as a no-op exactly as it compiles in a non-NGX_DEBUG * build (see src/core/ngx_log.h's third #define block). */ #include #include typedef unsigned long ngx_uint_t; typedef long time_t_dummy; /* unused; time_t comes from */ #include typedef struct ngx_log_s ngx_log_t; struct ngx_log_s { int unused; }; /* Test-controlled fake clock, standing in for ngx_cached_time->sec. */ static time_t fake_now; #define ngx_time() fake_now /* No-op, matching the non-NGX_DEBUG compiled shape of the real macro. */ #define ngx_log_debug1(level, log, err, fmt, arg1) #include "generated_dio_ratelimit.inc" #define NOT_SUPPRESSED ((ngx_uint_t) -1) static int failures; static int checks; static void reset_state(void) { ngx_http_zstd_static_dio_err_window_start = 0; ngx_http_zstd_static_dio_err_suppressed = 0; fake_now = 1000; } static void check(const char *what, ngx_uint_t got, ngx_uint_t want) { checks++; if (got != want) { printf("FAIL - %s: got=%lu want=%lu\n", what, (unsigned long) got, (unsigned long) want); failures++; } } int main(void) { ngx_uint_t r; int i; /* * 1. First call in a fresh worker (window_start == 0) must ALWAYS * log — this is the "do not swallow the first one" invariant. * It reports 0 prior occurrences. */ reset_state(); r = ngx_http_zstd_static_dio_err_should_log(NULL, -1); check("first call logs immediately", r, 0); /* * 2. Every subsequent call inside the same window is suppressed * (sentinel NOT_SUPPRESSED), and the internal counter increments * once per suppressed call — proven indirectly via what the NEXT * logged call reports. */ for (i = 0; i < 5; i++) { r = ngx_http_zstd_static_dio_err_should_log(NULL, -1); check("mid-window call is suppressed", r, NOT_SUPPRESSED); } /* * 3. Still inside the window (window boundary is a `>=` on the delta, * so window_start + WINDOW - 1 is still inside): still suppressed. */ fake_now = 1000 + NGX_HTTP_ZSTD_STATIC_DIO_ERR_WINDOW - 1; r = ngx_http_zstd_static_dio_err_should_log(NULL, -1); check("last second before rollover still suppressed", r, NOT_SUPPRESSED); /* * 4. At exactly window_start + WINDOW, the window rolls over: this * call logs again and reports every suppressed hit folded in — * 5 from step 2 plus 1 from step 3 = 6. */ fake_now = 1000 + NGX_HTTP_ZSTD_STATIC_DIO_ERR_WINDOW; r = ngx_http_zstd_static_dio_err_should_log(NULL, -1); check("window rollover reports folded suppressed count", r, 6); /* * 5. The rollover call itself resets the window and the counter, so * the immediately following call (still at the same fake_now) is * suppressed again with the count restarted at 0 prior — proven by * checking the NEXT rollover reports exactly 1, not 7. */ r = ngx_http_zstd_static_dio_err_should_log(NULL, -1); check("call right after rollover is suppressed", r, NOT_SUPPRESSED); fake_now += NGX_HTTP_ZSTD_STATIC_DIO_ERR_WINDOW; r = ngx_http_zstd_static_dio_err_should_log(NULL, -1); check("counter did not leak across the rollover", r, 1); /* * 6. A worker-cycle reset (a fresh process/window_start back to 0, * as happens on config reload's fork of new workers) always logs * immediately again, regardless of how much suppressed history the * OLD worker had accumulated — a reload must not hide a NEW * problem behind a stale suppression window. */ reset_state(); ngx_http_zstd_static_dio_err_should_log(NULL, -1); r = ngx_http_zstd_static_dio_err_should_log(NULL, -1); check("old worker has suppressed state", r, NOT_SUPPRESSED); reset_state(); r = ngx_http_zstd_static_dio_err_should_log(NULL, -1); check("fresh worker-cycle state logs immediately", r, 0); /* * 7. Monotonic-forward clock sanity: a window boundary exactly one * tick past window_start (the minimum non-negative delta reaching * WINDOW) rolls over; one tick short of it does not. This pins the * boundary condition the .c file's SAFETY comment claims never * underflows. */ reset_state(); ngx_http_zstd_static_dio_err_should_log(NULL, -1); /* opens window at 1000 */ fake_now = 1000 + NGX_HTTP_ZSTD_STATIC_DIO_ERR_WINDOW - 1; r = ngx_http_zstd_static_dio_err_should_log(NULL, -1); check("boundary-1 still suppressed", r, NOT_SUPPRESSED); fake_now = 1000 + NGX_HTTP_ZSTD_STATIC_DIO_ERR_WINDOW; r = ngx_http_zstd_static_dio_err_should_log(NULL, -1); check("boundary rolls over", r, 1); /* * 8. Backward wall-clock step (NTP correction, admin action): * ngx_time() going backward must force a rollover, not extend * suppression indefinitely. A signed `now - window_start` under a * backward step is negative and would otherwise stay under the * `>=` window threshold forever. */ reset_state(); ngx_http_zstd_static_dio_err_should_log(NULL, -1); /* opens window at 1000 */ r = ngx_http_zstd_static_dio_err_should_log(NULL, -1); check("mid-window before clock step is suppressed", r, NOT_SUPPRESSED); fake_now = 500; /* clock stepped backward past window_start */ r = ngx_http_zstd_static_dio_err_should_log(NULL, -1); check("backward clock step forces rollover, not extended suppression", r, 1); /* A malformed short read must not consume the hard-error window. */ reset_state(); r = ngx_http_zstd_static_dio_err_should_log(NULL, 0); check("zero-byte EOF logs without rate limiting", r, 0); r = ngx_http_zstd_static_dio_err_should_log(NULL, -1); check("first hard error after zero-byte EOF still logs", r, 0); reset_state(); r = ngx_http_zstd_static_dio_err_should_log(NULL, 3); check("short malformed read logs without rate limiting", r, 0); r = ngx_http_zstd_static_dio_err_should_log(NULL, -1); check("first hard error after malformed read still logs", r, 0); r = ngx_http_zstd_static_dio_err_should_log(NULL, -1); check("following hard error is rate limited", r, NOT_SUPPRESSED); if (failures) { printf("FAILED: %d check(s)\n", failures); return 1; } printf("OK: directio probe log rate limiter (%d checks)\n", checks); return 0; } myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_dio_err_ratelimit_unit.sh000077500000000000000000000124551525232545500277320ustar00rootroot00000000000000#!/bin/bash # Unit fixture for ngx_http_zstd_static_dio_err_should_log() (A33-F4(b) log # rate limit for the directio hard-error probe arm). # # The function is `static` inside src/ngx_http_zstd_static_module.c, so this # extracts it verbatim by line range (same technique already used by # ci/tools/test_ceil_log2_unit.sh for ngx_http_zstd_ceil_log2()) and compiles # it standalone against a fake, test-controlled ngx_time() clock. No nginx # tree, no zstd, no filesystem — pure C over a tiny counter. set -euo pipefail # ci/tools/ -> ci/ -> repo root: two levels since the ci/ move. cd "$(dirname "$0")/../.." SRC="src/ngx_http_zstd_static_module.c" CC="${CC:-cc}" OUT="$(mktemp -d "${TMPDIR:-/tmp}/zstd-dio-ratelimit-unit.XXXXXX")" trap 'rm -rf "$OUT"' EXIT # Locate the function by its exact signature line, then its opening and # closing braces, so the extraction survives unrelated edits elsewhere in # the file but fails loudly if the function itself is renamed or removed. START_LINE="$(grep -n '^static ngx_uint_t$' "$SRC" | head -1 | cut -d: -f1)" if [ -z "$START_LINE" ]; then echo "FAIL: could not locate ngx_http_zstd_static_dio_err_should_log() start in $SRC" >&2 exit 1 fi SIG_LINE=$((START_LINE + 1)) if ! sed -n "${SIG_LINE}p" "$SRC" \ | grep -q 'ngx_http_zstd_static_dio_err_should_log(ngx_log_t \*log, ssize_t n)'; then echo "FAIL: line $SIG_LINE in $SRC is not" \ "ngx_http_zstd_static_dio_err_should_log's signature" >&2 exit 1 fi # First "^}" at column 0 after START_LINE closes the function. REL_END="$(tail -n "+${START_LINE}" "$SRC" | grep -n '^}' | head -1 | cut -d: -f1)" END_LINE=$((START_LINE + REL_END - 1)) # Also pull the two file-scope static counters and the window constant that # the function reads/writes, plus the header comment block above them, so # the extraction stays self-contained and does not require hand-copying # state declarations that could drift from the shipped names. WINDOW_LINE="$(grep -n '^#define NGX_HTTP_ZSTD_STATIC_DIO_ERR_WINDOW' "$SRC" | head -1 | cut -d: -f1)" STATE_START="$(grep -n '^static time_t ngx_http_zstd_static_dio_err_window_start;' "$SRC" | head -1 | cut -d: -f1)" STATE_END="$(grep -n '^static ngx_uint_t ngx_http_zstd_static_dio_err_suppressed;' "$SRC" | head -1 | cut -d: -f1)" if [ -z "$WINDOW_LINE" ] || [ -z "$STATE_START" ] || [ -z "$STATE_END" ]; then echo "FAIL: could not locate the dio-err rate limiter's state declarations in $SRC" >&2 exit 1 fi mkdir -p "$OUT" { echo "/* Generated by test_dio_err_ratelimit_unit.sh — verbatim" echo " * extraction from $SRC. Do not edit; re-run to regenerate. */" sed -n "${WINDOW_LINE}p" "$SRC" echo sed -n "${STATE_START},${STATE_END}p" "$SRC" echo sed -n "${START_LINE},${END_LINE}p" "$SRC" } >"$OUT/generated_dio_ratelimit.inc" # Sanity: the extraction must contain the markers this test relies on, or a # source reflow silently produced an empty/garbage .inc. if ! grep -q 'ngx_http_zstd_static_dio_err_should_log(ngx_log_t \*log, ssize_t n)' \ "$OUT/generated_dio_ratelimit.inc"; then echo "FAIL: extracted fragment does not contain the function signature" >&2 cat "$OUT/generated_dio_ratelimit.inc" >&2 exit 1 fi if ! grep -q 'NGX_HTTP_ZSTD_STATIC_DIO_ERR_WINDOW' "$OUT/generated_dio_ratelimit.inc"; then echo "FAIL: extracted fragment missing the window constant — extraction range wrong" >&2 exit 1 fi # The call site must not describe a successful short read as an alignment or # device failure. Parse the sibling branches so moving the limiter call into # the n >= 0 arm (or the malformed diagnostic into the hard-error arm) fails. python3 - "$SRC" <<'PYEOF' import sys text = open(sys.argv[1], encoding="utf-8").read() anchor = text.index("ngx_uint_t dio_suppressed;") start = text.index("if (n >= 0) {", anchor) def block_at(pos): opening = text.index("{", pos) depth = 0 for i in range(opening, len(text)): if text[i] == "{": depth += 1 elif text[i] == "}": depth -= 1 if depth == 0: return text[opening + 1:i], i + 1 raise ValueError("unterminated call-site branch") short_read, after = block_at(start) else_pos = text.index("else {", after) hard_error, _ = block_at(else_pos) malformed = "frame header -- treating sidecar as " limiter = "ngx_http_zstd_static_dio_err_should_log(log, n);" alignment = "aligned probe " if malformed not in short_read or limiter in short_read or alignment in short_read: raise SystemExit("FAIL: n >= 0 branch is not malformed-only") if limiter not in hard_error or alignment not in hard_error or malformed in hard_error: raise SystemExit("FAIL: n < 0 branch is not the rate-limited alignment error") PYEOF cp ci/tools/test_dio_err_ratelimit_unit.c "$OUT/test_dio_err_ratelimit_unit.c" # -Wextra omitted deliberately: with ngx_log_debug1() stubbed as a no-op # (matching its real non-NGX_DEBUG compiled shape), the `log` parameter # goes genuinely unused in THIS build only — the same nginx calling- # convention exemption already applied to the module's other unit # fixtures and the repo's -Wextra lint policy. "$CC" -Wall -Werror -O1 -g -I"$OUT" \ -o "$OUT/dio_ratelimit_unit" "$OUT/test_dio_err_ratelimit_unit.c" "$OUT/dio_ratelimit_unit" echo "OK: directio probe log rate limiter unit fixture" \ "(extracted from current $SRC)" myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_docs_ci_drift.sh000077500000000000000000000223251525232545500257660ustar00rootroot00000000000000#!/usr/bin/env bash # # Regression test for audit sha e289021 F7: README/SECURITY/CONTRIBUTING # drifted materially from the live CI workflows (stale badge count, dead # AGENTS.md link, a workflow present in .github/workflows/ but undocumented, # a false -Werror claim). The full fix was a one-time manual re-sync; # nothing then stopped it drifting again on the next workflow add/rename. # # This does NOT try to keep exact TAP subtest counts in sync (that's # exactly the kind of brittle exact-count claim the F7 fix already # recommended dropping) -- it checks structural facts that are cheap to # keep true and expensive to silently get wrong: # - every workflow file under .github/workflows/ is referenced somewhere # in README.md (by filename), so a new/renamed workflow can't go # undocumented # - every workflow referenced in README.md's badges/table actually exists # under .github/workflows/, so a removed/renamed workflow can't leave a # dead link/badge behind # - SECURITY.md does not link the removed AGENTS.md # - CONTRIBUTING.md does not claim the fuzz harness builds with -Werror # unless fuzz/build.sh (or equivalent) actually passes it # # Usage: tools/test_docs_ci_drift.sh set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # ci/tools/ -> ci/ -> repo root: TWO levels since the ci/ move. One climb # lands in ci/, where README.md and .github/ do not exist -- and the loop # below would then report every workflow undocumented. MODULE_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")" cd "$MODULE_DIR" || exit 1 # shellcheck source=ci/linter/lib.sh . "$MODULE_DIR/ci/linter/lib.sh" collect_directives() { mapfile_checked DIRECTIVES sed -n \ 's/^### \(zstd[_a-z]*\)$/\1/p' README.md } produce_readme_workflows() { grep -oE 'workflows/[A-Za-z0-9_-]+\.yml' README.md \ | sed 's#workflows/##' \ | sort -u } collect_readme_workflows() { mapfile_checked README_WORKFLOWS produce_readme_workflows } case "${1:-}" in --selftest-directive-list) collect_directives || exit $? exit 0 ;; --selftest-workflow-list) collect_readme_workflows || exit $? exit 0 ;; esac fail=0 toc="$(sed -n '/^# Table of Contents$/,/^# Status$/p' README.md)" local_suites="$(sed -n '/^Run the suites locally:/,/^# Unit tests/p' README.md)" deep_suite_step="$(sed -n '/name: Run ci\/t\//,/^ fuzz:/p' .github/workflows/ci-deep.yml)" # Every public directive heading must be reachable from the README index. collect_directives || exit $? for directive in "${DIRECTIVES[@]}"; do if ! grep -Fq -- "[$directive](#$directive)" <<<"$toc"; then echo "FAIL: README directive section $directive is missing from the table of contents" fail=1 fi done # Commands advertised as the local/deep full suite must include every Perl suite. for suite in ci/t/*.t; do name="$(basename "$suite")" grep -Fq -- "$name" <<<"$local_suites" || { echo "FAIL: README local test instructions omit $name" fail=1 } if ! grep -F -- "$name" <<<"$deep_suite_step" \ | grep -Eq '^[[:space:]]*prove '; then echo "FAIL: CI Deep full-suite claim omits $name" fail=1 fi done # --- every real workflow file is mentioned in README.md --- for wf in .github/workflows/*.yml; do name="$(basename "$wf")" if ! grep -q "$name" README.md; then echo "FAIL: .github/workflows/$name exists but is not referenced in README.md (undocumented workflow)" fail=1 fi done # --- every workflow filename README.md references actually exists --- collect_readme_workflows || exit $? for name in "${README_WORKFLOWS[@]}"; do [ -f ".github/workflows/$name" ] || { echo "FAIL: README.md references .github/workflows/$name, which does not exist (stale/dead link or badge)" fail=1 } done # --- SECURITY.md must not link the removed AGENTS.md --- if [ -f SECURITY.md ] && grep -q 'AGENTS\.md' SECURITY.md; then echo "FAIL: SECURITY.md still links AGENTS.md, which was removed (PR #79)" fail=1 fi # --- README's -DZSTD_STATIC_LINKING_ONLY claim must match auto/zstd's # actual behavior (audit A31b-F5): the flag is committed to CFLAGS only # inside the `ZSTD_INC`/`ZSTD_LIB` static-archive branch (auto/zstd's # "we try the static library first" section) -- auto-discovery (no # ZSTD_INC/ZSTD_LIB) and pkg-config never define it. The documented plain # `./configure --add-dynamic-module=...` command therefore does not carry # the flag, so README.md must say so explicitly rather than imply the # flag is just "on" for anyone following the docs. This is a docs-drift # regression test, not a build-behavior change: it never touches # auto/zstd or the release build flags. --- # shellcheck disable=SC2016 # literal grep pattern: $ZSTD_INC/$ must not expand if ! grep -q '^[[:space:]]*ngx_zstd_opt_I="-I\$ZSTD_INC -DZSTD_STATIC_LINKING_ONLY"$' auto/zstd; then echo "FAIL: auto/zstd no longer stages -DZSTD_STATIC_LINKING_ONLY in the ZSTD_INC/ZSTD_LIB static-archive branch the way this test expects -- update auto/zstd's staging line or this test together with README.md's Installation/Compatibility wording" fail=1 fi # Extract the WHOLE auto-discovery `else` branch (from its marker comment to # the closing `fi` at the same indent) rather than a fixed window, so the flag # cannot be reintroduced anywhere inside it and still pass this test. zstd_autodisco_branch=$( sed -n '/^ # auto-discovery: prefer dynamic linking/,/^fi$/p' auto/zstd ) if [ -z "$zstd_autodisco_branch" ]; then echo "FAIL: could not locate auto/zstd's auto-discovery branch (the '# auto-discovery: prefer dynamic linking' marker or its closing 'fi' moved) -- update this test together with auto/zstd and README.md" fail=1 elif printf '%s\n' "$zstd_autodisco_branch" | grep -q 'DZSTD_STATIC_LINKING_ONLY'; then echo "FAIL: auto/zstd's auto-discovery branch now mentions -DZSTD_STATIC_LINKING_ONLY -- README.md's 'plain ./configure does not enable it' claim is stale and must be rewritten to match" fail=1 fi # shellcheck disable=SC2016 # literal README phrase, backticks must not run if ! grep -Fq 'does not enable `-DZSTD_STATIC_LINKING_ONLY`' README.md; then echo "FAIL: README.md Installation section no longer states that the plain ./configure command does not enable -DZSTD_STATIC_LINKING_ONLY (audit A31b-F5 regression)" fail=1 fi if grep -Fq 'production and CI builds enable that flag' README.md; then echo "FAIL: README.md re-introduced the 'production and CI builds enable that flag' phrasing, which reads as the flag being on by default rather than requiring ZSTD_INC/ZSTD_LIB pointed at a static libzstd (audit A31b-F5 regression)" fail=1 fi # --- CONTRIBUTING.md's -Werror claim about the fuzz harness must match # reality: only assert this if a fuzz build step actually exists and does # NOT pass -Werror, since a future CI change legitimately flipping this on # would make the check obsolete rather than wrong. --- if [ -f CONTRIBUTING.md ] && grep -qi 'fuzz.*-Werror\|-Werror.*fuzz' CONTRIBUTING.md; then if ! grep -rq -- '-Werror' fuzz/ 2>/dev/null && ! grep -rlq -- '-Werror' .github/workflows/*.yml 2>/dev/null; then echo "FAIL: CONTRIBUTING.md claims the fuzz harness builds with -Werror, but no fuzz build step passes it" fail=1 fi fi # --- README's static-probe skippable-chain bound must match # NGX_HTTP_ZSTD_STATIC_MAX_SKIP_FRAMES (audit A30-F6): the README states # a concrete "up to 4 leading skippable frames" bound and says the # declared-window check applies to the first REGULAR frame reached after # that chain. Anchor both on the real code so a future change to the # bound, or to which frame the window check targets, breaks this test # instead of silently drifting the prose again. --- skip_frames_define=$( grep -E '^#define[[:space:]]+NGX_HTTP_ZSTD_STATIC_MAX_SKIP_FRAMES[[:space:]]+[0-9]+' \ src/ngx_http_zstd_frame_probe.h ) if [ -z "$skip_frames_define" ]; then echo "FAIL: could not find NGX_HTTP_ZSTD_STATIC_MAX_SKIP_FRAMES in src/ngx_http_zstd_frame_probe.h -- update this test together with the source" fail=1 else skip_bound="$(printf '%s\n' "$skip_frames_define" | grep -oE '[0-9]+$')" if ! grep -Fq "up to $skip_bound leading skippable frames" README.md; then echo "FAIL: README.md's stated skippable-chain bound does not match NGX_HTTP_ZSTD_STATIC_MAX_SKIP_FRAMES ($skip_bound) in src/ngx_http_zstd_frame_probe.h -- update README.md's Declared-window/Magic-number validation prose" fail=1 fi fi if ! grep -q 'ngx_http_zstd_static_probe_verdict' src/ngx_http_zstd_static_module.c; then echo "FAIL: ngx_http_zstd_static_probe_verdict no longer exists in src/ngx_http_zstd_static_module.c -- the window-check anchor this test relies on moved or was renamed; update this test together with README.md" fail=1 fi if ! grep -Fq 'first regular frame' README.md; then echo "FAIL: README.md no longer states that the declared-window check applies to the first REGULAR frame reached after the leading skippable-frame chain (audit A30-F6 regression)" fail=1 fi if [ "$fail" -ne 0 ]; then echo "FAIL: docs/CI drift detected (audit sha e289021 F7 regression)" exit 1 fi echo "✓ README/SECURITY/CONTRIBUTING workflow references match .github/workflows/ contents" myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_encoding.py000077500000000000000000000530401525232545500247750ustar00rootroot00000000000000#!/usr/bin/env python3 import argparse import concurrent.futures import http.server import os import pathlib import re import shutil import socket import subprocess import sys import tempfile import threading import time import urllib.request from typing import ClassVar FIXTURE_SENTINEL = "END-OF-ZSTD-TRUNCATION-FIXTURE-9f4c3d1b" def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description="Run an end-to-end zstd encoding smoke test against nginx." ) parser.add_argument( "--nginx-binary", required=True, help="Path to the nginx binary to start for the smoke test.", ) parser.add_argument( "--filter-module", help=( "Optional path to ngx_http_zstd_filter_module.so. " "If omitted, the helper auto-detects a sibling module next to the nginx binary." ), ) parser.add_argument( "--static-module", help=( "Optional path to ngx_http_zstd_static_module.so. " "If omitted, the helper auto-detects a sibling module next to the nginx binary." ), ) parser.add_argument( "--port", type=int, default=18080, help="Local TCP port for the temporary nginx instance.", ) parser.add_argument( "--backend-port", type=int, help="Local TCP port for the temporary Cache-Control upstream.", ) parser.add_argument( "--zstd-bin", default=shutil.which("zstd") or "zstd", help="Path to the zstd CLI used for decompression.", ) parser.add_argument( "--gzip-vary", choices=("on", "off"), default="off", help="Whether to enable gzip_vary in the temporary nginx config.", ) parser.add_argument( "--expect-vary", action="store_true", help=( "Require the response to include Vary: Accept-Encoding. " "Useful when gzip_vary is enabled." ), ) parser.add_argument( "--fixture-lines", type=int, default=8192, help="Number of repeated lines to generate in the JavaScript fixture.", ) parser.add_argument( "--request-count", type=int, default=1, help="How many sequential requests to verify against the same nginx worker.", ) parser.add_argument( "--concurrent-requests", type=int, default=1, help="How many concurrent requests to verify against the same nginx worker.", ) return parser.parse_args() def wait_for_port(port: int, timeout: float = 10.0) -> None: deadline = time.time() + timeout while time.time() < deadline: try: with socket.create_connection(("127.0.0.1", port), timeout=0.5): return except OSError: time.sleep(0.1) raise RuntimeError(f"nginx did not start listening on 127.0.0.1:{port}") def build_fixture(path: pathlib.Path, lines: int) -> bytes: with path.open("w", encoding="utf-8", newline="\n") as handle: handle.write("// zstd truncation regression fixture\n") handle.write("const payload = [\n") for index in range(lines): handle.write( f' "line-{index:05d}: zstd-regression-payload-abcdefghijklmnopqrstuvwxyz0123456789",\n' ) handle.write("];\n") handle.write( "globalThis.__zstd_fixture_checksum = `${payload.length}:${payload[0]}:${payload[payload.length - 1]}`;\n" ) handle.write(f'globalThis.__zstd_fixture_end = "{FIXTURE_SENTINEL}";\n') handle.write("console.log(globalThis.__zstd_fixture_checksum);\n") handle.write("console.log(globalThis.__zstd_fixture_end);\n") data = path.read_bytes() if len(data) <= 131072: raise RuntimeError( f"fixture too small to catch truncation bugs: {len(data)} bytes" ) return data def detect_module_path( explicit_path: str | None, nginx_binary: pathlib.Path, module_name: str, ) -> pathlib.Path | None: if explicit_path: return pathlib.Path(explicit_path) sibling = nginx_binary.parent / module_name if sibling.exists(): return sibling return None def write_config( conf_path: pathlib.Path, root_dir: pathlib.Path, port: int, backend_port: int, gzip_vary: str, modules: list[pathlib.Path], ) -> None: load_modules = "".join(f"load_module {module};\n" for module in modules) conf_path.write_text( f""" {load_modules}worker_processes 1; error_log logs/error.log info; pid logs/nginx.pid; events {{ worker_connections 128; }} http {{ access_log logs/access.log; # $zstd_ratio is a log-phase variable; capture it to its own file so # the harness can assert it is a finite N.NNN value (regression for # 064895c, the compression-ratio integer-overflow fix). log_format zstd_ratio_fmt "$zstd_ratio"; default_type application/octet-stream; sendfile off; gzip_vary {gzip_vary}; keepalive_timeout 5; server {{ listen 127.0.0.1:{port}; server_name localhost; root {root_dir}; location = /test.js {{ types {{ application/javascript js; }} default_type application/javascript; zstd on; zstd_min_length 1; zstd_types application/javascript; access_log logs/zstd_ratio.log zstd_ratio_fmt; }} location = /cache-control-public.js {{ types {{ application/javascript js; }} default_type application/javascript; zstd on; zstd_min_length 1; zstd_types application/javascript; proxy_pass http://127.0.0.1:{backend_port}/cache-control-public.js; }} location = /cache-control-quoted-no-transform.js {{ types {{ application/javascript js; }} default_type application/javascript; zstd on; zstd_min_length 1; zstd_types application/javascript; proxy_pass http://127.0.0.1:{backend_port}/cache-control-quoted-no-transform.js; }} location = /cache-control-quoted-comma.js {{ types {{ application/javascript js; }} default_type application/javascript; zstd on; zstd_min_length 1; zstd_types application/javascript; proxy_pass http://127.0.0.1:{backend_port}/cache-control-quoted-comma.js; }} location = /cache-control-no-transform-comma.js {{ types {{ application/javascript js; }} default_type application/javascript; zstd on; zstd_min_length 1; zstd_types application/javascript; proxy_pass http://127.0.0.1:{backend_port}/cache-control-no-transform-comma.js; }} location = /cache-control-no-transform-ows.js {{ types {{ application/javascript js; }} default_type application/javascript; zstd on; zstd_min_length 1; zstd_types application/javascript; proxy_pass http://127.0.0.1:{backend_port}/cache-control-no-transform-ows.js; }} location = /cache-control-no-transform-arg.js {{ types {{ application/javascript js; }} default_type application/javascript; zstd on; zstd_min_length 1; zstd_types application/javascript; proxy_pass http://127.0.0.1:{backend_port}/cache-control-no-transform-arg.js; }} location = /cache-control-no-transform-repeat.js {{ types {{ application/javascript js; }} default_type application/javascript; zstd on; zstd_min_length 1; zstd_types application/javascript; proxy_pass http://127.0.0.1:{backend_port}/cache-control-no-transform-repeat.js; }} }} }} """.lstrip(), encoding="utf-8", ) class CacheControlHandler(http.server.BaseHTTPRequestHandler): payload = b"" cache_control: ClassVar[dict[str, list[str]]] = { "/cache-control-public.js": ["public"], "/cache-control-quoted-no-transform.js": ['public, extension="no-transform"'], "/cache-control-quoted-comma.js": ['public, extension="a,no-transform,b"'], "/cache-control-no-transform-comma.js": ["public, no-transform"], "/cache-control-no-transform-ows.js": [" public ; max-age=60 , No-Transform "], "/cache-control-no-transform-repeat.js": ["public", "no-transform"], "/cache-control-no-transform-arg.js": ["public, no-transform=1"], } def do_GET(self) -> None: values = self.cache_control.get(self.path) if values is None: self.send_error(404) return self.send_response(200) self.send_header("Content-Type", "application/javascript") self.send_header("Content-Length", str(len(self.payload))) for value in values: self.send_header("Cache-Control", value) self.end_headers() self.wfile.write(self.payload) def log_message(self, fmt: str, *args: object) -> None: return def fetch_response(port: int) -> tuple[bytes, str, str]: return fetch_path(port, "/test.js") def fetch_path(port: int, path: str) -> tuple[bytes, str, str]: request = urllib.request.Request( f"http://127.0.0.1:{port}{path}", headers={"Accept-Encoding": "zstd", "User-Agent": "zstd-ci-smoke-test/1.0"}, ) with urllib.request.urlopen(request, timeout=10) as response: compressed = response.read() content_encoding = response.headers.get("Content-Encoding", "") vary = response.headers.get("Vary", "") return compressed, content_encoding, vary def validate_cache_control_no_transform(port: int, expected: bytes) -> None: public_body, public_encoding, _ = fetch_path(port, "/cache-control-public.js") if public_encoding.lower() != "zstd": raise RuntimeError( "Cache-Control public negative control did not compress: " f"Content-Encoding={public_encoding!r}" ) quoted_body, quoted_encoding, _ = fetch_path( port, "/cache-control-quoted-no-transform.js" ) if quoted_encoding.lower() != "zstd": raise RuntimeError( "Cache-Control quoted no-transform parameter was treated as a " f"directive: Content-Encoding={quoted_encoding!r}" ) if quoted_body == expected: raise RuntimeError( "Cache-Control quoted no-transform negative control was not transformed" ) quoted_comma_body, quoted_comma_encoding, _ = fetch_path( port, "/cache-control-quoted-comma.js" ) if quoted_comma_encoding.lower() != "zstd": raise RuntimeError( "Cache-Control quoted comma extension was split into a directive: " f"Content-Encoding={quoted_comma_encoding!r}" ) if quoted_comma_body == expected: raise RuntimeError( "Cache-Control quoted comma negative control was not transformed" ) for path in ( "/cache-control-no-transform-comma.js", "/cache-control-no-transform-ows.js", "/cache-control-no-transform-repeat.js", # Malformed but intent-clear: the directive defines no argument # (RFC 9111 5.2.2.6), so an origin writing no-transform=1 gets # honored, not transformed -- the walker compares the directive # name, cut at '='. "/cache-control-no-transform-arg.js", ): body, encoding, _ = fetch_path(port, path) if encoding: raise RuntimeError( f"{path}: expected no Content-Encoding for no-transform, got {encoding!r}" ) if body != expected: raise RuntimeError(f"{path}: identity response body changed") if public_body == expected: raise RuntimeError("Cache-Control public negative control was not transformed") def decompress_payload(zstd_bin: str, compressed: bytes) -> bytes: result = subprocess.run( [zstd_bin, "-d", "-q", "-c"], input=compressed, capture_output=True, check=False, ) if result.returncode != 0: stderr = result.stderr.decode("utf-8", errors="replace") raise RuntimeError(f"zstd decompression failed: {stderr.strip()}") return result.stdout def read_if_exists(path: pathlib.Path) -> str: if not path.exists(): return "" return path.read_text(encoding="utf-8", errors="replace") def validate_response( port: int, zstd_bin: str, expected: bytes, expect_vary: bool, request_label: str, ) -> None: compressed, encoding, vary = fetch_response(port) if encoding.lower() != "zstd": raise RuntimeError(f"expected Content-Encoding=zstd, got {encoding!r}") if expect_vary: vary_values = { value.strip().lower() for value in vary.split(",") if value.strip() } if "accept-encoding" not in vary_values: raise RuntimeError( f"expected Vary to include Accept-Encoding, got {vary!r}" ) decoded = decompress_payload(zstd_bin, compressed) if decoded != expected: raise RuntimeError( "decompressed response does not match source fixture; " f"{request_label}, expected {len(expected)} bytes, got {len(decoded)} bytes" ) if FIXTURE_SENTINEL.encode("utf-8") not in decoded: raise RuntimeError("fixture sentinel missing from decoded response") RATIO_RE = re.compile(r"^\d+\.\d{3}$") def validate_ratio_log( log_path: pathlib.Path, request_count: int, timeout: float = 10.0 ) -> None: """Assert every logged $zstd_ratio is a finite N.NNN value. Regression for 064895c ("integer overflow in compression ratio calculation"): before the fix bytes_in*1000 could overflow and yield a garbage or zero ratio. The CI JavaScript fixture is highly compressible, so a correct ratio must parse AND be > 1.0 (output smaller than input). $zstd_ratio is written in nginx's *log phase*, which runs after the response body has already been flushed to the client. validate_response therefore returns before nginx has necessarily written the access_log line — a race that widens under the (much slower) ASAN/UBSAN binary and used to fail intermittently with "the ratio code path did not run". Poll for at least request_count non-empty lines before asserting, so a slow log-phase write is waited out rather than mistaken for a missing one. """ deadline = time.monotonic() + timeout lines: list[str] = [] while True: content = read_if_exists(log_path) lines = [line for line in content.splitlines() if line.strip()] if len(lines) >= request_count or time.monotonic() >= deadline: break time.sleep(0.05) if not lines: raise RuntimeError( f"$zstd_ratio log {log_path} is empty — the variable was not " "populated (the ratio code path did not run)" ) for line in lines: value = line.strip() if not RATIO_RE.match(value): raise RuntimeError( f"$zstd_ratio {value!r} is not a finite N.NNN value " "(integer-overflow / garbage regression, cf. 064895c)" ) if float(value) <= 1.0: raise RuntimeError( f"$zstd_ratio {value!r} <= 1.0 for a highly compressible " "fixture — ratio computed wrong" ) def main() -> int: args = parse_args() nginx_binary = pathlib.Path(args.nginx_binary) if not nginx_binary.exists(): raise FileNotFoundError(f"nginx binary not found: {nginx_binary}") if shutil.which(args.zstd_bin) is None and not pathlib.Path(args.zstd_bin).exists(): raise FileNotFoundError(f"zstd CLI not found: {args.zstd_bin}") if args.request_count < 1: raise ValueError(f"request-count must be >= 1, got {args.request_count}") if args.concurrent_requests < 1: raise ValueError( f"concurrent-requests must be >= 1, got {args.concurrent_requests}" ) backend_port = args.backend_port or args.port + 1 filter_module = detect_module_path( args.filter_module, nginx_binary, "ngx_http_zstd_filter_module.so", ) static_module = detect_module_path( args.static_module, nginx_binary, "ngx_http_zstd_static_module.so", ) modules = [ module for module in (filter_module, static_module) if module is not None ] for module in modules: if not module.exists(): raise FileNotFoundError(f"zstd module not found: {module}") # Everything below must stay readable by the workers when run as # root (they drop to the compiled-in nginx user): a restrictive # inherited umask (e.g. 077) would strip group/other bits from # every fixture and subdir created here, 403ing the workers even # with the scratch root itself opened up. os.umask(0o022) with tempfile.TemporaryDirectory(prefix="zstd-ci-smoke-") as temp_dir_str: # mkdtemp gives 0700: run as root, workers drop to the # compiled-in nginx user and cannot enter it -> 403s os.chmod(temp_dir_str, 0o755) temp_dir = pathlib.Path(temp_dir_str) html_dir = temp_dir / "html" conf_dir = temp_dir / "conf" logs_dir = temp_dir / "logs" html_dir.mkdir() conf_dir.mkdir() logs_dir.mkdir() fixture_path = html_dir / "test.js" expected = build_fixture(fixture_path, args.fixture_lines) conf_path = conf_dir / "nginx.conf" write_config( conf_path, html_dir, args.port, backend_port, args.gzip_vary, modules ) CacheControlHandler.payload = expected backend = http.server.ThreadingHTTPServer( ("127.0.0.1", backend_port), CacheControlHandler ) backend_thread = threading.Thread(target=backend.serve_forever, daemon=True) backend_thread.start() process = subprocess.Popen( [ str(nginx_binary), "-p", str(temp_dir), "-c", str(conf_path), "-g", "daemon off; master_process off;", ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, ) try: wait_for_port(args.port) for request_index in range(args.request_count): validate_response( args.port, args.zstd_bin, expected, args.expect_vary, f"request {request_index + 1}/{args.request_count}", ) if args.concurrent_requests > 1: with concurrent.futures.ThreadPoolExecutor( max_workers=args.concurrent_requests ) as executor: futures = [ executor.submit( validate_response, args.port, args.zstd_bin, expected, args.expect_vary, f"concurrent request {request_index + 1}/{args.concurrent_requests}", ) for request_index in range(args.concurrent_requests) ] for future in futures: future.result() validate_cache_control_no_transform(args.port, expected) validate_ratio_log( pathlib.Path(temp_dir) / "logs" / "zstd_ratio.log", args.request_count, ) print( "OK: verified zstd response integrity" f" for {len(expected)}-byte JavaScript fixture" + ( f" across {args.request_count} sequential requests" if args.request_count > 1 else "" ) + ( f" and {args.concurrent_requests} concurrent requests" if args.concurrent_requests > 1 else "" ) + (" with Vary: Accept-Encoding" if args.expect_vary else "") ) return 0 finally: backend.shutdown() backend.server_close() process.terminate() try: # 30s, not 5: gcov-instrumented nginx flushing .gcda # at exit on a loaded runner can outlive a tight reap # budget; teardown runs after the verdict, so patience # costs nothing when healthy. process.wait(timeout=30) except subprocess.TimeoutExpired: process.kill() process.wait(timeout=30) output = process.stdout.read() if process.stdout is not None else "" error_log = read_if_exists(logs_dir / "error.log") if process.returncode not in (0, -15): sys.stderr.write("nginx stdout/stderr:\n") sys.stderr.write(output) sys.stderr.write("\nnginx error log:\n") sys.stderr.write(error_log) if __name__ == "__main__": try: raise SystemExit(main()) except Exception as exc: print(f"ERROR: {exc}", file=sys.stderr) raise myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_evp_prepared_tree_probe.sh000077500000000000000000000051251525232545500300540ustar00rootroot00000000000000#!/bin/sh set -eu root=$(cd "$(dirname "$0")/../.." && pwd) tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT HUP INT TERM mkdir -p "$tmp/auto" cat >"$tmp/auto/feature" <<'EOF' ngx_found=no ngx_test_incs= for ngx_test_path in $ngx_feature_path; do ngx_test_incs="$ngx_test_incs -I$ngx_test_path" done cat >"$ngx_test_dir/probe.c" </dev/null 2>&1; then ngx_found=yes printf '#define %s 1\n' "$ngx_feature_name" >"$ngx_test_result" else : >"$ngx_test_result" fi EOF cat >"$tmp/auto/module" <<'EOF' : EOF run_probe() ( config=$1 prepared=$2 result=$3 case_dir=$tmp/$result mkdir -p "$case_dir" cd "$tmp" ngx_addon_dir=$root/filter ngx_zstd_incs= ngx_zstd_opt_L= ngx_module_link=ADDON USE_OPENSSL=YES OPENSSL=$prepared CORE_INCS= HTTP_FILTER_MODULES=ngx_http_range_header_filter_module HTTP_GZIP=NO _ngx_zstd_root=$root ngx_test_dir=$case_dir ngx_test_result=$case_dir/result export ngx_addon_dir ngx_zstd_incs ngx_zstd_opt_L ngx_module_link export USE_OPENSSL OPENSSL CORE_INCS HTTP_FILTER_MODULES HTTP_GZIP export _ngx_zstd_root ngx_test_dir ngx_test_result # shellcheck source=/dev/null . "$config" >/dev/null cp "$ngx_test_result" "$tmp/$result.out" ) prepared=$tmp/prepared mkdir -p "$prepared/openssl/include/openssl" cat >"$prepared/openssl/include/openssl/evp.h" <<'EOF' #define EVP_MAX_MD_SIZE 64 const void *EVP_sha256(void); EOF run_probe "$root/filter/config" "$prepared" prepared grep -q '^#define NGX_HTTP_ZSTD_HAVE_LIBCRYPTO 1$' "$tmp/prepared.out" run_probe "$root/filter/config" "$tmp/missing" missing if grep -q NGX_HTTP_ZSTD_HAVE_LIBCRYPTO "$tmp/missing.out"; then echo 'missing prepared OpenSSL tree unexpectedly enabled EVP' >&2 exit 1 fi # Negative control: the prepared MSVC layout must stop probing successfully if # its include spelling is removed from the production config. # The single quotes deliberately preserve the config's literal shell variable. # shellcheck disable=SC2016 sed 's| $OPENSSL/openssl/include||' "$root/filter/config" >"$tmp/config-mutated" run_probe "$tmp/config-mutated" "$prepared" mutated if grep -q NGX_HTTP_ZSTD_HAVE_LIBCRYPTO "$tmp/mutated.out"; then echo 'negative control still enabled EVP without the MSVC include path' >&2 exit 1 fi echo 'prepared OpenSSL EVP probe: PASS' myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_freecctxparams_unit.c000066400000000000000000000341431525232545500270470ustar00rootroot00000000000000/* * Unit fixture for ngx_http_zstd_estimate_cctx_memory(). * * The function is `static` inside src/ngx_http_zstd_filter_module.c, so it * cannot be linked from outside that TU. test_freecctxparams_unit.sh * extracts it verbatim by line range (the repo's established precedent -- * see test_read_dict_file_unit.sh and test_cctx_profile_pack.sh) and * #includes the generated file here, so this test always exercises the * CURRENT shipped implementation rather than a hand-copied duplicate. * * WHY A UNIT FIXTURE AND NOT THE HTTP FAULT-INJECTION PROBE. This function * takes an ngx_conf_t * and runs during "nginx -t" config load, before any * request exists -- the probe endpoint that arms the other libzstd fault * sites in this module has no request in flight through which to reach it. * A direct unit fixture that fakes the ZSTD_CCtx_params call chain is the * only way to force each of the six early-return error exits. * * THE DEFECT CLASS THIS GUARDS. Every one of the six early returns must * call ZSTD_freeCCtxParams(cp) before returning NGX_CONF_ERROR; a leak in * ANY ONE of the six passes CI silently because the other five still free * correctly and no per-request path ever touches this function again. * Six independent scenarios below each force exactly one exit and assert * ZSTD_freeCCtxParams was called EXACTLY ONCE on the SAME cp pointer the * function was handed -- a counting assertion, not a reachability check, * so deleting any single free call turns that scenario's count from 1 to 0. */ #include #include #include #include /* --- minimal nginx type/macro surface the extracted function needs ----- */ typedef long ngx_int_t; typedef long ngx_flag_t; typedef long ssize_t_ngx; /* unused placeholder, kept for clarity */ typedef struct { int unused; } ngx_conf_t; typedef struct { ngx_int_t level; ssize_t target_cblock_size; ngx_flag_t long_mode; } ngx_http_zstd_loc_conf_t; #define NGX_LOG_EMERG 2 #define NGX_CONF_OK ((char *) 0) #define NGX_CONF_ERROR ((char *) -1) /* * Diagnostics are counted, not printed: the assertions below care that each * failing exit REPORTS (the shipped code always does), not about wording. * Variadic and ignoring its arguments so any format string the shipped code * uses compiles unchanged. */ static int log_calls; static void ngx_conf_log_error(int level, ngx_conf_t *cf, int err, const char *fmt, ...) { (void) level; (void) cf; (void) err; (void) fmt; log_calls++; } /* --- real zstd.h for types/macros/enum only; every ZSTD_* function below * is faked in this file, not linked from libzstd. ZSTD_STATIC_LINKING_ONLY * (passed on the command line by test_freecctxparams_unit.sh) is required * for ZSTD_CCtx_params / ZSTD_createCCtxParams / ZSTD_freeCCtxParams / * ZSTD_CCtxParams_init / ZSTD_CCtxParams_setParameter / * ZSTD_estimateCStreamSize_usingCCtxParams to be declared at all; the * struct stays opaque, which is all the extracted function needs. */ #ifndef ZSTD_STATIC_LINKING_ONLY #define ZSTD_STATIC_LINKING_ONLY #endif #include /* * The production module correctly omits targetCBlockSize when built with * libzstd headers older than 1.5.6. This fixture links no real libzstd, * however: every call is faked below, and it must still compile and drive * that sixth error exit on CI hosts whose system zstd.h predates 1.5.6. * Retain an older header's experimental alias when present, otherwise supply * the public 1.5.6 value only to this fake translation unit, then raise the * extracted function's version gate accordingly. */ #if ZSTD_VERSION_NUMBER < 10506 #ifndef ZSTD_c_targetCBlockSize #define ZSTD_c_targetCBlockSize ((ZSTD_cParameter) 130) #endif #undef ZSTD_VERSION_NUMBER #define ZSTD_VERSION_NUMBER 10506 #endif /* --- scripted fake ZSTD_* layer ----------------------------------------- * * One sentinel ZSTD_CCtx_params object stands in for the real (opaque) * struct; the extracted function only ever holds a pointer to it, so * identity -- not contents -- is what "the same cp" means here. * * Each scenario configures which named call should fail (return an error * code) via `fail_call`; every other scripted call succeeds (returns 0). * free_count / freed_ptr record every ZSTD_freeCCtxParams() invocation so * the harness can assert both "freed exactly once" and "freed the right * pointer" -- a free on a different pointer, or a leak (0 frees), or a * double-free (2+ frees) all fail the assertion. */ struct ZSTD_CCtx_params_s { int tag; }; static struct ZSTD_CCtx_params_s sentinel_cp; static int create_returns_null; static int free_count; static void *freed_ptr; typedef enum { CALL_NONE = 0, CALL_INIT, CALL_WINDOWLOG, /* the plain zstd_window_log setParameter */ CALL_ENABLE_LDM, CALL_LDM_WINDOWLOG, /* first of the 5-step LDM derivation chain */ CALL_LDM_HASHLOG, CALL_LDM_MINMATCH, CALL_LDM_BUCKETSIZELOG, CALL_LDM_HASHRATELOG, CALL_TARGET_CBLOCK, } which_call_t; static which_call_t fail_call; ZSTD_CCtx_params * ZSTD_createCCtxParams(void) { if (create_returns_null) { return NULL; } sentinel_cp.tag = 0; return &sentinel_cp; } size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params *params) { if (params != NULL) { free_count++; freed_ptr = params; } return 0; } size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params *params, int level) { (void) params; (void) level; return (fail_call == CALL_INIT) ? (size_t) -1 : 0; } size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params *params, ZSTD_cParameter param, int value) { (void) params; (void) value; switch ((int) param) { case ZSTD_c_windowLog: /* * Two different call sites push windowLog: the plain * "zstd_window_log" push (only when long_mode is off) and the LDM * derivation's first push (only when long_mode is on). Only one * is reachable per scenario, so a single tag distinguishes them * without ambiguity. */ if (fail_call == CALL_WINDOWLOG || fail_call == CALL_LDM_WINDOWLOG) { return (size_t) -1; } return 0; case ZSTD_c_enableLongDistanceMatching: return (fail_call == CALL_ENABLE_LDM) ? (size_t) -1 : 0; case ZSTD_c_ldmHashLog: return (fail_call == CALL_LDM_HASHLOG) ? (size_t) -1 : 0; case ZSTD_c_ldmMinMatch: return (fail_call == CALL_LDM_MINMATCH) ? (size_t) -1 : 0; case ZSTD_c_ldmBucketSizeLog: return (fail_call == CALL_LDM_BUCKETSIZELOG) ? (size_t) -1 : 0; case ZSTD_c_ldmHashRateLog: return (fail_call == CALL_LDM_HASHRATELOG) ? (size_t) -1 : 0; #if ZSTD_VERSION_NUMBER >= 10506 case ZSTD_c_targetCBlockSize: return (fail_call == CALL_TARGET_CBLOCK) ? (size_t) -1 : 0; #endif default: return 0; } } size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params *params) { (void) params; return 12345; /* arbitrary non-error size_t */ } unsigned ZSTD_isError(size_t code) { return code == (size_t) -1; } const char * ZSTD_getErrorName(size_t code) { (void) code; return "fake-error"; } #include "generated_estimate_cctx_memory.inc" /* --- harness ------------------------------------------------------------ */ static int failures; static void reset(void) { create_returns_null = 0; free_count = 0; freed_ptr = NULL; fail_call = CALL_NONE; log_calls = 0; } /* * Asserts EXACTLY ONE free, on the SAME cp the function received. This is * the counting assertion the item requires: it distinguishes "freed once" * (1) from "leaked" (0) and from "double-freed" (2+), and it is falsified * by deleting the one ZSTD_freeCCtxParams(cp) call this scenario exercises * -- unlike a bare "was it called at all" check, which a stray free on an * unrelated exit could still satisfy. */ static void check_freed_once(const char *name, char *rc) { if (rc != NGX_CONF_ERROR) { printf("\xe2\x9c\x97 %s: rc=%p want NGX_CONF_ERROR\n", name, (void *) rc); failures++; return; } if (free_count != 1) { printf("\xe2\x9c\x97 %s: ZSTD_freeCCtxParams called %d time(s), " "want exactly 1\n", name, free_count); failures++; return; } if (freed_ptr != (void *) &sentinel_cp) { printf("\xe2\x9c\x97 %s: freed pointer %p != cp %p\n", name, freed_ptr, (void *) &sentinel_cp); failures++; return; } if (log_calls < 1) { printf("\xe2\x9c\x97 %s: rc ok but no diagnostic logged\n", name); failures++; return; } printf("\xe2\x9c\x93 %s\n", name); } int main(void) { ngx_conf_t cf; ngx_http_zstd_loc_conf_t conf; size_t est; char *rc; memset(&cf, 0, sizeof(cf)); /* 0. Baseline: the ordinary success path frees exactly once too. */ reset(); memset(&conf, 0, sizeof(conf)); conf.level = 3; conf.long_mode = 0; conf.target_cblock_size = 0; est = 0; rc = ngx_http_zstd_estimate_cctx_memory(&cf, &conf, 0, &est); if (rc != NGX_CONF_OK) { printf("\xe2\x9c\x97 success path: rc=%p want NGX_CONF_OK\n", (void *) rc); failures++; } else if (free_count != 1) { printf("\xe2\x9c\x97 success path: ZSTD_freeCCtxParams called %d " "time(s), want exactly 1\n", free_count); failures++; } else if (est != 12345) { printf("\xe2\x9c\x97 success path: *est=%zu want 12345\n", est); failures++; } else { printf("\xe2\x9c\x93 success path frees exactly once\n"); } /* 1. ZSTD_CCtxParams_init() fails -- first error exit. */ reset(); memset(&conf, 0, sizeof(conf)); conf.level = 3; fail_call = CALL_INIT; rc = ngx_http_zstd_estimate_cctx_memory(&cf, &conf, 0, &est); check_freed_once("exit 1: ZSTD_CCtxParams_init fails", rc); /* * 2. Plain zstd_window_log setParameter fails -- second error exit. * Only reached when window_log > 0 and long_mode is off. */ reset(); memset(&conf, 0, sizeof(conf)); conf.level = 3; fail_call = CALL_WINDOWLOG; rc = ngx_http_zstd_estimate_cctx_memory(&cf, &conf, 20, &est); check_freed_once("exit 2: windowLog setParameter fails", rc); /* * 3. enableLongDistanceMatching setParameter fails -- third error exit. * Only reached when long_mode is on. */ reset(); memset(&conf, 0, sizeof(conf)); conf.level = 3; conf.long_mode = 1; fail_call = CALL_ENABLE_LDM; rc = ngx_http_zstd_estimate_cctx_memory(&cf, &conf, 0, &est); check_freed_once("exit 3: enableLongDistanceMatching setParameter fails", rc); /* * 4. The 5-step LDM derivation chain fails -- fourth error exit. Each * of the five sub-calls is independently capable of triggering this * exact exit; exercise the LAST one (ldmHashRateLog) because it proves * the short-circuit chain still reaches all the way to the final push * before erroring, not just the first. */ reset(); memset(&conf, 0, sizeof(conf)); conf.level = 3; conf.long_mode = 1; fail_call = CALL_LDM_HASHRATELOG; rc = ngx_http_zstd_estimate_cctx_memory(&cf, &conf, 0, &est); check_freed_once("exit 4: LDM derivation chain fails (ldmHashRateLog)", rc); /* * 4b. Same exit, different failing sub-call (ldmMinMatch), to show the * chain's OWN short-circuit does not accidentally skip the free either. */ reset(); memset(&conf, 0, sizeof(conf)); conf.level = 3; conf.long_mode = 1; fail_call = CALL_LDM_MINMATCH; rc = ngx_http_zstd_estimate_cctx_memory(&cf, &conf, 0, &est); check_freed_once("exit 4b: LDM derivation chain fails (ldmMinMatch)", rc); /* * 5. Defence-in-depth: ldm_rlog <= 0 -- fifth error exit. This is NOT * a ZSTD_isError() branch; it is a derived-value guard. Drive it by * requesting a window_log so small that ldm_hlog clamps to * ZSTD_LDM_HASHLOG_MIN >= ldm_wlog, forcing ldm_rlog <= 0. * ZSTD_LDM_HASHLOG_MIN in the real libzstd headers is 6, so window_log * 6 makes ldm_wlog=6, ldm_hlog clamps to 6, ldm_rlog = 6-6 = 0. */ reset(); memset(&conf, 0, sizeof(conf)); conf.level = 3; conf.long_mode = 1; rc = ngx_http_zstd_estimate_cctx_memory(&cf, &conf, 6, &est); check_freed_once("exit 5: derived LDM hash rate is zero (ldm_rlog<=0)", rc); /* * 6. targetCBlockSize setParameter fails -- sixth error exit. Only * reached when target_cblock_size > 0 (and the libzstd version gate is * satisfied, which it is: this build's zstd.h reports * ZSTD_VERSION_NUMBER >= 10506). */ reset(); memset(&conf, 0, sizeof(conf)); conf.level = 3; conf.target_cblock_size = 512; fail_call = CALL_TARGET_CBLOCK; rc = ngx_http_zstd_estimate_cctx_memory(&cf, &conf, 0, &est); check_freed_once("exit 6: targetCBlockSize setParameter fails", rc); /* * 7. ZSTD_createCCtxParams() itself returns NULL. Not one of the six * counted exits (there is no cp to free), but worth a sanity check that * this path does NOT call ZSTD_freeCCtxParams on a NULL cp and does not * crash. */ reset(); memset(&conf, 0, sizeof(conf)); conf.level = 3; create_returns_null = 1; rc = ngx_http_zstd_estimate_cctx_memory(&cf, &conf, 0, &est); if (rc != NGX_CONF_ERROR) { printf("\xe2\x9c\x97 createCCtxParams NULL: rc=%p want " "NGX_CONF_ERROR\n", (void *) rc); failures++; } else if (free_count != 0) { printf("\xe2\x9c\x97 createCCtxParams NULL: ZSTD_freeCCtxParams " "called %d time(s) on a NULL cp, want 0\n", free_count); failures++; } else { printf("\xe2\x9c\x93 createCCtxParams NULL: no free attempted, " "no crash\n"); } if (failures) { printf("\xe2\x9d\x8c %d freecctxparams unit assertion(s) failed\n", failures); return 1; } printf("\xe2\x9c\x93 all freecctxparams unit assertions passed\n"); return 0; } myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_freecctxparams_unit.sh000077500000000000000000000102421525232545500272340ustar00rootroot00000000000000#!/bin/bash # Unit fixture for ngx_http_zstd_estimate_cctx_memory(). # # This is a CONFIG-TIME function (takes ngx_conf_t *cf, runs during # "nginx -t"), not a per-request path, so the HTTP fault-injection probe # endpoint cannot reach it at all -- there is no request in flight for it # to poison. A direct unit-test harness is the only way to force each of # its six early-return error exits and observe whether ZSTD_freeCCtxParams() # actually ran on that exit. # # Extraction, not duplication: the function is `static` inside # ngx_http_zstd_filter_module.c, so this extracts it verbatim by line range # (the same technique as test_read_dict_file_unit.sh and # test_cctx_profile_pack.sh) along with the three NGX_HTTP_ZSTD_LDM_* macros # it references, and compiles it standalone in test_freecctxparams_unit.c # against scripted ZSTD_* fakes. A hand-copied duplicate of the six exits # would drift from the shipped code and keep passing while a real one # regressed to leaking. set -euo pipefail # ci/tools/ -> ci/ -> repo root: TWO levels. cd "$(dirname "$0")/../.." SRC="src/ngx_http_zstd_filter_module.c" CC="${CC:-cc}" OUT="$(mktemp -d "${TMPDIR:-/tmp}/zstd-freecctxparams-unit.XXXXXX")" trap 'rm -rf "$OUT"' EXIT mkdir -p "$OUT" # --- the three LDM layout #defines the function references -------------- # Extracted, never re-typed: NGX_HTTP_ZSTD_LDM_WINDOWLOG feeds directly into # the divisor-derivation arithmetic under test. DEF_START="$(grep -n '^#define NGX_HTTP_ZSTD_LDM_WINDOWLOG' "$SRC" \ | head -1 | cut -d: -f1 || true)" if [ -z "$DEF_START" ]; then echo "FAIL: could not locate NGX_HTTP_ZSTD_LDM_WINDOWLOG in $SRC" >&2 exit 1 fi DEF_END="$(grep -n '^#define NGX_HTTP_ZSTD_LDM_BUCKETSIZELOG' "$SRC" \ | head -1 | cut -d: -f1 || true)" if [ -z "$DEF_END" ]; then echo "FAIL: could not locate NGX_HTTP_ZSTD_LDM_BUCKETSIZELOG in $SRC" >&2 exit 1 fi # --- the function itself ------------------------------------------------- # Locate by its signature line (the "static char *" return type sits on the # line above it, as in this file's house style), so extraction survives # unrelated edits but fails loudly if the function is renamed or removed. SIG_LINE="$(grep -n '^ngx_http_zstd_estimate_cctx_memory(ngx_conf_t \*cf,$' "$SRC" \ | head -1 | cut -d: -f1 || true)" if [ -z "$SIG_LINE" ]; then echo "FAIL: could not locate ngx_http_zstd_estimate_cctx_memory()'s definition in $SRC" >&2 exit 1 fi FUNC_START=$((SIG_LINE - 1)) if ! sed -n "${FUNC_START}p" "$SRC" | grep -q '^static char \*$'; then echo "FAIL: line $FUNC_START in $SRC is not the expected 'static char *' return-type line" >&2 exit 1 fi REL_END="$(tail -n "+${FUNC_START}" "$SRC" | grep -n '^}' | head -1 | cut -d: -f1 || true)" if [ -z "$REL_END" ]; then echo "FAIL: could not find the closing '}' for ngx_http_zstd_estimate_cctx_memory() in $SRC" >&2 exit 1 fi FUNC_END=$((FUNC_START + REL_END - 1)) { echo "/* Generated by test_freecctxparams_unit.sh — verbatim extraction of" echo " * NGX_HTTP_ZSTD_LDM_* from $SRC lines $DEF_START-$DEF_END and" echo " * ngx_http_zstd_estimate_cctx_memory() from lines $FUNC_START-$FUNC_END." echo " * Do not edit; regenerate by running the script. */" sed -n "${DEF_START},${DEF_END}p" "$SRC" echo sed -n "${FUNC_START},${FUNC_END}p" "$SRC" } > "$OUT/generated_estimate_cctx_memory.inc" # Non-vacuity: the extraction must actually contain all six error-exit frees # plus the one success-path free, i.e. seven ZSTD_freeCCtxParams call sites. # A mis-ranged .inc would compile to nothing (or a truncated function) and # silently "pass" every scenario since none of the fakes would ever fire. FREE_CALLS="$(grep -c 'ZSTD_freeCCtxParams(cp)' "$OUT/generated_estimate_cctx_memory.inc" || true)" if [ "$FREE_CALLS" -ne 7 ]; then echo "FAIL: extracted .inc contains $FREE_CALLS ZSTD_freeCCtxParams(cp) call(s), expected 7 (six error exits + one success exit) — extraction range is wrong" >&2 exit 1 fi $CC -std=gnu99 -Wall -Wextra -Werror -O1 \ -DZSTD_STATIC_LINKING_ONLY \ -I ci/tools -I "$OUT" \ -o "$OUT/test_freecctxparams_unit" \ ci/tools/test_freecctxparams_unit.c "$OUT/test_freecctxparams_unit" myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_fuzz_seconds_validation.sh000077500000000000000000000072561525232545500301270ustar00rootroot00000000000000#!/usr/bin/env bash # # Regression test for audit sha e289021 F1: ci-deep.yml's "Select fuzz # duration" step used to splice workflow_dispatch's fuzz_seconds input # directly into a run: shell command, letting a quoted payload (e.g. # `3600"; id; #`) execute on the self-hosted builder02 runner. The fix # passes the raw value through env: and validates it in Bash before use. # # This script extracts that exact validation logic (kept byte-identical # below to .github/workflows/ci-deep.yml's "Select fuzz duration" step) and # drives it against malicious/malformed/boundary inputs, asserting each is # rejected, plus a couple of valid inputs that must be accepted. # # Usage: tools/test_fuzz_seconds_validation.sh set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" WORKFLOW="$SCRIPT_DIR/../../.github/workflows/ci-deep.yml" fail=0 # Runs the same validation the workflow step performs, given # RAW_FUZZ_SECONDS, and prints either "FUZZ_SECS=" (accept) or nothing # with a nonzero exit (reject) -- mirrors the workflow's own exit-1-on-error # shape, minus the GitHub Actions ::error:: annotation syntax. validate() { local RAW_FUZZ_SECONDS="$1" case "$RAW_FUZZ_SECONDS" in '' | *[!0-9]*) return 1 ;; esac if [ "$RAW_FUZZ_SECONDS" -gt 14400 ]; then return 1 fi printf 'FUZZ_SECS=%s\n' "$RAW_FUZZ_SECONDS" } check_rejected() { local input="$1" desc="$2" if out="$(validate "$input" 2>/dev/null)"; then echo "FAIL: expected reject for $desc (input=$input), got accepted: $out" fail=1 else echo "✓ rejected: $desc" fi } check_accepted() { local input="$1" expect="$2" out="$(validate "$input" 2>/dev/null)" if [ "$out" != "FUZZ_SECS=$expect" ]; then echo "FAIL: expected accept for input=$input -> FUZZ_SECS=$expect, got: $out" fail=1 else echo "✓ accepted: $input -> $out" fi } # --- injection payloads (the actual F1 exploit shape) --- check_rejected '3600"; id; #' "quote-breakout shell injection" # shellcheck disable=SC2016 # literal payload fixture, must not expand check_rejected '$(id)' "command substitution" # shellcheck disable=SC2016 # literal payload fixture, must not expand check_rejected '`id`' "backtick substitution" check_rejected '3600; rm -rf /' "semicolon-chained command" check_rejected '3600 && id' "shell-and injection" check_rejected $'3600\nid' "embedded newline" # --- malformed / non-numeric --- check_rejected '' "empty string" check_rejected 'abc' "non-numeric" check_rejected '-100' "negative number" check_rejected '3600.5' "decimal" check_rejected ' 3600' "leading whitespace" check_rejected '3600 ' "trailing whitespace" # --- boundary: over budget --- check_rejected '14401' "one over the 14400s budget" check_rejected '999999999' "way over budget" # --- must still accept legitimate values --- check_accepted '3600' 3600 check_accepted '0' 0 check_accepted '14400' 14400 # --- drift guard: fail loudly if the workflow's validation logic changes # shape without this test being updated to match --- if ! grep -q "'\*\[!0-9\]\*'" "$WORKFLOW" 2>/dev/null; then if ! grep -q '\*\[!0-9\]\*' "$WORKFLOW"; then echo "FAIL: ci-deep.yml's non-numeric guard pattern not found -- validate() above may have drifted from the live workflow" fail=1 fi fi if ! grep -q '14400' "$WORKFLOW"; then echo "FAIL: ci-deep.yml no longer mentions the 14400s budget -- validate() above may have drifted from the live workflow" fail=1 fi if [ "$fail" -ne 0 ]; then echo "FAIL: fuzz_seconds validation regression" exit 1 fi echo "✓ all fuzz_seconds validation cases pass (injection rejected, valid values accepted)" myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_gitattributes.sh000077500000000000000000000131761525232545500260710ustar00rootroot00000000000000#!/bin/sh # Gates .gitattributes byte-exactness for the fuzz/test fixture dirs # (A30-F5): corpus_dcz/*, regressions/*, regressions_dcz/*, plus the # previously-covered corpus/*, fuzz.dict and t/suite/*. core.autocrlf=true # smudges any path git treats as text, corrupting raw fuzzer inputs and # saved crash reproducers -- there was no test anywhere pinning this # before. Also checks the ci-linkcheck-module archive-exclusion rule. # # Each assertion below has a paired negative control: a known-text file # that must report "text: set", and a hash-under-autocrlf check against a # deliberately text-treated temp file that must differ. Both prove the # assertion can fail, not just that it happens to pass today. set -eu cd "$(dirname "$0")/../.." REPO_ROOT="$(pwd)" FAIL=0 fail() { echo "FAIL: $1" >&2 FAIL=1 } # --- a) byte-exact dirs report text: unset --------------------------------- BYTE_EXACT_PATHS=" ci/fuzz/corpus/01_bare ci/fuzz/corpus_dcz/bad_hash ci/fuzz/fuzz.dict ci/fuzz/regressions/crash-wildcard-precedence-flip ci/fuzz/regressions_dcz/crash-dcz-framing-ok-payload-bad ci/t/suite/test ci/t/suite/dcz-dict " for p in $BYTE_EXACT_PATHS; do [ -n "$p" ] || continue attr="$(git check-attr text -- "$p" | sed 's/.*: text: //')" if [ "$attr" != "unset" ]; then fail "expected 'text: unset' for byte-exact fixture $p, got '$attr'" fi done # Negative control for (a): a known-text file must NOT be unset. attr="$(git check-attr text -- ci/tools/patches/README 2>/dev/null | sed 's/.*: text: //')" if [ -z "$attr" ] || [ "$attr" = "unset" ]; then # patches/README may not exist; fall back to any *.sh, which is # unconditionally "text eol=lf" in .gitattributes. attr="$(git check-attr text -- ci/tools/test_sha256_unit.sh | sed 's/.*: text: //')" fi if [ "$attr" != "set" ]; then fail "negative control: expected 'text: set' for a known-text file, got '$attr'" fi # regressions READMEs must stay text (the exception rule must win). for p in ci/fuzz/regressions/README.md ci/fuzz/regressions_dcz/README.md; do if [ -f "$p" ]; then attr="$(git check-attr text -- "$p" | sed 's/.*: text: //')" if [ "$attr" != "set" ]; then fail "expected 'text: set' for $p (README exception), got '$attr'" fi fi done # --- b) autocrlf hash control ----------------------------------------------- TMPDIR_WORK="$(mktemp -d "${TMPDIR:-/tmp}/zstd-gitattributes-test.XXXXXX")" cleanup() { rm -rf "$TMPDIR_WORK" } trap cleanup EXIT INT TERM # Pick a byte-exact fixture that is plausible to contain a bare LF (so an # autocrlf smudge would actually change it): the dcz corpus fixtures are # fixed-format hash strings without embedded newlines in some cases, so # use the fuzz corpus entry that is known to include control bytes/LF in # its Accept-Encoding header samples. HASH_FIXTURE="ci/fuzz/corpus_dcz/valid_hash" if [ ! -f "$HASH_FIXTURE" ]; then fail "expected fixture missing: $HASH_FIXTURE" else committed_sha="$(git cat-file blob "HEAD:$HASH_FIXTURE" | sha256sum | cut -d' ' -f1)" checkout_dir="$TMPDIR_WORK/autocrlf-checkout" mkdir -p "$checkout_dir" ( cd "$checkout_dir" && git -C "$REPO_ROOT" -c core.autocrlf=true archive HEAD "$HASH_FIXTURE" | tar -x ) smudged_sha="$(sha256sum "$checkout_dir/$HASH_FIXTURE" | cut -d' ' -f1)" if [ "$committed_sha" != "$smudged_sha" ]; then fail "byte-exact fixture $HASH_FIXTURE changed under core.autocrlf=true smudging (committed=$committed_sha smudged=$smudged_sha)" fi fi # Negative control for (b): a deliberately text-treated file with CRLF # content in its committed blob MUST change hash under autocrlf smudging, # proving the hash comparison itself is capable of catching a real # regression (git's autocrlf=true checkout normalizes CRLF -> LF for any # path git considers text). control_repo="$TMPDIR_WORK/control-repo" mkdir -p "$control_repo" ( cd "$control_repo" git init -q git config user.email test@example.invalid git config user.name test printf 'line one\r\nline two\r\n' > crlf.txt printf '*.txt text\n' > .gitattributes git add crlf.txt .gitattributes git commit -q -m control ) control_committed_sha="$(git -C "$control_repo" cat-file blob HEAD:crlf.txt | sha256sum | cut -d' ' -f1)" control_checkout="$TMPDIR_WORK/control-checkout" mkdir -p "$control_checkout" ( cd "$control_checkout" && git -C "$control_repo" -c core.autocrlf=true archive HEAD crlf.txt | tar -x ) control_smudged_sha="$(sha256sum "$control_checkout/crlf.txt" | cut -d' ' -f1)" if [ "$control_committed_sha" = "$control_smudged_sha" ]; then fail "negative control: expected a text-attributed CRLF file to change hash under core.autocrlf=true smudging, but it did not (test cannot detect a real regression)" fi # --- c) archive excludes ci-linkcheck-module -------------------------------- leaked="$(git archive HEAD | tar -t | grep -c '^ci/tools/ci-linkcheck-module/' || true)" if [ "$leaked" -ne 0 ]; then fail "git archive HEAD leaked $leaked ci-linkcheck-module entries (export-ignore rule broken)" fi # Negative control for (c): a path NOT covered by export-ignore must # still appear in the archive, proving the tar listing/grep can see a # real entry when one exists. present="$(git archive HEAD | tar -t | grep -c '^ci/tools/test_sha256_unit.sh$' || true)" if [ "$present" -eq 0 ]; then fail "negative control: expected ci/tools/test_sha256_unit.sh present in git archive HEAD, found none" fi if [ "$FAIL" -ne 0 ]; then echo "FAIL: .gitattributes byte-exactness / archive-exclusion checks failed" >&2 exit 1 fi echo "OK: .gitattributes byte-exact fixture rules and archive exclusion verified (with negative controls)" myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_input_chain_cursor_unit.c000066400000000000000000000210201525232545500277240ustar00rootroot00000000000000/* * Copyright (C) 2026 Thijs Eilander * SPDX-License-Identifier: BSD-2-Clause * * Deterministic ownership oracle for the body-filter input cursor. The * production functions are included verbatim from generated_input_cursor.inc; * only the nginx types and pool operations are shimmed here. */ #include #include #include #include typedef intptr_t ngx_int_t; typedef uintptr_t ngx_uint_t; #define NGX_OK 0 #define NGX_ERROR -1 #define NGX_AGAIN -2 #define NGX_DECLINED -5 typedef struct ngx_buf_s ngx_buf_t; typedef struct ngx_chain_s ngx_chain_t; typedef struct ngx_pool_s ngx_pool_t; struct ngx_buf_s { unsigned char *pos; unsigned char *last; unsigned flush; unsigned last_buf; }; struct ngx_chain_s { ngx_buf_t *buf; ngx_chain_t *next; }; struct ngx_pool_s { size_t calls; size_t fail_at; size_t frees; ngx_chain_t *free; }; typedef struct { const void *src; size_t size; size_t pos; } ZSTD_inBuffer; typedef struct { ngx_chain_t *in; ngx_chain_t **last_in; ngx_buf_t *in_buf; ZSTD_inBuffer buffer_in; uint64_t bytes_in; unsigned flush; unsigned last; unsigned redo; } ngx_http_zstd_ctx_t; typedef struct { ngx_pool_t *pool; } ngx_http_request_t; static ngx_chain_t * mock_alloc_chain_link(ngx_pool_t *pool) { ngx_chain_t *cl; pool->calls++; if (pool->fail_at != 0 && pool->calls == pool->fail_at) { return NULL; } if (pool->free != NULL) { cl = pool->free; pool->free = cl->next; return cl; } return calloc(1, sizeof(*cl)); } static void mock_free_chain(ngx_pool_t *pool, ngx_chain_t *cl) { pool->frees++; cl->next = pool->free; pool->free = cl; } #define ngx_http_zstd_alloc_chain_link(pool) mock_alloc_chain_link(pool) #define ngx_free_chain(pool, cl) mock_free_chain(pool, cl) #define ngx_buf_size(buf) ((size_t) ((buf)->last - (buf)->pos)) #define ngx_log_debug1(...) ((void) 0) #include "generated_input_cursor.inc" static unsigned char storage[8192]; static int failures; static void check(int condition, const char *message) { if (!condition) { fprintf(stderr, "FAIL: %s\n", message); failures++; } } static void init_ctx(ngx_http_zstd_ctx_t *ctx) { memset(ctx, 0, sizeof(*ctx)); ctx->last_in = &ctx->in; } static void make_chain(ngx_chain_t *links, ngx_buf_t *bufs, size_t count, size_t width) { size_t i; for (i = 0; i < count; i++) { bufs[i].pos = storage + (i % (sizeof(storage) - width)); bufs[i].last = bufs[i].pos + width; bufs[i].flush = 0; bufs[i].last_buf = 0; links[i].buf = &bufs[i]; links[i].next = i + 1 < count ? &links[i + 1] : NULL; } } static void test_fully_drained_no_copy(void) { ngx_buf_t bufs[8]; ngx_chain_t links[8]; ngx_chain_t *cursor; ngx_http_request_t r; ngx_http_zstd_ctx_t ctx; ngx_pool_t pool = {0}; size_t i; make_chain(links, bufs, 8, 3); cursor = links; r.pool = &pool; init_ctx(&ctx); for (i = 0; i < 8; i++) { check(ngx_http_zstd_filter_add_data(&r, &ctx, &cursor) == NGX_OK, "fully drained caller link loads"); check(ctx.in_buf == &bufs[i], "caller links stay in order"); ctx.buffer_in.pos = ctx.buffer_in.size; } check(cursor == NULL, "fully drained local cursor reaches NULL"); check(ngx_http_zstd_filter_copy_input(&r, &ctx, cursor) == NGX_OK, "empty lazy copy succeeds"); check(pool.calls == 0, "fully drained path allocates zero chain links"); check(pool.frees == 0, "caller-owned links are never freed"); for (i = 0; i < 7; i++) { check(links[i].next == &links[i + 1], "caller-owned next pointers remain unchanged"); } } static void test_partial_backpressure_and_retained_first(void) { ngx_buf_t bufs[4]; ngx_chain_t links[4]; ngx_chain_t *cursor; ngx_http_request_t r; ngx_http_zstd_ctx_t ctx; ngx_pool_t pool = {0}; make_chain(links, bufs, 4, 4); cursor = links; r.pool = &pool; init_ctx(&ctx); check(ngx_http_zstd_filter_add_data(&r, &ctx, &cursor) == NGX_OK, "partial case loads first caller buffer"); ctx.buffer_in.pos = 1; check(ngx_http_zstd_filter_add_data(&r, &ctx, &cursor) == NGX_OK, "backpressure leaves partial input selected"); check(cursor == &links[1], "partial input does not advance the tail cursor"); check(ngx_http_zstd_filter_copy_input(&r, &ctx, cursor) == NGX_OK, "unconsumed tail is copied lazily"); check(pool.calls == 3, "only the three-link tail is copied"); check(ctx.in != NULL && ctx.in->buf == &bufs[1], "copied tail begins after active buffer"); cursor = &links[0]; ctx.buffer_in.pos = ctx.buffer_in.size; check(ngx_http_zstd_filter_add_data(&r, &ctx, &cursor) == NGX_OK, "next callback drains retained input first"); check(ctx.in_buf == &bufs[1] && cursor == &links[0], "retained link wins over new caller input"); check(pool.frees == 1, "consumed retained wrapper returns to pool"); } static void test_many_links_and_lazy_failure(void) { enum { MANY = 2048 }; ngx_buf_t *bufs; ngx_chain_t *links, *cl; ngx_http_request_t r; ngx_http_zstd_ctx_t ctx; ngx_pool_t pool = {0}; size_t i, copied; bufs = calloc(MANY, sizeof(*bufs)); links = calloc(MANY, sizeof(*links)); check(bufs != NULL && links != NULL, "many-link fixture allocation"); if (bufs == NULL || links == NULL) { free(bufs); free(links); return; } make_chain(links, bufs, MANY, 1); r.pool = &pool; init_ctx(&ctx); check(ngx_http_zstd_filter_copy_input(&r, &ctx, links) == NGX_OK, "many-link lazy copy succeeds"); copied = 0; for (cl = ctx.in; cl != NULL; cl = cl->next) { check(cl->buf == &bufs[copied], "many-link copy preserves order"); copied++; } check(copied == MANY, "many-link copy retains every tail link"); pool.calls = 0; pool.fail_at = 4; pool.free = NULL; init_ctx(&ctx); check(ngx_http_zstd_filter_copy_input(&r, &ctx, links) == NGX_ERROR, "lazy tail allocation failure is reported"); copied = 0; for (cl = ctx.in; cl != NULL; cl = cl->next) { copied++; } check(copied == 3, "failure retains only pool-owned prefix copies"); check(pool.calls == 4, "fault fires at the requested lazy allocation"); for (i = 0; i + 1 < MANY; i++) { check(links[i].next == &links[i + 1], "lazy failure never rewrites caller-owned links"); } free(bufs); free(links); } static void test_flush_and_last_carriers(void) { ngx_buf_t bufs[3]; ngx_chain_t links[3]; ngx_chain_t *cursor; ngx_http_request_t r; ngx_http_zstd_ctx_t ctx; ngx_pool_t pool = {0}; make_chain(links, bufs, 3, 0); r.pool = &pool; init_ctx(&ctx); bufs[0].flush = 1; cursor = links; check(ngx_http_zstd_filter_add_data(&r, &ctx, &cursor) == NGX_OK, "zero-size flush carrier schedules compression"); check(ctx.flush == 1 && ctx.last == 0, "flush carrier sets flush without last"); init_ctx(&ctx); bufs[0].flush = 1; bufs[0].last_buf = 1; cursor = links; check(ngx_http_zstd_filter_add_data(&r, &ctx, &cursor) == NGX_OK, "zero-size terminal carrier schedules compression"); check(ctx.last == 1 && ctx.flush == 0, "last_buf keeps priority over a co-carried flush"); init_ctx(&ctx); cursor = links; bufs[0].flush = 0; bufs[0].last_buf = 0; check(ngx_http_zstd_filter_add_data(&r, &ctx, &cursor) == NGX_AGAIN, "ordinary empty carrier is skipped"); check(cursor == &links[1], "empty carrier advances only its local link"); } int main(void) { test_fully_drained_no_copy(); test_partial_backpressure_and_retained_first(); test_many_links_and_lazy_failure(); test_flush_and_last_carriers(); if (failures != 0) { fprintf(stderr, "FAIL: %d input-cursor assertions failed\n", failures); return 1; } puts("OK: input cursor (fully drained, partial/backpressure, retained-first, " "lazy-failure, many-link, flush/last carrier)"); return 0; } myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_input_chain_cursor_unit.sh000077500000000000000000000051561525232545500301330ustar00rootroot00000000000000#!/bin/bash # Copyright (C) 2026 Thijs Eilander # SPDX-License-Identifier: BSD-2-Clause # # Unit fixture for the body filter's caller-owned input cursor and lazy tail # copy. The two shipped helpers are extracted verbatim, then compiled against # a minimal nginx pool/chain shim. This makes ownership, ordering and the # allocation-failure boundary deterministic without needing socket pressure to # land at one exact event-loop iteration. set -euo pipefail cd "$(dirname "$0")/../.." SRC=src/ngx_http_zstd_filter_module.c CC="${CC:-cc}" OUT="$(mktemp -d "${TMPDIR:-/tmp}/zstd-input-cursor-unit.XXXXXX")" trap 'rm -rf "$OUT"' EXIT extract_function() { local signature="$1" label="$2" sig start rel end sig="$(grep -n "^${signature}" "$SRC" | head -1 | cut -d: -f1)" if [ -z "$sig" ]; then echo "FAIL: could not locate $label in $SRC" >&2 exit 1 fi start=$((sig - 1)) rel="$(tail -n "+${start}" "$SRC" | grep -n '^}' | head -1 | cut -d: -f1)" end=$((start + rel - 1)) sed -n "${start},${end}p" "$SRC" } { echo "/* Generated by test_input_chain_cursor_unit.sh from $SRC. */" extract_function 'ngx_http_zstd_filter_copy_input(' copy_input echo extract_function 'ngx_http_zstd_filter_add_data(' add_data } >"$OUT/generated_input_cursor.inc" for marker in ngx_http_zstd_filter_copy_input ngx_http_zstd_filter_add_data \ 'retained = ctx->in != NULL' 'ngx_free_chain(r->pool, cl)'; do if ! grep -q "$marker" "$OUT/generated_input_cursor.inc"; then echo "FAIL: extracted fragment is missing $marker" >&2 exit 1 fi done # The body-filter return boundary is part of the contract but deliberately is # not duplicated in the C shim. Pin the production wiring: an unconsumed local # cursor must be copied immediately before the ordinary pause return, and a # copy failure must enter the existing terminal label. if ! sed -n '/if (ctx->out == NULL && !flush)/,/return ctx->busy/p' "$SRC" \ | grep -A4 'ngx_http_zstd_filter_copy_input' \ | grep -q 'goto failed;' then echo "FAIL: body-filter pause return no longer terminally handles lazy-copy failure" >&2 exit 1 fi if ! sed -n '/ngx_http_zstd_filter_compress(r, ctx,/,/);/p' "$SRC" \ | grep -q 'ctx->in == NULL && in == NULL' then echo "FAIL: compressor terminal decision does not include both input queues" >&2 exit 1 fi cp ci/tools/test_input_chain_cursor_unit.c "$OUT/test_input_chain_cursor_unit.c" "$CC" -std=c99 -O2 -Wall -Wextra -Werror -I"$OUT" \ -o "$OUT/test_input_chain_cursor_unit" \ "$OUT/test_input_chain_cursor_unit.c" timeout 30s "$OUT/test_input_chain_cursor_unit" myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_ldm_budget_config.py000077500000000000000000000226361525232545500266510ustar00rootroot00000000000000#!/usr/bin/env python3 """Config-load regression: ``zstd_long`` + ``zstd_max_cctx_memory``. ``ZSTD_estimateCStreamSize_usingCCtxParams()`` divides by ``params.ldmParams.hashRateLog``. libzstd derives the LDM sub-parameters lazily during compression setup, so a ``ZSTD_CCtx_params`` that has only had ``ZSTD_c_enableLongDistanceMatching`` set still carries ``hashRateLog == 0``. The config-load budget check called the estimator on exactly such a params object, so ``nginx -t`` died with **SIGFPE** (exit 136) for any configuration combining ``zstd_long on`` with ``zstd_max_cctx_memory``. Setting ``zstd_window_log`` did not avoid it -- the divisor is unrelated to the window. Why this test is not folded into ``ci/t/00-filter.t``: that harness's ``--- must_die`` cannot tell a crash from a diagnostic. A SIGFPE and a clean ``[emerg]`` both "die", so a ``must_die`` block would have passed against the crashing build and proved nothing. This test asserts on the **exit status and its sign** -- a negative ``returncode`` (or 128+N) is a signal and always a failure, whatever the message says. Arms ---- 1. ``zstd_long on`` + ``zstd_max_cctx_memory 256m`` -> must PASS. The regression arm. Pre-fix: SIGFPE. 2. ``zstd_long on`` + ``zstd_max_cctx_memory 64m`` -> must be REFUSED with a diagnostic naming the budget. This is the arm that keeps the fix honest: it proves the estimate is still *enforced* under LDM rather than skipped, so the operator does not silently lose the bound they asked for. It also pins the magnitude -- the seeded estimate must land in the >=128 MB range that LDM's 128 MB default window implies, which is what distinguishes a correctly seeded estimate from one computed against the level's much smaller default window. 3. ``zstd_long on`` + ``zstd_window_log 20`` + ``64m`` -> must PASS. A capped window brings the same profile back under the same budget, proving arm 2's refusal is a real measurement and not a blanket "LDM always fails". 4. ``zstd_max_cctx_memory 256m`` alone (no LDM) -> must PASS. 5. ``zstd_long on`` alone (no budget) -> must PASS. Arms 4 and 5 are the controls: both passed on the crashing build too, so they are what stops this test from going vacuous if the estimator block is ever short-circuited or compiled out entirely. """ import argparse import pathlib import re import subprocess import sys import tempfile # LDM raises libzstd's default windowLog to 27 (128 MB). A correctly # seeded estimate for level 3 + LDM with no zstd_window_log is therefore # ~144 MB (measured 144328225 against libzstd 1.5.7; a real streaming # ZSTD_CCtx over 256 MB of input reported ZSTD_sizeof_CCtx() 144262689). # An estimate computed against the level's default window instead would # be ~3.6 MB, two orders of magnitude low, and would wrongly fit in the # 64 MB budget of arm 2. MIN_EXPECTED_LDM_ESTIMATE = 128 * 1024 * 1024 def parse_args(): p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) p.add_argument("--nginx-binary", required=True) p.add_argument("--filter-module") p.add_argument("--port", type=int, default=18099) return p.parse_args() def detect_module(explicit, nginx: pathlib.Path): if explicit: return pathlib.Path(explicit) sib = nginx.parent / "ngx_http_zstd_filter_module.so" return sib if sib.exists() else None def config_test(nginx: pathlib.Path, module, port: int, directives: str): """Run ``nginx -t`` on a minimal config; return (returncode, output).""" with tempfile.TemporaryDirectory() as td: root = pathlib.Path(td) (root / "conf").mkdir() (root / "logs").mkdir() load = f"load_module {module};\n" if module else "" (root / "conf" / "nginx.conf").write_text( f"{load}" "events {}\n" "http {\n" f" server {{\n" f" listen 127.0.0.1:{port};\n" " zstd on;\n" f" {directives}\n" " }\n" "}\n" ) proc = subprocess.run( [str(nginx), "-p", str(root), "-c", "conf/nginx.conf", "-t"], capture_output=True, text=True, timeout=60, check=False, ) return proc.returncode, proc.stdout + proc.stderr def check_not_a_signal(name: str, rc: int, out: str) -> list: """The core assertion: nginx -t must never terminate on a signal.""" if rc < 0: return [ ( f"{name}: nginx -t died on signal {-rc} " f"(SIGFPE is 8); output: {out.strip()!r}" ) ] if rc > 128: return [ ( f"{name}: nginx -t exited {rc} (128+{rc - 128}), which is " f"a signal death, not a diagnostic; output: " f"{out.strip()!r}" ) ] return [] def main() -> int: args = parse_args() nginx = pathlib.Path(args.nginx_binary).resolve() if not nginx.exists(): print(f"FAIL: nginx binary not found: {nginx}", file=sys.stderr) return 1 module = detect_module(args.filter_module, nginx) if module is not None: module = module.resolve() if not module.exists(): print(f"FAIL: filter module not found: {module}", file=sys.stderr) return 1 failures = [] # --- Arm 1: the regression. Pre-fix this was SIGFPE / exit 136. ----- rc, out = config_test( nginx, module, args.port, "zstd_long on; zstd_max_cctx_memory 256m;" ) failures += check_not_a_signal("arm1 ldm+256m", rc, out) if rc != 0: failures.append( f"arm1 ldm+256m: expected nginx -t to succeed (level 3 + LDM " f"needs ~144 MB, which fits in 256m) but it exited {rc}; " f"output: {out.strip()!r}" ) print(f" arm1 zstd_long + 256m -> exit {rc} (want 0)") # --- Arm 2: the budget is still ENFORCED under LDM. ---------------- rc, out = config_test( nginx, module, args.port, "zstd_long on; zstd_max_cctx_memory 64m;" ) failures += check_not_a_signal("arm2 ldm+64m", rc, out) if rc == 0: failures.append( "arm2 ldm+64m: nginx -t SUCCEEDED, but level 3 + LDM needs " "~144 MB, which exceeds the 64m budget. The operator asked " "for a bound and silently did not get one -- the estimator " "is either skipping LDM profiles or sizing against the " "wrong window." ) else: if "zstd_max_cctx_memory" not in out: failures.append( f"arm2 ldm+64m: refused, but the diagnostic does not name " f'"zstd_max_cctx_memory", so the operator cannot tell why: ' f"{out.strip()!r}" ) m = re.search(r"need ~(\d+) bytes", out) if not m: failures.append( f"arm2 ldm+64m: diagnostic does not report the estimated " f"byte count: {out.strip()!r}" ) elif int(m.group(1)) < MIN_EXPECTED_LDM_ESTIMATE: failures.append( f"arm2 ldm+64m: estimate {m.group(1)} bytes is below " f"{MIN_EXPECTED_LDM_ESTIMATE}. LDM raises the default " f"windowLog to 27 (128 MB), so a correctly seeded estimate " f"cannot be this small -- the LDM sub-parameters were not " f"seeded and the estimate was computed against the level's " f"default window." ) else: print(f" arm2 estimate reported -> {m.group(1)} bytes") print(f" arm2 zstd_long + 64m -> exit {rc} (want non-zero)") # --- Arm 3: a capped window fits the same budget. ------------------ rc, out = config_test( nginx, module, args.port, "zstd_long on; zstd_window_log 20; zstd_max_cctx_memory 64m;", ) failures += check_not_a_signal("arm3 ldm+wl20+64m", rc, out) if rc != 0: failures.append( f"arm3 ldm+wl20+64m: expected success (a 2^20 window under LDM " f"needs ~2.7 MB, well inside 64m) but nginx -t exited {rc}. " f"Arm 2's refusal is therefore a blanket rejection of LDM, not " f"a measurement; output: {out.strip()!r}" ) print(f" arm3 zstd_long + wl20 + 64m -> exit {rc} (want 0)") # --- Arm 4 (control): budget alone, no LDM. ----------------------- rc, out = config_test(nginx, module, args.port, "zstd_max_cctx_memory 256m;") failures += check_not_a_signal("arm4 256m only", rc, out) if rc != 0: failures.append( f"arm4 256m only (control): exited {rc}; output: {out.strip()!r}" ) print(f" arm4 256m alone (control) -> exit {rc} (want 0)") # --- Arm 5 (control): LDM alone, no budget. ----------------------- rc, out = config_test(nginx, module, args.port, "zstd_long on;") failures += check_not_a_signal("arm5 ldm only", rc, out) if rc != 0: failures.append( f"arm5 ldm only (control): exited {rc}; output: {out.strip()!r}" ) print(f" arm5 zstd_long alone (control) -> exit {rc} (want 0)") if failures: print("\nFAIL: zstd_long + zstd_max_cctx_memory config test", file=sys.stderr) for f in failures: print(f" - {f}", file=sys.stderr) return 1 print("\nOK: zstd_long + zstd_max_cctx_memory config validation (5/5 arms)") return 0 if __name__ == "__main__": sys.exit(main()) myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_lsan_positive_control_unit.sh000077500000000000000000000100251525232545500306430ustar00rootroot00000000000000#!/usr/bin/env bash # Copyright (C) 2026 Thijs Eilander # SPDX-License-Identifier: BSD-2-Clause # # Unit and negative-control fixture for lsan_log_verdict.sh and the shared # deliberate-leak canary. No nginx, libzstd, network, or runner privileges. # Usage: ci/tools/test_lsan_positive_control_unit.sh set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=ci/tools/lsan_log_verdict.sh . "$SCRIPT_DIR/lsan_log_verdict.sh" if ! command -v cc >/dev/null 2>&1; then echo "SKIP: no cc on PATH -- LSan verdict fixture needs one" >&2 exit 0 fi WORK="$(mktemp -d "${TMPDIR:-/tmp}/lsan-log-verdict-unit.XXXXXX")" trap 'rm -rf "$WORK"' EXIT fail=0 run_case() { local want_rc="$1" label="$2" control_rc="$3" shift 3 local rc if lsan_log_verdict "$control_rc" "$@" >/dev/null 2>&1; then rc=0 else rc=$? fi if [ "$rc" -ne "$want_rc" ]; then echo "FAIL: $label -- rc=$rc, want $want_rc" >&2 fail=1 else echo "OK: $label (rc=$rc)" fi } cat >"$WORK/finding.log" <<'EOF' ==10==LeakSanitizer: checking for leaks ==10==ERROR: LeakSanitizer: detected memory leaks SUMMARY: AddressSanitizer: 1234 byte(s) leaked in 1 allocation(s). EOF cat >"$WORK/actual-failure.log" <<'EOF' ==11==LeakSanitizer: checking for leaks ==11==Failed suspending threads. EOF cat >"$WORK/thread-exit-retry.log" <<'EOF' ==111==Could not attach to thread 123 (errno 3). ==111==LeakSanitizer: checking for leaks EOF cat >"$WORK/preflight-with-check.log" <<'EOF' ==12==WARNING: ptrace appears to be blocked (is seccomp enabled?). LeakSanitizer may hang. ==12==Child exited with signal 112. ==12==LeakSanitizer: checking for leaks EOF cat >"$WORK/preflight-without-check.log" <<'EOF' ==13==WARNING: ptrace appears to be blocked (is seccomp enabled?). LeakSanitizer may hang. ==13==Child exited with signal 31. EOF cat >"$WORK/clean.log" <<'EOF' ==14==LeakSanitizer: checking for leaks EOF run_case 1 "real sanitizer finding fails" 0 "$WORK/finding.log" run_case 2 "real stop-the-world failure is indeterminate" 0 "$WORK/actual-failure.log" run_case 0 "a thread-exit attach retry is non-fatal" 0 "$WORK/thread-exit-retry.log" run_case 0 "preflight heuristic is non-fatal when real check and canary pass" \ 0 "$WORK/preflight-with-check.log" run_case 2 "preflight warning without exit-hook marker is indeterminate" \ 0 "$WORK/preflight-without-check.log" run_case 2 "clean marker with failed canary is indeterminate" 1 "$WORK/clean.log" run_case 2 "clean marker with unavailable canary is indeterminate" 2 "$WORK/clean.log" run_case 2 "missing logs are indeterminate" 0 run_case 0 "clean marker with passing canary passes" 0 "$WORK/clean.log" run_case 2 "every process log must contain the exit-hook marker" 0 \ "$WORK/clean.log" "$WORK/preflight-without-check.log" MAPFILE_CHECKED_FAULT=partial-error export MAPFILE_CHECKED_FAULT if lsan_check_log_glob "$WORK/*.log" >/dev/null 2>&1 then list_rc=0 else list_rc=$? fi unset MAPFILE_CHECKED_FAULT if [ "$list_rc" -eq 23 ]; then echo "OK: partial log-list producer failure is propagated (rc=23)" else echo "FAIL: partial log-list producer failure -- rc=$list_rc, want 23" >&2 fail=1 fi # Negative control: the old gate failed solely on LLVM's heuristic warning, # even when the same log proved the real leak check started and the deliberate # leak control passed. Reproduce that exact false-red decision. if grep -qE 'ptrace appears to be blocked|LeakSanitizer may hang' \ "$WORK/preflight-with-check.log" then echo "OK: negative control reproduces the old false-red preflight gate" else echo "FAIL: negative control did not reproduce the old false-red gate" >&2 fail=1 fi if lsan_positive_control; then echo "OK: deliberate leak canary is detected on this host" else echo "FAIL: deliberate leak canary was not detected on this host" >&2 fail=1 fi if [ "$fail" -ne 0 ]; then echo "FAIL: LSan verdict fixture had failures" >&2 exit 1 fi echo "OK: LSan verdict fixture -- all cases and negative control passed" myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_max_length_cap_unit.c000066400000000000000000000153731525232545500270150ustar00rootroot00000000000000/* * Copyright (C) 2026 Thijs Eilander * SPDX-License-Identifier: BSD-2-Clause * * Unit oracle for ngx_http_zstd_max_length_exceeded() -- the * length-independent zstd_max_length cap checked per iteration of the * streaming body filter (see ci/t/00-filter.t TEST 42/111/112, which cover * the same cap end-to-end through a mock chunked upstream on the native * build; this fixture covers the one comparison-width defect those tests * cannot reach without a 32-bit off_t). * * The function is EXTRACTED VERBATIM from * src/ngx_http_zstd_filter_module.c by test_max_length_cap_unit.sh into * generated_max_length_cap.inc -- this file never re-implements the * comparison, so it cannot quietly agree with a stale copy of itself. * * What is asserted, and why each one matters: * * 1. NO CAP CONFIGURED (max_length == NGX_CONF_UNSET) never fires, * regardless of bytes_in. zstd_max_length is opt-in; an operator who * never set it must not see requests aborted. * * 2. AT THE CAP does not fire (bytes_in == max_length): the check is a * strict "exceeded", not "at or above". A body landing exactly on the * configured size must still be allowed to finish compressing -- * this is TEST 111's boundary, asserted here directly against the * predicate rather than through a live upstream. * * 3. ONE BYTE OVER THE CAP fires (bytes_in == max_length + 1): the * complementary boundary to (2), and TEST 112's case. * * 4. A LARGE bytes_in (well past INT32_MAX, still a legal uint64_t) must * still correctly exceed a small max_length. On a genuine 32-bit * off_t build (plain -m32, no _FILE_OFFSET_BITS override -- the * second pass in test_max_length_cap_unit.sh) off_t is 4 bytes, same * as ssize_t; the historical `(off_t) bytes_in > (off_t) max_length` * form cast the unsigned accumulator through that narrow off_t * instead of comparing unsigned-to-unsigned as this predicate does, * truncating a large bytes_in and losing the comparison. Sweeping * max_length up through and past INT32_MAX in assertions 2 and 3 * above, together with this large-bytes_in probe, is what makes this * fixture catch a regression back to that signed-cast form on that * build; an ILP32+LFS build (off_t widened to 64 bits) or a native * 64-bit build cannot distinguish the two forms at all. * * The sweep is exhaustive over the interesting boundary values rather than * a handful of hand-picked cases. */ #include #include #include #include #include #include /* Minimal nginx shim: the extracted code uses these names only. */ typedef intptr_t ngx_flag_t; #define NGX_CONF_UNSET -1 #include "generated_max_length_cap.inc" static long long failures; static void fail(const char *what, uint64_t bytes_in, ssize_t max_length, int got, int want) { failures++; if (failures <= 20) { fprintf(stderr, "FAIL %s: bytes_in=%" PRIu64 " max_length=%zd -> %d, want %d\n", what, bytes_in, (ssize_t) max_length, got, want); } } int main(void) { /* * max_length values spanning the legal ssize_t range: unset, tiny, * astride INT32_MAX (the width a truncating/narrowing cast would * disagree at), and the largest value ssize_t can hold on this build. */ static const ssize_t max_lengths[] = { (ssize_t) NGX_CONF_UNSET, 0, 1, 100, 4999, 5000, (ssize_t) INT32_MAX - 1, (ssize_t) INT32_MAX, #if SSIZE_MAX > INT32_MAX /* Only representable when ssize_t is wider than 32 bits (the * native 64-bit build); on ILP32, INT32_MAX is already SSIZE_MAX * and one past it is not a legal max_length. */ (ssize_t) INT32_MAX + 1, #endif }; size_t mi; long long swept = 0; for (mi = 0; mi < sizeof(max_lengths) / sizeof(max_lengths[0]); mi++) { ssize_t max_length = max_lengths[mi]; /* 1. no cap configured never fires, at any bytes_in tried below. */ if (max_length == NGX_CONF_UNSET) { static const uint64_t probes[] = { 0, 1, 5000, (uint64_t) INT32_MAX, (uint64_t) INT32_MAX + 1, (uint64_t) UINT32_MAX + 1000, }; size_t pi; for (pi = 0; pi < sizeof(probes) / sizeof(probes[0]); pi++) { int got = ngx_http_zstd_max_length_exceeded(probes[pi], max_length); swept++; if (got) { fail("unset-never-fires", probes[pi], max_length, got, 0); } } continue; } /* 2. at the cap: must NOT fire. */ { uint64_t bytes_in = (uint64_t) max_length; int got = ngx_http_zstd_max_length_exceeded(bytes_in, max_length); swept++; if (got) { fail("at-cap", bytes_in, max_length, got, 0); } } /* 3. one byte over: must fire. Computed in uint64_t so max_length * == SSIZE_MAX (the ILP32 build's INT32_MAX case) cannot overflow * the ssize_t addition before widening. */ { uint64_t bytes_in = (uint64_t) max_length + 1; int got = ngx_http_zstd_max_length_exceeded(bytes_in, max_length); swept++; if (!got) { fail("one-over-cap", bytes_in, max_length, got, 1); } } /* * 4. bytes_in far past INT32_MAX must still correctly compare * against a small max_length -- the 32-bit off_t regression case. * Skip when max_length itself is already >= this probe, since then * the probe would be at-or-under the cap and the expected answer * would flip. */ { uint64_t bytes_in = (uint64_t) INT32_MAX + 1000000; if ((uint64_t) max_length < bytes_in) { int got = ngx_http_zstd_max_length_exceeded(bytes_in, max_length); swept++; if (!got) { fail("large-bytes-in-exceeds-small-cap", bytes_in, max_length, got, 1); } } } } printf("swept %lld combinations\n", swept); if (failures) { printf("FAILED: %lld assertion(s)\n", failures); return 1; } printf("OK: max_length cap (unset never fires, at-cap allows, " "over-cap fires, large bytes_in vs small cap fires)\n"); return 0; } myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_max_length_cap_unit.sh000077500000000000000000000122611525232545500272010ustar00rootroot00000000000000#!/bin/bash # Copyright (C) 2026 Thijs Eilander # SPDX-License-Identifier: BSD-2-Clause # # Unit fixture for ngx_http_zstd_max_length_exceeded() -- the # length-independent zstd_max_length cap checked per iteration of the # streaming body filter. # # The predicate compares ctx->bytes_in (uint64_t) against zlcf->max_length # (ssize_t). A signed-cast form of this comparison used to read # `(off_t) bytes_in > (off_t) max_length`: that cast only loses bits when # off_t itself is narrower than the accumulator, which needs a genuine # 32-bit off_t (glibc's legacy default with no _FILE_OFFSET_BITS override). # A native 64-bit host build has an 8-byte off_t regardless, and so does an # ILP32+LFS build (-m32 -D_FILE_OFFSET_BITS=64): neither can distinguish the # signed-cast form from the fixed unsigned form, both give the same answer. # Plain -m32 with NO _FILE_OFFSET_BITS override is the one shape that # reproduces the truncation, so it is not optional coverage; a native or # ILP32+LFS run alone would let a regression back into the signed-cast form # pass silently. # # The function is extracted VERBATIM by line range and compiled standalone, # so a hand-copied duplicate of the arithmetic cannot quietly drift from # production. # # No nginx tree and no libzstd needed: pure C, self-contained. # # REQUIRE_M32: unset/"0"/""/false/no/off (case insensitive) keeps the # default local-dev behaviour -- SKIP with exit 0 when no -m32 toolchain is # present. Any other value (1/true/yes/on, case insensitive, or anything # else non-empty) makes a missing -m32 toolchain a hard FAILURE instead. set -euo pipefail # ci/tools/ -> ci/ -> repo root. cd "$(dirname "$0")/../.." SRC="src/ngx_http_zstd_filter_module.c" CC="${CC:-cc}" OUT="$(mktemp -d "${TMPDIR:-/tmp}/zstd-max-length-cap-unit.XXXXXX")" trap 'rm -rf "$OUT"' EXIT mkdir -p "$OUT" extract() { # $1 = start line, $2 = label; prints "start end" local start="$1" label="$2" rel end if [ -z "$start" ]; then echo "FAIL: could not locate $label in $SRC" >&2 exit 1 fi rel="$(tail -n "+${start}" "$SRC" | grep -n '^}' | head -1 | cut -d: -f1)" end=$((start + rel - 1)) echo "$start $end" } # --- ngx_http_zstd_max_length_exceeded() (the predicate under test) ------ SIG="$(grep -n '^ngx_http_zstd_max_length_exceeded(uint64_t bytes_in' "$SRC" \ | head -1 | cut -d: -f1)" if [ -z "$SIG" ]; then echo "FAIL: could not locate ngx_http_zstd_max_length_exceeded() in $SRC" >&2 exit 1 fi read -r START END < <(extract "$((SIG - 1))" "max_length_exceeded") { echo "/* Generated by test_max_length_cap_unit.sh -- verbatim extraction" echo " * from $SRC:" echo " * ngx_http_zstd_max_length_exceeded lines ${START}-${END}" echo " * Do not edit; re-run the script to regenerate. */" sed -n "${START},${END}p" "$SRC" } > "$OUT/generated_max_length_cap.inc" # Sanity: a source reflow must fail loudly here, not produce a garbage .inc # that still compiles and "passes". for marker in 'ngx_http_zstd_max_length_exceeded(uint64_t bytes_in' \ 'NGX_CONF_UNSET'; do if ! grep -q "$marker" "$OUT/generated_max_length_cap.inc"; then echo "FAIL: extracted fragment is missing \"$marker\"" >&2 cat "$OUT/generated_max_length_cap.inc" >&2 exit 1 fi done cp ci/tools/test_max_length_cap_unit.c "$OUT/test_max_length_cap_unit.c" "$CC" -Wall -Wextra -Werror -O2 -I"$OUT" \ -o "$OUT/max_length_cap_unit" "$OUT/test_max_length_cap_unit.c" "$OUT/max_length_cap_unit" echo "OK: max_length cap unit fixture (extracted from current $SRC)" # 32-bit off_t pass: a plain native build on this (64-bit) host, and even an # ILP32+LFS build (-m32 -D_FILE_OFFSET_BITS=64), both give off_t 8 bytes -- # casting bytes_in through it never truncates there, so the historical # `(off_t) bytes_in > (off_t) max_length` form cannot be distinguished from # the fixed unsigned comparison on either. The defect needs off_t ITSELF # narrow: plain -m32 with NO _FILE_OFFSET_BITS override gives a 4-byte # off_t (glibc's legacy 32-bit off_t), which is what this cap's comment # describes and what reproduces the cast losing bits. Not optional # coverage; skipped (not failed) when the runner has no 32-bit toolchain # at all -- UNLESS REQUIRE_M32 is set to a truthy value (1/true/yes/on, # case insensitive; anything else non-empty and not "0" also counts as # set), in which case a missing toolchain is a hard failure rather than a # silent skip (set in CI so a runner image that drops multilib loses this # lens loudly, not quietly). require_m32="${REQUIRE_M32:-0}" case "${require_m32,,}" in 0 | "" | false | no | off) require_m32_set=0 ;; *) require_m32_set=1 ;; esac if "$CC" -m32 -x c -o /dev/null - <<<'int main(void){return 0;}' \ >/dev/null 2>&1 then "$CC" -m32 -Wall -Wextra -Werror -O2 -I"$OUT" \ -o "$OUT/max_length_cap_unit_32off" "$OUT/test_max_length_cap_unit.c" "$OUT/max_length_cap_unit_32off" echo "OK: max_length cap unit fixture (32-bit off_t, bytes_in > INT32_MAX)" elif [ "$require_m32_set" = "1" ]; then echo "FAIL: no -m32 toolchain available and REQUIRE_M32 is set; 32-bit off_t pass cannot run" >&2 exit 1 else echo "SKIP: no -m32 toolchain available; 32-bit off_t pass not run" >&2 fi myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_max_length_diagnostics.sh000077500000000000000000000021541525232545500277060ustar00rootroot00000000000000#!/usr/bin/env bash # Pin distinct, truthful diagnostics for known- and unknown-length overruns. set -euo pipefail root=$(git rev-parse --show-toplevel) src="$root/src/ngx_http_zstd_filter_module.c" assert_contract() { local file=$1 grep -Fq 'if (ctx->pledged_size >= 0)' "$file" || return 1 grep -Fq 'after %uL bytes on a response with declared ' "$file" || return 1 grep -Fq 'Content-Length %O; aborting to protect the ' "$file" || return 1 grep -Fq 'after %uL bytes on a response with no ' "$file" || return 1 grep -Fq 'ctx->bytes_in,' "$file" || return 1 grep -Fq 'ctx->pledged_size);' "$file" || return 1 } assert_contract "$src" work=$(mktemp -d "${TMPDIR:-/tmp}/max-length-diagnostics.XXXXXX") trap 'rm -rf "$work"' EXIT cp "$src" "$work/filter.c" sed -i 's/after %uL bytes on a response with declared /after %uL bytes on a response with no /' "$work/filter.c" if assert_contract "$work/filter.c" >/dev/null 2>&1; then echo 'FAIL: lying-known-length diagnostic mutant did not make contract red' >&2 exit 1 fi echo 'OK: known/unknown max-length abort diagnostics and negative control' myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_nginx_releases.sh000077500000000000000000000151111525232545500261740ustar00rootroot00000000000000#!/usr/bin/env bash # # Hermetic controls for ci/tools/nginx-releases.sh, the one reader of the # nginx GitHub releases feed that the CI workflows, ci-build.sh and # bump-versions.sh share. The feed's parsing rules (parity, decoys, paging, # the cap) are covered case by case in test_bump_versions.sh through the # same functions; this file pins the command-line contract every workflow # step relies on: what is printed, what is fatal, and that an authenticated # call keeps the token out of curl's argv. # # Usage: ci/tools/test_nginx_releases.sh set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" RESOLVER="$SCRIPT_DIR/nginx-releases.sh" fail=0 say() { printf '%s\n' "$*"; } ok() { printf 'OK: %s\n' "$*"; } bad() { printf 'FAIL: %s\n' "$*" fail=1 } work="$(mktemp -d "${TMPDIR:-/tmp}/http-zstd-resolver-test.XXXXXX")" trap 'rm -rf -- "$work"' EXIT mkdir -p "$work/bin" # curl stub: answers the releases endpoint from NGINX_FEED, logs argv and any # stdin config so the token path can be checked. cat >"$work/bin/curl" <<'EOF' #!/bin/bash [ -z "${CURL_ARGV_LOG:-}" ] || printf '%s\n' "$@" >>"$CURL_ARGV_LOG" url="" prev="" for a in "$@"; do case "$a" in http*) url="$a" ;; esac [ "$prev" = "--config" ] && [ "$a" = "-" ] && cat >>"${CURL_HEADER_LOG:-/dev/null}" prev="$a" done case "$url" in *api.github.com/repos/nginx/nginx/releases?per_page=30\&page=*) case "${NGINX_FEED:-json}" in no-even) echo '[{"tag_name": "release-1.31.5", "draft": false, "prerelease": false}]' ;; no-odd) echo '[{"tag_name": "release-1.30.4", "draft": false, "prerelease": false},' echo ' {"tag_name": "release-1.28.0", "draft": false, "prerelease": false}]' ;; dead) echo "curl: (22) stub: feed unreachable" >&2 exit 22 ;; endless) echo "[" for i in $(seq 1 30); do echo " {\"tag_name\": \"release-1.29.$i\", \"draft\": false, \"prerelease\": false}$([ "$i" -lt 30 ] && echo ,)" done echo "]" ;; *) echo '[{"tag_name": "release-1.28.0", "draft": false, "prerelease": false},' echo ' {"tag_name": "release-1.33.0", "draft": true, "prerelease": false},' echo ' {"tag_name": "release-1.30.4", "draft": false, "prerelease": false},' echo ' {"tag_name": "release-1.31.5", "draft": false, "prerelease": false}]' ;; esac exit 0 ;; esac echo "stub: unexpected url $url" >&2 exit 22 EOF chmod +x "$work/bin/curl" export PATH="$work/bin:$PATH" run() { # NGINX_FEED-mode args... -> stdout in $out, stderr in $err, status in $rc local mode="$1" shift rc=0 out="$(NGINX_FEED="$mode" bash "$RESOLVER" "$@" 2>"$work/err")" || rc=$? err="$(cat "$work/err")" } say "== mainline / stable / both print the newest release of each parity ==" run json mainline [ "$rc" -eq 0 ] && [ "$out" = "1.31.5" ] && ok "mainline -> 1.31.5" || bad "mainline: rc=$rc out='$out' $err" run json stable [ "$rc" -eq 0 ] && [ "$out" = "1.30.4" ] && ok "stable -> 1.30.4 (the legacy 1.28.0 and the draft 1.33.0 ignored)" || bad "stable: rc=$rc out='$out' $err" run json both [ "$rc" -eq 0 ] && [ "$out" = "1.30.4 1.31.5" ] && ok "both -> '1.30.4 1.31.5'" || bad "both: rc=$rc out='$out' $err" run json [ "$rc" -eq 0 ] && [ "$out" = "1.31.5" ] && ok "no argument defaults to mainline" || bad "default: rc=$rc out='$out' $err" say "== a line the feed does not show is fatal by name, and prints nothing ==" run no-even stable [ "$rc" -ne 0 ] && [ -z "$out" ] && grep -q 'shows no stable' <<<"$err" && ok "stable absent: fatal, nothing printed" || bad "stable absent: rc=$rc out='$out' $err" run no-even mainline [ "$rc" -eq 0 ] && [ "$out" = "1.31.5" ] && ok "mainline still resolves when only the stable line is absent" || bad "mainline with no stable: rc=$rc out='$out' $err" run no-even both [ "$rc" -ne 0 ] && [ -z "$out" ] && ok "both with a line absent: fatal, nothing printed" || bad "both with no stable: rc=$rc out='$out' $err" # The other line, with its own guard and its own message: a feed of only # even-minor releases must refuse mainline by name, still serve stable, and # refuse both. (A guard that fell through to the stable one would print the # wrong diagnostic; a guard that fell through to the echo would print "-".) run no-odd mainline [ "$rc" -eq 1 ] && [ -z "$out" ] && grep -q 'shows no mainline (odd-minor) release' <<<"$err" && ok "mainline absent: fatal by name, status 1, nothing printed" || bad "mainline absent: rc=$rc out='$out' $err" run no-odd stable [ "$rc" -eq 0 ] && [ "$out" = "1.30.4" ] && ok "stable still resolves when only the mainline line is absent" || bad "stable with no mainline: rc=$rc out='$out' $err" run no-odd both [ "$rc" -eq 1 ] && [ -z "$out" ] && grep -q 'shows no mainline' <<<"$err" && ok "both with no mainline: fatal by name, nothing printed" || bad "both with no mainline: rc=$rc out='$out' $err" say "== the page cap is a refusal ==" run endless mainline [ "$rc" -ne 0 ] && [ -z "$out" ] && grep -q 'did not end within' <<<"$err" && ok "endless feed: refused at the cap, nothing printed" || bad "endless: rc=$rc out='$out' $err" say "== bad argument, and an unreachable feed ==" run json weekly [ "$rc" -eq 2 ] && grep -q '^usage:' <<<"$err" && ok "unknown line name: usage, status 2" || bad "usage: rc=$rc $err" run dead weekly [ "$rc" -eq 2 ] && grep -q '^usage:' <<<"$err" && ok "unknown line name is usage (2) even when the feed is unreachable: the selector is checked first" || bad "usage-dead: rc=$rc $err" run dead mainline [ "$rc" -eq 1 ] && [ -z "$out" ] && grep -q 'could not query' <<<"$err" && ok "unreachable feed: status 1, named, nothing printed" || bad "dead: rc=$rc out='$out' $err" say "== GH_TOKEN goes through curl's stdin, never its argv ==" argv_log="$work/argv" header_log="$work/header" token='resolver-test-token-not-for-output' rc=0 out="$(GH_TOKEN="$token" CURL_ARGV_LOG="$argv_log" CURL_HEADER_LOG="$header_log" bash "$RESOLVER" mainline 2>"$work/err")" || rc=$? if [ "$rc" -ne 0 ] || [ "$out" != "1.31.5" ]; then bad "token: rc=$rc out='$out' $(cat "$work/err")" elif grep -Fq "$token" "$argv_log"; then bad "token: the token appeared in curl's arguments" elif ! grep -Fq "Authorization: Bearer $token" "$header_log"; then bad "token: the Authorization header never reached curl's stdin" else ok "token: sent through stdin, absent from argv" fi if [ "$fail" -eq 0 ]; then say "all nginx-releases.sh cases pass" else say "one or more nginx-releases.sh cases FAILED" fi exit "$fail" myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_no_http_block.sh000077500000000000000000000025721525232545500260220ustar00rootroot00000000000000#!/bin/sh # An nginx that carries this module must still start without an http{} # block: a stream-only or mail-only deployment compiles the module in # (distribution builds enable every module) and configures none of it. # The #284 init-module hook treated the absent http main conf as an # error, so such a binary failed nginx -t -- and, because init-module # failures are silent, with no diagnostic at all. set -eu nginx_bin=${1:?usage: test_no_http_block.sh } work=$(mktemp -d "${TMPDIR:-/tmp}/no-http-block.XXXXXX") trap 'rm -rf "$work"' EXIT mkdir -p "$work/logs" printf '%s\n' \ 'pid logs/nginx.pid; error_log logs/error.log; events {}' \ > "$work/nohttp.conf" # The regression gate: an http-less configuration must validate. if ! "$nginx_bin" -t -p "$work" -c nohttp.conf; then echo 'FAIL: an http-less configuration was refused' >&2 exit 1 fi # Sanity control on the same binary: a configured http block still # validates, so the pass above is the no-op branch, not a disabled # check. (The refusal arms themselves need a runtime libzstd older # than the build headers; ci/tests/unit/test_version_policy.c covers # the policy table.) printf '%s\n' \ 'pid logs/nginx.pid; error_log logs/error.log; events {}' \ 'http { zstd on; }' \ > "$work/http.conf" "$nginx_bin" -t -p "$work" -c http.conf echo 'OK: http-less configuration validates' myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_package_artifact.py000077500000000000000000000150201525232545500264530ustar00rootroot00000000000000#!/usr/bin/env python3 import argparse import dataclasses import pathlib import shutil import subprocess import sys import tempfile MODULE_DIR = pathlib.Path(__file__).resolve().parent TEST_ENCODING = MODULE_DIR / "test_encoding.py" TEST_TERMINAL_FRAME = MODULE_DIR / "test_terminal_frame.py" @dataclasses.dataclass(frozen=True) class PackageLayout: package_root: pathlib.Path flavor: str modules_dir: pathlib.Path filter_module: pathlib.Path static_module: pathlib.Path load_snippet: pathlib.Path def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description=( "Validate a built nginx/angie http-zstd package artifact by checking " "its installed files and re-running the zstd smoke tests against the " "packaged module .so files." ) ) parser.add_argument( "--nginx-binary", required=True, help="Path to the nginx or angie binary used to run the smoke tests.", ) parser.add_argument( "--package-root", help="Path to an already extracted package root (contains usr/lib/... and usr/share/...).", ) parser.add_argument( "--deb-path", help="Path to a .deb package to extract and validate.", ) parser.add_argument( "--zstd-bin", default=shutil.which("zstd") or "zstd", help="Path to the zstd CLI used by the child smoke tests.", ) parser.add_argument( "--request-count", type=int, default=8, help="Sequential request count for the large-response smoke test.", ) parser.add_argument( "--concurrent-requests", type=int, default=8, help="Concurrent request count for the large-response smoke test.", ) parser.add_argument( "--terminal-request-count", type=int, default=12, help="Sequential request count for the terminal-frame regression smoke.", ) return parser.parse_args() def extract_deb(deb_path: pathlib.Path, destination: pathlib.Path) -> pathlib.Path: subprocess.run( ["dpkg-deb", "-x", str(deb_path), str(destination)], check=True, capture_output=True, text=True, ) return destination def detect_package_layout(package_root: pathlib.Path) -> PackageLayout: candidates = ( ( "angie", package_root / "usr/lib/angie/modules", package_root / "usr/share/angie/modules-available/mod-http-zstd.conf", ), ( "nginx", package_root / "usr/lib/nginx/modules", package_root / "usr/share/nginx/modules-available/mod-http-zstd.conf", ), ) for flavor, modules_dir, load_snippet in candidates: filter_module = modules_dir / "ngx_http_zstd_filter_module.so" static_module = modules_dir / "ngx_http_zstd_static_module.so" if filter_module.exists() and static_module.exists() and load_snippet.exists(): return PackageLayout( package_root=package_root, flavor=flavor, modules_dir=modules_dir, filter_module=filter_module, static_module=static_module, load_snippet=load_snippet, ) raise FileNotFoundError( "could not find packaged zstd modules under usr/lib/{angie,nginx}/modules " "with matching mod-http-zstd.conf load snippet" ) def verify_load_snippet(layout: PackageLayout) -> None: snippet = layout.load_snippet.read_text(encoding="utf-8", errors="replace") for module in (layout.filter_module.name, layout.static_module.name): if module not in snippet: raise RuntimeError( f"load snippet {layout.load_snippet} does not reference {module}" ) def run_child(command: list[str]) -> None: subprocess.run(command, check=True) def run_smoke_tests( layout: PackageLayout, nginx_binary: pathlib.Path, zstd_bin: str, request_count: int, concurrent_requests: int, terminal_request_count: int, ) -> None: run_child( [ sys.executable, str(TEST_ENCODING), "--nginx-binary", str(nginx_binary), "--filter-module", str(layout.filter_module), "--static-module", str(layout.static_module), "--zstd-bin", zstd_bin, "--request-count", str(request_count), "--concurrent-requests", str(concurrent_requests), ] ) run_child( [ sys.executable, str(TEST_TERMINAL_FRAME), "--nginx-binary", str(nginx_binary), "--filter-module", str(layout.filter_module), "--static-module", str(layout.static_module), "--zstd-bin", zstd_bin, "--request-count", str(terminal_request_count), ] ) def main() -> int: args = parse_args() if bool(args.package_root) == bool(args.deb_path): raise ValueError("provide exactly one of --package-root or --deb-path") nginx_binary = pathlib.Path(args.nginx_binary) if not nginx_binary.exists(): raise FileNotFoundError(f"nginx binary not found: {nginx_binary}") managed_tempdir: tempfile.TemporaryDirectory[str] | None = None try: if args.package_root: package_root = pathlib.Path(args.package_root) else: deb_path = pathlib.Path(args.deb_path) if not deb_path.exists(): raise FileNotFoundError(f"package not found: {deb_path}") managed_tempdir = tempfile.TemporaryDirectory(prefix="http-zstd-deb-") package_root = extract_deb(deb_path, pathlib.Path(managed_tempdir.name)) layout = detect_package_layout(package_root) verify_load_snippet(layout) run_smoke_tests( layout, nginx_binary, args.zstd_bin, args.request_count, args.concurrent_requests, args.terminal_request_count, ) print( "OK: verified packaged http-zstd artifact " f"for {layout.flavor} using {layout.load_snippet.name} and packaged modules" ) return 0 finally: if managed_tempdir is not None: managed_tempdir.cleanup() if __name__ == "__main__": try: raise SystemExit(main()) except Exception as exc: print(f"ERROR: {exc}", file=sys.stderr) raise myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_perf_micro_witnesses.c000066400000000000000000000233511525232545500272310ustar00rootroot00000000000000/* * Deterministic behavior and work-count witnesses for A33 P7/P9/P10. * test_perf_micro_witnesses.sh extracts the production helpers verbatim. */ #include #include #include #include #include #include #include typedef unsigned char u_char; typedef long ngx_int_t; typedef unsigned long ngx_uint_t; typedef unsigned long ngx_file_uniq_t; typedef struct { size_t len; u_char *data; } ngx_str_t; typedef struct ngx_list_part_s ngx_list_part_t; struct ngx_list_part_s { void *elts; ngx_uint_t nelts; ngx_list_part_t *next; }; typedef struct { ngx_list_part_t part; } ngx_list_t; typedef struct { ngx_uint_t hash; ngx_str_t key; ngx_str_t value; } ngx_table_elt_t; typedef struct { ngx_list_t headers; } ngx_http_headers_out_t; typedef struct { ngx_http_headers_out_t headers_out; } ngx_http_request_t; typedef struct { ngx_file_uniq_t uniq; time_t mtime; off_t size; } ngx_open_file_info_t; #define ngx_inline inline #define ngx_strncasecmp(s1, s2, n) \ strncasecmp((const char *) (s1), (const char *) (s2), (n)) #define NGX_MAX_PATH 256 #define NGX_HTTP_ZSTD_STATIC_VERDICT_CACHE_SLOTS 64 #define NGX_HTTP_ZSTD_STATIC_VERDICT_NONE 0 #define NGX_HTTP_ZSTD_STATIC_VERDICT_BAD 1 #define NGX_HTTP_ZSTD_STATIC_VERDICT_GOOD 2 typedef struct { ngx_file_uniq_t uniq; time_t mtime; time_t checked; off_t size; size_t len; ngx_uint_t valid; u_char path[NGX_MAX_PATH]; } ngx_http_zstd_static_verdict_cache_t; static ngx_http_zstd_static_verdict_cache_t ngx_http_zstd_static_verdict_cache[ NGX_HTTP_ZSTD_STATIC_VERDICT_CACHE_SLOTS]; static ngx_uint_t ngx_http_zstd_static_verdict_cache_next; static ngx_uint_t ngx_http_zstd_static_verdict_cache_count; static ngx_uint_t memcmp_calls; static int counted_memcmp(const void *a, const void *b, size_t n) { memcmp_calls++; return memcmp(a, b, n); } #define ngx_memcmp counted_memcmp #define ngx_memcpy memcpy #ifdef TEST_VERDICT_CACHE_MUTANT #include "generated_verdict_cache_work_mutant.inc" #else #include "generated_verdict_cache_work.inc" #endif #include "generated_vary_find_tokens.inc" #include "../../src/ngx_http_zstd_cache_control.h" static int failures; static void check(int condition, const char *name) { if (!condition) { fprintf(stderr, "FAIL: %s\n", name); failures++; } } static int ascii_equal(const u_char *p, size_t len, const char *token) { size_t i; if (len != strlen(token)) { return 0; } for (i = 0; i < len; i++) { if (tolower((unsigned char) p[i]) != tolower((unsigned char) token[i])) { return 0; } } return 1; } /* Independent, deliberately simple Vary-list oracle for one field value. */ static void reference_vary(const char *value, const char *a, const char *b, ngx_uint_t *has_a, ngx_uint_t *has_b) { const u_char *p, *last, *start, *end; *has_a = 0; *has_b = 0; p = (const u_char *) value; last = p + strlen(value); while (p < last) { while (p < last && (*p == ',' || *p == ' ' || *p == '\t')) { p++; } start = p; while (p < last && *p != ',') { p++; } end = p; while (end > start && (end[-1] == ' ' || end[-1] == '\t')) { end--; } *has_a |= ascii_equal(start, (size_t) (end - start), a); *has_b |= ascii_equal(start, (size_t) (end - start), b); } } static void test_vary_behavior(void) { static const char *cases[] = { "", "gzip", "foo , bar", " available-dictionary\t, x", ",,Sec-Fetch-Site,,Available-Dictionary ", "AVAILABLE-DICTIONARY, sec-fetch-site", "foo,bar " }; ngx_http_request_t r; ngx_table_elt_t h; ngx_uint_t got_a, got_b, want_a, want_b; size_t i; memset(&r, 0, sizeof(r)); memset(&h, 0, sizeof(h)); h.hash = 1; h.key.data = (u_char *) "Vary"; h.key.len = sizeof("Vary") - 1; r.headers_out.headers.part.elts = &h; r.headers_out.headers.part.nelts = 1; for (i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { h.value.data = (u_char *) cases[i]; h.value.len = strlen(cases[i]); reference_vary(cases[i], "Available-Dictionary", "Sec-Fetch-Site", &want_a, &want_b); ngx_http_zstd_vary_find_tokens( &r, "Available-Dictionary", sizeof("Available-Dictionary") - 1, "Sec-Fetch-Site", sizeof("Sec-Fetch-Site") - 1, NULL, 0, &got_a, &got_b, NULL); check(got_a == want_a && got_b == want_b, "Vary helper agrees with independent token oracle"); } } static const u_char * reference_directive_end(const u_char *p, const u_char *last, const u_char **name_end) { int quoted; *name_end = NULL; quoted = 0; while (p < last) { if (*p == '\\' && quoted && p + 1 < last) { p += 2; continue; } if (*p == '"') { quoted = !quoted; } else if (!quoted && *p == ',') { break; } else if (!quoted && *name_end == NULL && (*p == ';' || *p == '=')) { *name_end = p; } p++; } if (*name_end == NULL) { *name_end = p; } return p; } /* Independent quote-aware Cache-Control oracle. */ static int reference_no_transform(const char *value) { const u_char *p, *last, *start, *name_end; p = (const u_char *) value; last = p + strlen(value); while (p < last) { while (p < last && (*p == ' ' || *p == '\t' || *p == ',')) { p++; } start = p; p = reference_directive_end(p, last, &name_end); while (name_end > start && (name_end[-1] == ' ' || name_end[-1] == '\t')) { name_end--; } if (ascii_equal(start, (size_t) (name_end - start), "no-transform")) { return 1; } if (p < last) { p++; } } return 0; } static void test_cache_control_behavior(void) { static const char *cases[] = { "", "max-age=60", "no-transform", " NO-TRANSFORM = arg", "extension=\"no-transform\"", "foo=\"a,b\", no-transform", "foo=\"a,\\\"b\", public", "no-transform; x=y", ",, private" }; ngx_table_elt_t cc; size_t i; memset(&cc, 0, sizeof(cc)); for (i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { cc.value.data = (u_char *) cases[i]; cc.value.len = strlen(cases[i]); check(ngx_http_zstd_cache_control_value_no_transform(&cc) == reference_no_transform(cases[i]), "Cache-Control helper agrees with independent directive oracle"); } } static void fill_verdict_entry(ngx_uint_t i, const ngx_str_t *path, const ngx_open_file_info_t *of) { ngx_http_zstd_static_verdict_cache[i].valid = NGX_HTTP_ZSTD_STATIC_VERDICT_BAD; ngx_http_zstd_static_verdict_cache[i].uniq = of->uniq; ngx_http_zstd_static_verdict_cache[i].mtime = of->mtime; ngx_http_zstd_static_verdict_cache[i].size = of->size; ngx_http_zstd_static_verdict_cache[i].len = path->len; memcpy(ngx_http_zstd_static_verdict_cache[i].path, "other", path->len); } static void test_verdict_cache_work_count(void) { ngx_open_file_info_t of; ngx_str_t path; ngx_uint_t i; memset(ngx_http_zstd_static_verdict_cache, 0, sizeof(ngx_http_zstd_static_verdict_cache)); path.data = (u_char *) "target"; path.len = sizeof("target") - 1; of.uniq = 7; of.mtime = 11; of.size = 13; /* Poison every slot beyond count. A count-bounded implementation must * not inspect them; the former fixed-64 walk makes 64 memcmp calls. */ for (i = 0; i < NGX_HTTP_ZSTD_STATIC_VERDICT_CACHE_SLOTS; i++) { fill_verdict_entry(i, &path, &of); } ngx_http_zstd_static_verdict_cache_count = 0; memcmp_calls = 0; check(ngx_http_zstd_static_cached_verdict(&path, &of, 100, 60) == NGX_HTTP_ZSTD_STATIC_VERDICT_NONE, "empty verdict cache misses"); check(memcmp_calls == 0, "empty verdict cache inspects zero slots"); ngx_http_zstd_static_verdict_cache_count = 3; memcmp_calls = 0; check(ngx_http_zstd_static_cached_verdict(&path, &of, 100, 60) == NGX_HTTP_ZSTD_STATIC_VERDICT_NONE, "partially populated verdict cache misses"); check(memcmp_calls == 3, "partial verdict cache inspects only populated slots"); memset(ngx_http_zstd_static_verdict_cache, 0, sizeof(ngx_http_zstd_static_verdict_cache)); ngx_http_zstd_static_verdict_cache_count = 0; ngx_http_zstd_static_verdict_cache_next = 0; for (i = 0; i < NGX_HTTP_ZSTD_STATIC_VERDICT_CACHE_SLOTS + 2; i++) { ngx_http_zstd_static_remember(&path, &of, NGX_HTTP_ZSTD_STATIC_VERDICT_BAD, 100, 60); } check(ngx_http_zstd_static_verdict_cache_count == NGX_HTTP_ZSTD_STATIC_VERDICT_CACHE_SLOTS, "remember count saturates at the ring bound"); check(ngx_http_zstd_static_verdict_cache_next == 2, "remember cursor wraps independently of populated count"); } int main(void) { test_vary_behavior(); test_cache_control_behavior(); test_verdict_cache_work_count(); if (failures != 0) { fprintf(stderr, "FAILED: %d performance witness check(s)\n", failures); return 1; } puts("OK: production helpers match references and bounded-work witnesses"); return 0; } myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_perf_micro_witnesses.sh000077500000000000000000000113661525232545500274270ustar00rootroot00000000000000#!/bin/bash # Deterministic witnesses for the A33 P7/P9/P10 scan reductions. # # The runtime fixture compiles verbatim extracts of the production helpers; # this script separately pins the source shape that makes P9/P10 single-pass. # The two checks are complementary: behavior alone cannot observe a redundant # walk, while source shape alone cannot prove the parser still answers right. set -euo pipefail cd "$(dirname "$0")/../.." CC="${CC:-cc}" OUT="$(mktemp -d "${TMPDIR:-/tmp}/zstd-perf-witnesses.XXXXXX")" trap 'rm -rf "$OUT"' EXIT extract_function() { local src=$1 signature=$2 output=$3 sig_line start_line rel_end end_line sig_line=$(grep -n -m1 "^${signature}$" "$src" | cut -d: -f1 || true) if [ -z "$sig_line" ]; then echo "FAIL: could not locate $signature in $src" >&2 return 1 fi start_line=$((sig_line - 1)) rel_end=$(awk -v start="$start_line" \ 'NR >= start && /^}$/ { print NR - start + 1; exit }' "$src") if [ -z "$rel_end" ]; then echo "FAIL: could not find the end of $signature in $src" >&2 return 1 fi end_line=$((start_line + rel_end - 1)) sed -n "${start_line},${end_line}p" "$src" >>"$output" } extract_function src/ngx_http_zstd_common.h \ 'ngx_http_zstd_vary_find_tokens(ngx_http_request_t \*r,' \ "$OUT/generated_vary_find_tokens.inc" extract_function src/ngx_http_zstd_static_module.c \ 'ngx_http_zstd_static_cached_verdict(ngx_str_t \*path, ngx_open_file_info_t \*of,' \ "$OUT/generated_verdict_cache_work.inc" extract_function src/ngx_http_zstd_static_module.c \ 'ngx_http_zstd_static_remember(ngx_str_t \*path, ngx_open_file_info_t \*of,' \ "$OUT/generated_verdict_cache_work.inc" check_vary_single_scan() { local input=$1 count count=$(grep -Fc "while (p < end && *p != ',')" "$input" || true) if [ "$count" -ne 1 ] || ! grep -Fq 'p = tok_end;' "$input"; then echo "FAIL: Vary helper no longer has one comma scan plus cursor reuse" >&2 return 1 fi } check_cache_control_single_scan() { local input=$1 value_body calls if ! grep -Fq 'u_char **name_end' "$input"; then echo "FAIL: Cache-Control segment helper lost its name-end out parameter" >&2 return 1 fi value_body=$(sed -n \ '/^ngx_http_zstd_cache_control_value_no_transform(ngx_table_elt_t \*cc)$/,/^}/p' \ "$input") calls=$(printf '%s\n' "$value_body" \ | grep -c 'ngx_http_zstd_cache_control_directive_end' || true) if [ "$calls" -ne 1 ] || printf '%s\n' "$value_body" \ | grep -Eq 'while \([^)]*< directive_end'; then echo "FAIL: Cache-Control value helper reintroduced a second segment scan" >&2 return 1 fi } check_vary_single_scan "$OUT/generated_vary_find_tokens.inc" check_cache_control_single_scan src/ngx_http_zstd_cache_control.h # Controls for the two structural detectors. Recreate the removed re-walks # and require each detector to reject its own old-cost shape. sed "/p = tok_end;/i\\ while (p < end && *p != ',') { p++; }" \ "$OUT/generated_vary_find_tokens.inc" >"$OUT/vary-mutant.inc" if check_vary_single_scan "$OUT/vary-mutant.inc" >/dev/null 2>&1; then echo "FAIL: Vary redundant-scan control escaped the source-shape gate" >&2 exit 1 fi sed '/ end = name_end;/i\ while (name_end < directive_end) { name_end++; }' \ src/ngx_http_zstd_cache_control.h >"$OUT/cache-control-mutant.h" if check_cache_control_single_scan "$OUT/cache-control-mutant.h" \ >/dev/null 2>&1; then echo "FAIL: Cache-Control redundant-scan control escaped the source-shape gate" >&2 exit 1 fi "$CC" -std=c99 -Wall -Wextra -Wshadow -Werror -O2 -I"$OUT" \ -o "$OUT/test_perf_micro_witnesses" \ ci/tools/test_perf_micro_witnesses.c "$OUT/test_perf_micro_witnesses" # P7 control: compile the same fixture with the old 64-slot loop bound. The # poison entries beyond count make that redundant walk observable as extra # ngx_memcmp calls without timing noise. sed 's/i < ngx_http_zstd_static_verdict_cache_count/i < NGX_HTTP_ZSTD_STATIC_VERDICT_CACHE_SLOTS/' \ "$OUT/generated_verdict_cache_work.inc" \ >"$OUT/generated_verdict_cache_work_mutant.inc" if ! grep -Fq 'i < NGX_HTTP_ZSTD_STATIC_VERDICT_CACHE_SLOTS' \ "$OUT/generated_verdict_cache_work_mutant.inc"; then echo "FAIL: verdict-cache loop-bound control did not apply" >&2 exit 1 fi "$CC" -std=c99 -Wall -Wextra -Wshadow -Werror -O2 -I"$OUT" \ -DTEST_VERDICT_CACHE_MUTANT \ -o "$OUT/test_perf_micro_witnesses_mutant" \ ci/tools/test_perf_micro_witnesses.c if "$OUT/test_perf_micro_witnesses_mutant" >/dev/null 2>&1; then echo "FAIL: 64-slot verdict-cache mutant passed the call-count witness" >&2 exit 1 fi echo 'OK: A33 P7/P9/P10 performance witnesses and controls' myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_perf_stat_recipe.py000077500000000000000000000540671525232545500265370ustar00rootroot00000000000000#!/usr/bin/env python3 import importlib.util import io import json import pathlib import sys import tempfile import unittest from typing import Any from unittest import mock TOOLS_DIR = pathlib.Path(__file__).resolve().parent def load_tool(name: str) -> Any: path = TOOLS_DIR / f"{name}.py" spec = importlib.util.spec_from_file_location(name, path) if spec is None or spec.loader is None: raise RuntimeError(f"cannot load test module {path}") module = importlib.util.module_from_spec(spec) sys.modules[name] = module spec.loader.exec_module(module) return module ab_bench = load_tool("ab_bench") perf_recipe = load_tool("perf_stat_recipe") class DczWorkloadTests(unittest.TestCase): def test_nginx_starts_in_its_own_process_group(self) -> None: arm = ab_bench.Arm("baseline", pathlib.Path("/nginx")) with ( tempfile.TemporaryDirectory() as temp_dir, mock.patch("builtins.open", mock.mock_open()), mock.patch.object(ab_bench.subprocess, "Popen") as popen, ): ab_bench.start_nginx( arm, pathlib.Path("nginx.conf"), pathlib.Path(temp_dir) ) self.assertTrue(popen.call_args.kwargs["start_new_session"]) def test_hit_configures_requested_dictionaries_and_canonical_headers(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: root = pathlib.Path(temp_dir) workload = ab_bench.prepare_workload(root, "dcz", 4, "hit") self.assertEqual(workload.config.count("zstd_dcz_dict_file "), 4) self.assertIn("zstd_dcz_assume_secure_transport on;", workload.config) self.assertNotIn("zstd_dcz_dict_trust_hashes", workload.config) self.assertEqual(dict(workload.headers)["Accept-Encoding"], "zstd, dcz") self.assertEqual(dict(workload.headers)["Sec-Fetch-Site"], "same-origin") self.assertRegex( dict(workload.headers)["Available-Dictionary"], r"^:[A-Za-z0-9+/]{43}=:$", ) self.assertEqual(workload.expected_encoding, "dcz") self.assertEqual(len(list((root / "dictionaries").iterdir())), 4) def test_miss_advertises_no_configured_dictionary(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: root = pathlib.Path(temp_dir) workload = ab_bench.prepare_workload(root, "dcz", 16, "miss") advertised = dict(workload.headers)["Available-Dictionary"] configured = { ab_bench._available_dictionary(path.read_bytes()) for path in (root / "dictionaries").iterdir() } self.assertNotIn(advertised, configured) self.assertEqual(workload.expected_encoding, "zstd") self.assertEqual(len(configured), 16) def test_default_zstd_workload_stays_dictionary_free(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: workload = ab_bench.prepare_workload( pathlib.Path(temp_dir), "zstd", 1, "hit" ) self.assertEqual(workload.headers, (("Accept-Encoding", "zstd"),)) self.assertEqual(workload.config, "") self.assertEqual(workload.expected_encoding, "zstd") def test_rejects_unsupported_dictionary_count(self) -> None: with ( tempfile.TemporaryDirectory() as temp_dir, self.assertRaisesRegex(ValueError, "must be one of"), ): ab_bench.prepare_workload(pathlib.Path(temp_dir), "dcz", 2, "hit") def test_preflight_requires_the_expected_content_encoding(self) -> None: response = mock.Mock(status=200) response.getheader.return_value = "zstd" connection = mock.Mock() connection.getresponse.return_value = response workload = ab_bench.Workload( mode="dcz", headers=(("Accept-Encoding", "zstd, dcz"),), expected_encoding="dcz", ) with ( mock.patch.object( ab_bench.http.client, "HTTPConnection", return_value=connection ), self.assertRaisesRegex(RuntimeError, "Content-Encoding='zstd'"), ): ab_bench.verify_workload(18400, workload) connection.request.assert_called_once_with( "GET", "/8kb", headers={"Accept-Encoding": "zstd, dcz"} ) response.read.assert_called_once_with() def test_readiness_failure_reaps_the_spawned_nginx_process(self) -> None: process = mock.Mock() process.pid = 4321 process.wait.side_effect = [ ab_bench.subprocess.TimeoutExpired(["nginx"], 10), 0, ] backend = mock.Mock() workload = ab_bench.Workload( mode="zstd", headers=(("Accept-Encoding", "zstd"),), expected_encoding="zstd", ) arm = ab_bench.Arm("baseline", pathlib.Path("/nginx")) with ( mock.patch.object(ab_bench, "PacedBackend", return_value=backend), mock.patch.object(ab_bench, "prepare_workload", return_value=workload), mock.patch.object( ab_bench, "write_conf", return_value=pathlib.Path("/nginx.conf") ), mock.patch.object(ab_bench, "start_nginx", return_value=process), mock.patch.object( ab_bench, "require_own_nginx_ready", side_effect=RuntimeError("readiness failed"), ), mock.patch.object(ab_bench.os, "killpg") as kill_group, self.assertRaisesRegex(RuntimeError, "readiness failed"), ): ab_bench.run_release_pass( [arm], [1], 1, "1s", 1, 18400, 6, "zstd", 1, "hit" ) process.terminate.assert_called_once_with() self.assertEqual( process.wait.call_args_list, [mock.call(timeout=10), mock.call(timeout=5)], ) kill_group.assert_called_once_with(4321, ab_bench.signal.SIGKILL) backend.stop.assert_called_once_with() def test_nginx_group_cleanup_reaps_when_group_is_already_gone(self) -> None: process = mock.Mock(pid=4321) process.wait.side_effect = [ ab_bench.subprocess.TimeoutExpired(["nginx"], 10), 0, ] with mock.patch.object(ab_bench.os, "killpg", side_effect=ProcessLookupError): ab_bench.stop_nginx(process) self.assertEqual(process.wait.call_args_list[-1], mock.call(timeout=5)) def test_perf_attaches_only_to_resolved_nginx_workers(self) -> None: process = mock.Mock() process.poll.return_value = None attachment = ab_bench.PerfAttachment( pathlib.Path("/result.perf"), "cpu_core/cache-misses/" ) with ( mock.patch.object(ab_bench, "nginx_worker_pids", return_value=[1234]), mock.patch.object( ab_bench.subprocess, "Popen", return_value=process ) as popen, mock.patch.object(ab_bench.time, "sleep"), ): self.assertIs(ab_bench.start_worker_perf(99, 1, attachment), process) command = popen.call_args.args[0] self.assertEqual(command[command.index("-p") + 1], "1234") self.assertNotIn("99", command) def test_perf_attachment_rejects_multiple_workers(self) -> None: attachment = ab_bench.PerfAttachment( pathlib.Path("/result.perf"), "cpu_core/cache-misses/" ) with ( mock.patch.object(ab_bench.subprocess, "Popen") as popen, self.assertRaisesRegex(RuntimeError, "requires --workers 1"), ): ab_bench.start_worker_perf(99, 2, attachment) popen.assert_not_called() def test_perf_stop_accepts_the_expected_sigint_exit(self) -> None: for returncode in (-ab_bench.signal.SIGINT, 128 + ab_bench.signal.SIGINT): with self.subTest(returncode=returncode): process = mock.Mock() process.poll.return_value = None process.wait.return_value = returncode ab_bench.stop_worker_perf(process) process.send_signal.assert_called_once_with(ab_bench.signal.SIGINT) def test_perf_stop_rejects_an_early_clean_exit(self) -> None: process = mock.Mock() process.poll.return_value = 0 with self.assertRaisesRegex(RuntimeError, "exited before"): ab_bench.stop_worker_perf(process) process.send_signal.assert_not_called() class PerfRecipeTests(unittest.TestCase): @staticmethod def _matrix_result( counters: Any, successful_requests: int, *, core_type: str = "P", cpu_mask: str = "0-7", cpu_identity: str = "vendor/model/stepping", ) -> dict: scenarios = {} for dictionary_count, case in perf_recipe.dcz_scenarios(): scenarios[f"dicts-{dictionary_count}-{case}"] = { "dictionary_count": dictionary_count, "case": case, "measured_successful_requests": successful_requests, "counters": counters.to_dict(), "engagement": {"baseline": "dcz" if case == "hit" else "zstd"}, } return { "schema_version": 2, "workload": "dcz", "core_type": core_type, "cpu_mask": cpu_mask, "cpu_identity": cpu_identity, "normalization": "measured_successful_requests", "scenarios": scenarios, } def test_documented_flag_first_cli_spellings_remain_supported(self) -> None: self.assertEqual( perf_recipe.normalize_cli_argv(["--nginx", "/nginx"]), ["measure", "--nginx", "/nginx"], ) self.assertEqual( perf_recipe.normalize_cli_argv(["--compare", "base.json", "opt.json"]), ["compare", "base.json", "opt.json"], ) self.assertEqual(perf_recipe.normalize_cli_argv(["--help"]), ["--help"]) def test_matrix_contains_required_hit_and_miss_cells(self) -> None: self.assertEqual( perf_recipe.dcz_scenarios(), ( (1, "hit"), (1, "miss"), (4, "hit"), (4, "miss"), (16, "hit"), (16, "miss"), ), ) def test_matrix_measure_orchestrates_every_worker_only_scenario(self) -> None: perf_text = "\n".join( f"100 cpu_core/{event}/" for event in ( "cache-misses", "cache-references", "instructions", "cycles", ) ) def fake_run(command: list[str], cpu_mask: str) -> None: self.assertEqual(cpu_mask, "0 1") count = command[command.index("--dcz-dictionaries") + 1] case = command[command.index("--dcz-case") + 1] json_path = pathlib.Path(command[command.index("--json") + 1]) perf_path = pathlib.Path(command[command.index("--perf-stat-output") + 1]) json_path.write_text( json.dumps( { "meta": { "successful_requests": 10, "perf_scope": "nginx-workers-measured-release", "engagement": { "baseline": "dcz" if case == "hit" else "zstd" }, } } ), encoding="utf-8", ) perf_path.write_text(perf_text, encoding="utf-8") self.assertIn(count, {"1", "4", "16"}) with ( tempfile.TemporaryDirectory() as temp_dir, mock.patch.object( perf_recipe, "detect_hybrid_cores", return_value=("0 1", "2 3", 0) ), mock.patch.object(perf_recipe, "cpu_identity", return_value="test-cpu"), mock.patch.object( perf_recipe, "run_pinned_bench", side_effect=fake_run ) as run, ): result_path = pathlib.Path(temp_dir) / "matrix.json" scenarios = perf_recipe.measure_dcz_matrix( pathlib.Path(sys.executable), str(result_path), "P" ) saved = json.loads(result_path.read_text()) self.assertEqual(run.call_count, 6) self.assertEqual( set(scenarios), {f"dicts-{n}-{c}" for n, c in perf_recipe.dcz_scenarios()}, ) self.assertEqual(saved["normalization"], "measured_successful_requests") def test_matrix_identity_failure_prevents_scenario_execution(self) -> None: with ( mock.patch.object( perf_recipe, "detect_hybrid_cores", return_value=("0 1", "2 3", 0) ), mock.patch.object( perf_recipe, "cpu_identity", side_effect=RuntimeError("missing CPU identity"), ), mock.patch.object(perf_recipe, "run_pinned_bench") as run, self.assertRaisesRegex(RuntimeError, "missing CPU identity"), ): perf_recipe.measure_dcz_matrix(pathlib.Path(sys.executable)) run.assert_not_called() def test_pinned_bench_timeout_kills_the_process_group(self) -> None: process = mock.Mock(pid=4321) with ( mock.patch.object(perf_recipe.subprocess, "Popen", return_value=process), mock.patch.object( perf_recipe, "wait_process_without_reaping", side_effect=perf_recipe.subprocess.TimeoutExpired(["taskset"], 600), ), mock.patch.object(perf_recipe, "kill_process_group") as kill_group, self.assertRaisesRegex(RuntimeError, "process group killed"), ): perf_recipe.run_pinned_bench(["bench"], "0 1") kill_group.assert_called_once_with(process) def test_pinned_bench_interrupt_kills_the_process_group(self) -> None: process = mock.Mock(pid=4321) with ( mock.patch.object(perf_recipe.subprocess, "Popen", return_value=process), mock.patch.object( perf_recipe, "wait_process_without_reaping", side_effect=KeyboardInterrupt, ), mock.patch.object(perf_recipe, "kill_process_group") as kill_group, self.assertRaises(KeyboardInterrupt), ): perf_recipe.run_pinned_bench(["bench"], "0 1") kill_group.assert_called_once_with(process) def test_pinned_bench_nonzero_exit_kills_the_process_group(self) -> None: process = mock.Mock(returncode=1) with ( mock.patch.object(perf_recipe.subprocess, "Popen", return_value=process), mock.patch.object( perf_recipe, "wait_process_without_reaping", return_value=False ), mock.patch.object(perf_recipe, "kill_process_group") as kill_group, self.assertRaisesRegex(RuntimeError, "exit code 1"), ): perf_recipe.run_pinned_bench(["bench"], "0 1") kill_group.assert_called_once_with(process) def test_failed_status_is_observed_without_reaping_the_group_leader(self) -> None: process = mock.Mock(pid=4321, args=["bench"]) status = mock.Mock(si_code=perf_recipe.os.CLD_EXITED, si_status=1) with mock.patch.object(perf_recipe.os, "waitid", return_value=status) as waitid: self.assertFalse(perf_recipe.wait_process_without_reaping(process, 1)) waitid.assert_called_once_with( perf_recipe.os.P_PID, 4321, perf_recipe.os.WEXITED | perf_recipe.os.WNOHANG | perf_recipe.os.WNOWAIT, ) process.wait.assert_not_called() def test_clean_status_is_observed_without_reaping_the_group_leader(self) -> None: process = mock.Mock(pid=4321, args=["bench"]) status = mock.Mock(si_code=perf_recipe.os.CLD_EXITED, si_status=0) with mock.patch.object(perf_recipe.os, "waitid", return_value=status): self.assertTrue(perf_recipe.wait_process_without_reaping(process, 1)) process.wait.assert_not_called() def test_pinned_bench_success_reaps_without_group_kill(self) -> None: process = mock.Mock(pid=4321) with ( mock.patch.object(perf_recipe.subprocess, "Popen", return_value=process), mock.patch.object( perf_recipe, "wait_process_without_reaping", return_value=True ), mock.patch.object(perf_recipe, "kill_process_group") as kill_group, ): perf_recipe.run_pinned_bench(["bench"], "0 1") process.wait.assert_called_once_with(timeout=10) kill_group.assert_not_called() def test_dcz_command_selects_workload_and_evidence_json(self) -> None: command = perf_recipe.build_dcz_bench_command( pathlib.Path("/nginx"), 16, "miss", pathlib.Path("/result.json"), pathlib.Path("/result.perf"), "cpu_core/cache-misses/", ) self.assertEqual( command, [ "python3", str(perf_recipe.TOOLS_DIR / "ab_bench.py"), "--arm-a", "/nginx:baseline", "--rounds", "1", "--workers", "1", "--duration", "5s", "--workload", "dcz", "--dcz-dictionaries", "16", "--dcz-case", "miss", "--json", "/result.json", "--perf-stat-output", "/result.perf", "--perf-events", "cpu_core/cache-misses/", ], ) def test_successful_request_denominator_excludes_errors(self) -> None: result = { "release": { "baseline": { "8": { "8kb": [ {"requests": 120, "errors": 20}, {"requests": 80, "errors": 0}, ] } } } } self.assertEqual(perf_recipe.successful_requests_from_bench(result), 180) def test_measured_request_denominator_excludes_setup_traffic(self) -> None: result = { "meta": { "successful_requests": 100, "completed_successful_requests": 141, } } self.assertEqual(perf_recipe.successful_requests_from_bench(result), 100) def test_normalization_reverses_misleading_raw_counter_delta(self) -> None: baseline = perf_recipe.PerfCounters(1000, 2000, 3000, 4000) optimized = perf_recipe.PerfCounters(1500, 3000, 4500, 6000) self.assertGreater(optimized.cache_misses, baseline.cache_misses) baseline_norm = perf_recipe.normalize_counters(baseline, 100) optimized_norm = perf_recipe.normalize_counters(optimized, 300) self.assertLess(optimized_norm["cache_misses"], baseline_norm["cache_misses"]) def test_matrix_compare_uses_normalized_counts_and_prints_all_deltas(self) -> None: baseline = self._matrix_result( perf_recipe.PerfCounters(1000, 2000, 3000, 4000), 100 ) optimized = self._matrix_result( perf_recipe.PerfCounters(1500, 3000, 4500, 6000), 300 ) output = io.StringIO() with mock.patch("sys.stdout", output): perf_recipe.compare_dcz_results(baseline, optimized) rendered = output.getvalue() self.assertIn("dicts-16-miss", rendered) self.assertIn("miss delta", rendered) self.assertIn("ref delta", rendered) self.assertIn("insn delta", rendered) self.assertIn("cycle delta", rendered) self.assertEqual(rendered.count("-50.0%"), 24) def test_matrix_compare_rejects_incompatible_measurements(self) -> None: baseline = self._matrix_result(perf_recipe.PerfCounters(1, 2, 3, 4), 1) optimized = self._matrix_result( perf_recipe.PerfCounters(1, 2, 3, 4), 1, core_type="E", cpu_mask="8-15", ) with self.assertRaisesRegex(RuntimeError, "matching core_type"): perf_recipe.compare_dcz_results(baseline, optimized) optimized = self._matrix_result( perf_recipe.PerfCounters(1, 2, 3, 4), 1, cpu_identity="different-cpu", ) with self.assertRaisesRegex(RuntimeError, "matching cpu_identity"): perf_recipe.compare_dcz_results(baseline, optimized) def test_matrix_compare_rejects_missing_identity_and_wrong_engagement(self) -> None: baseline = self._matrix_result(perf_recipe.PerfCounters(1, 2, 3, 4), 1) optimized = self._matrix_result(perf_recipe.PerfCounters(1, 2, 3, 4), 1) del baseline["core_type"] del optimized["core_type"] with self.assertRaisesRegex(RuntimeError, "core_type P or E"): perf_recipe.compare_dcz_results(baseline, optimized) baseline = self._matrix_result(perf_recipe.PerfCounters(1, 2, 3, 4), 1) optimized = self._matrix_result(perf_recipe.PerfCounters(1, 2, 3, 4), 1) del baseline["cpu_identity"] del optimized["cpu_identity"] with self.assertRaisesRegex(RuntimeError, "non-empty cpu_identity"): perf_recipe.compare_dcz_results(baseline, optimized) baseline = self._matrix_result(perf_recipe.PerfCounters(1, 2, 3, 4), 1) optimized = self._matrix_result(perf_recipe.PerfCounters(1, 2, 3, 4), 1) optimized["scenarios"]["dicts-16-hit"]["engagement"] = {"baseline": "zstd"} with self.assertRaisesRegex(RuntimeError, "requires dcz engagement"): perf_recipe.compare_dcz_results(baseline, optimized) def test_normalization_rejects_zero_successful_requests(self) -> None: counters = perf_recipe.PerfCounters(1, 2, 3, 4) with self.assertRaisesRegex(RuntimeError, "without successful requests"): perf_recipe.normalize_counters(counters, 0) if __name__ == "__main__": unittest.main() myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_pledge_bound_input_unit.c000066400000000000000000000261001525232545500277000ustar00rootroot00000000000000/* * Copyright (C) 2026 Thijs Eilander * SPDX-License-Identifier: BSD-2-Clause * * Unit oracle for ngx_http_zstd_pledge_to_bound_input() -- the guard that * decides whether an upstream-declared (off_t) body length may be handed to * ZSTD_compressBound() (a size_t) to size the FIRST output buffer. * * The function is EXTRACTED VERBATIM from * src/ngx_http_zstd_filter_module.c by test_pledge_bound_input_unit.sh into * generated_pledge_bound_input.inc. This file never re-implements the * arithmetic, so it cannot quietly agree with a stale copy of itself. * * The defect it guards: on an ILP32 build with large-file support * (sizeof(size_t) == 4, sizeof(off_t) == 8 -- the ordinary shape once * _FILE_OFFSET_BITS=64 is in effect) a bare `(size_t) pledged_size` cast * truncates modulo 2^32. A declared length of 4 GiB + 389 becomes 389, * ZSTD_compressBound() returns ~389, and the caller's clamp shrinks the * first output buffer to ~389 bytes for a 4 GiB body -- an undersized * destination that libzstd is then free to overrun. * * What is asserted, and why each one matters: * * 1. ORACLE AGREEMENT over the whole domain. An independent uintmax_t * oracle -- deliberately written in a different shape from the * function, comparing at a width no build narrows -- decides accept * vs decline for every case, and *out is checked to equal the input * exactly whenever the function accepted. The oracle is what catches * a truncation: a wrong accept shows up as *out != input, which is * precisely the heap-corrupting outcome. * * 2. THE FAST PATH SURVIVES. Every value representable in size_t and * below ZSTD_MAX_INPUT_SIZE must be ACCEPTED. A "fix" that declines * broadly would make the truncation unreachable while silently * disabling the first-buffer sizing the call site exists for; case 2 * is the regression guard against exactly that shortcut, and it is * why the sweep includes ordinary small lengths and not only the * pathological ones. * * 3. THE THREE DECLINE REASONS, each hit by a named case: negative * (unknown length), above SIZE_MAX (ILP32 truncation), and at or * above ZSTD_MAX_INPUT_SIZE (where ZSTD_compressBound() returns 0 -- * an error, not a bound; a 0 propagating into the caller's clamp * floors buf_size to 256 bytes, which is WORSE than truncation). * * 4. BOUNDARIES, exhaustively walked rather than sampled: SIZE_MAX - 1, * SIZE_MAX, SIZE_MAX + 1, ZSTD_MAX_INPUT_SIZE - 1, * ZSTD_MAX_INPUT_SIZE, ZSTD_MAX_INPUT_SIZE + 1, 4 GiB - 1, 4 GiB, * 4 GiB + 389 (the item's fixture length), 0, and every power of two * +/- 1 that fits in off_t. Plus a deterministic pseudo-random sweep * so a hand-picked table cannot be the only coverage. * * ZSTD_COMPRESSBOUND() (the macro) is used rather than ZSTD_compressBound() * (the function) deliberately: zstd.h documents them as computing the same * value, and the macro needs only the header. That keeps this fixture * header-only and linkable under -m32 on a host that has no 32-bit libzstd * -- and the ILP32 pass is the only one that can observe the defect, so it * must not be the pass that gets skipped for a packaging reason. * * Case labels are compile-time string literals rather than formatted * per-case text: check() already prints the offending pledged value in * every failure message, so an snprintf()-built label added nothing but a * dynamic format string in a test binary. * * The interesting width (sizeof(off_t) > sizeof(size_t)) does not exist on * an LP64 host, so the .sh driver compiles this file TWICE: natively, and * again with -m32 -D_FILE_OFFSET_BITS=64. Only the second run reaches the * SIZE_MAX-overflow branch; the native run proves the LP64 behaviour is * unchanged. Both are required for this fixture to mean anything. */ #include #include #include #include #include #include /* Minimal nginx shim: the extracted code uses these names only. */ typedef long ngx_int_t; typedef unsigned long ngx_uint_t; #define ngx_inline inline #include "generated_pledge_bound_input.inc" static long long failures; static long long accepts; static long long declines; /* * Independent oracle. Written against uintmax_t -- at least as wide as both * off_t and size_t on every conforming implementation -- so its decision is * made at a width nothing here narrows. Deliberately NOT structured like the * function under test: it computes the two ceilings first and compares once, * rather than short-circuiting through a sizeof() test. */ static int oracle_accepts(off_t pledged, uintmax_t *want_out) { uintmax_t v, size_ceiling, zstd_ceiling; if (pledged < 0) { return 0; } v = (uintmax_t) pledged; size_ceiling = (uintmax_t) SIZE_MAX; zstd_ceiling = (uintmax_t) ZSTD_MAX_INPUT_SIZE; if (v > size_ceiling) { return 0; } if (v >= zstd_ceiling) { return 0; } *want_out = v; return 1; } static void check(off_t pledged, const char *label) { size_t got_out = (size_t) 0xA5A5A5A5u; /* poison */ ngx_uint_t got; uintmax_t want_out = 0; int want; want = oracle_accepts(pledged, &want_out); got = ngx_http_zstd_pledge_to_bound_input(pledged, &got_out); if (!!got != !!want) { fprintf(stderr, "FAIL %s: pledged=%" PRIdMAX " -> accepted=%d, oracle=%d" " (sizeof off_t=%zu size_t=%zu)\n", label, (intmax_t) pledged, (int) !!got, want, sizeof(off_t), sizeof(size_t)); failures++; return; } if (want) { accepts++; if ((uintmax_t) got_out != want_out) { fprintf(stderr, "FAIL %s: pledged=%" PRIdMAX " accepted but *out=%" PRIuMAX " != %" PRIuMAX " -- NARROWING\n", label, (intmax_t) pledged, (uintmax_t) got_out, want_out); failures++; return; } /* * The whole point of accepting: ZSTD_compressBound() must return a * real bound (non-zero) that is at least the input. A zero here * would mean the guard let an over-ceiling value through and the * caller would floor buf_size to 256 bytes. */ { size_t bound = ZSTD_COMPRESSBOUND(got_out); if (bound == 0 || bound < got_out) { fprintf(stderr, "FAIL %s: pledged=%" PRIdMAX " accepted but" " ZSTD_COMPRESSBOUND(%zu)=%zu\n", label, (intmax_t) pledged, got_out, bound); failures++; } } } else { declines++; } } /* Deterministic xorshift64* -- no libc RNG, identical on every runner. */ static uint64_t rng_state = 0x9E3779B97F4A7C15ull; static uint64_t rng_next(void) { rng_state ^= rng_state >> 12; rng_state ^= rng_state << 25; rng_state ^= rng_state >> 27; return rng_state * 0x2545F4914F6CDD1Dull; } int main(void) { uintmax_t off_max; int i; /* * The largest positive off_t, computed without overflow: all value bits * of the signed type set. */ off_max = ((uintmax_t) 1 << (sizeof(off_t) * 8 - 1)) - 1; printf("sizeof(off_t)=%zu sizeof(size_t)=%zu SIZE_MAX=%" PRIuMAX " ZSTD_MAX_INPUT_SIZE=%" PRIuMAX " OFF_MAX=%" PRIuMAX "\n", sizeof(off_t), sizeof(size_t), (uintmax_t) SIZE_MAX, (uintmax_t) ZSTD_MAX_INPUT_SIZE, off_max); /* --- 3: negative / unknown length ---------------------------------- */ check((off_t) -1, "unknown(-1)"); check((off_t) -2, "negative(-2)"); check((off_t) (-(intmax_t) off_max - 1), "off_t min"); /* --- 2: the fast path must survive --------------------------------- */ check((off_t) 0, "zero"); check((off_t) 1, "one"); check((off_t) 389, "389 (the small-body case the sizing exists for)"); check((off_t) 131072, "default bufs.size"); check((off_t) (1024 * 1024), "1 MiB"); /* --- 4: named boundaries ------------------------------------------- */ { static const uintmax_t interesting[] = { 0xFFFFFFFFull - 1, /* 4 GiB - 2 */ 0xFFFFFFFFull, /* 4 GiB - 1: last 32-bit value */ 0x100000000ull, /* 4 GiB exactly */ 0x100000185ull, /* 4 GiB + 389: the item's fixture */ 0x100000000ull + 131072, /* 4 GiB + a full buffer */ 0xFF00FF00ull - 1, /* ILP32 ZSTD ceiling - 1 */ 0xFF00FF00ull, /* ILP32 ZSTD ceiling */ 0xFF00FF00ull + 1, 0xFF00FF00FF00FF00ull - 1, /* LP64 ZSTD ceiling - 1 */ 0xFF00FF00FF00FF00ull, /* LP64 ZSTD ceiling */ 0xFF00FF00FF00FF00ull + 1, }; size_t n; for (n = 0; n < sizeof(interesting) / sizeof(interesting[0]); n++) { uintmax_t v = interesting[n]; if (v > off_max) { continue; /* not representable as a positive off_t here */ } check((off_t) v, "named boundary"); } } /* SIZE_MAX -1 / SIZE_MAX / SIZE_MAX +1, when they fit in off_t. */ { uintmax_t sm = (uintmax_t) SIZE_MAX; int d; for (d = -1; d <= 1; d++) { uintmax_t v; if (d < 0 && sm == 0) { continue; } v = sm + (uintmax_t) d; if (v > off_max) { continue; } check((off_t) v, "SIZE_MAX neighbourhood"); } } /* Every power of two, +/- 1, that fits in off_t. */ for (i = 0; i < (int) (sizeof(off_t) * 8 - 1); i++) { uintmax_t p = (uintmax_t) 1 << i; int d; for (d = -1; d <= 1; d++) { uintmax_t v; if (d < 0 && p == 0) { continue; } v = p + (uintmax_t) d; if (v > off_max) { continue; } check((off_t) v, "power of two neighbourhood"); } } /* --- 1: randomized sweep over the full off_t range ------------------ */ for (i = 0; i < 200000; i++) { uint64_t r = rng_next(); off_t v; if (sizeof(off_t) == 4) { v = (off_t) (int32_t) (uint32_t) r; } else { v = (off_t) (int64_t) r; } check(v, "random"); } if (failures) { fprintf(stderr, "FAILED: %lld case(s)\n", failures); return 1; } /* * Both outcomes must actually have been exercised. A build where every * case declined (or every case accepted) would report a vacuous pass. */ if (accepts == 0 || declines == 0) { fprintf(stderr, "FAILED: vacuous sweep -- accepts=%lld declines=%lld;" " one branch was never reached\n", accepts, declines); return 1; } printf("OK: %lld accepted, %lld declined, oracle agreed on all\n", accepts, declines); return 0; } myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_pledge_bound_input_unit.sh000077500000000000000000000104301525232545500300720ustar00rootroot00000000000000#!/bin/bash # Copyright (C) 2026 Thijs Eilander # SPDX-License-Identifier: BSD-2-Clause # # Unit fixture for ngx_http_zstd_pledge_to_bound_input() -- the guard that # decides whether an upstream-declared off_t body length may be narrowed to # size_t and handed to ZSTD_compressBound() to size the FIRST output buffer. # # The whole defect this closes lives in a width relationship # (sizeof(off_t) > sizeof(size_t)) that DOES NOT EXIST on the LP64 hosts CI # runs on. A test that only ran natively would pass on a truncating # implementation, so this script compiles the fixture TWICE: # # 1. natively (LP64 here): proves the 64-bit behaviour is unchanged and # that the fast path still accepts ordinary lengths; # 2. -m32 -D_FILE_OFFSET_BITS=64: the ILP32-with-large-file-support shape, # 32-bit size_t under a 64-bit off_t. This is the ONLY pass that # reaches the SIZE_MAX-overflow branch, i.e. the only one that can # observe the truncation. It is not optional coverage. # # The function is extracted VERBATIM by line range from src/ rather than # re-typed, so a future edit to the production arithmetic cannot leave a # hand-copied duplicate here passing forever. # # REQUIRE_M32: unset/"0"/""/false/no/off (case insensitive) keeps the # default local-dev behaviour -- SKIP with exit 0 when no -m32 toolchain is # present. Any other value (1/true/yes/on, case insensitive, or anything # else non-empty) makes a missing -m32 toolchain a hard FAILURE instead. set -euo pipefail # ci/tools/ -> ci/ -> repo root. cd "$(dirname "$0")/../.." SRC="src/ngx_http_zstd_filter_module.c" CC="${CC:-cc}" OUT="$(mktemp -d "${TMPDIR:-/tmp}/zstd-pledge-bound-unit.XXXXXX")" trap 'rm -rf "$OUT"' EXIT mkdir -p "$OUT" SIG="$(grep -n '^ngx_http_zstd_pledge_to_bound_input(off_t pledged_size' \ "$SRC" | head -1 | cut -d: -f1)" if [ -z "$SIG" ]; then echo "FAIL: could not locate ngx_http_zstd_pledge_to_bound_input() in $SRC" >&2 exit 1 fi # Signature line is preceded by the storage-class/return-type line. START=$((SIG - 1)) REL_END="$(tail -n "+${START}" "$SRC" | grep -n '^}' | head -1 | cut -d: -f1)" if [ -z "$REL_END" ]; then echo "FAIL: could not find the end of ngx_http_zstd_pledge_to_bound_input()" >&2 exit 1 fi END=$((START + REL_END - 1)) { echo "/* Generated by test_pledge_bound_input_unit.sh -- verbatim" echo " * extraction from $SRC lines ${START}-${END}." echo " * Do not edit; re-run the script to regenerate. */" sed -n "${START},${END}p" "$SRC" } > "$OUT/generated_pledge_bound_input.inc" # Sanity: a source reflow or a regression to a bare cast must fail loudly # here, not silently produce an .inc that still compiles and "passes". for marker in 'ngx_http_zstd_pledge_to_bound_input(off_t pledged_size' \ 'sizeof(off_t) > sizeof(size_t)' \ 'SIZE_MAX' \ 'ZSTD_MAX_INPUT_SIZE'; do if ! grep -q "$marker" "$OUT/generated_pledge_bound_input.inc"; then echo "FAIL: extracted fragment is missing \"$marker\" --" \ "extraction range wrong, or the full-width guard regressed" >&2 cat "$OUT/generated_pledge_bound_input.inc" >&2 exit 1 fi done cp ci/tools/test_pledge_bound_input_unit.c "$OUT/" # --- native pass --------------------------------------------------------- "$CC" -Wall -Wextra -Werror -O2 -I"$OUT" \ -o "$OUT/pledge_bound_unit" "$OUT/test_pledge_bound_input_unit.c" "$OUT/pledge_bound_unit" echo "OK: pledged-length bound guard (native, extracted from current $SRC)" # --- ILP32 + LFS pass ---------------------------------------------------- require_m32="${REQUIRE_M32:-0}" case "${require_m32,,}" in 0 | "" | false | no | off) require_m32_set=0 ;; *) require_m32_set=1 ;; esac if "$CC" -m32 -D_FILE_OFFSET_BITS=64 -Wall -Wextra -Werror -O2 -I"$OUT" \ -o "$OUT/pledge_bound_unit_ilp32" \ "$OUT/test_pledge_bound_input_unit.c" >/dev/null 2>&1 then "$OUT/pledge_bound_unit_ilp32" echo "OK: pledged-length bound guard (ILP32 + LFS, off_t wider than size_t)" elif [ "$require_m32_set" = "1" ]; then echo "FAIL: no 32-bit toolchain available and REQUIRE_M32 is set;" \ "the ILP32/LFS pass -- the only one that can observe the truncation" \ "-- cannot run" >&2 exit 1 else echo "SKIP: no 32-bit toolchain available; ILP32/LFS pass not run" >&2 fi myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_probe_header_seam.sh000077500000000000000000000113431525232545500266150ustar00rootroot00000000000000#!/bin/sh set -eu root=$(cd "$(dirname "$0")/../.." && pwd) # Production must consume the authoritative header, not a copy. grep -Fq '#include "ngx_http_zstd_frame_probe.h"' \ "$root/src/ngx_http_zstd_static_module.c" check_definition() { expected=$1 fn=$2 tree=$3 if ! defs=$(grep -r -n -h "^$fn(" \ "$tree/src" "$tree/filter" "$tree/static" "$tree/ci"); then echo "probe seam: could not find $fn definitions" >&2 return 1 fi if ! sites=$(grep -r -l "^$fn(" \ "$tree/src" "$tree/filter" "$tree/static" "$tree/ci"); then echo "probe seam: could not find $fn definition sites" >&2 return 1 fi count=$(printf '%s\n' "$defs" | wc -l) if [ "$count" -ne 1 ] || [ "$sites" != "$expected" ]; then echo "probe seam: unexpected definition sites for $fn:" >&2 echo "$sites" >&2 echo "probe seam: expected 1 definition, found $count" >&2 return 1 fi } # Each probe function is defined exactly once in the tree. A second # column-0 definition anywhere in the module sources is the # synchronized-copy drift #270 removed coming back -- possible while # the unit tests keep exercising only the header. for fn in ngx_http_zstd_static_probe_frame ngx_http_zstd_static_probe_reuse; do check_definition "$root/src/ngx_http_zstd_frame_probe.h" "$fn" \ "$root" done # Same seam, second family: the Cache-Control no-transform helpers # moved to their own authoritative header; production must include it # and each helper must keep exactly one definition. grep -Fq '#include "ngx_http_zstd_cache_control.h"' \ "$root/src/ngx_http_zstd_filter_module.c" for fn in ngx_http_zstd_cache_control_directive_end \ ngx_http_zstd_cache_control_value_no_transform \ ngx_http_zstd_cache_control_no_transform; do check_definition "$root/src/ngx_http_zstd_cache_control.h" "$fn" \ "$root" done # Same seam, the ratio helper: production must include its header and # the split must keep exactly one definition. grep -Fq '#include "ngx_http_zstd_ratio.h"' \ "$root/src/ngx_http_zstd_filter_module.c" check_definition "$root/src/ngx_http_zstd_ratio.h" \ ngx_http_zstd_ratio_parts "$root" # Same seam, the dictionary-file I/O family (#312): production must # include its header, and the read loop and the hex nibble decoder must # each keep exactly one definition. The module's logging shell around # the loop (ngx_http_zstd_read_dict_file) keeps its name and stays in # the module; it is not part of the family. if ! grep -Fq '#include "ngx_http_zstd_dict_file.h"' \ "$root/src/ngx_http_zstd_filter_module.c"; then echo 'probe seam: src/ngx_http_zstd_filter_module.c no longer includes ngx_http_zstd_dict_file.h' >&2 exit 1 fi for fn in ngx_http_zstd_dict_file_read ngx_http_zstd_hex_nibble \ ngx_http_zstd_dict_file_check_dir ngx_http_zstd_dict_file_next_component \ ngx_http_zstd_dict_file_open_strict; do check_definition "$root/src/ngx_http_zstd_dict_file.h" "$fn" "$root" done # Detection control: redirect the real unit fixture to a copied probe # implementation under ci/. The fixture must stay green while the seam # check turns red, reproducing the drift this gate exists to catch. tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT mkdir -p "$tmp/src" "$tmp/filter" "$tmp/static" "$tmp/ci/tools" cp "$root/src/ngx_http_zstd_frame_probe.h" \ "$tmp/src/ngx_http_zstd_frame_probe.h" cp "$root/src/ngx_http_zstd_frame_probe.h" \ "$tmp/ci/tools/generated_static_probe.inc" sed 's|#include "../../src/ngx_http_zstd_frame_probe.h"|#include "generated_static_probe.inc"|' \ "$root/ci/tools/test_static_probe_unit.c" \ >"$tmp/ci/tools/test_static_probe_unit.c" if ! grep -Fq '#include "generated_static_probe.inc"' \ "$tmp/ci/tools/test_static_probe_unit.c"; then echo 'probe seam: detection control did not redirect the unit fixture' >&2 exit 1 fi cc=${CC:-cc} "$cc" -std=gnu99 -Wall -Wextra -Werror -O1 \ -I "$tmp/ci/tools" -o "$tmp/test_static_probe_unit" \ "$tmp/ci/tools/test_static_probe_unit.c" "$tmp/test_static_probe_unit" >/dev/null if check_definition "$tmp/src/ngx_http_zstd_frame_probe.h" \ ngx_http_zstd_static_probe_frame "$tmp" >/dev/null 2>&1; then echo 'probe seam: detection control missed a copied CI implementation' >&2 exit 1 fi # Cache-control control: the same staged-drift shape -- the header # copied back under ci/ (no unit fixture exists for this family, so # the copy alone is the scenario) -- must turn the definition count # red. cp "$root/src/ngx_http_zstd_cache_control.h" \ "$tmp/src/ngx_http_zstd_cache_control.h" cp "$root/src/ngx_http_zstd_cache_control.h" \ "$tmp/ci/tools/copied_cache_control.h" if check_definition "$tmp/src/ngx_http_zstd_cache_control.h" \ ngx_http_zstd_cache_control_value_no_transform "$tmp" \ >/dev/null 2>&1; then echo 'probe seam: detection control missed a copied cache-control header' >&2 exit 1 fi echo 'probe header seam: PASS' myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_proxy_unbuffered_truncation.py000077500000000000000000000335371525232545500310540ustar00rootroot00000000000000#!/usr/bin/env python3 """Regression test for bug B: silent truncation / aborted response when zstd compresses a proxied response with ``proxy_buffering off``. Background ---------- Production (deb.myguard.nl, ``proxy_buffering off`` in the WordPress vhost) served truncated / empty zstd responses. Root cause: under ``proxy_buffering off`` the upstream forces a flush around body data on a compress call that feeds libzstd nothing (or sub-block-size input it buffers internally); the filter then forwarded a *zero-size* buffer, which nginx's ``ngx_http_write_filter`` rejects with ``zero size buf in writer`` and aborts the request. The body filter's compress-directive selection ignored ``ctx->flush`` and the empty-buffer emit guard had a ``!(rc == 0 && ctx->flush)`` exception that let the empty buffer through. Why the existing suite missed it -------------------------------- ``t/00-filter.t`` chunked cases use ``--- ignore_response`` (assert a log line, never decode the body). ``test_terminal_frame.py`` uses a tiny body that never crosses an output buffer. None used ``proxy_buffering off``. This test closes that gap: it reproduces the exact production transport (proxy_pass -> chunked, no Content-Length, ``proxy_buffering off``) and **decodes + byte-compares the full body** across the buffer-boundary size matrix. It MUST fail on a module without the fix (truncated/aborted -> decoded bytes differ or curl fails) and pass with it. Hard test-rig discipline (learned the hard way during the bug-B hunt) -------------------------------------------------------------------- * Decode and byte-compare the *whole* body and assert exact length — never trust an HTTP 200 or a non-empty body alone. * Assert the response for size N actually has length N: guards against a stale / wrong-body false-green. * Threaded backend: a single-threaded http.server is OOM/stall-killed under the boundary matrix and produces mass false failures. * Verify the backend answers correctly *before* starting nginx. * Fresh output file per request; ``curl -f`` so an HTTP error is a failure, not a silently reused stale file. Self-contained: stdlib + the ``zstd`` CLI only (no PHP-FPM/fcgiwrap), so it runs anywhere the rest of the Python suite does. """ from __future__ import annotations import argparse import http.server import os import pathlib import socket import socketserver import subprocess import sys import tempfile import threading import time import urllib.request # Sizes straddle ZSTD_CStreamInSize (~131072 = 128 KiB), the historical # boundary for this module's truncation bug class. 1 and 1024 are below # one zstd block (output buffered internally -> the exact forced-flush # zero-output path); the rest force multi-buffer streaming. SIZES = [1, 1024, 131071, 131072, 131073, 200000, 1048576] def parse_args() -> argparse.Namespace: p = argparse.ArgumentParser( description=( "Regression test for the proxy_buffering-off zstd truncation bug (bug B)." ) ) p.add_argument( "--nginx-binary", required=True, help="nginx (or angie) binary to launch." ) p.add_argument( "--filter-module", help="Path to ngx_http_zstd_filter_module.so " "(auto-detected next to the binary if omitted).", ) p.add_argument( "--static-module", help="Path to ngx_http_zstd_static_module.so " "(auto-detected next to the binary if omitted).", ) p.add_argument( "--zstd-bin", default="zstd", help="zstd CLI used for decompression." ) p.add_argument("--port", type=int, default=18086, help="Front-end nginx port.") p.add_argument( "--backend-port", type=int, default=18087, help="Mock chunked upstream port." ) p.add_argument( "--repeat", type=int, default=5, help="Times to re-request each size (the bug is " "intermittent at the exact boundary).", ) return p.parse_args() def detect_module( explicit: str | None, nginx: pathlib.Path, name: str ) -> pathlib.Path | None: if explicit: return pathlib.Path(explicit) sib = nginx.parent / name return sib if sib.exists() else None class _Handler(http.server.BaseHTTPRequestHandler): """Serve /b/ as exactly deterministic bytes, **chunked, no Content-Length**, written in small records so the response is a genuine streamed chunked body (the production shape).""" protocol_version = "HTTP/1.1" fixture_dir: pathlib.Path = pathlib.Path() def log_message(self, *a): pass def do_GET(self): name = self.path.rsplit("/", 1)[-1].split("?")[0] path = self.fixture_dir / name if not path.is_file(): self.send_response(404) self.send_header("Content-Length", "0") self.end_headers() return data = path.read_bytes() self.send_response(200) self.send_header("Content-Type", "application/octet-stream") self.send_header("Transfer-Encoding", "chunked") self.end_headers() mv = memoryview(data) step = 16384 for i in range(0, len(mv), step): chunk = bytes(mv[i : i + step]) self.wfile.write(b"%X\r\n" % len(chunk) + chunk + b"\r\n") self.wfile.write(b"0\r\n\r\n") class _ThreadingHTTPServer(socketserver.ThreadingMixIn, socketserver.TCPServer): # Single-threaded server is OOM/stall-killed by the matrix and # yields mass false failures — must be threaded. allow_reuse_address = True daemon_threads = True def wait_port(port: int, timeout: float = 10.0) -> None: deadline = time.time() + timeout while time.time() < deadline: try: with socket.create_connection(("127.0.0.1", port), 0.5): return except OSError: time.sleep(0.1) raise RuntimeError(f"nothing listening on 127.0.0.1:{port}") def http_get(port: int, path: str, timeout: float = 20.0) -> bytes: req = urllib.request.Request( f"http://127.0.0.1:{port}{path}", headers={"Accept-Encoding": "zstd", "User-Agent": "zstd-bugb-regression/1.0"}, ) with urllib.request.urlopen(req, timeout=timeout) as resp: if resp.headers.get("Content-Encoding", "").lower() != "zstd": raise RuntimeError( f"{path}: expected Content-Encoding=zstd, got " f"{resp.headers.get('Content-Encoding')!r}" ) return resp.read() def zstd_decompress(zstd_bin: str, blob: bytes) -> bytes: if blob[:4] != b"\x28\xb5\x2f\xfd": raise RuntimeError( f"missing zstd magic; first 16B hex={blob[:16].hex()} " f"(truncated/aborted response)" ) r = subprocess.run( [zstd_bin, "-d", "-q", "-c"], input=blob, capture_output=True, check=False ) if r.returncode != 0: raise RuntimeError( "zstd -d failed (premature end / corrupt frame): " + r.stderr.decode("utf-8", "replace").strip() ) return r.stdout def main() -> int: args = parse_args() nginx = pathlib.Path(args.nginx_binary) if not nginx.exists(): raise FileNotFoundError(f"nginx binary not found: {nginx}") filt = detect_module(args.filter_module, nginx, "ngx_http_zstd_filter_module.so") stat = detect_module(args.static_module, nginx, "ngx_http_zstd_static_module.so") modules = [m for m in (filt, stat) if m is not None] for m in modules: if not m.exists(): raise FileNotFoundError(f"module not found: {m}") # Everything below must stay readable by the workers when run as # root (they drop to the compiled-in nginx user): a restrictive # inherited umask (e.g. 077) would strip group/other bits from # every fixture and subdir created here, 403ing the workers even # with the scratch root itself opened up. os.umask(0o022) with tempfile.TemporaryDirectory(prefix="zstd-bugb-") as td: # mkdtemp gives 0700: run as root, workers drop to the # compiled-in nginx user and cannot enter it -> 403s os.chmod(td, 0o755) root = pathlib.Path(td) fixtures = root / "fix" logs = root / "logs" fixtures.mkdir() logs.mkdir() # Deterministic, low-compressibility fixtures (like real CSS: # ~1:1, so the compressed stream genuinely crosses buffers). # Distinct per size so a wrong-body response is detectable. for n in SIZES: buf = bytearray(n) x = (n * 2654435761) & 0xFFFFFFFF for i in range(n): x = (x * 1103515245 + 12345) & 0xFFFFFFFF buf[i] = (x >> 16) & 0xFF (fixtures / str(n)).write_bytes(bytes(buf)) _Handler.fixture_dir = fixtures backend = _ThreadingHTTPServer(("127.0.0.1", args.backend_port), _Handler) bt = threading.Thread(target=backend.serve_forever, daemon=True) bt.start() try: wait_port(args.backend_port) # Verify the backend itself is correct BEFORE trusting any # end-to-end result (rig-discipline rule). for n in (1, 131072, 1048576): raw = urllib.request.urlopen( f"http://127.0.0.1:{args.backend_port}/b/{n}", timeout=10 ).read() if len(raw) != n: raise RuntimeError( f"backend self-check failed: /b/{n} returned " f"{len(raw)} bytes, expected {n}" ) load = "".join(f"load_module {m};\n" for m in modules) conf = root / "nginx.conf" conf.write_text( f"""{load}worker_processes 1; error_log {logs}/error.log warn; pid {root}/nginx.pid; events {{ worker_connections 64; }} http {{ access_log off; default_type application/octet-stream; zstd on; zstd_comp_level 3; zstd_min_length 1; zstd_types application/octet-stream; server {{ listen 127.0.0.1:{args.port}; location /b/ {{ # The exact production trigger: unbuffered proxying of a # chunked, no-Content-Length upstream through the zstd # body filter. proxy_pass http://127.0.0.1:{args.backend_port}; proxy_http_version 1.1; proxy_set_header Connection ""; proxy_buffering off; }} }} }} """, encoding="utf-8", ) proc = subprocess.Popen( [ str(nginx), "-p", str(root), "-c", str(conf), "-g", "daemon off; master_process off;", ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, ) try: wait_port(args.port) failures: list[str] = [] for n in SIZES: expected = (fixtures / str(n)).read_bytes() for attempt in range(1, args.repeat + 1): label = f"size={n} attempt={attempt}/{args.repeat}" try: blob = http_get(args.port, f"/b/{n}") decoded = zstd_decompress(args.zstd_bin, blob) except Exception as exc: # noqa: BLE001 failures.append(f"{label}: {exc}") continue if len(decoded) != n: failures.append( f"{label}: decoded {len(decoded)} bytes, " f"expected {n} (TRUNCATION)" ) elif decoded != expected: failures.append( f"{label}: decoded body differs from " f"origin at equal length (CORRUPTION)" ) if failures: sys.stderr.write( f"bug-B regression FAILED ({len(failures)} failing checks):\n" ) for f in failures[:20]: sys.stderr.write(f" - {f}\n") elog = logs / "error.log" if elog.exists(): zsb = [ ln for ln in elog.read_text("utf-8", "replace").splitlines() if "zero size buf in writer" in ln ] if zsb: sys.stderr.write( f" nginx logged {len(zsb)} " f"'zero size buf in writer' " f"(the bug-B signature)\n" ) return 1 total = len(SIZES) * args.repeat print( f"OK: {total} proxy_buffering-off zstd responses " f"across {len(SIZES)} boundary sizes round-tripped " f"byte-exact (no truncation, no zero-size-buf abort)" ) return 0 finally: proc.terminate() try: # 30s, not 5: gcov-instrumented nginx flushing # .gcda at exit on a loaded runner can outlive a # tight reap budget; teardown runs after the # verdict, so patience costs nothing when healthy. proc.wait(timeout=30) except subprocess.TimeoutExpired: proc.kill() proc.wait(timeout=30) finally: backend.shutdown() backend.server_close() if __name__ == "__main__": try: raise SystemExit(main()) except Exception as exc: print(f"ERROR: {exc}", file=sys.stderr) raise myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_ratio_scaling_unit.c000066400000000000000000000205761525232545500266630ustar00rootroot00000000000000/* * Copyright (C) 2026 Thijs Eilander * SPDX-License-Identifier: BSD-2-Clause * * Unit oracle for ngx_http_zstd_ratio_parts() -- the $zstd_ratio scaling * helper. The previous form computed `bytes_in * 1000 / bytes_out` in one * step: for a bytes_in near UINT64_MAX the *1000 multiplication overflows * the uint64_t accumulator and wraps, corrupting both the integer and * fractional digits reported to the client/log. The fixed form divides * first and only scales the (necessarily smaller) remainder, so the * multiplication cannot overflow. * * The function is EXTRACTED VERBATIM from * src/ngx_http_zstd_filter_module.c by test_ratio_scaling_unit.sh into * generated_ratio_parts.inc -- this file never re-implements the scaling, * so it cannot quietly agree with a stale copy of itself. * * What is asserted, and why: * * 1. THRESHOLD/BOUNDARY cases at ordinary byte counts (e.g. bytes_in == * bytes_out, 2x, non-exact ratios) must keep producing the same * integer and fractional digits as before -- rounding/output below * the overflow threshold is a preserved invariant, not something this * row is allowed to change. * * 2. OVERFLOW NEGATIVE CONTROL: bytes_in near UINT64_MAX with a small * bytes_out must still produce the mathematically correct integer * part (bytes_in / bytes_out, verified via widening arithmetic in the * test itself) rather than a wrapped/corrupted value. This is the * case the old `* 1000` first form got wrong. */ #include #include #include #include #include typedef unsigned long ngx_uint_t; /* THE authoritative copy, included directly -- no extracted duplicate. */ #include "../../src/ngx_http_zstd_ratio.h" static long long failures; static void fail(const char *what, uint64_t bytes_in, uint64_t bytes_out, ngx_uint_t got_int, ngx_uint_t got_frac, ngx_uint_t want_int, ngx_uint_t want_frac) { failures++; if (failures <= 20) { fprintf(stderr, "FAIL %s: bytes_in=%" PRIu64 " bytes_out=%" PRIu64 " -> %lu.%03lu, want %lu.%03lu\n", what, bytes_in, bytes_out, got_int, got_frac, want_int, want_frac); } } static void check(const char *what, uint64_t bytes_in, uint64_t bytes_out, ngx_uint_t want_int, ngx_uint_t want_frac) { ngx_uint_t got_int, got_frac; ngx_http_zstd_ratio_parts(bytes_in, bytes_out, &got_int, &got_frac); if (got_int != want_int || got_frac != want_frac) { fail(what, bytes_in, bytes_out, got_int, got_frac, want_int, want_frac); } } int main(void) { /* 1. Ordinary threshold/boundary cases -- output must be unchanged. */ check("equal", 1000, 1000, 1, 0); check("2x-exact", 2000, 1000, 2, 0); check("half", 500, 1000, 0, 500); check("one-third", 1000, 3000, 0, 333); check("small-remainder", 1001, 1000, 1, 1); check("large-ordinary", 987654321ULL, 123456789ULL, 8, 0); /* 2. Overflow negative control: bytes_in near UINT64_MAX. The old * `bytes_in * 1000 / bytes_out` form would overflow computing * bytes_in * 1000 here and wrap to a wrong (and non-deterministic * looking) integer part; the fixed divide-first form must still * report the true quotient. */ { uint64_t bytes_in = UINT64_MAX - 3; uint64_t bytes_out = 7; uint64_t want_int = bytes_in / bytes_out; uint64_t remainder = bytes_in % bytes_out; uint64_t want_frac = remainder * 1000 / bytes_out; check("near-uint64-max-small-divisor", bytes_in, bytes_out, (ngx_uint_t) want_int, (ngx_uint_t) want_frac); } { uint64_t bytes_in = UINT64_MAX; uint64_t bytes_out = UINT64_MAX - 1; /* bytes_in / bytes_out == 1, remainder == 1, frac == 1000/(MAX-1) * which truncates to 0. */ check("near-uint64-max-both", bytes_in, bytes_out, 1, 0); } /* 3. Expanding stream: bytes_out > bytes_in, which happens whenever * zstd expands incompressible input. This is the case a divide-first * form still gets wrong -- the remainder is then bytes_in itself, so * scaling it by 1000 in one step overflows exactly as the original * single-expression form did. Reported by CodeRabbit against the first * version of this fix; the digit-at-a-time long division must report * 0.999 rather than wrapping to 0.000. */ check("large-expanding-stream", UINT64_MAX - 1, UINT64_MAX, 0, 999); check("expanding-half", 500, 1000, 0, 500); check("expanding-near-max-third", UINT64_MAX / 3, UINT64_MAX, 0, 333); /* 4. Differential sweep against an unsigned __int128 oracle over a * mix of ordinary and extreme magnitudes. The oracle computes the * exact bytes_in*1000/bytes_out in 128-bit, so any wrap or precision * loss in the 64-bit implementation shows up as a mismatch rather * than needing a hand-picked witness per case. */ { static const uint64_t vals[] = { 1, 2, 3, 7, 999, 1000, 1001, 65535, 4294967295ULL, 4294967296ULL, 987654321987654321ULL, UINT64_MAX / 3, UINT64_MAX / 2, UINT64_MAX - 1, UINT64_MAX }; size_t n = sizeof(vals) / sizeof(vals[0]); size_t i, j; for (i = 0; i < n; i++) { for (j = 0; j < n; j++) { unsigned __int128 scaled; uint64_t bi = vals[i]; uint64_t bo = vals[j]; ngx_uint_t gi, gf; char name[64]; scaled = ((unsigned __int128) bi * 1000) / (unsigned __int128) bo; ngx_http_zstd_ratio_parts(bi, bo, &gi, &gf); if (gi != (ngx_uint_t) (scaled / 1000) || gf != (ngx_uint_t) (scaled % 1000)) { snprintf(name, sizeof(name), "oracle[%zu][%zu]", i, j); printf("FAIL %s: bytes_in=%llu bytes_out=%llu -> " "%llu.%03llu, want %llu.%03llu\n", name, (unsigned long long) bi, (unsigned long long) bo, (unsigned long long) gi, (unsigned long long) gf, (unsigned long long) (scaled / 1000), (unsigned long long) (scaled % 1000)); failures++; } } } } /* 5. Randomized differential sweep against the same 128-bit oracle, * with a fixed seed so a failure is reproducible. Covers magnitude * combinations the hand-picked table does not enumerate, including * many bytes_out > bytes_in (expanding) pairs. */ { unsigned long long state = 0x9E3779B97F4A7C15ULL; int iter; for (iter = 0; iter < 200000; iter++) { unsigned __int128 scaled; uint64_t bi, bo; ngx_uint_t gi, gf; state = state * 6364136223846793005ULL + 1442695040888963407ULL; bi = state; state = state * 6364136223846793005ULL + 1442695040888963407ULL; bo = state; /* exercise small magnitudes too, not only full-width values */ if (iter % 3 == 1) { bi >>= (iter % 63); } if (iter % 3 == 2) { bo >>= (iter % 63); } if (bo == 0) { bo = 1; } scaled = ((unsigned __int128) bi * 1000) / (unsigned __int128) bo; ngx_http_zstd_ratio_parts(bi, bo, &gi, &gf); if (gi != (ngx_uint_t) (scaled / 1000) || gf != (ngx_uint_t) (scaled % 1000)) { printf("FAIL random[%d]: bytes_in=%llu bytes_out=%llu -> " "%llu.%03llu, want %llu.%03llu\n", iter, (unsigned long long) bi, (unsigned long long) bo, (unsigned long long) gi, (unsigned long long) gf, (unsigned long long) (scaled / 1000), (unsigned long long) (scaled % 1000)); failures++; break; } } } if (failures) { printf("FAILED: %lld assertion(s)\n", failures); return 1; } printf("OK: ratio scaling (ordinary thresholds unchanged, " "near-UINT64_MAX overflow-safe)\n"); return 0; } myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_ratio_scaling_unit.sh000077500000000000000000000071421525232545500270500ustar00rootroot00000000000000#!/bin/bash # Copyright (C) 2026 Thijs Eilander # SPDX-License-Identifier: BSD-2-Clause # # Unit fixture for ngx_http_zstd_ratio_parts() -- the overflow-safe # $zstd_ratio integer/fractional scaling helper. # # The helper lives in src/ngx_http_zstd_ratio.h, THE authoritative copy, # and the fixture includes that header directly (the frame-probe fixture's # shape): there is no extracted or hand-copied duplicate here to drift from # production. No nginx tree and no libzstd needed: pure C, self-contained. set -euo pipefail # ci/tools/ -> ci/ -> repo root. cd "$(dirname "$0")/../.." HDR="src/ngx_http_zstd_ratio.h" SRC="src/ngx_http_zstd_filter_module.c" CC="${CC:-cc}" OUT="$(mktemp -d "${TMPDIR:-/tmp}/zstd-ratio-scaling-unit.XXXXXX")" trap 'rm -rf "$OUT"' EXIT for marker in 'ngx_http_zstd_ratio_parts(uint64_t bytes_in' \ 'remainder <= UINT64_MAX / 10' 'ratio_int' 'ratio_frac'; do if ! grep -q "$marker" "$HDR"; then echo "FAIL: $HDR lacks marker: $marker" >&2 exit 1 fi done # Production must route $zstd_ratio through the header's helper. The # assertion is bound to the variable HANDLER's body, not the whole file: a # helper call anywhere else plus inline arithmetic back in the handler # must still fail. The body is cut from the handler's column-0 signature # to its closing brace. if ! grep -Fq '#include "ngx_http_zstd_ratio.h"' "$SRC"; then echo "FAIL: $SRC no longer includes $HDR" >&2 exit 1 fi handler_body() { sed -n '/^ngx_http_zstd_ratio_variable(ngx_http_request_t \*r,$/,/^}/p' "$1" } assert_routes_through_helper() { local body body="$(handler_body "$1")" if [ -z "$body" ]; then echo "FAIL: could not cut ngx_http_zstd_ratio_variable() out of $1" >&2 return 1 fi if ! printf '%s\n' "$body" | grep -q 'ngx_http_zstd_ratio_parts(ctx->bytes_in, ctx->bytes_out'; then echo "FAIL: \$zstd_ratio no longer calls ngx_http_zstd_ratio_parts()" >&2 return 1 fi # Any multiply or divide on the byte counters inside the handler is # inline ratio arithmetic; the helper call carries neither operator and # the bytes_out == 0 guard is a comparison. if printf '%s\n' "$body" | grep -Eq 'bytes_(in|out)[^;]*[*/]|[*/][^;]*bytes_(in|out)'; then echo "FAIL: \$zstd_ratio computes the ratio inline again" >&2 return 1 fi } assert_routes_through_helper "$SRC" # Negative control: put the old arithmetic back into the handler (the # helper call stays elsewhere in the file) and the check must go red. sed 's|^ ngx_http_zstd_ratio_parts(ctx->bytes_in, ctx->bytes_out, &ratio_int,$| ratio_int = (ngx_uint_t) (ctx->bytes_in * 1000 / ctx->bytes_out / 1000);|' \ "$SRC" >"$OUT/mutant.c" if ! grep -q 'bytes_in \* 1000' "$OUT/mutant.c"; then echo "FAIL: mutant did not apply (handler call site moved?)" >&2 exit 1 fi if assert_routes_through_helper "$OUT/mutant.c" >/dev/null 2>&1; then echo "FAIL: inline-arithmetic mutant passed the routing check" >&2 exit 1 fi "$CC" -std=gnu99 -Wall -Wextra -Werror -O2 -I ci/tools \ -o "$OUT/ratio_scaling_unit" ci/tools/test_ratio_scaling_unit.c "$OUT/ratio_scaling_unit" # The header itself must stay C89-clean for the portable build, on its own # fallback (ngx_inline empty; no -D mask). The fixture's 128-bit oracle and # ULL literals are gnu99 and stay out of this pass. printf '#include \ntypedef uintptr_t ngx_uint_t;\n#include "../../src/ngx_http_zstd_ratio.h"\nint main(void) { ngx_uint_t a, b; ngx_http_zstd_ratio_parts(3, 2, &a, &b); return a == 1 && b == 500 ? 0 : 1; }\n' \ >"$OUT/c89.c" "$CC" -std=c89 -pedantic-errors -Wall -Wextra -Werror -O1 \ -I ci/tools -o "$OUT/ratio_c89" "$OUT/c89.c" "$OUT/ratio_c89" echo "OK: ratio scaling unit fixture (against $HDR)" myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_read_dict_file_unit.c000066400000000000000000000317401525232545500267550ustar00rootroot00000000000000/* * Unit fixture for the dictionary read loop, in both its layers: * * - ngx_http_zstd_dict_file_read(), THE authoritative loop in * src/ngx_http_zstd_dict_file.h, included directly (the frame-probe * and ratio fixtures' shape): no extracted or hand-copied duplicate * here to drift from production. * - ngx_http_zstd_read_dict_file(), the filter module's logging shell * around it, still `static` inside src/ngx_http_zstd_filter_module.c * and so extracted verbatim by test_read_dict_file_unit.sh into the * generated file included below. That pins the shell's contract -- * NGX_OK only for a complete read, exactly one diagnostic per * failure, the failing read's errno handed to the logger -- and, * because the script refuses an extraction that still contains a * read call, that the module routes through the header. * * WHY A UNIT FIXTURE AND NOT A CONFIG FIXTURE. The three behaviours that * matter here are short reads, EINTR, and early EOF. None is reachable * from ci/tools/test_dict_path_hardening.sh: read() on a local tmpfs/ext4 * regular file never returns a short count, a signal cannot be steered * into the master's read window from a shell, and shrinking the file * between fstat() and read() is a race, not a fixture. Measured: with the * loop reverted to a single read whose short count is fatal, EVERY * config-level fixture still passed -- including a 1 MiB dictionary. A * config fixture cannot discriminate this change; a stubbed read can. * * ngx_read_fd is a macro over read(2) in the shipped tree. Here it is a * scripted stub so each call's return value and errno are chosen by the * test, which is what makes the loop's three exits (continue-on-short, * retry-on-EINTR, stop-on-EOF) directly observable. */ #include #include #include #include #include /* ssize_t */ #include /* close(), for the walk's contract */ /* --- minimal nginx type/macro surface the header and the shell need --- */ typedef intptr_t ngx_int_t; typedef uintptr_t ngx_uint_t; typedef int ngx_fd_t; typedef unsigned char u_char; #define NGX_OK 0 #define NGX_ERROR -1 #define NGX_EINTR EINTR #define ngx_errno errno #define ngx_read_fd_n "read()" /* the strict walk's contract: compiled in with the header, unused here */ #define NGX_INVALID_FILE -1 #define NGX_MAX_PATH 4096 #define ngx_memcpy memcpy #define ngx_strcmp(a, b) strcmp((const char *) (a), (const char *) (b)) #define ngx_close_file close typedef struct { size_t len; u_char *data; } ngx_str_t; typedef struct { int unused; } ngx_conf_t; #define NGX_LOG_EMERG 2 /* * The loop guards its EINTR retry with #if !(NGX_WIN32). Define it to 0 * explicitly: an undefined macro already evaluates to 0 in #if, so this * changes nothing, but it makes the POSIX build the stated intent rather * than a side effect of the macro being absent -- and it is the branch * the EINTR assertion below exercises. */ #define NGX_WIN32 0 /* * Diagnostics are counted, not printed: the assertions below care that * the shell REPORTS (exactly once, on the right exit, with the right * errno), not about the wording. Variadic and ignoring its format so any * message the shipped code uses compiles unchanged. */ static int log_calls; static int log_last_err; static void ngx_conf_log_error(int level, ngx_conf_t *cf, int err, const char *fmt, ...) { (void) level; (void) cf; (void) fmt; log_calls++; log_last_err = err; } /* --- scripted ngx_read_fd stub --------------------------------------- * * Each entry is one call's outcome: n >= 0 is a byte count to deliver * (0 = EOF), n < 0 is a failure carrying `err` in errno. The stub fills * the destination with a byte pattern derived from the running offset so * the test can prove the bytes landed CONTIGUOUSLY and in order -- a loop * that re-read into the wrong offset would still satisfy a length check. */ typedef struct { ssize_t n; int err; } step_t; /* declared before the stub: an exhausted script must fail the run */ static int failures; static step_t steps[16]; static int nsteps; static int step_i; static size_t delivered; static ssize_t ngx_read_fd(ngx_fd_t fd, void *buf, size_t size) { step_t *s; size_t i, give; (void) fd; if (step_i >= nsteps) { /* a read the script did not expect is a failed assertion, not * just a message: count it so the run cannot exit 0 */ fprintf(stderr, "FAIL: stub ran out of scripted steps " "(loop called read() more times than expected)\n"); failures++; errno = EIO; return -1; } s = &steps[step_i++]; if (s->n < 0) { errno = s->err; return -1; } give = (size_t) s->n; if (give > size) { fprintf(stderr, "FAIL: scripted step %d returns %zu for a %zu-byte " "request -- the loop asked for less than it should\n", step_i - 1, give, size); return -1; } for (i = 0; i < give; i++) { ((u_char *) buf)[i] = (u_char) ((delivered + i) & 0xff); } delivered += give; return (ssize_t) give; } /* * The header's members are `static ngx_inline`; this fixture uses the * loop and the hex decoder but not the strict walk, so ngx_inline must * be a real `inline` here or -Werror=unused-function fires on the walk. * (The C89 pass in the .sh keeps the empty fallback and allows unused.) */ #define ngx_inline inline #include "../../src/ngx_http_zstd_dict_file.h" #include "generated_read_dict_file.inc" /* --- harness ---------------------------------------------------------- */ static void reset(void) { nsteps = 0; step_i = 0; delivered = 0; log_calls = 0; log_last_err = 0; } static void push(ssize_t n, int err) { steps[nsteps].n = n; steps[nsteps].err = err; nsteps++; } static void check(const char *name, long got, long want) { if (got != want) { printf("✗ %s: got %ld want %ld\n", name, got, want); failures++; return; } printf("✓ %s\n", name); } /* Verify the buffer holds the contiguous 0,1,2,... pattern for `size`. */ static int pattern_ok(const u_char *buf, size_t size) { size_t i; for (i = 0; i < size; i++) { if (buf[i] != (u_char) (i & 0xff)) { printf(" buffer diverges at offset %zu: got %u want %u\n", i, buf[i], (unsigned) (i & 0xff)); return 0; } } return 1; } int main(void) { u_char buf[4096]; ngx_str_t path; ngx_conf_t cf; ssize_t n; ngx_int_t rc; int read_errno; path.len = 4; path.data = (u_char *) "d.dc"; printf("# ngx_http_zstd_dict_file_read() -- the header's loop\n"); /* 1. One full read: the ordinary case must still work. */ reset(); memset(buf, 0xee, sizeof(buf)); push(1024, 0); n = ngx_http_zstd_dict_file_read(3, buf, 1024); check("single complete read returns size", (long) n, 1024); if (n == 1024 && !pattern_ok(buf, 1024)) { failures++; } /* * 2. THE REGRESSION THE LOOP EXISTS FOR. Three short counts that add * up to the requested size must complete. The single-read form * stopped at the first partial count. */ reset(); memset(buf, 0xee, sizeof(buf)); push(100, 0); push(400, 0); push(524, 0); n = ngx_http_zstd_dict_file_read(3, buf, 1024); check("short reads are resumed, not fatal", (long) n, 1024); if (n == 1024 && !pattern_ok(buf, 1024)) { failures++; } check("short-read case issued exactly three reads", step_i, 3); /* 3. A single-byte dribble still completes (worst-case chunking). */ reset(); memset(buf, 0xee, sizeof(buf)); push(1, 0); push(1, 0); push(1, 0); push(1, 0); push(1, 0); n = ngx_http_zstd_dict_file_read(3, buf, 5); check("one-byte-at-a-time reads complete", (long) n, 5); if (n == 5 && !pattern_ok(buf, 5)) { failures++; } /* * 4. EINTR is retried and must NOT lose progress. The interrupt lands * BETWEEN two partial reads, which is the arrangement that catches a * retry that restarts at offset 0. */ reset(); memset(buf, 0xee, sizeof(buf)); push(600, 0); push(-1, EINTR); push(424, 0); n = ngx_http_zstd_dict_file_read(3, buf, 1024); check("EINTR is retried", (long) n, 1024); if (n == 1024 && !pattern_ok(buf, 1024)) { failures++; } check("EINTR retry resumed at the right offset (three reads)", step_i, 3); /* * 5. Early EOF with bytes still owed returns the PARTIAL count, not * an error: the caller compares it with the size it validated and * names the file that changed under it. The bytes that did land are * intact. */ reset(); memset(buf, 0xee, sizeof(buf)); push(500, 0); push(0, 0); n = ngx_http_zstd_dict_file_read(3, buf, 1024); check("early EOF returns the partial count", (long) n, 500); if (n == 500 && !pattern_ok(buf, 500)) { failures++; } /* * 6. A hard read error returns -1 with the failing read's errno * intact: the shell logs ngx_errno after the call, so the loop must * return without touching it. */ reset(); errno = 0; push(500, 0); push(-1, EIO); n = ngx_http_zstd_dict_file_read(3, buf, 1024); read_errno = errno; /* before check() prints: stdio may set errno */ check("read error returns -1", (long) n, -1); check("read error leaves the failing read's errno in place", read_errno, EIO); check("read error is not retried", step_i, 2); /* * 7. An immediate EOF on a size the caller believed non-zero is the * "file shrank to nothing between fstat() and read()" case. */ reset(); push(0, 0); n = ngx_http_zstd_dict_file_read(3, buf, 1024); check("immediate EOF returns 0", (long) n, 0); printf("# ngx_http_zstd_hex_nibble() -- the supplied-hash digit decoder\n"); /* * 13. Every byte value against a reference: the sixteen digits in * either case decode to their value, everything else -- including the * bytes adjacent to each run, and the high half -- to 0xff. This is * the decoder that decides whether a supplied hash literal is trusted * verbatim, so a stray accepted byte is a silently wrong key. */ { unsigned c, bad = 0; for (c = 0; c < 256; c++) { unsigned want = 0xff; if (c >= '0' && c <= '9') { want = c - '0'; } else if (c >= 'a' && c <= 'f') { want = c - 'a' + 10; } else if (c >= 'A' && c <= 'F') { want = c - 'A' + 10; } if (ngx_http_zstd_hex_nibble((u_char) c) != want) { if (bad < 5) { printf(" hex_nibble(0x%02x) = 0x%02x, want 0x%02x\n", c, ngx_http_zstd_hex_nibble((u_char) c), want); } bad++; } } check("hex_nibble decodes all 256 byte values as the reference does", (long) bad, 0); } printf("# ngx_http_zstd_read_dict_file() -- the module's logging shell\n"); /* 8. A complete read is NGX_OK and logs nothing. */ reset(); push(100, 0); push(924, 0); rc = ngx_http_zstd_read_dict_file(&cf, 3, &path, buf, 1024); check("shell: complete read is NGX_OK", (long) rc, NGX_OK); check("shell: complete read logs nothing", log_calls, 0); /* 9. EINTR mid-file is transparent to the shell: no diagnostic. */ reset(); push(600, 0); push(-1, EINTR); push(424, 0); rc = ngx_http_zstd_read_dict_file(&cf, 3, &path, buf, 1024); check("shell: EINTR retry is NGX_OK", (long) rc, NGX_OK); check("shell: EINTR retry logs nothing", log_calls, 0); /* 10. Early EOF is NGX_ERROR with exactly one diagnostic, errno 0. */ reset(); push(500, 0); push(0, 0); rc = ngx_http_zstd_read_dict_file(&cf, 3, &path, buf, 1024); check("shell: early EOF is NGX_ERROR", (long) rc, NGX_ERROR); check("shell: early EOF logs exactly once", log_calls, 1); check("shell: early EOF is logged without an errno", log_last_err, 0); /* 11. A read error is NGX_ERROR, logged once WITH the read's errno. */ reset(); push(500, 0); push(-1, EIO); rc = ngx_http_zstd_read_dict_file(&cf, 3, &path, buf, 1024); check("shell: read error is NGX_ERROR", (long) rc, NGX_ERROR); check("shell: read error logs exactly once", log_calls, 1); check("shell: read error is logged with the read's errno", log_last_err, EIO); /* 12. Immediate EOF: the shell reports it like any other short file. */ reset(); push(0, 0); rc = ngx_http_zstd_read_dict_file(&cf, 3, &path, buf, 1024); check("shell: immediate EOF is NGX_ERROR", (long) rc, NGX_ERROR); check("shell: immediate EOF logs exactly once", log_calls, 1); if (failures) { printf("❌ %d read_dict_file unit assertion(s) failed\n", failures); return 1; } printf("✓ all read_dict_file unit assertions passed\n"); return 0; } myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_read_dict_file_unit.sh000077500000000000000000000125021525232545500271430ustar00rootroot00000000000000#!/bin/bash # Unit fixture for the dictionary read loop, both layers. # # The loop, ngx_http_zstd_dict_file_read(), lives in # src/ngx_http_zstd_dict_file.h, THE authoritative copy, and the fixture # includes that header directly (the frame-probe and ratio fixtures' # shape): no extracted or hand-copied duplicate to drift from production. # # The filter module's logging shell around it, ngx_http_zstd_read_dict_file(), # is still `static` inside src/ngx_http_zstd_filter_module.c, so it is # extracted verbatim by line range and compiled against the same scripted # ngx_read_fd stub. An extraction that still contains a read call is # refused: that is the loop restored inside the module, the copy drift the # header exists to end. No nginx tree, no zstd needed -- pure C. set -euo pipefail # ci/tools/ -> ci/ -> repo root: TWO levels. cd "$(dirname "$0")/../.." HDR="src/ngx_http_zstd_dict_file.h" SRC="src/ngx_http_zstd_filter_module.c" CC="${CC:-cc}" OUT="$(mktemp -d "${TMPDIR:-/tmp}/zstd-read-dict-unit.XXXXXX")" trap 'rm -rf "$OUT"' EXIT for marker in 'ngx_http_zstd_dict_file_read(ngx_fd_t fd, u_char \*buf, size_t size)' \ 'ngx_errno == NGX_EINTR' 'ngx_http_zstd_hex_nibble(u_char c)'; do if ! grep -q "$marker" "$HDR"; then echo "FAIL: $HDR lacks marker: $marker" >&2 exit 1 fi done if ! grep -Fq '#include "ngx_http_zstd_dict_file.h"' "$SRC"; then echo "FAIL: $SRC no longer includes $HDR" >&2 exit 1 fi # Locate the shell by its signature line (the `static ngx_int_t` return # type sits on the line above it, as in this file's house style), so the # extraction survives unrelated edits but fails loudly if the function is # renamed or removed. Each locate pipeline ends in `|| true`: under # pipefail a missing pattern would otherwise kill the script at the # assignment, one line before the diagnostic that names what went missing. SIG_LINE="$(grep -n '^ngx_http_zstd_read_dict_file(ngx_conf_t \*cf, ngx_fd_t fd, ngx_str_t \*path,$' "$SRC" | head -1 | cut -d: -f1 || true)" if [ -z "$SIG_LINE" ]; then echo "FAIL: could not locate ngx_http_zstd_read_dict_file()'s definition in $SRC" >&2 exit 1 fi START_LINE=$((SIG_LINE - 1)) if ! sed -n "${START_LINE}p" "$SRC" | grep -q '^static ngx_int_t$'; then echo "FAIL: line $START_LINE in $SRC is not the expected 'static ngx_int_t' return-type line" >&2 exit 1 fi # First "^}" at column 0 after START_LINE closes the function. REL_END="$(tail -n "+${START_LINE}" "$SRC" | grep -n '^}' | head -1 | cut -d: -f1 || true)" if [ -z "$REL_END" ]; then echo "FAIL: could not find the closing brace of ngx_http_zstd_read_dict_file() in $SRC" >&2 exit 1 fi END_LINE=$((START_LINE + REL_END - 1)) { echo "/* Generated by test_read_dict_file_unit.sh -- verbatim extraction of" echo " * ngx_http_zstd_read_dict_file() from $SRC lines $START_LINE-$END_LINE." echo " * Do not edit; regenerate by running the script. */" sed -n "${START_LINE},${END_LINE}p" "$SRC" } > "$OUT/generated_read_dict_file.inc" # Routing: the shell must call the header's loop and must not read on its # own. A mis-ranged extraction fails the first test; the loop restored # inside the module fails the second. if ! grep -q 'ngx_http_zstd_dict_file_read(fd, buf, size)' "$OUT/generated_read_dict_file.inc"; then echo "FAIL: extracted shell does not call ngx_http_zstd_dict_file_read() -- extraction range is wrong, or the module no longer routes through $HDR" >&2 sed -n '1,30p' "$OUT/generated_read_dict_file.inc" >&2 exit 1 fi if grep -q 'ngx_read_fd(' "$OUT/generated_read_dict_file.inc"; then echo "FAIL: ngx_http_zstd_read_dict_file() reads on its own again; the loop belongs in $HDR" >&2 exit 1 fi $CC -std=gnu99 -Wall -Wextra -Werror -O1 \ -I ci/tools -I "$OUT" \ -o "$OUT/test_read_dict_file_unit" \ ci/tools/test_read_dict_file_unit.c "$OUT/test_read_dict_file_unit" # The header itself must stay C89-clean for the portable build, on its own # fallback (ngx_inline empty; no -D mask), against the real read(2). With # the fallback empty every member is a plain static function, so the # ones this program does not call are allowed to go unused. printf '%s\n' \ '#include ' \ '#include ' \ '#include ' \ '#include ' \ 'typedef unsigned char u_char;' \ 'typedef int ngx_fd_t;' \ 'typedef struct { size_t len; u_char *data; } ngx_str_t;' \ '#define ngx_read_fd(fd, buf, n) read(fd, buf, n)' \ '#define ngx_errno errno' \ '#define NGX_EINTR EINTR' \ '#define NGX_INVALID_FILE -1' \ '#define NGX_MAX_PATH 4096' \ '#define ngx_memcpy memcpy' \ '#define ngx_strcmp(a, b) strcmp((const char *) (a), (const char *) (b))' \ '#define ngx_close_file close' \ '#include "../../src/ngx_http_zstd_dict_file.h"' \ 'int main(void) {' \ ' u_char b[8]; int fd = open("/dev/zero", O_RDONLY);' \ ' if (fd < 0) return 2;' \ ' if (ngx_http_zstd_dict_file_read(fd, b, sizeof(b)) != (ssize_t) sizeof(b)) return 1;' \ ' return ngx_http_zstd_hex_nibble(0x41) == 10 && ngx_http_zstd_hex_nibble(0x47) == 0xff ? 0 : 1;' \ '}' \ >"$OUT/c89.c" "$CC" -std=c89 -pedantic-errors -Wall -Wextra -Werror -Wno-unused-function \ -O1 -I ci/tools -o "$OUT/dict_file_c89" "$OUT/c89.c" "$OUT/dict_file_c89" echo "OK: read_dict_file unit fixture (loop from $HDR, shell extracted from $SRC)" myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_reload_leak.sh000077500000000000000000000104431525232545500254330ustar00rootroot00000000000000#!/usr/bin/env bash # # Reload-under-ASAN leak regression for the zstd_dict_file ZSTD_CDict # lifecycle. Targets the bug class fixed in 0fb40d9 ("Free ZSTD_CDict on # configuration cleanup to prevent memory leak") and f735a5d ("pass size=0 # to ngx_pool_cleanup_add for dict cleanup handler"): the CDict leak only # manifests when the configuration is reloaded (SIGHUP), which the normal # smoke tests never do. # # Strategy: start nginx (built with -fsanitize=address) using a dictionary, # SIGHUP it several times so the old cycle's dict cleanup runs repeatedly, # then stop nginx cleanly. ASAN's leak detector reports on exit; a leak # report in the log fails this script. # # LeakSanitizer uses ptrace for its exit-time stop-the-world phase. LLVM first # runs a heuristic preflight which may print "ptrace appears to be blocked" # even though it then continues and the real check succeeds. Verbose logs plus # ci/tools/lsan_log_verdict.sh distinguish that false warning from an actual # attach failure, require an exit-hook marker for every process, and run a # deliberate-leak control. The shutdown watchdog still bounds a real hang. # # Usage: tools/test_reload_leak.sh [reloads] set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" NGINX="${1:?usage: test_reload_leak.sh [reloads]}" RELOADS="${2:-5}" WORK="$(mktemp -d)" cleanup() { rm -rf "$WORK"; } trap cleanup EXIT mkdir -p "$WORK/conf" "$WORK/logs" "$WORK/html" # A non-trivial dictionary so ZSTD_createCDict() actually allocates. head -c 8192 /dev/urandom | base64 >"$WORK/html/zstd.dict" cat >"$WORK/conf/nginx.conf" </dev/null; then ready=1 break fi sleep 0.1 done if [ "$ready" -ne 1 ]; then echo "❌ nginx under ASAN did not accept connections within 10s" kill -9 "$NGINX_PID" 2>/dev/null || true exit 1 fi for i in $(seq 1 "$RELOADS"); do curl -fsS -o /dev/null -H 'Accept-Encoding: zstd' "http://127.0.0.1:18099/" kill -HUP "$NGINX_PID" sleep 0.5 echo " reload $i/$RELOADS done" done # One final request against the latest cycle, then a clean shutdown so # every cycle's pool cleanup (including the dict cleanup handler) runs. # The watchdog bounds the wait: a ptrace-blocked LSan can hang at exit, # which would otherwise eat the job timeout instead of yielding a # verdict. And wait must not run under set -e: a leak makes nginx exit # 23 (exitcode above), which used to abort the script right here, # skipping the report that says WHY. curl -fsS -o /dev/null -H 'Accept-Encoding: zstd' "http://127.0.0.1:18099/" kill -QUIT "$NGINX_PID" ( sleep 90; kill -9 "$NGINX_PID" 2>/dev/null ) & WATCHDOG=$! set +e wait "$NGINX_PID" rc=$? set -e kill "$WATCHDOG" 2>/dev/null || true if [ "$rc" -eq 137 ]; then echo "::warning::nginx under ASAN had to be killed by the shutdown" \ "watchdog (LeakSanitizer hang?). Leak check INDETERMINATE and" \ "therefore failed." exit 1 fi if [ "$rc" -ne 0 ]; then echo "❌ nginx exited non-zero ($rc) under ASAN after reloads" tail -50 "$WORK/logs/error.log" || true exit 1 fi "$SCRIPT_DIR/lsan_log_verdict.sh" "$WORK/logs/asan*" echo "✓ No CDict leak across $RELOADS config reloads under ASAN" myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_reload_under_load.py000077500000000000000000000320201525232545500266440ustar00rootroot00000000000000#!/usr/bin/env python3 """Regression test: response correctness across nginx reload under load. Bug class --------- Commit 2c538e0 ("fix: register CDict cleanup handler to prevent leak on reload") and the broader CDict / config-cycle lifetime history: a ``zstd_dict_file`` ``ZSTD_CDict`` is owned by the configuration cycle. On ``nginx -s reload`` the old cycle is retired while in-flight requests on old workers are still compressing with the old cycle's CDict. A mishandled lifetime here can leak (covered by the existing ``tools/test_reload_leak.sh``) **or** corrupt / truncate responses that straddle the reload — which nothing currently asserts. This test keeps a steady request stream running, issues repeated ``SIGHUP`` reloads while requests are in flight, and verifies **every** response (old-cycle, straddling, and new-cycle) still decompresses byte-exact to its origin, the master stays up, and the error log stays clean (no ``[alert]``/``[emerg]``/``zero size buf``/sanitizer). Rig discipline (same hard rules as the other regression tests): * threaded backend; backend self-check before trusting results * distinct per-request bodies so a wrong/stale body is detectable * decode-and-diff the full body; HTTP error is a failure * a real ``zstd_dict_file`` so the CDict lifetime path is exercised * overall timeout so a hang/livelock is a hard FAIL Self-contained: stdlib + the ``zstd`` CLI only. """ from __future__ import annotations import argparse import http.server import os import pathlib import signal import socket import socketserver import subprocess import sys import tempfile import threading import time import urllib.request CSTREAM_IN = 131072 SIZES = [4096, 65000, CSTREAM_IN - 1, CSTREAM_IN + 1, 200000] def parse_args() -> argparse.Namespace: p = argparse.ArgumentParser( description="Response correctness across reload under load " "(2c538e0 CDict-cycle class)." ) p.add_argument("--nginx-binary", required=True) p.add_argument("--filter-module") p.add_argument("--static-module") p.add_argument("--zstd-bin", default="zstd") p.add_argument("--port", type=int, default=18100) p.add_argument("--backend-port", type=int, default=18101) p.add_argument("--reloads", type=int, default=6) p.add_argument( "--workers", type=int, default=4, help="Concurrent client workers driving load." ) p.add_argument( "--duration", type=float, default=18.0, help="Seconds of sustained load (reloads spread across it).", ) return p.parse_args() def detect(explicit, nginx: pathlib.Path, name: str): if explicit: return pathlib.Path(explicit) sib = nginx.parent / name return sib if sib.exists() else None def payload_for(rid: int) -> bytes: size = SIZES[rid % len(SIZES)] head = f"RELOAD-REQ-{rid:09d}-".encode() head = head + b"." * (48 - len(head)) buf = bytearray(head) x = (rid * 2246822519 + 7) & 0xFFFFFFFF while len(buf) < size: x = (x * 1103515245 + 12345) & 0xFFFFFFFF buf.append((x >> 16) & 0xFF) return bytes(buf[:size]) class _Handler(http.server.BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" def log_message(self, *a): pass def do_GET(self): seg = self.path.split("?")[0].rsplit("/", 1)[-1] try: rid = int(seg) except ValueError: self.send_response(404) self.send_header("Content-Length", "0") self.end_headers() return data = payload_for(rid) self.send_response(200) self.send_header("Content-Type", "application/octet-stream") self.send_header("Transfer-Encoding", "chunked") self.end_headers() mv = memoryview(data) for i in range(0, len(mv), 16384): c = bytes(mv[i : i + 16384]) self.wfile.write(b"%X\r\n" % len(c) + c + b"\r\n") self.wfile.write(b"0\r\n\r\n") class _Srv(socketserver.ThreadingMixIn, socketserver.TCPServer): allow_reuse_address = True daemon_threads = True def wait_port(port: int, timeout: float = 10.0) -> None: end = time.time() + timeout while time.time() < end: try: with socket.create_connection(("127.0.0.1", port), 0.5): return except OSError: time.sleep(0.1) raise RuntimeError(f"nothing listening on 127.0.0.1:{port}") def one(port: int, rid: int, zstd_bin: str, dict_path: pathlib.Path) -> None: req = urllib.request.Request( f"http://127.0.0.1:{port}/r/{rid}", headers={"Accept-Encoding": "zstd", "User-Agent": "zstd-reload/1.0"}, ) with urllib.request.urlopen(req, timeout=20) as resp: if (resp.headers.get("Content-Encoding") or "").lower() != "zstd": raise RuntimeError( f"rid={rid}: not zstd (C-E={resp.headers.get('Content-Encoding')!r})" ) blob = resp.read() if blob[:4] != b"\x28\xb5\x2f\xfd": raise RuntimeError(f"rid={rid}: no zstd magic") # nginx.conf below sets zstd_dict_file, so every response is compressed # against that dictionary (ZSTD_CCtx_refCDict in the filter module) -- # the decoder needs the same dictionary via -D, or libzstd correctly # rejects the frame as "Data corruption detected" even though nothing is # actually corrupt. Omitting -D here previously made this test fail on # every run regardless of reload timing (any response, any size). r = subprocess.run( [zstd_bin, "-dq", "-c", "-D", str(dict_path)], input=blob, capture_output=True, check=False, ) if r.returncode != 0: raise RuntimeError( f"rid={rid}: zstd -d failed: " + r.stderr.decode("utf-8", "replace") ) if r.stdout != payload_for(rid): raise RuntimeError( f"rid={rid}: body mismatch across reload " f"(got {len(r.stdout)}B want {len(payload_for(rid))}B)" ) def main() -> int: args = parse_args() nginx = pathlib.Path(args.nginx_binary) if not nginx.exists(): raise FileNotFoundError(nginx) mods = [ m for m in ( detect(args.filter_module, nginx, "ngx_http_zstd_filter_module.so"), detect(args.static_module, nginx, "ngx_http_zstd_static_module.so"), ) if m ] # Everything below must stay readable by the workers when run as # root (they drop to the compiled-in nginx user): a restrictive # inherited umask (e.g. 077) would strip group/other bits from # every fixture and subdir created here, 403ing the workers even # with the scratch root itself opened up. os.umask(0o022) with tempfile.TemporaryDirectory(prefix="zstd-reload-") as td: # mkdtemp gives 0700: run as root, workers drop to the # compiled-in nginx user and cannot enter it -> 403s os.chmod(td, 0o755) root = pathlib.Path(td) logs = root / "logs" logs.mkdir() # A non-trivial dictionary so ZSTD_createCDict actually # allocates and the per-cycle CDict lifetime is exercised. dict_path = root / "zstd.dict" dseed = bytearray() x = 0x9E3779B1 while len(dseed) < 16384: x = (x * 1103515245 + 12345) & 0xFFFFFFFF dseed.append((x >> 16) & 0xFF) dict_path.write_bytes(bytes(dseed)) backend = _Srv(("127.0.0.1", args.backend_port), _Handler) threading.Thread(target=backend.serve_forever, daemon=True).start() try: wait_port(args.backend_port) for rid in (0, 2, 4): if urllib.request.urlopen( f"http://127.0.0.1:{args.backend_port}/r/{rid}", timeout=10 ).read() != payload_for(rid): raise RuntimeError(f"backend self-check failed rid={rid}") load = "".join(f"load_module {m};\n" for m in mods) pid_file = root / "nginx.pid" conf = root / "nginx.conf" conf.write_text( f"""{load}worker_processes 2; error_log {logs}/error.log warn; pid {pid_file}; events {{ worker_connections 256; }} http {{ access_log off; default_type application/octet-stream; zstd_dict_file_unsafe on; zstd_dict_file {dict_path}; zstd on; zstd_comp_level 6; zstd_min_length 1; zstd_types application/octet-stream; server {{ listen 127.0.0.1:{args.port}; location /r/ {{ proxy_pass http://127.0.0.1:{args.backend_port}/r/; proxy_http_version 1.1; proxy_set_header Connection ""; }} }} }} """, encoding="utf-8", ) # Master process mode (reload needs a master to signal). proc = subprocess.Popen( [str(nginx), "-p", str(root), "-c", str(conf), "-g", "daemon off;"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, ) try: wait_port(args.port) stop = threading.Event() failures: list[str] = [] counter = {"n": 0} clock = threading.Lock() def worker(): while not stop.is_set(): with clock: rid = counter["n"] counter["n"] += 1 try: one(args.port, rid, args.zstd_bin, dict_path) except Exception as e: # noqa: BLE001 failures.append(str(e)) threads = [ threading.Thread(target=worker, daemon=True) for _ in range(args.workers) ] for t in threads: t.start() # Spread SIGHUP reloads across the load window. deadline = time.time() + args.duration interval = args.duration / (args.reloads + 1) master_pid = int(pid_file.read_text().strip()) done_reloads = 0 while time.time() < deadline and not failures: time.sleep(interval) if done_reloads < args.reloads: os.kill(master_pid, signal.SIGHUP) done_reloads += 1 stop.set() for t in threads: t.join(timeout=10) # Final clean shutdown so all cycle cleanups run # (incl. the CDict cleanup handler from 2c538e0). os.kill(master_pid, signal.SIGQUIT) try: # 30s, not 10/5: gcov-instrumented nginx flushing # .gcda at exit on a loaded runner can outlive a # tight reap budget; teardown runs after the # verdict, so patience costs nothing when healthy. proc.wait(timeout=30) except subprocess.TimeoutExpired: proc.kill() proc.wait(timeout=30) el = logs / "error.log" logtext = el.read_text("utf-8", "replace") if el.exists() else "" bad = [ ln for ln in logtext.splitlines() if ( "[alert]" in ln or "[emerg]" in ln or "zero size buf in writer" in ln or "AddressSanitizer" in ln or "LeakSanitizer" in ln or "runtime error:" in ln ) ] if failures or bad: sys.stderr.write( f"RELOAD-UNDER-LOAD FAILED: " f"{len(failures)} response failures, " f"{len(bad)} log issues, " f"{counter['n']} reqs, " f"{done_reloads} reloads:\n" ) for f in failures[:15]: sys.stderr.write(f" resp: {f}\n") for b in bad[:10]: sys.stderr.write(f" log: {b}\n") return 1 print( f"OK: {counter['n']} requests across " f"{done_reloads} SIGHUP reloads under " f"{args.workers}-way load — every response " f"decoded byte-exact, master survived, error log " f"clean (CDict cycle lifetime intact)" ) return 0 finally: if proc.poll() is None: proc.terminate() try: proc.wait(timeout=30) except subprocess.TimeoutExpired: proc.kill() proc.wait(timeout=30) finally: backend.shutdown() backend.server_close() if __name__ == "__main__": try: raise SystemExit(main()) except Exception as exc: print(f"ERROR: {exc}", file=sys.stderr) raise myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_run_testkit_scenarios.sh000077500000000000000000000105531525232545500276140ustar00rootroot00000000000000#!/usr/bin/env bash # Offline positive and negative controls for run-testkit-scenarios.sh. set -euo pipefail ROOT="$(git rev-parse --show-toplevel)" TOOL="$ROOT/ci/tools/run-testkit-scenarios.sh" WORK="$(mktemp -d "${TMPDIR:-/tmp}/testkit-runner-unit.XXXXXX")" trap 'rm -rf "$WORK"' EXIT mkdir -p "$WORK/root/ci/t/harness/ci/prober" "$WORK/root/ci/t/harness-scenarios" \ "$WORK/logs" "$WORK/build" for scenario in fault-arms alloc-neutral fault-palloc codec-call-count \ setparam-call-count setparam-call-count-nodict; do mkdir -p "$WORK/root/ci/t/harness-scenarios/$scenario" done FAKE="$WORK/fake-run-scenario" cat >"$FAKE" <<'SH' #!/usr/bin/env bash set -euo pipefail scenario="$(basename "$1")" printf '%s\t%s\n' "$scenario" "${PROBER_ALLOW_LOG-unset}" >> "$CALLS" case "$scenario" in fault-arms) printf '1..1\nok 1 - fault reached\n' ;; alloc-neutral) printf '1..5\nok 1 - boot\nok 2 - probe\nok 3 - cycle used\nok 4 - cycle blocks\nok 5 - worker fds\n' ;; fault-palloc) printf '1..2\nok 1 - boot\nok 2 - PALLOC site accepted the arm\n' ;; codec-call-count) printf '1..10\n' for n in 1 3 5 7 9; do printf 'ok %s - decoded\n' "$n"; done for n in 2 4 6 8 10; do printf 'ok %s - total codec calls is 1\n' "$n"; done ;; setparam-call-count) printf '1..7\nok 1 - boot\nok 2 - decoded\nok 3 - setParameter call count is 1\nok 4 - frame window size is 1024\nok 5 - decoded\nok 6 - ready\nok 7 - setParameter call count is 2\n' ;; setparam-call-count-nodict) printf '1..4\nok 1 - boot\nok 2 - decoded\nok 3 - setParameter call count is 3\nok 4 - frame window size is 2048\n' ;; esac SH chmod +x "$FAKE" run() { CALLS="$WORK/calls" PROBER_RUN_SCENARIO="$FAKE" "$TOOL" \ --root "$WORK/root" --build "$WORK/build" --version 1.31.4 \ --log-dir "$WORK/logs" } run [ "$(wc -l < "$WORK/calls")" -eq 6 ] grep -q $'^fault-arms\tzstd: ZSTD_' "$WORK/calls" grep -q $'^fault-palloc\tzstd: ' "$WORK/calls" grep -q $'^alloc-neutral\tunset$' "$WORK/calls" # A skipped property witness must fail even when the fake runner exits zero. cat >"$WORK/skip-runner" <<'SH' #!/usr/bin/env bash printf '1..1\nok 1 - nothing # SKIP unavailable\n' SH chmod +x "$WORK/skip-runner" if CALLS="$WORK/calls" PROBER_RUN_SCENARIO="$WORK/skip-runner" "$TOOL" \ --root "$WORK/root" --build "$WORK/build" --version 1.31.4 \ fault-arms >/dev/null 2>&1; then echo "FAIL: all-skip scenario was accepted" >&2 exit 1 fi # The specific call-count witness is stronger than generic green TAP. cat >"$WORK/weak-runner" <<'SH' #!/usr/bin/env bash printf '1..10\n' for n in $(seq 1 10); do printf 'ok %s - decoded\n' "$n"; done SH chmod +x "$WORK/weak-runner" if CALLS="$WORK/calls" PROBER_RUN_SCENARIO="$WORK/weak-runner" "$TOOL" \ --root "$WORK/root" --build "$WORK/build" --version 1.31.4 \ codec-call-count >/dev/null 2>&1; then echo "FAIL: codec scenario without count witnesses was accepted" >&2 exit 1 fi # A green prefix is not a complete TAP result: every planned number is owed. cat >"$WORK/incomplete-runner" <<'SH' #!/usr/bin/env bash printf '1..2\nok 1 - only half the plan ran\n' SH chmod +x "$WORK/incomplete-runner" if PROBER_RUN_SCENARIO="$WORK/incomplete-runner" "$TOOL" \ --root "$WORK/root" --build "$WORK/build" --version 1.31.4 \ fault-arms >/dev/null 2>&1; then echo "FAIL: incomplete TAP plan was accepted" >&2 exit 1 fi # --help and an explicit --root must not require the caller to be in a Git tree. (cd "$WORK" && "$TOOL" --help >/dev/null) # The requires gates run with pipefail. A grep -q probe exits on its first # match and can SIGPIPE a still-writing nm, turning a real symbol into a false # "packaged build" skip. Make nm deterministically outlive that first match; # the shipped counted/full-consumption check must still accept the module. mkdir -p "$WORK/build/objs" "$WORK/bin" : >"$WORK/build/objs/ngx_http_zstd_filter_module.so" cat >"$WORK/bin/nm" <<'SH' #!/usr/bin/env bash printf '00000000 T ngx_test_probe_arm\n' for n in $(seq 1 20000); do printf '%08d T unrelated_symbol_%d\n' "$n" "$n" done SH chmod +x "$WORK/bin/nm" PATH="$WORK/bin:$PATH" PROBER_BUILD="$WORK/build" \ bash "$ROOT/ci/t/harness-scenarios/alloc-neutral/requires" \ ignored nginx 1.31.4 echo "OK: testkit scenario runner positive/negative controls passed" myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_sanitizer_suite_verdict.sh000077500000000000000000000053631525232545500301370ustar00rootroot00000000000000#!/usr/bin/env bash set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" VERDICT="$ROOT/ci/tools/sanitizer_suite_verdict.py" WORK="$(mktemp -d "$ROOT/.sanitizer-verdict-test.XXXXXX")" trap 'rm -rf "$WORK"' EXIT cat >"$WORK/suite.tap" <<'EOF' ok 1 - first ok 2 - second 1..2 EOF cat >"$WORK/core.log" <<'EOF' nginx.c:10:3: runtime error: core-only undefined behavior #0 0x123 in ngx_worker_process_cycle nginx/src/os/unix/ngx_process_cycle.c:1 EOF python3 "$VERDICT" --suite "fixture:$WORK/suite.tap:0" \ --sanitizer-log "$WORK/core.log" >/dev/null cat >"$WORK/core-with-later-module-log.log" <<'EOF' nginx.c:10:3: runtime error: core-only undefined behavior #0 0x123 in ngx_worker_process_cycle nginx/src/os/unix/ngx_process_cycle.c:1 2026/08/31 12:00:00 [notice] 12#12: harmless ngx_http_zstd_body_filter diagnostic EOF python3 "$VERDICT" --suite "fixture:$WORK/suite.tap:0" \ --sanitizer-log "$WORK/core-with-later-module-log.log" >/dev/null cat >"$WORK/module.log" <<'EOF' ==12==ERROR: AddressSanitizer: heap-use-after-free on address 0x123 #0 0x123 in memcpy sanitizer_common_interceptors.inc:1 #1 0x456 in ngx_http_zstd_body_filter src/ngx_http_zstd_filter_module.c:512 SUMMARY: AddressSanitizer: heap-use-after-free sanitizer_common_interceptors.inc:1 EOF if python3 "$VERDICT" --suite "fixture:$WORK/suite.tap:0" \ --sanitizer-log "$WORK/module.log" >/dev/null 2>&1; then echo "FAIL: split-line-module-frame mutant was not rejected" >&2 exit 1 fi cat >"$WORK/lsan-module.log" <<'EOF' ==12==ERROR: LeakSanitizer: detected memory leaks Direct leak of 64 byte(s) in 1 object(s) allocated from: #0 0x123 in malloc sanitizer_common_interceptors.inc:1 #1 0x456 in ngx_http_zstd_body_filter src/ngx_http_zstd_filter_module.c:512 SUMMARY: AddressSanitizer: 64 byte(s) leaked in 1 allocation(s). EOF if python3 "$VERDICT" --suite "fixture:$WORK/suite.tap:0" \ --sanitizer-log "$WORK/lsan-module.log" >/dev/null 2>&1; then echo "FAIL: split-line LSan module frame was not rejected" >&2 exit 1 fi cat "$WORK/core.log" "$WORK/module.log" >"$WORK/multiple.log" if python3 "$VERDICT" --suite "fixture:$WORK/suite.tap:0" \ --sanitizer-log "$WORK/multiple.log" >/dev/null 2>&1; then echo "FAIL: later module-attributable report was not rejected" >&2 exit 1 fi if python3 "$VERDICT" --suite "fixture:$WORK/suite.tap:23" \ --sanitizer-log "$WORK/core.log" >/dev/null 2>&1; then echo "FAIL: ignored-suite-exit mutant was not rejected" >&2 exit 1 fi sed 's/^1\.\.2$/1..3/' "$WORK/suite.tap" >"$WORK/truncated.tap" if python3 "$VERDICT" --suite "fixture:$WORK/truncated.tap:0" \ --sanitizer-log "$WORK/core.log" >/dev/null 2>&1; then echo "FAIL: truncated TAP plan was not rejected" >&2 exit 1 fi echo "sanitizer suite verdict tests passed" myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_sha256_unit.c000066400000000000000000000237771525232545500250630ustar00rootroot00000000000000/* * Unit fixture for ngx_http_zstd_sha256.h — issue #100 item 3. * * Compiled twice by tools/test_sha256_unit.sh against the shim headers * in tools/sha256-unit/ (no nginx tree, no OpenSSL needed): * * 1. Portable build (NGX_HTTP_ZSTD_HAVE_LIBCRYPTO undefined): the * local SHA-256 against NIST FIPS 180-4 vectors, plus * incremental-vs-one-shot equivalence at block-boundary split * sizes (the update() partial-block paths). * * 2. EVP-fallback build (-DNGX_HTTP_ZSTD_HAVE_LIBCRYPTO=1): the * include resolves to the FAKE header in the shim * directory, and this file supplies EVP_Digest()/EVP_sha256() * implementations with scripted behaviour. Exercises the wrapper's * three paths deterministically: EVP failure after a PARTIAL * digest write (the fallback must overwrite all 32 bytes with the * correct value), EVP "success" with a wrong output length (the * mdlen guard must reject it and fall back), and EVP success * (whose scripted digest must be returned verbatim, proving the * EVP path is actually taken). */ #include #include #include "../../src/ngx_http_zstd_sha256.h" static int failures = 0; /* TAP-style assertion: print ok/FAIL for `name` and count failures for the process exit code; `detail` (may be NULL) is appended to failures to aid diagnosis. */ static void check(const char *name, int cond, const char *detail) { if (cond) { printf("ok - %s\n", name); } else { printf("FAIL - %s %s\n", name, detail ? detail : ""); failures++; } } /* Render a 32-byte digest as 64 lowercase hex characters plus NUL, for comparison against the published vector strings. */ static void to_hex(const u_char digest[NGX_HTTP_ZSTD_SHA256_DIGEST_LEN], char out[65]) { static const char hex[] = "0123456789abcdef"; int i; for (i = 0; i < NGX_HTTP_ZSTD_SHA256_DIGEST_LEN; i++) { out[i * 2] = hex[digest[i] >> 4]; out[i * 2 + 1] = hex[digest[i] & 0x0f]; } out[64] = '\0'; } #if (NGX_HTTP_ZSTD_HAVE_LIBCRYPTO) /* * Scripted fake EVP. Modes 0/1/2 fail init/update/final, mode 3 reports a * wrong digest length, and mode 4 succeeds. */ static int fake_evp_mode; static int fake_evp_calls; static unsigned char fake_evp_ctx; /* Fake EVP_sha256(): an opaque non-NULL token — the wrapper under test only passes it through, never dereferences it. */ const EVP_MD * EVP_sha256(void) { return (const EVP_MD *) "fake-sha256"; } int EVP_DigestInit_ex(EVP_MD_CTX *ctx, const EVP_MD *type, ENGINE *impl) { (void) ctx; (void) type; (void) impl; fake_evp_calls++; return fake_evp_mode != 0; } int EVP_DigestUpdate(EVP_MD_CTX *ctx, const void *data, size_t count) { (void) ctx; (void) data; (void) count; fake_evp_calls++; return fake_evp_mode != 1; } int EVP_DigestFinal_ex(EVP_MD_CTX *ctx, unsigned char *md, unsigned int *size) { (void) ctx; fake_evp_calls++; if (fake_evp_mode == 2) { memset(md, 0x5a, 17); return 0; } if (fake_evp_mode == 3) { memset(md, 0x5a, NGX_HTTP_ZSTD_SHA256_DIGEST_LEN); *size = 16; return 1; } memset(md, 0xaa, NGX_HTTP_ZSTD_SHA256_DIGEST_LEN); *size = NGX_HTTP_ZSTD_SHA256_DIGEST_LEN; return 1; } /* EVP-fallback build: drive the wrapper through the three scripted EVP behaviours and assert the digest that comes out of each. */ int main(void) { u_char digest[NGX_HTTP_ZSTD_SHA256_DIGEST_LEN]; char hexdigest[65]; int i, all_aa; static const char abc[] = "abc"; static const char abc_hex[] = "ba7816bf8f01cfea414140de5dae2223" "b00361a396177a9cb410ff61f20015ad"; fake_evp_calls = 0; memset(digest, 0, sizeof(digest)); ngx_http_zstd_sha256((const u_char *) abc, 3, digest, NULL); to_hex(digest, hexdigest); check("missing cycle EVP context falls back to portable SHA-256", strcmp(hexdigest, abc_hex) == 0, hexdigest); check("missing cycle EVP context makes no EVP calls", fake_evp_calls == 0, NULL); for (fake_evp_mode = 0; fake_evp_mode < 4; fake_evp_mode++) { fake_evp_calls = 0; memset(digest, 0, sizeof(digest)); ngx_http_zstd_sha256((const u_char *) abc, 3, digest, (EVP_MD_CTX *) &fake_evp_ctx); to_hex(digest, hexdigest); check("EVP failure falls back to the portable implementation", strcmp(hexdigest, abc_hex) == 0, hexdigest); check("fallback consulted the expected EVP stages", fake_evp_calls == (fake_evp_mode < 3 ? fake_evp_mode + 1 : 3), NULL); } /* Mode 2: scripted success -> the EVP digest is used verbatim, proving the accelerated path is actually taken on success. */ fake_evp_mode = 4; memset(digest, 0, sizeof(digest)); ngx_http_zstd_sha256((const u_char *) abc, 3, digest, (EVP_MD_CTX *) &fake_evp_ctx); all_aa = 1; for (i = 0; i < NGX_HTTP_ZSTD_SHA256_DIGEST_LEN; i++) { if (digest[i] != 0xaa) { all_aa = 0; } } check("successful EVP digest is returned verbatim", all_aa, NULL); if (failures) { printf("FAILED: %d EVP-fallback check(s)\n", failures); return 1; } printf("OK: EVP failure-injection fixture\n"); return 0; } #else /* portable build */ typedef struct { const char *input; size_t len; const char *hex; } sha256_vector_t; /* NIST FIPS 180-4 / SHA-2 test vectors. */ static const sha256_vector_t vectors[] = { { "", 0, "e3b0c44298fc1c149afbf4c8996fb924" "27ae41e4649b934ca495991b7852b855" }, { "abc", 3, "ba7816bf8f01cfea414140de5dae2223" "b00361a396177a9cb410ff61f20015ad" }, { "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", 56, "248d6a61d20638b8e5c026930c3e6039" "a33ce45964ff2167f6ecedd419db06c1" }, }; /* Portable build: the local SHA-256 against the published vectors, then incremental-vs-one-shot equivalence at block-boundary splits. */ int main(void) { u_char digest[NGX_HTTP_ZSTD_SHA256_DIGEST_LEN]; u_char ref[NGX_HTTP_ZSTD_SHA256_DIGEST_LEN]; char hexdigest[65]; char name[128]; size_t i, off, n; u_char *buf; uint32_t lcg; ngx_http_zstd_sha256_t c; static const size_t buf_len = 100000; static const size_t splits[] = { 1, 7, 63, 64, 65, 4096 }; /* One-shot API against the published vectors. */ for (i = 0; i < sizeof(vectors) / sizeof(vectors[0]); i++) { ngx_http_zstd_sha256((const u_char *) vectors[i].input, vectors[i].len, digest, NULL); to_hex(digest, hexdigest); snprintf(name, sizeof(name), "NIST vector (%zu bytes)", vectors[i].len); check(name, strcmp(hexdigest, vectors[i].hex) == 0, hexdigest); } /* Remainder 55 mod 64 — the one residue the other totals (0, 3, 56, 32 mod 64) never reach, and chunked updates cannot change a total's residue. It pins final()'s padding boundary exactly: 55 content bytes + the 0x80 pad byte fill block_len to 56, the largest value that must still fit the 8-byte length in the same block ("block_len > 56" spills; mutating it to ">= 56" corrupts precisely this residue and nothing else in this file — review planted that mutation and the prior vectors all survived). */ { u_char a55[55]; static const char a55_hex[] = "9f4390f8d30c2dd92ec9f095b65e2b9a" "e9b0a925a5258e241c9f1e910f734318"; memset(a55, 'a', sizeof(a55)); ngx_http_zstd_sha256(a55, sizeof(a55), digest, NULL); to_hex(digest, hexdigest); check("padding-boundary vector (55 x 'a', remainder 55 mod 64)", strcmp(hexdigest, a55_hex) == 0, hexdigest); } /* The million-'a' vector, fed in 1000-byte updates: exercises the full-block fast path and the length accounting at scale. */ { u_char a1000[1000]; static const char million_hex[] = "cdc76e5c9914fb9281a1c7e284d73e67" "f1809a48a497200e046d39ccc7112cd0"; memset(a1000, 'a', sizeof(a1000)); ngx_http_zstd_sha256_init(&c); for (i = 0; i < 1000; i++) { ngx_http_zstd_sha256_update(&c, a1000, sizeof(a1000)); } ngx_http_zstd_sha256_final(&c, digest); to_hex(digest, hexdigest); check("NIST vector (1,000,000 x 'a', incremental)", strcmp(hexdigest, million_hex) == 0, hexdigest); } /* Incremental updates at block-boundary split sizes must agree with the one-shot digest of the same pseudo-random buffer (fixed LCG, so the input is deterministic without external files). */ buf = malloc(buf_len); if (buf == NULL) { printf("FAIL - malloc(%zu)\n", buf_len); return 1; } lcg = 0x12345678; for (i = 0; i < buf_len; i++) { lcg = lcg * 1664525 + 1013904223; buf[i] = (u_char) (lcg >> 24); } ngx_http_zstd_sha256(buf, buf_len, ref, NULL); for (i = 0; i < sizeof(splits) / sizeof(splits[0]); i++) { ngx_http_zstd_sha256_init(&c); for (off = 0; off < buf_len; off += n) { n = splits[i]; if (n > buf_len - off) { n = buf_len - off; } ngx_http_zstd_sha256_update(&c, buf + off, n); } ngx_http_zstd_sha256_final(&c, digest); snprintf(name, sizeof(name), "incremental chunks of %zu match one-shot", splits[i]); check(name, memcmp(digest, ref, NGX_HTTP_ZSTD_SHA256_DIGEST_LEN) == 0, NULL); } free(buf); if (failures) { printf("FAILED: %d portable check(s)\n", failures); return 1; } printf("OK: portable SHA-256 fixture (vectors + incremental)\n"); return 0; } #endif myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_sha256_unit.sh000077500000000000000000000016551525232545500252450ustar00rootroot00000000000000#!/bin/bash # Unit fixture for ngx_http_zstd_sha256.h (issue #100 item 3): the # portable implementation against NIST vectors, and the EVP-to-portable # fallback with an injectable fake EVP_Digest. Self-contained — shim # headers in ci/tools/sha256-unit/ stand in for the ngx types and for # , so this needs a C compiler and nothing else. set -euo pipefail # ci/tools/ -> ci/ -> repo root: TWO levels since the ci/ move. cd "$(dirname "$0")/../.." CC="${CC:-cc}" OUT="$(mktemp -d "${TMPDIR:-/tmp}/zstd-sha256-unit.XXXXXX")" trap 'rm -rf "$OUT"' EXIT "$CC" -Wall -Wextra -Werror -O2 -Ici/tools/sha256-unit \ -o "$OUT/portable" ci/tools/test_sha256_unit.c "$OUT/portable" "$CC" -Wall -Wextra -Werror -O2 -Ici/tools/sha256-unit \ -DNGX_HTTP_ZSTD_HAVE_LIBCRYPTO=1 \ -o "$OUT/evp-fallback" ci/tools/test_sha256_unit.c "$OUT/evp-fallback" echo "OK: sha256 unit fixture (portable vectors + EVP failure injection)" myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_slow_drain.py000077500000000000000000000440671525232545500253610ustar00rootroot00000000000000#!/usr/bin/env python3 """Deterministic drivers for the two timing-dependent filter paths. Ported from the unified-module branch (#117), where the same tool pins the recycling pause path. The trigger was the coverage floor: the gcov job's 82% gate assumes two code paths that only ran when CI load happened to produce the right races, and one run (PR #124, nginx pin bump) lost every race at once — 8 lines vanished, 82.6% fell to 81.9%, and an unrelated two-line env change failed the floor. Comparing that run's lcov tracefile against the previous green run identified exactly the lines; this tool makes both paths run on purpose so the floor measures tests, not scheduler luck. The two paths, and how they are forced: 1. **The nomem entry block** (busy-buffer recovery at the top of the body filter): needs ``zstd_buffers`` exhausted while the client cannot drain, then a writer-driven re-invocation. A local socket drains instantly, which is why no Test::Nginx block can reach it. Forced here with ``zstd_buffers 2 8k``, a ~1 MB incompressible body, ``sndbuf=16384`` on the listener, a small client-side SO_RCVBUF, and deliberately slow reads. Witness: "zstd get_buf: no free buffer, nomem set" — plus a byte-exact decode proving the pause/resume seams preserved the stream. 2. **The content-less flush completion** (empty out_buf, flush pending, encoder fully drained — the #116-era livelock guard): needs a flush to reach the filter while the encoder holds nothing. nginx's non-buffered upstream path sends exactly that — ``ngx_http_upstream_send_response`` emits a data-less NGX_HTTP_FLUSH special before relaying any body — but only a backend that sends its headers alone and its body strictly later makes the special arrive with an empty encoder. The mock backend here sleeps between headers and body to guarantee it. Witness: "content-less flush completed", once per proxied response. Requires an nginx binary built ``--with-debug`` (the witnesses are ngx_log_debug lines) with the zstd filter module, plus the ``zstd`` CLI for decoding. """ from __future__ import annotations import argparse import os import pathlib import re import socket import subprocess import sys import tempfile import threading import time import typing DRAIN_SIZE = 1_000_000 READ_CHUNK = 16384 READ_DELAY = 0.02 PROXY_PART = 40_000 PROXY_REPEAT = 8 HEADER_BODY_GAP = 0.3 INTER_PART_GAP = 0.15 NOMEM_WITNESS = "no free buffer, nomem set" FLUSH_WITNESS = "content-less flush completed" class WitnessFloor(typing.NamedTuple): """One (log substring, human label, minimum occurrence count) row.""" witness: str path_name: str floor: int def parse_args() -> argparse.Namespace: """Parse the CLI arguments (binary, module path, ports, zstd CLI).""" p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) p.add_argument( "--nginx-binary", required=True, help="nginx (or angie) binary to launch." ) p.add_argument( "--filter-module", help="Path to ngx_http_zstd_filter_module.so " "(auto-detected next to the binary if omitted).", ) p.add_argument( "--zstd-bin", default="zstd", help="zstd CLI used for decompression." ) p.add_argument("--port", type=int, default=18095, help="Front-end nginx port.") p.add_argument( "--backend-port", type=int, default=18096, help="Mock slow-headers upstream port.", ) p.add_argument( "--log-level", choices=("debug", "warn"), default="debug", help="Location error_log level. Witnesses are only " "asserted at debug. Use warn under sanitizer " "builds: UBSAN (-fno-sanitize-recover) fatally " "traps nginx core's own debug logging — every " '"%%V?%%V" line passes r->args={0,NULL} into ' "ngx_sprintf_str's nonnull argument on " "query-less URIs (ngx_string.c:586) — so a " "sanitized nginx cannot log at debug at all. " "The forced paths still run; only their log " "proof is skipped.", ) return p.parse_args() def detect_module( explicit: str | None, nginx: pathlib.Path, name: str ) -> pathlib.Path | None: """Resolve a dynamic-module .so: explicit path, or beside the binary.""" if explicit: return pathlib.Path(explicit) sib = nginx.parent / name return sib if sib.exists() else None def fixture_bytes(n: int) -> bytes: """Deterministic pseudorandom (incompressible) fixture of n bytes.""" buf = bytearray(n) x = (n * 2654435761) & 0xFFFFFFFF for i in range(n): x = (x * 1103515245 + 12345) & 0xFFFFFFFF buf[i] = (x >> 16) & 0xFF return bytes(buf) def wait_port(port: int, timeout: float = 10.0) -> None: """Block until something accepts connections on 127.0.0.1:port.""" deadline = time.time() + timeout while time.time() < deadline: try: with socket.create_connection(("127.0.0.1", port), 0.5): return except OSError: time.sleep(0.1) raise RuntimeError(f"nothing listening on 127.0.0.1:{port}") class SlowHeaderBackend(threading.Thread): """Sends 200 + headers immediately, then the body only after a pause — so nginx's non-buffered data-less FLUSH special always reaches the filter before a single body byte does.""" def __init__(self, port: int, body: bytes) -> None: """Bind the listener up front so wait-for-port has a target.""" super().__init__(daemon=True) self.port = port self.body = body self.srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.srv.bind(("127.0.0.1", port)) self.srv.listen(8) def run(self) -> None: """Accept loop: headers immediately, body in two paused halves.""" while True: try: conn, _ = self.srv.accept() except OSError: return try: conn.settimeout(10) buf = b"" while b"\r\n\r\n" not in buf: piece = conn.recv(4096) if not piece: break buf += piece conn.sendall( b"HTTP/1.1 200 OK\r\n" b"Content-Type: application/octet-stream\r\n" b"Connection: close\r\n\r\n" ) time.sleep(HEADER_BODY_GAP) half = len(self.body) // 2 conn.sendall(self.body[:half]) time.sleep(INTER_PART_GAP) conn.sendall(self.body[half:]) except OSError: pass finally: conn.close() def stop(self) -> None: """Close the listener; the accept loop exits on the next OSError.""" self.srv.close() def http_get(port: int, path: str, slow: bool, timeout: float = 120.0) -> bytes: """GET path expecting Content-Encoding: zstd; slow=True throttles reads through a small SO_RCVBUF to create real backpressure.""" s = socket.create_connection(("127.0.0.1", port), timeout) s.settimeout(timeout) if slow: # Without a small SO_RCVBUF the kernel buffers the whole # compressed response client-side and nginx never feels the # backpressure this test exists to create. s.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 16384) req = f"GET {path} HTTP/1.0\r\nHost: t\r\nAccept-Encoding: zstd\r\n\r\n" s.sendall(req.encode("latin1")) raw = b"" while True: piece = s.recv(READ_CHUNK) if not piece: break raw += piece if slow: time.sleep(READ_DELAY) s.close() head, _, body = raw.partition(b"\r\n\r\n") headers = head.decode("latin1", "replace") status = headers.splitlines()[0] if headers else "" if " 200 " not in f"{status} ": raise RuntimeError(f"{path}: expected 200, got {status!r} ({len(body)}B body)") m = re.search(r"(?im)^content-encoding:\s*(\S+)", headers) got = m.group(1) if m else None if got != "zstd": raise RuntimeError( f"{path}: Content-Encoding {got!r}, " f"wanted 'zstd' (status {status!r}, " f"{len(body)}B body)" ) return body def main() -> int: """Launch nginx + mock backend, run both scenarios, check witnesses.""" args = parse_args() nginx = pathlib.Path(args.nginx_binary) if not nginx.exists(): raise FileNotFoundError(nginx) v = subprocess.run([str(nginx), "-V"], capture_output=True, text=True, check=False) if "zstd" not in v.stderr: raise RuntimeError("nginx -V shows no zstd module") if "--with-debug" not in v.stderr: raise RuntimeError( "the witnesses are ngx_log_debug lines: " "this tool needs an nginx built --with-debug" ) filter_so = detect_module( args.filter_module, nginx, "ngx_http_zstd_filter_module.so" ) load = f"load_module {filter_so};\n" if filter_so else "" def decode(blob: bytes) -> bytes: """Decompress with the zstd CLI; a nonzero exit means truncation.""" r = subprocess.run( [args.zstd_bin, "-d", "-q", "-c"], input=blob, capture_output=True, check=False, ) if r.returncode != 0: raise RuntimeError( "zstd decode failed (truncated/corrupt stream): " + r.stderr.decode("utf-8", "replace").strip() ) return r.stdout drain_body = fixture_bytes(DRAIN_SIZE) proxy_body = fixture_bytes(2 * PROXY_PART) backend = SlowHeaderBackend(args.backend_port, proxy_body) backend.start() # mkdtemp gives 0700: when nginx runs as root (the ASAN job) the # workers drop privileges and cannot traverse into the fixture # tree — /drain/big then serves uncompressed instead of erroring. # Same lines the sibling tools carry, dropped once in the port and # caught by exactly that job. os.umask(0o022) with tempfile.TemporaryDirectory(prefix="zstd-drain-") as td: os.chmod(td, 0o755) root = pathlib.Path(td) (root / "logs").mkdir() html = root / "html" / "d" html.mkdir(parents=True) (html / "big").write_bytes(drain_body) # The location error_log level is a flag (see --log-level): # sanitizer builds fatally trap nginx core's own debug logging # (the "%V?%V" lines pass NULL for empty args — latent in core, # first hit by this tool because nothing else in CI ran a # sanitized binary at debug level), so the ASAN job runs at # warn and skips the witness assertions. The forced paths run # regardless of log level; the roundtrip oracles keep gating. lvl = args.log_level conf = root / "nginx.conf" conf.write_text( f"""worker_processes 1; {load}error_log {root}/logs/error.log warn; pid {root}/nginx.pid; events {{ worker_connections 64; }} http {{ access_log off; default_type application/octet-stream; zstd on; zstd_min_length 1; zstd_types application/octet-stream; gzip_vary on; server {{ listen 127.0.0.1:{args.port} sndbuf=16384; location /drain/ {{ error_log {root}/logs/error.log {lvl}; alias {html}/; zstd_buffers 2 8k; }} location /px {{ error_log {root}/logs/error.log {lvl}; proxy_pass http://127.0.0.1:{args.backend_port}; proxy_buffering off; }} }} }} """, encoding="utf-8", ) # nginx's own stdout/stderr to a file, not a PIPE: nothing # drains a pipe here (deadlock risk on a chatty ASan abort), # and on failure the tail is the diagnosis. nlog_path = root / "logs" / "nginx-stdout.log" # SIM115 suppressed above: the handle must outlive this block: it is Popen's # stdout for the whole server lifetime. A with-block would close it # while nginx is still writing. nlog = open(nlog_path, "w", encoding="utf-8") # noqa: SIM115 proc = subprocess.Popen( [ str(nginx), "-p", str(root), "-c", str(conf), "-g", "daemon off; master_process off;", ], stdout=nlog, stderr=subprocess.STDOUT, text=True, ) def alive_or_die(when: str) -> None: """Fail with nginx's captured output if the process exited — a bind conflict or sanitizer abort otherwise masquerades as responses from a stale listener on the same port.""" if proc.poll() is not None: nlog.flush() tail = nlog_path.read_text("utf-8", "replace")[-2000:] raise RuntimeError( f"nginx exited (rc={proc.returncode}) {when}; output tail:\n{tail}" ) try: wait_port(args.port) alive_or_die("during startup") failures: list[str] = [] # 1. slow-drain: force the buffer cap and the writer-driven # re-entry through the nomem entry block try: t0 = time.time() body = http_get(args.port, "/drain/big", slow=True) took = time.time() - t0 plain = decode(body) if plain != drain_body: failures.append( f"slow-drain: decoded {len(plain)}B, expected " f"{len(drain_body)}B — the pause/resume seams " f"corrupted or truncated the stream" ) else: print( f" slow-drain: {len(body)}B compressed drained " f"in {took:.1f}s, decoded byte-exact" ) except Exception as exc: # noqa: BLE001 failures.append(f"slow-drain: {exc}") alive_or_die("after the slow-drain scenario") # 2. proxied data-less flush: every response starts with the # upstream FLUSH special hitting an empty encoder ok = 0 for i in range(PROXY_REPEAT): try: body = http_get(args.port, "/px", slow=False) if decode(body) != proxy_body: failures.append(f"proxy #{i}: decode mismatch") break ok += 1 except Exception as exc: # noqa: BLE001 failures.append(f"proxy #{i}: {exc}") break if ok: print( f" proxy: {ok}/{PROXY_REPEAT} unbuffered responses " f"decoded byte-exact" ) elog = (root / "logs" / "error.log").read_text("utf-8", "replace") if args.log_level != "debug": print( " witnesses skipped (--log-level warn: sanitizer " "builds cannot log at debug — paths still forced, " "roundtrips still gate)" ) # The upstream FLUSH special is unconditional per response, # so every successful proxied request owes one content-less # completion — a lower bound, not an exact pin, because # nginx may legally emit further data-less specials # mid-relay (and those only log content-less when they too # meet an empty encoder). The nomem count depends on drain # scheduling, so only its existence is pinned. witness_floors: tuple[WitnessFloor, ...] = () if args.log_level == "debug": witness_floors = ( WitnessFloor(NOMEM_WITNESS, "buffer-cap/nomem", 1), WitnessFloor(FLUSH_WITNESS, "content-less flush", PROXY_REPEAT), ) for witness, path_name, floor in witness_floors: n = elog.count(witness) if n < floor: failures.append( f"witness short: {witness!r} x{n}, expected at " f"least x{floor} — the {path_name} path did not " f"run as forced (geometry drift, or the " f"filter's debug lines changed)" ) else: print(f" witness: {witness!r} x{n}") if failures: sys.stderr.write(f"slow-drain FAILED ({len(failures)}):\n") for f in failures: sys.stderr.write(f" - {f}\n") # CI runs are not reproducible interactively: ship the # evidence with the verdict. nlog.flush() sys.stderr.write("--- nginx stdout/stderr tail:\n") sys.stderr.write(nlog_path.read_text("utf-8", "replace")[-2000:] + "\n") sys.stderr.write("--- error.log tail (non-debug):\n") sys.stderr.write( "\n".join(ln for ln in elog.splitlines() if "[debug]" not in ln)[ -3000: ] + "\n" ) return 1 proof = ( "witnessed" if args.log_level == "debug" else "roundtripped (witnesses skipped)" ) print( f"OK: forced backpressure and data-less flushes both " f"{proof}, streams byte-exact" ) return 0 finally: proc.terminate() try: proc.wait(timeout=30) except subprocess.TimeoutExpired: proc.kill() proc.wait(timeout=30) nlog.close() backend.stop() if __name__ == "__main__": try: raise SystemExit(main()) except Exception as exc: print(f"ERROR: {exc}", file=sys.stderr) raise SystemExit(2) from exc myguard-labs-nginx-zstd-module-3486b5b/ci/tools/test_soak_asan_verdict_unit.sh000077500000000000000000000121251525232545500277060ustar00rootroot00000000000000#!/usr/bin/env bash # # Unit fixture for soak_asan_verdict() (ci/tools/soak_asan_verdict.sh). # # A27-F9: ci/tools/soak.sh used to fail the ASAN/UBSAN soak when # "$WORK/logs/asan*" merely EXISTED, never reading its content -- so a # LeakSanitizer that could not even start (ptrace blocked by this runner # pool's LXC seccomp/yama profile -- see soak_asan_verdict.sh's header) # looked identical to a real finding. Observed in CI: PR #214 run # 33117464973, job "A/UBSan / ASan/UBSan soak (60s)" attempt 1 failed # with "FAIL: ASAN/UBSAN report:" whose entire payload was # ==4635==WARNING: ptrace appears to be blocked (is seccomp enabled?). # ==4635==Child exited with signal 32. # with all four soak workers having reported clean in the same run. # Attempt 2 (identical binary, identical soak) passed with no asan* file # written at all. # # This drives soak_asan_verdict() directly against four fixtures, sources # the SAME file soak.sh sources (no hand-copied logic to drift), and ends # with a mandatory negative control: the ORIGINAL `ls`-only gate applied to # the ptrace-only fixture, proving that gate fails on exactly this input -- # the false-failure this whole row exists to fix. # # No nginx, no libzstd, no network -- pure Bash. # # Usage: tools/test_soak_asan_verdict_unit.sh set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=ci/tools/soak_asan_verdict.sh . "$SCRIPT_DIR/soak_asan_verdict.sh" WORK="$(mktemp -d "${TMPDIR:-/tmp}/soak-asan-verdict-unit.XXXXXX")" trap 'rm -rf "$WORK"' EXIT fail=0 # assert_verdict