astral_async_http_range_reader-0.11.0/.cargo_vcs_info.json0000644000000001361046102023000173550ustar { "git": { "sha1": "13ee8bb5bc903595f773508b001afbb906eb262d" }, "path_in_vcs": "" }astral_async_http_range_reader-0.11.0/.github/depandabot.yml000064400000000000000000000002321046102023000223040ustar 00000000000000version: 2 updates: - package-ecosystem: "github-actions" directory: "/" # Check for updates every Monday schedule: interval: "weekly"astral_async_http_range_reader-0.11.0/.github/workflows/lint-pr.yml000064400000000000000000000005231046102023000236300ustar 00000000000000name: "Lint PR" on: pull_request_target: types: - opened - edited - synchronize permissions: pull-requests: read jobs: main: name: Validate PR title runs-on: ubuntu-latest steps: - uses: amannn/action-semantic-pull-request@v5 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}astral_async_http_range_reader-0.11.0/.github/workflows/release-plz.yml000064400000000000000000000023041046102023000244650ustar 00000000000000name: Release-plz on: push: branches: - main jobs: release-plz-release: name: Release-plz release runs-on: ubuntu-latest if: ${{ github.repository_owner == 'astral-sh' }} permissions: contents: write id-token: write steps: - &checkout name: Checkout repository uses: actions/checkout@v6 with: fetch-depth: 0 persist-credentials: true - &install-rust name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable - name: Run release-plz uses: release-plz/action@v0.5 with: command: release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} release-plz-pr: name: Release-plz PR runs-on: ubuntu-latest if: ${{ github.repository_owner == 'astral-sh' }} permissions: pull-requests: write contents: write concurrency: group: release-plz-${{ github.ref }} cancel-in-progress: false steps: - *checkout - *install-rust - name: Run release-plz uses: release-plz/action@v0.5 with: command: release-pr env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} astral_async_http_range_reader-0.11.0/.github/workflows/rust-compile.yml000064400000000000000000000024061046102023000246700ustar 00000000000000on: push: branches: - "main" pull_request: name: Rust concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true env: RUST_LOG: info RUST_BACKTRACE: 1 RUSTFLAGS: "-D warnings" CARGO_TERM_COLOR: always jobs: check-rustdoc-links: name: Check intra-doc links runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: actions-rust-lang/setup-rust-toolchain@v1 - run: cargo rustdoc --all-features -- -D warnings -W unreachable-pub format_and_lint: name: Format and Lint runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: actions-rust-lang/setup-rust-toolchain@v1 with: components: clippy, rustfmt - name: Run rustfmt uses: actions-rust-lang/rustfmt@v1 - name: Run clippy run: cargo clippy build: name: ubuntu-latest runs-on: ubuntu-latest needs: [ format_and_lint ] steps: - name: Checkout source code uses: actions/checkout@v4 - name: Install Rust toolchain uses: actions-rust-lang/setup-rust-toolchain@v1 with: components: rustfmt - name: Build run: cargo build - name: Run tests run: cargo test -- --nocapture astral_async_http_range_reader-0.11.0/.gitignore000064400000000000000000000002701046102023000201120ustar 00000000000000# Generated by Cargo # will have compiled files and executables /target/ # These are backup files generated by rustfmt **/*.rs.bk # These are files generated by IntelliJ IDEA .idea/ astral_async_http_range_reader-0.11.0/CHANGELOG.md000064400000000000000000000054671046102023000177500ustar 00000000000000# Changelog All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] ## [0.11.0](https://github.com/astral-sh/async_http_range_reader/compare/v0.10.0...v0.11.0) - 2026-03-31 ### Added - Migrate off bisect ([#10](https://github.com/astral-sh/async_http_range_reader/pull/10)) ### Other - Merge upstream main ([#12](https://github.com/astral-sh/async_http_range_reader/pull/12)) - Switch async_zip dev-dependency to astral_async_zip - Update rstest dev-dependency to 0.26 - Merge pull request #8 from astral-sh/konsti/bounds-checks - Fix clippy - release v0.10.0 ## [0.10.0](https://github.com/astral-sh/async_http_range_reader/compare/v0.9.1...v0.10.0) - 2026-02-09 ### Other - Modernize release-plz setup - Update thiserror and itertools - Update lockfile - Rustfmt - Revert "update lib.rs set accept-encoding: identity to prevent double-compression" - update lib.rs set accept-encoding: identity to prevent double-compression - update README to reflect astral-forked package - new clippy notes: port updates from prefix-dev/async_http_range_reader - update reqwest to 0.13.1, reqwest-middleware to 0.5.0 ## [0.9.1](https://github.com/prefix-dev/async_http_range_reader/compare/v0.9.0...v0.9.1) - 2024-11-12 ### Other - Update reqwest-middleware from 0.3.0 to 0.4.0 ([#21](https://github.com/prefix-dev/async_http_range_reader/pull/21)) ## [0.9.0](https://github.com/prefix-dev/async_http_range_reader/compare/v0.8.0...v0.9.0) - 2024-10-28 ### Other - Update dependencies ([#19](https://github.com/prefix-dev/async_http_range_reader/pull/19)) ## [0.8.0](https://github.com/prefix-dev/async_http_range_reader/compare/v0.7.1...v0.8.0) - 2024-05-06 ### Fixed - always use requested URL, rather than response URL ([#16](https://github.com/prefix-dev/async_http_range_reader/pull/16)) ## [0.7.1](https://github.com/prefix-dev/async_http_range_reader/compare/v0.7.0...v0.7.1) - 2024-04-10 ### Other - Upgrade `reqwest` to v0.12.3 ([#12](https://github.com/prefix-dev/async_http_range_reader/pull/12)) ## [0.7.0](https://github.com/prefix-dev/async_http_range_reader/compare/v0.6.1...v0.7.0) - 2024-02-23 ### Added - lint pr titles for better changelogs ([#7](https://github.com/prefix-dev/async_http_range_reader/pull/7)) ### Fixed - tests ### Other - Allow extra headers to be attached to requests ([#9](https://github.com/prefix-dev/async_http_range_reader/pull/9)) ## [0.6.1](https://github.com/prefix-dev/async_http_range_reader/compare/v0.6.0...v0.6.1) - 2024-02-17 ### Added - added releaze-pls to the repository. ### Other - Propagate unsupported error when server returns 200 ([#5](https://github.com/prefix-dev/async_http_range_reader/pull/5)) astral_async_http_range_reader-0.11.0/Cargo.lock0000644000001027101046102023000153310ustar # This file is automatically @generated by Cargo. # It is not intended for manual editing. version = 4 [[package]] name = "addr2line" version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] [[package]] name = "adler2" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" [[package]] name = "aho-corasick" version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] [[package]] name = "anyhow" version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c042108f3ed77fd83760a5fd79b53be043192bb3b9dba91d8c574c0ada7850c8" [[package]] name = "assert_matches" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" [[package]] name = "astral-reqwest-middleware" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98e1c6be25cfbf1bb4fea1a9da51bc05d3259a9062df4e53f54e5607895e33c9" dependencies = [ "anyhow", "async-trait", "http", "reqwest", "thiserror 2.0.18", "tower-service", ] [[package]] name = "astral_async_http_range_reader" version = "0.11.0" dependencies = [ "assert_matches", "astral-reqwest-middleware", "astral_async_zip", "axum", "futures", "http-content-range", "itertools", "memmap2", "reqwest", "rstest", "thiserror 2.0.18", "tokio", "tokio-stream", "tokio-util", "tower-http", "tracing", "url", ] [[package]] name = "astral_async_zip" version = "0.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab72a761e6085828cc8f0e05ed332b2554701368c5dc54de551bfaec466518ba" dependencies = [ "crc32fast", "futures-lite", "pin-project", "thiserror 1.0.65", "tokio", "tokio-util", ] [[package]] name = "async-trait" version = "0.1.83" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", "syn", ] [[package]] name = "atomic-waker" version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "axum" version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "504e3947307ac8326a5437504c517c4b56716c9d98fac0028c2acc7ca47d70ae" dependencies = [ "async-trait", "axum-core", "bytes", "futures-util", "http", "http-body", "http-body-util", "hyper", "hyper-util", "itoa", "matchit", "memchr", "mime", "percent-encoding", "pin-project-lite", "rustversion", "serde", "sync_wrapper", "tokio", "tower", "tower-layer", "tower-service", ] [[package]] name = "axum-core" version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" dependencies = [ "async-trait", "bytes", "futures-util", "http", "http-body", "http-body-util", "mime", "pin-project-lite", "rustversion", "sync_wrapper", "tower-layer", "tower-service", ] [[package]] name = "backtrace" version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ "addr2line", "cfg-if", "libc", "miniz_oxide", "object", "rustc-demangle", "windows-targets", ] [[package]] name = "base64" version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bitflags" version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" [[package]] name = "bumpalo" version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytes" version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" [[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "crc32fast" version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ "cfg-if", ] [[package]] name = "either" version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" [[package]] name = "equivalent" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "fastrand" version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "form_urlencoded" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] [[package]] name = "futures" version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", "futures-executor", "futures-io", "futures-sink", "futures-task", "futures-util", ] [[package]] name = "futures-channel" version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", ] [[package]] name = "futures-core" version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", "futures-util", ] [[package]] name = "futures-io" version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-lite" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52527eb5074e35e9339c6b4e8d12600c7128b68fb25dcb9fa9dec18f7c25f3a5" dependencies = [ "fastrand", "futures-core", "futures-io", "parking", "pin-project-lite", ] [[package]] name = "futures-macro" version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", "syn", ] [[package]] name = "futures-sink" version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-timer" version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" [[package]] name = "futures-util" version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", "futures-io", "futures-macro", "futures-sink", "futures-task", "memchr", "pin-project-lite", "pin-utils", "slab", ] [[package]] name = "gimli" version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "hashbrown" version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" [[package]] name = "hermit-abi" version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "http" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" dependencies = [ "bytes", "fnv", "itoa", ] [[package]] name = "http-body" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", "http", ] [[package]] name = "http-body-util" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", "futures-util", "http", "http-body", "pin-project-lite", ] [[package]] name = "http-content-range" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91314cc9d86f625097a3365cab4e4b6f190eac231650f8f41c1edd8080cea1d0" [[package]] name = "http-range-header" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08a397c49fec283e3d6211adbe480be95aae5f304cfb923e9970e08956d5168a" [[package]] name = "httparse" version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" [[package]] name = "httpdate" version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" dependencies = [ "atomic-waker", "bytes", "futures-channel", "futures-core", "http", "http-body", "httparse", "httpdate", "itoa", "pin-project-lite", "pin-utils", "smallvec", "tokio", "want", ] [[package]] name = "hyper-util" version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ "base64", "bytes", "futures-channel", "futures-util", "http", "http-body", "hyper", "ipnet", "libc", "percent-encoding", "pin-project-lite", "socket2 0.6.3", "tokio", "tower-service", "tracing", ] [[package]] name = "idna" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ "unicode-bidi", "unicode-normalization", ] [[package]] name = "indexmap" version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", "hashbrown", ] [[package]] name = "ipnet" version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" [[package]] name = "iri-string" version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8e7418f59cc01c88316161279a7f665217ae316b388e58a0d10e29f54f1e5eb" dependencies = [ "memchr", "serde", ] [[package]] name = "itertools" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" dependencies = [ "either", ] [[package]] name = "itoa" version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "js-sys" version = "0.3.91" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" dependencies = [ "once_cell", "wasm-bindgen", ] [[package]] name = "libc" version = "0.2.183" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" [[package]] name = "log" version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "matchit" version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] name = "memchr" version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memmap2" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f" dependencies = [ "libc", ] [[package]] name = "mime" version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "mime_guess" version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" dependencies = [ "mime", "unicase", ] [[package]] name = "miniz_oxide" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" dependencies = [ "adler2", ] [[package]] name = "mio" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" dependencies = [ "hermit-abi", "libc", "wasi", "windows-sys 0.52.0", ] [[package]] name = "object" version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "memchr", ] [[package]] name = "once_cell" version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "parking" version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" [[package]] name = "percent-encoding" version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pin-project" version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", "syn", ] [[package]] name = "pin-project-lite" version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" [[package]] name = "pin-utils" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "proc-macro-crate" version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" dependencies = [ "toml_edit", ] [[package]] name = "proc-macro2" version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" dependencies = [ "unicode-ident", ] [[package]] name = "quote" version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" dependencies = [ "proc-macro2", ] [[package]] name = "regex" version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", "regex-automata", "regex-syntax", ] [[package]] name = "regex-automata" version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" dependencies = [ "aho-corasick", "memchr", "regex-syntax", ] [[package]] name = "regex-syntax" version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "relative-path" version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "reqwest" version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801" dependencies = [ "base64", "bytes", "futures-core", "futures-util", "http", "http-body", "http-body-util", "hyper", "hyper-util", "js-sys", "log", "percent-encoding", "pin-project-lite", "sync_wrapper", "tokio", "tokio-util", "tower", "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "wasm-streams", "web-sys", ] [[package]] name = "rstest" version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f5a3193c063baaa2a95a33f03035c8a72b83d97a54916055ba22d35ed3839d49" dependencies = [ "futures-timer", "futures-util", "rstest_macros", ] [[package]] name = "rstest_macros" version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" dependencies = [ "cfg-if", "glob", "proc-macro-crate", "proc-macro2", "quote", "regex", "relative-path", "rustc_version", "syn", "unicode-ident", ] [[package]] name = "rustc-demangle" version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustc_version" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ "semver", ] [[package]] name = "rustversion" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" [[package]] name = "semver" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" version = "1.0.213" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ea7893ff5e2466df8d720bb615088341b295f849602c6956047f8f80f0e9bc1" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" version = "1.0.213" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e85ad2009c50b58e87caa8cd6dac16bdf511bbfb7af6c33df902396aa480fa5" dependencies = [ "proc-macro2", "quote", "syn", ] [[package]] name = "slab" version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ "autocfg", ] [[package]] name = "smallvec" version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "socket2" version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" dependencies = [ "libc", "windows-sys 0.52.0", ] [[package]] name = "socket2" version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", "windows-sys 0.61.2", ] [[package]] name = "syn" version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] [[package]] name = "sync_wrapper" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" dependencies = [ "futures-core", ] [[package]] name = "thiserror" version = "1.0.65" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d11abd9594d9b38965ef50805c5e469ca9cc6f197f883f717e0269a3057b3d5" dependencies = [ "thiserror-impl 1.0.65", ] [[package]] name = "thiserror" version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ "thiserror-impl 2.0.18", ] [[package]] name = "thiserror-impl" version = "1.0.65" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae71770322cbd277e69d762a16c444af02aa0575ac0d174f0b9562d3b37f8602" dependencies = [ "proc-macro2", "quote", "syn", ] [[package]] name = "thiserror-impl" version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", "syn", ] [[package]] name = "tinyvec" version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" dependencies = [ "tinyvec_macros", ] [[package]] name = "tinyvec_macros" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" version = "1.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "145f3413504347a2be84393cc8a7d2fb4d863b375909ea59f2158261aa258bbb" dependencies = [ "backtrace", "bytes", "libc", "mio", "pin-project-lite", "socket2 0.5.7", "tokio-macros", "windows-sys 0.52.0", ] [[package]] name = "tokio-macros" version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", "syn", ] [[package]] name = "tokio-stream" version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" dependencies = [ "futures-core", "pin-project-lite", "tokio", "tokio-util", ] [[package]] name = "tokio-util" version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" dependencies = [ "bytes", "futures-core", "futures-io", "futures-sink", "pin-project-lite", "tokio", ] [[package]] name = "toml_datetime" version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" [[package]] name = "toml_edit" version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ "indexmap", "toml_datetime", "winnow", ] [[package]] name = "tower" version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", "pin-project-lite", "sync_wrapper", "tokio", "tower-layer", "tower-service", ] [[package]] name = "tower-http" version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ "bitflags", "bytes", "futures-core", "futures-util", "http", "http-body", "http-body-util", "http-range-header", "httpdate", "iri-string", "mime", "mime_guess", "percent-encoding", "pin-project-lite", "tokio", "tokio-util", "tower", "tower-layer", "tower-service", "tracing", ] [[package]] name = "tower-layer" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-service" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ "pin-project-lite", "tracing-attributes", "tracing-core", ] [[package]] name = "tracing-attributes" version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", "syn", ] [[package]] name = "tracing-core" version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", ] [[package]] name = "try-lock" version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "unicase" version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df" [[package]] name = "unicode-bidi" version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" [[package]] name = "unicode-ident" version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-normalization" version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" dependencies = [ "tinyvec", ] [[package]] name = "url" version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" dependencies = [ "form_urlencoded", "idna", "percent-encoding", ] [[package]] name = "want" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" dependencies = [ "try-lock", ] [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" dependencies = [ "cfg-if", "once_cell", "rustversion", "wasm-bindgen-macro", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" version = "0.4.64" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8" dependencies = [ "cfg-if", "futures-util", "js-sys", "once_cell", "wasm-bindgen", "web-sys", ] [[package]] name = "wasm-bindgen-macro" version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" dependencies = [ "quote", "wasm-bindgen-macro-support", ] [[package]] name = "wasm-bindgen-macro-support" version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" dependencies = [ "bumpalo", "proc-macro2", "quote", "syn", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" dependencies = [ "unicode-ident", ] [[package]] name = "wasm-streams" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" dependencies = [ "futures-util", "js-sys", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", ] [[package]] name = "web-sys" version = "0.3.91" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" dependencies = [ "js-sys", "wasm-bindgen", ] [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" [[package]] name = "windows-sys" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ "windows-targets", ] [[package]] name = "windows-sys" version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ "windows-link", ] [[package]] name = "windows-targets" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", "windows_i686_gnu", "windows_i686_gnullvm", "windows_i686_msvc", "windows_x86_64_gnu", "windows_x86_64_gnullvm", "windows_x86_64_msvc", ] [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" dependencies = [ "memchr", ] astral_async_http_range_reader-0.11.0/Cargo.toml0000644000000042331046102023000153550ustar # THIS FILE IS AUTOMATICALLY GENERATED BY CARGO # # When uploading crates to the registry Cargo will automatically # "normalize" Cargo.toml files for maximal compatibility # with all versions of Cargo and also rewrite `path` dependencies # to registry (e.g., crates.io) dependencies. # # If you are reading this file be aware that the original Cargo.toml # will likely look very different (and much more reasonable). # See Cargo.toml.orig for the original contents. [package] edition = "2021" rust-version = "1.74" name = "astral_async_http_range_reader" version = "0.11.0" authors = ["Bas Zalmstra "] build = false exclude = ["test-data/*"] autolib = false autobins = false autoexamples = false autotests = false autobenches = false description = "A library for streaming reading of files over HTTP using range requests" readme = "README.md" license = "MIT" repository = "https://github.com/astral-sh/async_http_range_reader" [lib] name = "async_http_range_reader" path = "src/lib.rs" [dependencies.futures] version = "0.3.28" [dependencies.http-content-range] version = "0.2.0" [dependencies.itertools] version = "0.14.0" [dependencies.memmap2] version = "0.9.0" [dependencies.reqwest] version = "0.13.1" features = ["stream"] default-features = false [dependencies.reqwest-middleware] version = "0.5.1" package = "astral-reqwest-middleware" [dependencies.thiserror] version = "2.0.18" [dependencies.tokio] version = "1.33.0" default-features = false [dependencies.tokio-stream] version = "0.1.14" features = ["sync"] [dependencies.tokio-util] version = "0.7.9" [dependencies.tracing] version = "0.1.40" [dev-dependencies.assert_matches] version = "1.5.0" [dev-dependencies.astral_async_zip] version = "0.0.17" features = ["tokio"] default-features = false [dev-dependencies.axum] version = "0.7.5" features = [ "tokio", "http1", ] default-features = false [dev-dependencies.rstest] version = "0.26.0" [dev-dependencies.tokio] version = "1.33.0" features = [ "macros", "test-util", ] default-features = false [dev-dependencies.tower-http] version = "0.6.1" features = ["fs"] default-features = false [dev-dependencies.url] version = "2.4.1" astral_async_http_range_reader-0.11.0/Cargo.toml.orig000064400000000000000000000024201046102023000210100ustar 00000000000000[package] name = "astral_async_http_range_reader" authors = ["Bas Zalmstra "] version = "0.11.0" edition = "2021" description = "A library for streaming reading of files over HTTP using range requests" license = "MIT" repository = "https://github.com/astral-sh/async_http_range_reader" exclude = ["test-data/*"] rust-version = "1.74" [lib] name = "async_http_range_reader" [dependencies] futures = "0.3.28" http-content-range = "0.2.0" itertools = "0.14.0" memmap2 = "0.9.0" reqwest = { version = "0.13.1", default-features = false, features = ["stream"] } reqwest-middleware = { version = "0.5.1", package = "astral-reqwest-middleware" } tokio = { version = "1.33.0", default-features = false } tokio-stream = { version = "0.1.14", features = ["sync"] } tokio-util = "0.7.9" thiserror = "2.0.18" tracing = "0.1.40" [dev-dependencies] axum = { version = "0.7.5", default-features = false, features = ["tokio", "http1"] } tokio = { version = "1.33.0", default-features = false, features = ["macros", "test-util"] } tower-http = { version = "0.6.1", default-features = false, features = ["fs"] } astral_async_zip = { version = "0.0.17", default-features = false, features = ["tokio"] } assert_matches = "1.5.0" rstest = { version = "0.26.0" } url = { version = "2.4.1" } astral_async_http_range_reader-0.11.0/LICENSE000064400000000000000000000020551046102023000171320ustar 00000000000000MIT License Copyright (c) 2023 Bas Zalmstra Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. astral_async_http_range_reader-0.11.0/README.md000064400000000000000000000015571046102023000174120ustar 00000000000000# Async HTTP Range Reader [![Crates.io](https://img.shields.io/crates/v/astral_async_http_range_reader?style=flat-square)](https://crates.io/crates/astral_async_http_range_reader) [![docs.rs](https://img.shields.io/docsrs/astral_async_http_range_reader?style=flat-square)](https://docs.rs/astral_async_http_range_reader/) [![GitHub Workflow Status (branch)](https://img.shields.io/github/actions/workflow/status/astral-sh/async_http_range_reader/rust-compile.yml?branch=main&style=flat-square)](https://github.com/astral-sh/async_http_range_reader/actions?query=branch%3Amain) [![GitHub](https://img.shields.io/github/license/astral-sh/astral_async_http_range_reader?style=flat-square)](https://github.com/astral-sh/async_http_range_reader/blob/main/LICENSE) A crate that provides the `AsyncHttpRangeReader` struct, which allows streaming files over HTTP using range requests. astral_async_http_range_reader-0.11.0/release-plz.toml000064400000000000000000000000451046102023000212420ustar 00000000000000[workspace] git_release_enable = trueastral_async_http_range_reader-0.11.0/src/error.rs000064400000000000000000000052161046102023000204150ustar 00000000000000use std::sync::Arc; /// Error type used for [`crate::AsyncHttpRangeReader`] #[derive(Clone, Debug, thiserror::Error)] #[non_exhaustive] pub enum AsyncHttpRangeReaderError { /// The server does not support range requests #[error("range requests are not supported")] HttpRangeRequestUnsupported, /// Other HTTP error #[error(transparent)] HttpError(#[from] Arc), /// An error occurred during transport #[error("an error occurred during transport: {0}")] TransportError(#[source] Arc), /// An IO error occurred #[error("io error occurred: {0}")] IoError(#[source] Arc), /// Content-Range header is missing from response #[error("content-range header is missing from response")] ContentRangeMissing, /// Content-Length header is missing from response #[error("content-length header is missing from response")] ContentLengthMissing, /// Memory mapping the file failed #[error("memory mapping the file failed")] MemoryMapError(#[source] Arc), /// Error from `http-content-range` #[error("invalid Content-Range header: {0}")] ContentRangeParser(String), /// The server returned an invalid range response #[error( "request and response range mismatch, \ expected {expected_start}-{expected_end_inclusive}/{expected_complete_length}, \ got {actual_start}-{actual_end_inclusive}/{actual_complete_length}" )] RangeMismatch { expected_start: u64, expected_end_inclusive: u64, expected_complete_length: usize, actual_start: u64, actual_end_inclusive: u64, actual_complete_length: u64, }, /// The server returned more bytes than the range request asked for #[error("range response returned more than the expected {expected} bytes")] ResponseTooLong { expected: u64 }, /// The server returned fewer bytes than the range request asked for #[error("expected {expected} bytes from range response, got {actual}")] ResponseTooShort { expected: u64, actual: u64 }, } impl From for AsyncHttpRangeReaderError { fn from(err: std::io::Error) -> Self { AsyncHttpRangeReaderError::IoError(Arc::new(err)) } } impl From for AsyncHttpRangeReaderError { fn from(err: reqwest_middleware::Error) -> Self { AsyncHttpRangeReaderError::TransportError(Arc::new(err)) } } impl From for AsyncHttpRangeReaderError { fn from(err: reqwest::Error) -> Self { AsyncHttpRangeReaderError::TransportError(Arc::new(err.into())) } } astral_async_http_range_reader-0.11.0/src/lib.rs000064400000000000000000001216321046102023000200330ustar 00000000000000//! This library provides the [`AsyncHttpRangeReader`] type. //! //! It allows streaming a file over HTTP while also allow random access. The type implements both //! [`AsyncRead`] as well as [`AsyncSeek`]. This is supported through the use of range requests. //! Each individual read will request a portion of the file using an HTTP range request. //! //! Requesting numerous small reads might turn out to be relatively slow because each reads needs to //! perform an HTTP request. To alleviate this issue [`AsyncHttpRangeReader::prefetch`] is provided. //! Using this method you can *prefect* a number of bytes which will be streamed in on the //! background. If a read operation is reading from already (pre)fetched ranges it will stream from //! the internal cache instead. //! //! Internally the [`AsyncHttpRangeReader`] stores a memory map which allows sparsely reading the //! data into memory without actually requiring all memory for file to be resident in memory. //! //! The primary use-case for this library is to be able to sparsely stream a zip archive over HTTP //! but its designed in a generic fashion. mod error; mod sparse_range; use futures::{FutureExt, Stream, StreamExt}; use http_content_range::{ContentRange, ContentRangeBytes}; use memmap2::MmapMut; use reqwest::header::HeaderMap; use reqwest::{Response, Url}; use sparse_range::SparseRange; use std::{ io::{self, SeekFrom}, ops::Range, pin::Pin, sync::Arc, task::{ready, Context, Poll}, }; use tokio::{ io::{AsyncRead, AsyncSeek, ReadBuf}, sync::watch::Sender, sync::{watch, Mutex}, }; use tokio_stream::wrappers::WatchStream; use tokio_util::sync::PollSender; use tracing::{info_span, Instrument}; pub use error::AsyncHttpRangeReaderError; /// An `AsyncRangeReader` enables reading from a file over HTTP using range requests. /// /// See the [`crate`] level documentation for more information. /// /// The general entrypoint is [`AsyncHttpRangeReader::new`]. Depending on the /// [`CheckSupportMethod`], this will either call [`AsyncHttpRangeReader::initial_tail_request`] or /// [`AsyncHttpRangeReader::initial_head_request`] to send the initial request and then /// [`AsyncHttpRangeReader::from_range_response`] or [`AsyncHttpRangeReader::from_head_response`] to /// initialize the async reader. If you want to apply a caching layer, you can send the initial head /// (or tail) request yourself with your cache headers (e.g. through the /// [http-cache-semantics](https://docs.rs/http-cache-semantics) crate): /// /// ```rust /// # use url::Url; /// # use async_http_range_reader::{AsyncHttpRangeReader, AsyncHttpRangeReaderError}; /// # use reqwest::header::HeaderMap; /// async fn get_reader_cached( /// url: Url, /// ) -> Result, AsyncHttpRangeReaderError> { /// let etag = "63c550e8-5ae"; /// let client = reqwest::Client::new(); /// let response = client /// .head(url.clone()) /// .header(reqwest::header::IF_NONE_MATCH, etag) /// .send() /// .await?; /// if response.status() == reqwest::StatusCode::NOT_MODIFIED { /// Ok(None) /// } else { /// let reader = AsyncHttpRangeReader::from_head_response(client, response, url, HeaderMap::default()).await?; /// Ok(Some(reader)) /// } /// } /// ``` #[derive(Debug)] pub struct AsyncHttpRangeReader { inner: Mutex, len: u64, } #[derive(Default, Clone, Debug)] struct StreamerState { resident_range: SparseRange, requested_ranges: Vec>, error: Option, } #[derive(Debug)] struct Inner { /// A read-only view on the memory mapped data. The `downloaded_range` indicates the regions of /// memory that contain bytes that have been downloaded. data: &'static [u8], /// The current read position in the stream pos: u64, /// The range of bytes that have been requested for download requested_range: SparseRange, /// The range of bytes that have actually been downloaded to `data`. streamer_state: StreamerState, /// A channel receiver that holds the last downloaded range (or an error) from the background /// task. streamer_state_rx: WatchStream, /// A channel sender to send range requests to the background task /// /// Contract: All ranges sent must be inside the range of the memory map request_tx: tokio::sync::mpsc::Sender>, /// An optional object to reserve a slot in the `request_tx` sender. When in the process of /// sending a requests this contains an actual value. poll_request_tx: Option>>, } /// For the initial request, we support either directly requesting N bytes from the end for file /// or, if you the server doesn't support negative byte offsets, starting with a HEAD request /// instead pub enum CheckSupportMethod { /// Perform a range request with a negative byte range. This will return the N bytes from the /// *end* of the file as well as the file-size. This is especially useful to also immediately /// get some bytes from the end of the file. NegativeRangeRequest(u64), /// Perform a head request to get the length of the file and check if the server supports range /// requests. Head, } fn error_for_status(response: reqwest::Response) -> reqwest_middleware::Result { response .error_for_status() .map_err(reqwest_middleware::Error::Reqwest) } impl AsyncHttpRangeReader { /// Construct a new `AsyncHttpRangeReader`. pub async fn new( client: impl Into, url: Url, check_method: CheckSupportMethod, extra_headers: HeaderMap, ) -> Result<(Self, HeaderMap), AsyncHttpRangeReaderError> { let client = client.into(); match check_method { CheckSupportMethod::NegativeRangeRequest(initial_chunk_size) => { let response = Self::initial_tail_request( client.clone(), url.clone(), initial_chunk_size, HeaderMap::default(), ) .await?; let response_headers = response.headers().clone(); let self_ = Self::from_range_response(client, response, url, extra_headers).await?; Ok((self_, response_headers)) } CheckSupportMethod::Head => { let response = Self::initial_head_request(client.clone(), url.clone(), HeaderMap::default()) .await?; let response_headers = response.headers().clone(); let self_ = Self::from_head_response(client, response, url, extra_headers).await?; Ok((self_, response_headers)) } } } /// Send an initial range request to determine if the remote accepts range /// requests. This will return a number of bytes from the end of the stream. Use the /// `initial_chunk_size` parameter to define how many bytes should be requested from the end. pub async fn initial_tail_request( client: impl Into, url: reqwest::Url, initial_chunk_size: u64, extra_headers: HeaderMap, ) -> Result { let client = client.into(); let tail_response = client .get(url) .header( reqwest::header::RANGE, format!("bytes=-{initial_chunk_size}"), ) .headers(extra_headers) .send() .await .and_then(error_for_status) .map_err(Arc::new) .map_err(AsyncHttpRangeReaderError::HttpError)?; Ok(tail_response) } #[deprecated(note = "use `from_range_response` instead")] pub async fn from_tail_response( client: impl Into, tail_request_response: Response, url: Url, extra_headers: HeaderMap, ) -> Result { Self::from_range_response(client, tail_request_response, url, extra_headers).await } /// Initialize the reader from [`AsyncHttpRangeReader::initial_tail_request`] (or a user /// provided range response) pub async fn from_range_response( client: impl Into, response: Response, url: Url, extra_headers: HeaderMap, ) -> Result { let client = client.into(); // Get the size of the file from this initial request let content_range_header = response .headers() .get(reqwest::header::CONTENT_RANGE) .ok_or(AsyncHttpRangeReaderError::ContentRangeMissing)? .to_str() .map_err(|_err| AsyncHttpRangeReaderError::ContentRangeMissing)?; // The parser ensures finish < complete_length let content_range = ContentRange::parse(content_range_header).ok_or_else(|| { AsyncHttpRangeReaderError::ContentRangeParser(content_range_header.to_string()) })?; let (start, end_inclusive, complete_length) = match content_range { ContentRange::Bytes(ContentRangeBytes { first_byte, last_byte, complete_length, }) => (first_byte, last_byte, complete_length), _ => return Err(AsyncHttpRangeReaderError::HttpRangeRequestUnsupported), }; // Allocate a memory map to hold the data let memory_map = memmap2::MmapOptions::new() .len(complete_length as usize) .map_anon() .map_err(Arc::new) .map_err(AsyncHttpRangeReaderError::MemoryMapError)?; // SAFETY: Get a read-only slice to the memory. This is safe because the memory map is never // reallocated and we keep track of the initialized part. let memory_map_slice = unsafe { std::slice::from_raw_parts(memory_map.as_ptr(), memory_map.len()) }; let requested_range = SparseRange::from_range(start..end_inclusive + 1); // adding more than 2 entries to the channel would block the sender. I assumed two would // suffice because I would want to 1) prefetch a certain range and 2) read stuff via the // AsyncRead implementation. Any extra would simply have to wait for one of these to // succeed. I eventually used 10 because who cares. let (request_tx, request_rx) = tokio::sync::mpsc::channel(10); let (state_tx, state_rx) = watch::channel(StreamerState::default()); tokio::spawn(run_streamer( client, url, extra_headers, Some((response, start, end_inclusive + 1)), memory_map, state_tx, request_rx, )); // Configure the initial state of the streamer. let mut streamer_state = StreamerState::default(); streamer_state .requested_ranges .push(start..end_inclusive + 1); let reader = Self { len: memory_map_slice.len() as u64, inner: Mutex::new(Inner { data: memory_map_slice, pos: 0, requested_range, streamer_state, streamer_state_rx: WatchStream::new(state_rx), request_tx, poll_request_tx: None, }), }; Ok(reader) } /// Send an initial range request to determine if the remote accepts range /// requests and get the content length pub async fn initial_head_request( client: impl Into, url: reqwest::Url, extra_headers: HeaderMap, ) -> Result { let client = client.into(); // Perform a HEAD request to get the content-length. let head_response = client .head(url.clone()) .headers(extra_headers) .send() .await .and_then(error_for_status) .map_err(Arc::new) .map_err(AsyncHttpRangeReaderError::HttpError)?; Ok(head_response) } /// Initialize the reader from [`AsyncHttpRangeReader::initial_head_request`] (or a user /// provided response) pub async fn from_head_response( client: impl Into, head_response: Response, url: Url, extra_headers: HeaderMap, ) -> Result { let client = client.into(); // Are range requests supported? if head_response .headers() .get(reqwest::header::ACCEPT_RANGES) .and_then(|h| h.to_str().ok()) != Some("bytes") { return Err(AsyncHttpRangeReaderError::HttpRangeRequestUnsupported); } let content_length: u64 = head_response .headers() .get(reqwest::header::CONTENT_LENGTH) .ok_or(AsyncHttpRangeReaderError::ContentLengthMissing)? .to_str() .map_err(|_err| AsyncHttpRangeReaderError::ContentLengthMissing)? .parse() .map_err(|_err| AsyncHttpRangeReaderError::ContentLengthMissing)?; // Allocate a memory map to hold the data let memory_map = memmap2::MmapOptions::new() .len(content_length as _) .map_anon() .map_err(Arc::new) .map_err(AsyncHttpRangeReaderError::MemoryMapError)?; // SAFETY: Get a read-only slice to the memory. This is safe because the memory map is never // reallocated and we keep track of the initialized part. let memory_map_slice = unsafe { std::slice::from_raw_parts(memory_map.as_ptr(), memory_map.len()) }; let requested_range = SparseRange::default(); // adding more than 2 entries to the channel would block the sender. I assumed two would // suffice because I would want to 1) prefetch a certain range and 2) read stuff via the // AsyncRead implementation. Any extra would simply have to wait for one of these to // succeed. I eventually used 10 because who cares. let (request_tx, request_rx) = tokio::sync::mpsc::channel(10); let (state_tx, state_rx) = watch::channel(StreamerState::default()); tokio::spawn(run_streamer( client, url, extra_headers, None, memory_map, state_tx, request_rx, )); // Configure the initial state of the streamer. let streamer_state = StreamerState::default(); let reader = Self { len: memory_map_slice.len() as u64, inner: Mutex::new(Inner { data: memory_map_slice, pos: 0, requested_range, streamer_state, streamer_state_rx: WatchStream::new(state_rx), request_tx, poll_request_tx: None, }), }; Ok(reader) } /// Returns the ranges that this instance actually performed HTTP requests for. pub async fn requested_ranges(&self) -> Vec> { let mut inner = self.inner.lock().await; if let Some(Some(new_state)) = inner.streamer_state_rx.next().now_or_never() { inner.streamer_state = new_state; } inner.streamer_state.requested_ranges.clone() } /// Prefetches a range of bytes from the remote. When specifying a large range this can /// drastically reduce the number of requests required to the server. pub async fn prefetch(&mut self, bytes: Range) { let inner = self.inner.get_mut(); // Ensure the range is withing the file size and non-zero of length. let range = bytes.start..(bytes.end.min(inner.data.len() as u64)); if range.start >= range.end { return; } // Check if the range has been requested or not. let inner = self.inner.get_mut(); if let Some((new_range, _)) = inner.requested_range.cover(range.clone()) { let _ = inner.request_tx.send(range).await; inner.requested_range = new_range; } } /// Returns the length of the stream in bytes #[allow(clippy::len_without_is_empty)] pub fn len(&self) -> u64 { self.len } } /// A task that will download parts from the remote archive and "send" them to the frontend as they /// become available. #[tracing::instrument(name = "fetch_ranges", skip_all, fields(url))] async fn run_streamer( client: reqwest_middleware::ClientWithMiddleware, url: Url, extra_headers: HeaderMap, response: Option<(Response, u64, u64)>, mut memory_map: MmapMut, mut state_tx: Sender, mut request_rx: tokio::sync::mpsc::Receiver>, ) { let mut state = StreamerState::default(); if let Some((response, start, end_exclusive)) = response { // Add the initial range to the state state.requested_ranges.push(start..end_exclusive); // Stream the initial data in memory if !stream_response( response, start, end_exclusive, &mut memory_map, &mut state_tx, &mut state, ) .await { return; } } // Listen for any new incoming requests 'outer: loop { let range = match request_rx.recv().await { Some(range) => range, None => { break 'outer; } }; // Determine the range that we need to cover let uncovered_ranges = match state.resident_range.cover(range) { None => continue, Some((_, uncovered_ranges)) => uncovered_ranges, }; // Download and stream each range. for range in uncovered_ranges { // Update the requested ranges state .requested_ranges .push(*range.start()..*range.end() + 1); // Execute the request let range_string = format!("bytes={}-{}", range.start(), range.end()); let span = info_span!("fetch_range", range = range_string.as_str()); let response = match client .get(url.clone()) .header(reqwest::header::RANGE, range_string) .headers(extra_headers.clone()) .send() .instrument(span) .await .and_then(error_for_status) .map_err(std::io::Error::other) { Err(e) => { state.error = Some(e.into()); let _ = state_tx.send(state); break 'outer; } Ok(response) => response, }; if let Err(err) = validate_content_range(&response, *range.start(), *range.end(), memory_map.len()) { state.error = Some(err); let _ = state_tx.send(state); break 'outer; } // If the server returns a successful, but non-206 response (e.g., 200), then it // doesn't support range requests (even if the `Accept-Ranges` header is set). if response.status() != reqwest::StatusCode::PARTIAL_CONTENT { state.error = Some(AsyncHttpRangeReaderError::HttpRangeRequestUnsupported); let _ = state_tx.send(state); break 'outer; } if !stream_response( response, *range.start(), *range.end() + 1, &mut memory_map, &mut state_tx, &mut state, ) .await { break 'outer; } } } } /// Ensure that the response range headers match the request range headers fn validate_content_range( response: &Response, expected_start: u64, expected_end_inclusive: u64, expected_complete_length: usize, ) -> Result<(), AsyncHttpRangeReaderError> { let content_range_header = response .headers() .get(reqwest::header::CONTENT_RANGE) .ok_or(AsyncHttpRangeReaderError::ContentRangeMissing)? .to_str() .map_err(|_err| AsyncHttpRangeReaderError::ContentRangeMissing)?; let content_range = ContentRange::parse(content_range_header).ok_or_else(|| { AsyncHttpRangeReaderError::ContentRangeParser(content_range_header.to_string()) })?; let (actual_start, actual_end_inclusive, actual_complete_length) = match content_range { ContentRange::Bytes(ContentRangeBytes { first_byte, last_byte, complete_length, }) => (first_byte, last_byte, complete_length), _ => return Err(AsyncHttpRangeReaderError::HttpRangeRequestUnsupported), }; if expected_start != actual_start || expected_end_inclusive != actual_end_inclusive || expected_complete_length as u64 != actual_complete_length { return Err(AsyncHttpRangeReaderError::RangeMismatch { expected_start, expected_end_inclusive, expected_complete_length, actual_start, actual_end_inclusive, actual_complete_length, }); } Ok(()) } /// Streams the data from the specified response to the memory map updating progress in between. /// Returns `true` if everything went fine, `false` if anything went wrong. The error state, if any, /// is stored in `state_tx` so the "frontend" will consume it. /// /// The response must return bytes for the range of precisely `start..end_exclusive`. async fn stream_response( tail_request_response: Response, start: u64, end_exclusive: u64, memory_map: &mut MmapMut, state_tx: &mut Sender, state: &mut StreamerState, ) -> bool { // Enforce request channel contract assert!( (end_exclusive as usize) <= memory_map.len(), "end is outside of memory map {} > {}", end_exclusive, memory_map.len() ); let mut offset = start; let mut byte_stream = tail_request_response.bytes_stream(); while let Some(bytes) = byte_stream.next().await { let bytes = match bytes { Err(e) => { state.error = Some(e.into()); let _ = state_tx.send(state.clone()); return false; } Ok(bytes) => bytes, }; // Determine the range of these bytes in the complete file let byte_range = offset..offset + bytes.len() as u64; // Update the offset offset += bytes.len() as u64; // Prevent the server from sending more bytes than advertised in a response if offset > end_exclusive { state.error = Some(AsyncHttpRangeReaderError::ResponseTooLong { expected: end_exclusive - start, }); let _ = state_tx.send(state.clone()); return false; } // Copy the data from the stream to memory memory_map[byte_range.start as usize..byte_range.end as usize] .copy_from_slice(bytes.as_ref()); // Update the range of bytes that have been downloaded state.resident_range.update(byte_range); // Notify anyone that's listening that we have downloaded some extra data if state_tx.send(state.clone()).is_err() { // If we failed to set the state it means there is no receiver. In that case we should // just exit. return false; } } // Prevent the server from sending less bytes than advertised in a response if offset != end_exclusive { state.error = Some(AsyncHttpRangeReaderError::ResponseTooShort { expected: end_exclusive - start, actual: offset - start, }); let _ = state_tx.send(state.clone()); return false; } true } impl AsyncSeek for AsyncHttpRangeReader { fn start_seek(self: Pin<&mut Self>, position: SeekFrom) -> io::Result<()> { let me = self.get_mut(); let inner = me.inner.get_mut(); inner.pos = match position { SeekFrom::Start(pos) => pos, SeekFrom::End(relative) => (inner.data.len() as i64).saturating_add(relative) as u64, SeekFrom::Current(relative) => (inner.pos as i64).saturating_add(relative) as u64, }; Ok(()) } fn poll_complete(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { let inner = self.inner.get_mut(); Poll::Ready(Ok(inner.pos)) } } impl AsyncRead for AsyncHttpRangeReader { fn poll_read( self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>, ) -> Poll> { let me = self.get_mut(); let inner = me.inner.get_mut(); // If a previous error occurred we return that. if let Some(e) = inner.streamer_state.error.as_ref() { return Poll::Ready(Err(io::Error::other(e.clone()))); } // Determine the range to be fetched let range = inner.pos..(inner.pos + buf.remaining() as u64).min(inner.data.len() as u64); if range.start >= range.end { return Poll::Ready(Ok(())); } // Ensure we requested the required bytes while !inner.requested_range.is_covered(range.clone()) { // If there is an active range request wait for it to complete if let Some(mut poll) = inner.poll_request_tx.take() { match poll.poll_reserve(cx) { Poll::Ready(_) => { let _ = poll.send_item(range.clone()); inner.requested_range.update(range.clone()); break; } Poll::Pending => { inner.poll_request_tx = Some(poll); return Poll::Pending; } } } // Request the range inner.poll_request_tx = Some(PollSender::new(inner.request_tx.clone())); } // If there is still a request poll open but there is no need for a request, abort it. if let Some(mut poll) = inner.poll_request_tx.take() { poll.abort_send(); } loop { // Is the range already available? if inner .streamer_state .resident_range .is_covered(range.clone()) { let len = (range.end - range.start) as usize; buf.initialize_unfilled_to(len) .copy_from_slice(&inner.data[range.start as usize..range.end as usize]); buf.advance(len); inner.pos += len as u64; return Poll::Ready(Ok(())); } // Otherwise wait for new data to come in match ready!(Pin::new(&mut inner.streamer_state_rx).poll_next(cx)) { None => unreachable!(), Some(state) => { inner.streamer_state = state; if let Some(e) = inner.streamer_state.error.as_ref() { return Poll::Ready(Err(io::Error::other(e.clone()))); } } } } } } #[cfg(test)] mod static_directory_server; #[cfg(test)] mod test { use super::*; use crate::static_directory_server::StaticDirectoryServer; use assert_matches::assert_matches; use async_zip::tokio::read::seek::ZipFileReader; use axum::body::Body; use axum::extract::Request; use axum::response::IntoResponse; use futures::AsyncReadExt; use reqwest::header; use reqwest::Method; use reqwest::{Client, StatusCode}; use rstest::*; use std::path::Path; use tokio::io::AsyncReadExt as _; #[rstest] #[case(CheckSupportMethod::Head)] #[case(CheckSupportMethod::NegativeRangeRequest(8192))] #[tokio::test] async fn async_range_reader_zip(#[case] check_method: CheckSupportMethod) { // Spawn a static file server let path = Path::new(&std::env::var("CARGO_MANIFEST_DIR").unwrap()).join("test-data"); let server = StaticDirectoryServer::new(&path) .await .expect("could not initialize server"); // check that file is there and has the right size let filepath = path.join("andes-1.8.3-pyhd8ed1ab_0.conda"); assert!( filepath.exists(), "The conda package is not there yet. Did you run `git lfs pull`?" ); let file_size = std::fs::metadata(&filepath).unwrap().len(); assert_eq!( file_size, 2_463_995, "The conda package is not there yet. Did you run `git lfs pull`?" ); // Construct an AsyncRangeReader let (mut range, _) = AsyncHttpRangeReader::new( Client::new(), server.url().join("andes-1.8.3-pyhd8ed1ab_0.conda").unwrap(), check_method, HeaderMap::default(), ) .await .expect("Could not download range - did you run `git lfs pull`?"); // Make sure we have read the last couple of bytes range.prefetch(range.len() - 8192..range.len()).await; assert_eq!(range.len(), file_size); let mut reader = ZipFileReader::with_tokio(tokio::io::BufReader::with_capacity(0, range)) .await .unwrap(); assert_eq!( reader .file() .entries() .iter() .map(|e| e.filename().as_str().unwrap_or("")) .collect::>(), vec![ "metadata.json", "info-andes-1.8.3-pyhd8ed1ab_0.tar.zst", "pkg-andes-1.8.3-pyhd8ed1ab_0.tar.zst", ] ); // Get the number of performed requests so far let request_ranges = reader .inner_mut() .get_mut() .get_mut() .requested_ranges() .await; assert_eq!(request_ranges.len(), 1); assert_eq!( request_ranges[0].end - request_ranges[0].start, 8192, "first request should be the size of the initial chunk size" ); assert_eq!( request_ranges[0].end, file_size, "first request should be at the end" ); // Prefetch the data for the metadata.json file let entry = reader.file().entries().first().unwrap(); let offset = entry.header_offset(); // Get the size of the entry plus the header + size of the filename. We should also actually // include bytes for the extra fields but we don't have that information. let size = entry.compressed_size() + 30 + entry.filename().as_bytes().len() as u64; // The zip archive uses as BufReader which reads in chunks of 8192. To ensure we prefetch // enough data we round the size up to the nearest multiple of the buffer size. let buffer_size = 8192; let size = size.div_ceil(buffer_size) * buffer_size; // Fetch the bytes from the zip archive that contain the requested file. reader .inner_mut() .get_mut() .get_mut() .prefetch(offset..offset + size as u64) .await; // Read the contents of the metadata.json file let mut contents = String::new(); reader .reader_with_entry(0) .await .unwrap() .read_to_string(&mut contents) .await .unwrap(); // Get the number of performed requests let request_ranges = reader .inner_mut() .get_mut() .get_mut() .requested_ranges() .await; assert_eq!(contents, r#"{"conda_pkg_format_version": 2}"#); assert_eq!(request_ranges.len(), 2); assert_eq!( request_ranges[1], 0..size, "expected only two range requests" ); } #[rstest] #[case(CheckSupportMethod::Head)] #[case(CheckSupportMethod::NegativeRangeRequest(8192))] #[tokio::test] async fn async_range_reader(#[case] check_method: CheckSupportMethod) { // Spawn a static file server let path = Path::new(&std::env::var("CARGO_MANIFEST_DIR").unwrap()).join("test-data"); let server = StaticDirectoryServer::new(&path) .await .expect("could not initialize server"); // Construct an AsyncRangeReader let (mut range, _) = AsyncHttpRangeReader::new( Client::new(), server.url().join("andes-1.8.3-pyhd8ed1ab_0.conda").unwrap(), check_method, HeaderMap::default(), ) .await .expect("bla"); // Also open a simple file reader let mut file = tokio::fs::File::open(path.join("andes-1.8.3-pyhd8ed1ab_0.conda")) .await .unwrap(); // Read until the end and make sure that the contents matches let mut range_read = vec![0; 64 * 1024]; let mut file_read = vec![0; 64 * 1024]; loop { // Read with the async reader let range_read_bytes = range.read(&mut range_read).await.unwrap(); // Read directly from the file let file_read_bytes = file .read_exact(&mut file_read[0..range_read_bytes]) .await .unwrap(); assert_eq!(range_read_bytes, file_read_bytes); assert_eq!( range_read[0..range_read_bytes], file_read[0..file_read_bytes] ); if file_read_bytes == 0 && range_read_bytes == 0 { break; } } } #[tokio::test] async fn test_not_found() { let server = StaticDirectoryServer::new(Path::new(env!("CARGO_MANIFEST_DIR"))) .await .expect("could not initialize server"); let err = AsyncHttpRangeReader::new( Client::new(), server.url().join("not-found").unwrap(), CheckSupportMethod::Head, HeaderMap::default(), ) .await .expect_err("expected an error"); assert_matches!( err, AsyncHttpRangeReaderError::HttpError(err) if err.status() == Some(StatusCode::NOT_FOUND) ); } /// Spawn a server where the HEAD response reports `head_size` bytes, and range requests always /// claim to be `pretend_size` bytes, while actually serving `actual_size`. async fn spawn_mismatch_server( head_content_length: usize, pretend_size: usize, actual_size: usize, ) -> Url { let app = axum::Router::new().fallback(async move |request: Request| match *request.method() { Method::HEAD => { let headers = [ (header::CONTENT_LENGTH, head_content_length.to_string()), (header::ACCEPT_RANGES, "bytes".to_string()), ]; (StatusCode::OK, headers).into_response() } Method::GET => { let range_header = request .headers() .get(header::RANGE) .unwrap() .to_str() .unwrap() .to_string(); let range_spec = range_header.strip_prefix("bytes=").unwrap(); let (start_str, _end_str) = range_spec.split_once('-').unwrap(); let start = start_str.parse::().unwrap(); // The end is inclusive let end = start + pretend_size - 1; axum::response::Response::builder() .status(StatusCode::PARTIAL_CONTENT) // Note that the client ignores this value currently, it only checks the // actual size .header( header::CONTENT_RANGE, format!("bytes {start}-{end}/{head_content_length}"), ) .body(Body::from(vec![1u8; actual_size])) .unwrap() .into_response() } _ => StatusCode::METHOD_NOT_ALLOWED.into_response(), }); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let local_addr = listener.local_addr().unwrap(); tokio::spawn(async move { axum::serve(listener, app.into_make_service()) .await .unwrap(); }); Url::parse(&format!("http://localhost:{}/file", local_addr.port())).unwrap() } /// HEAD says 512 bytes, but range responses return 1024 bytes — overflows /// the memory map. #[tokio::test] async fn test_content_length_response_beyond_content_length() { /// Extract the [`AsyncHttpRangeReaderError`] from an `io::Error` returned by `read`. fn into_range_error(err: std::io::Error) -> AsyncHttpRangeReaderError { err.into_inner() .unwrap() .downcast::() .map(|e| *e) .unwrap() } let cases: Vec<(usize, usize, usize, Option)> = vec![ // Baseline (512, 512, 512, None), // The requested and declared length is 512, while the actual content is 1024 ( 512, 512, 1024, Some(AsyncHttpRangeReaderError::ResponseTooLong { expected: 512 }), ), // The declared total length is 512, but it says and sends a range of 1024 ( 512, 1024, 1024, Some(AsyncHttpRangeReaderError::ContentRangeParser( "bytes 0-1023/512".to_string(), )), ), // The declared total length is 512, but it says a range of 1024 ( 512, 1024, 512, Some(AsyncHttpRangeReaderError::ContentRangeParser( "bytes 0-1023/512".to_string(), )), ), // Baseline (1024, 512, 512, None), // We requested 512, but we're getting 1024 ( 1024, 512, 1024, Some(AsyncHttpRangeReaderError::ResponseTooLong { expected: 512 }), ), // We requested 512, but we're getting 1024 ( 1024, 1024, 1024, Some(AsyncHttpRangeReaderError::RangeMismatch { expected_start: 0, expected_end_inclusive: 511, expected_complete_length: 1024, actual_start: 0, actual_end_inclusive: 1023, actual_complete_length: 1024, }), ), // We requested 512, but the header says 1024 ( 1024, 1024, 512, Some(AsyncHttpRangeReaderError::RangeMismatch { expected_start: 0, expected_end_inclusive: 511, expected_complete_length: 1024, actual_start: 0, actual_end_inclusive: 1023, actual_complete_length: 1024, }), ), ]; for (head_content_length, range_header_length, range_actual_length, expected_error) in cases { let url = spawn_mismatch_server( head_content_length, range_header_length, range_actual_length, ) .await; let (mut reader, _) = AsyncHttpRangeReader::new( Client::new(), url, CheckSupportMethod::Head, HeaderMap::default(), ) .await .unwrap(); assert_eq!(reader.len(), head_content_length as u64); reader.prefetch(0..512).await; let mut buf = vec![0u8; 512]; let result = reader.read(&mut buf).await; let label = format!("{head_content_length} {range_header_length} {range_actual_length}"); match expected_error { None => { assert_matches!(result, Ok(_), "{label}"); } Some(expected) => { // The nested error don't support `PartialEq` assert_eq!( into_range_error(result.unwrap_err()).to_string(), expected.to_string(), "{label}" ); } } } } } astral_async_http_range_reader-0.11.0/src/sparse_range.rs000064400000000000000000000137671046102023000217470ustar 00000000000000use itertools::Itertools; use std::{ fmt::{Debug, Display, Formatter}, ops::{Range, RangeInclusive}, }; // A data structure that keeps track of a range of values with potential holes in them. #[derive(Default, Clone, Eq, PartialEq)] pub struct SparseRange { left: Vec, right: Vec, } impl Display for SparseRange { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( f, "{}", self.covered_ranges() .format_with(", ", |elt, f| f(&format_args!( "{}..={}", elt.start(), elt.end() ))) ) } } impl Debug for SparseRange { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "{self}",) } } impl SparseRange { // Construct a new SparseRange from an initial covered range. pub fn from_range(range: Range) -> Self { Self { left: vec![range.start], right: vec![range.end - 1], // -1 because the stored range are inclusive } } /// Returns the covered ranges pub fn covered_ranges(&self) -> impl Iterator> + '_ { self.left .iter() .zip(self.right.iter()) .map(|(&left, &right)| RangeInclusive::new(left, right)) } pub fn is_covered(&self, range: Range) -> bool { let range_start = range.start; let range_end = range.end - 1; // Compute the indices of the ranges that are covered by the request let left_index = self.right.partition_point(|&e| e < range_start); let right_index = self.left.partition_point(|&e| e <= range_end + 1); // Get all the range bounds that are covered let left_slice = &self.left[left_index..right_index]; let right_slice = &self.right[left_index..right_index]; // Compute the bounds of covered range taking into account existing covered ranges. let start = left_slice .first() .map_or(range_start, |&left_bound| left_bound.min(range_start)); // Get the ranges that are missing let mut bound = start; for (&left_bound, &right_bound) in left_slice.iter().zip(right_slice.iter()) { if left_bound > bound { return false; } bound = right_bound + 1; } let end = right_slice .last() .map_or(range_end, |&right_bound| right_bound.max(range_end)); bound > end } /// Updates the current range to also cover the specified range. pub fn update(&mut self, range: Range) { if let Some((new_range, _)) = self.cover(range) { *self = new_range; } } /// Find the ranges that are uncovered for the specified range together with what the /// [`SparseRange`] would look like if we covered that range. pub fn cover(&self, range: Range) -> Option<(SparseRange, Vec>)> { let range_start = range.start; let range_end = range.end - 1; // Compute the indices of the ranges that are covered by the request let left_index = self.right.partition_point(|&e| e < range_start); let right_index = self.left.partition_point(|&e| e <= range_end + 1); // Get all the range bounds that are covered let left_slice = &self.left[left_index..right_index]; let right_slice = &self.right[left_index..right_index]; // Compute the bounds of covered range taking into account existing covered ranges. let start = left_slice .first() .map_or(range_start, |&left_bound| left_bound.min(range_start)); let end = right_slice .last() .map_or(range_end, |&right_bound| right_bound.max(range_end)); // Get the ranges that are missing let mut ranges = Vec::new(); let mut bound = start; for (&left_bound, &right_bound) in left_slice.iter().zip(right_slice.iter()) { if left_bound > bound { ranges.push(bound..=(left_bound - 1)); } bound = right_bound + 1; } if bound <= end { ranges.push(bound..=end); } if ranges.is_empty() { None } else { let mut new_left = self.left.clone(); new_left.splice(left_index..right_index, [start]); let mut new_right = self.right.clone(); new_right.splice(left_index..right_index, [end]); Some(( Self { left: new_left, right: new_right, }, ranges, )) } } } #[cfg(test)] mod test { use super::SparseRange; #[test] fn test_sparse_range() { let range = SparseRange::default(); assert!(range.covered_ranges().next().is_none()); assert_eq!( range.cover(5..10).unwrap().0, SparseRange::from_range(5..10) ); let range = SparseRange::from_range(5..10); assert_eq!(range.covered_ranges().collect::>(), vec![5..=9]); assert!(range.is_covered(5..10)); assert!(range.is_covered(6..9)); assert!(!range.is_covered(5..11)); assert!(!range.is_covered(3..8)); assert_eq!( range.cover(3..5), Some((SparseRange::from_range(3..10), vec![3..=4])) ); let (range, missing) = range.cover(12..15).unwrap(); assert_eq!( range.covered_ranges().collect::>(), vec![5..=9, 12..=14] ); assert_eq!(missing, vec![12..=14]); assert!(range.is_covered(5..10)); assert!(range.is_covered(12..15)); assert!(!range.is_covered(5..15)); assert!(!range.is_covered(11..12)); let (range, missing) = range.cover(8..14).unwrap(); assert_eq!(range.covered_ranges().collect::>(), vec![5..=14]); assert_eq!(missing, vec![10..=11]); } } astral_async_http_range_reader-0.11.0/src/static_directory_server.rs000064400000000000000000000045641046102023000242320ustar 00000000000000use axum::routing::get_service; use reqwest::Url; use std::net::SocketAddr; use std::path::Path; use tokio::sync::oneshot; use tower_http::services::ServeDir; /// A convenient async HTTP server that serves the content of a folder. The server only listens to /// `127.0.0.1` and uses a random port. This makes it safe to run multiple instances. Its perfect to /// use for testing HTTP file requests. pub struct StaticDirectoryServer { local_addr: SocketAddr, shutdown_sender: Option>, } impl StaticDirectoryServer { /// Returns the root `Url` to the server. pub fn url(&self) -> Url { Url::parse(&format!("http://localhost:{}", self.local_addr.port())).unwrap() } } impl StaticDirectoryServer { pub async fn new(path: impl AsRef) -> Result { let service = get_service(ServeDir::new(path)); // Create a router that will serve the static files let app = axum::Router::new().nest_service("/", service); // Construct the server that will listen on localhost but with a *random port*. The random // port is very important because it enables creating multiple instances at the same time. // We need this to be able to run tests in parallel. let addr = SocketAddr::new([127, 0, 0, 1].into(), 0); let listener = tokio::net::TcpListener::bind(addr).await?; // Get the address of the server so we can bind to it at a later stage. let addr = listener.local_addr()?; // Setup a graceful shutdown trigger which is fired when this instance is dropped. let (tx, rx) = oneshot::channel(); // Spawn the server in the background. tokio::spawn(async move { let _ = axum::serve(listener, app.into_make_service()) .with_graceful_shutdown(async { rx.await.ok(); }) .await; }); Ok(Self { local_addr: addr, shutdown_sender: Some(tx), }) } } impl Drop for StaticDirectoryServer { fn drop(&mut self) { if let Some(tx) = self.shutdown_sender.take() { let _ = tx.send(()); } } } /// Error type used for [`StaticDirectoryServerError`] #[derive(Debug, thiserror::Error)] pub enum StaticDirectoryServerError { #[error(transparent)] Io(#[from] std::io::Error), }