libcramjam-0.8.0/.cargo_vcs_info.json0000644000000001360000000000100131420ustar { "git": { "sha1": "2337ac269dea3f9c795fe504e08d11012b59d5df" }, "path_in_vcs": "" }libcramjam-0.8.0/.github/FUNDING.yml000064400000000000000000000013441046102023000151110ustar 00000000000000# These are supported funding model platforms github: [milesgranger] patreon: # Replace with a single Patreon username open_collective: # Replace with a single Open Collective username ko_fi: # Replace with a single Ko-fi username tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry liberapay: # Replace with a single Liberapay username issuehunt: # Replace with a single IssueHunt username otechie: # Replace with a single Otechie username lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] libcramjam-0.8.0/.github/workflows/CI.yml000064400000000000000000000063021046102023000163460ustar 00000000000000name: CI on: push: branches: - main pull_request: concurrency: group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }} cancel-in-progress: true jobs: build-test: name: ${{ matrix.os }}-capi=${{ matrix.capi }}-codec=${{ matrix.codec || 'all' }} runs-on: ${{ matrix.os }} strategy: fail-fast: false #${{ !( startsWith(github.ref, 'refs/heads/master') || startsWith(github.ref, 'refs/tags/') ) }} matrix: os: - macos-14 # arm64 - macos-13 # x86_64 - windows-latest - ubuntu-latest capi: - true - false codec: - zstd - blosc2 - gzip - brotli - lz4 - xz - deflate - bzip2 - null # Use all codecs # TODO: codecs not implemented in capi feature exclude: - capi: true codec: blosc2 - capi: true codec: xz - capi: true codec: deflate # TODO: capi feature fails with all enabled due to duplicate export of 'compress' symbol # first from the capi here, then from blosc2 internal zlib-ng; probably just rename ours # prefixed with 'cramjam_' or something. - capi: true codec: null os: ubuntu-latest - capi: true codec: null os: windows-latest steps: - uses: actions/checkout@v4 - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 name: Rust Cache - name: Install nasm if: runner.os != 'macOS' uses: ilammy/setup-nasm@v1 - name: Install build deps (OSX) if: runner.os == 'macOS' run: brew install automake autoconf coreutils libtool nasm - name: Set MSVC developer prompt if: runner.os == 'Windows' uses: ilammy/msvc-dev-cmd@v1.6.0 - name: Audit if: | !matrix.codec && matrix.capi run: cargo install cargo-audit && cargo audit - name: Test capi and single codec if: matrix.capi && matrix.codec run: cargo test -p libcramjam --no-default-features --features ${{ matrix.codec }} --features capi --lib - name: Test capi and all codecs if: matrix.capi && !matrix.codec run: cargo test -p libcramjam --features capi --lib - name: Test no capi and all codecs if: | !matrix.capi && !matrix.codec run: cargo test -p libcramjam --lib - name: Test no capi and single codec if: | !matrix.capi && matrix.codec run: cargo test -p libcramjam --lib --no-default-features --features ${{ matrix.codec }} # build-wasm32: # runs-on: ubuntu-latest # steps: # - uses: actions/checkout@v4 # - name: Install Rust toolchain # uses: dtolnay/rust-toolchain@stable # with: # target: wasm32-unknown-emscripten # - name: Install Emscripten # uses: mymindstorm/setup-emsdk@v14 # - name: Build # run: cargo build --target wasm32-unknown-emscripten --features wasm32-compat libcramjam-0.8.0/.gitignore000064400000000000000000000035401046102023000137240ustar 00000000000000# Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so .idea/ benchenv/ cramjam-python/_build Cargo.lock # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ pip-wheel-metadata/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 db.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # PEP 582; used by e.g. github.com/David-OConnor/pyflow __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ #Added by cargo /target .vscode/ libcramjam-0.8.0/Cargo.lock0000644000000312520000000000100111200ustar # This file is automatically @generated by Cargo. # It is not intended for manual editing. version = 4 [[package]] name = "adler2" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" [[package]] name = "alloc-no-stdlib" version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" [[package]] name = "alloc-stdlib" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" dependencies = [ "alloc-no-stdlib", ] [[package]] name = "autocfg" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "bitflags" version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" [[package]] name = "blosc2-rs" version = "0.4.0+2.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a752bab53ac66bf6d49fafda48e56ee637ca496ab54be10f54f59a8698363cb3" dependencies = [ "blosc2-sys", "parking_lot", ] [[package]] name = "blosc2-sys" version = "0.4.0+2.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a780fec12b1352d5d7d87858d192ecba0bd220310a35dacc18756fe58139871" dependencies = [ "cmake", "copy_dir", "libc", "pkg-config", ] [[package]] name = "brotli" version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", "brotli-decompressor", ] [[package]] name = "brotli-decompressor" version = "4.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a334ef7c9e23abf0ce748e8cd309037da93e606ad52eb372e4ce327a0dcfbdfd" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", ] [[package]] name = "bzip2" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" dependencies = [ "bzip2-sys", ] [[package]] name = "bzip2-sys" version = "0.1.13+1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" dependencies = [ "cc", "pkg-config", ] [[package]] name = "cc" version = "1.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0fc897dc1e865cc67c0e05a836d9d3f1df3cbe442aa4a9473b18e12624a4951" dependencies = [ "jobserver", "libc", "shlex", ] [[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cmake" version = "0.1.54" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" dependencies = [ "cc", ] [[package]] name = "copy_dir" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "543d1dd138ef086e2ff05e3a48cf9da045da2033d16f8538fd76b86cd49b2ca3" dependencies = [ "walkdir", ] [[package]] name = "crc32fast" version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ "cfg-if", ] [[package]] name = "flate2" version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" dependencies = [ "crc32fast", "miniz_oxide", ] [[package]] name = "getrandom" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", "libc", "r-efi", "wasi", ] [[package]] name = "isal-rs" version = "0.5.3+496255c" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ec7734f9db7ef4c18bac0e94210aaa717c149b168e076ff681a56b342fca9ed" dependencies = [ "isal-sys", ] [[package]] name = "isal-sys" version = "0.5.3+496255c" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aefc9239959a60eaba201ccdd99897b5270be98d01f561c2166f5e3343e5a29b" dependencies = [ "cc", "copy_dir", ] [[package]] name = "jobserver" version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" dependencies = [ "getrandom", "libc", ] [[package]] name = "libc" version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" [[package]] name = "libcramjam" version = "0.8.0" dependencies = [ "blosc2-rs", "brotli", "bzip2", "flate2", "isal-rs", "libc", "libdeflate-sys", "lz4", "snap", "xz2", "zstd", ] [[package]] name = "libdeflate-sys" version = "1.19.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc9caa76c8cc6ee8c4efcf8f4514a812ebcad3aa7d3b548efe4d26da1203f177" dependencies = [ "cc", "pkg-config", ] [[package]] name = "lock_api" version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" dependencies = [ "autocfg", "scopeguard", ] [[package]] name = "lz4" version = "1.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" dependencies = [ "lz4-sys", ] [[package]] name = "lz4-sys" version = "1.11.1+lz4-1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" dependencies = [ "cc", "libc", ] [[package]] name = "lzma-sys" version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" dependencies = [ "cc", "libc", "pkg-config", ] [[package]] name = "miniz_oxide" version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" dependencies = [ "adler2", ] [[package]] name = "parking_lot" version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" dependencies = [ "lock_api", "parking_lot_core", ] [[package]] name = "parking_lot_core" version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", "windows-targets", ] [[package]] name = "pkg-config" version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" [[package]] name = "r-efi" version = "5.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" [[package]] name = "redox_syscall" version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "928fca9cf2aa042393a8325b9ead81d2f0df4cb12e1e24cef072922ccd99c5af" dependencies = [ "bitflags", ] [[package]] name = "same-file" version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" dependencies = [ "winapi-util", ] [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "shlex" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "smallvec" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" [[package]] name = "snap" version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] name = "walkdir" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ "same-file", "winapi-util", ] [[package]] name = "wasi" version = "0.14.2+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" dependencies = [ "wit-bindgen-rt", ] [[package]] name = "winapi-util" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ "windows-sys", ] [[package]] name = "windows-sys" version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ "windows-targets", ] [[package]] name = "windows-targets" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", "windows_i686_gnu", "windows_i686_gnullvm", "windows_i686_msvc", "windows_x86_64_gnu", "windows_x86_64_gnullvm", "windows_x86_64_msvc", ] [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "wit-bindgen-rt" version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ "bitflags", ] [[package]] name = "xz2" version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" dependencies = [ "lzma-sys", ] [[package]] name = "zstd" version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" version = "7.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" version = "2.0.15+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" dependencies = [ "cc", "pkg-config", ] libcramjam-0.8.0/Cargo.toml0000644000000070720000000000100111460ustar # THIS FILE IS AUTOMATICALLY GENERATED BY CARGO # # When uploading crates to the registry Cargo will automatically # "normalize" Cargo.toml files for maximal compatibility # with all versions of Cargo and also rewrite `path` dependencies # to registry (e.g., crates.io) dependencies. # # If you are reading this file be aware that the original Cargo.toml # will likely look very different (and much more reasonable). # See Cargo.toml.orig for the original contents. [package] edition = "2021" name = "libcramjam" version = "0.8.0" build = "build.rs" autolib = false autobins = false autoexamples = false autotests = false autobenches = false description = "Compression library combining a plethora of algorithms in a similar as possible API" readme = "README.md" license = "MIT" [package.metadata.capi.header] name = "cramjam" subdirectory = "cramjam" [package.metadata.capi.library] name = "cramjam" rustflags = "-Cpanic=abort" [package.metadata.capi.pkg_config] strip_include_path_components = 1 [features] blosc2 = ["blosc2-static"] blosc2-shared = [ "dep:blosc2-rs", "blosc2-rs/shared", ] blosc2-static = [ "dep:blosc2-rs", "blosc2-rs/static", ] brotli = ["dep:brotli"] bzip2 = ["dep:bzip2"] capi = ["dep:libc"] default = [ "snappy", "lz4", "bzip2", "brotli", "xz", "zstd", "gzip", "deflate", "zlib", ] deflate = ["deflate-static"] deflate-shared = [ "dep:libdeflate-sys", "dep:flate2", "libdeflate-sys/dynamic", ] deflate-static = [ "dep:libdeflate-sys", "dep:flate2", ] gzip = ["gzip-static"] gzip-shared = [ "dep:libdeflate-sys", "dep:flate2", "libdeflate-sys/dynamic", "deflate-shared", ] gzip-static = [ "dep:libdeflate-sys", "dep:flate2", "deflate-static", ] ideflate = ["isal-static"] ideflate-shared = ["isal-shared"] ideflate-static = ["isal-static"] igzip = ["isal-static"] igzip-shared = ["isal-shared"] igzip-static = ["isal-static"] isal-shared = [ "dep:isal-rs", "isal-rs/shared", ] isal-static = [ "dep:isal-rs", "isal-rs/static", ] izlib = ["isal-static"] izlib-shared = ["isal-shared"] izlib-static = ["isal-static"] lz4 = ["dep:lz4"] snappy = ["dep:snap"] use-system-blosc2 = [ "dep:blosc2-rs", "blosc2-rs/use-system-blosc2", ] use-system-isal = [ "dep:isal-rs", "isal-rs/use-system-isal", ] wasm32-compat = ["blosc2-rs/deactivate-zlib-optim"] xz = ["xz-static"] xz-shared = ["dep:xz2"] xz-static = [ "dep:xz2", "xz2/static", ] zlib = ["zlib-static"] zlib-shared = [ "dep:libdeflate-sys", "dep:flate2", "libdeflate-sys/dynamic", "deflate-shared", ] zlib-static = [ "dep:libdeflate-sys", "dep:flate2", "deflate-static", ] zstd = ["dep:zstd"] [lib] name = "libcramjam" path = "src/lib.rs" [dependencies.blosc2-rs] version = "0.4.0+2.15.2" optional = true default-features = false [dependencies.brotli] version = "^7" features = [ "std", "ffi-api", ] optional = true default-features = false [dependencies.bzip2] version = ">=0.4,<0.6" optional = true [dependencies.flate2] version = "^1" optional = true [dependencies.libc] version = "0.2" optional = true [dependencies.libdeflate-sys] version = "<1.20.0" optional = true [dependencies.lz4] version = "^1" optional = true [dependencies.snap] version = "^1" optional = true [dependencies.xz2] version = "0.1.7" optional = true [dependencies.zstd] version = "^0.13" optional = true [target.'cfg(target_pointer_width = "64")'.dependencies.isal-rs] version = "^0.5" optional = true default-features = false [profile.release] opt-level = 3 lto = "fat" codegen-units = 1 strip = true libcramjam-0.8.0/Cargo.toml.orig000064400000000000000000000066541046102023000146340ustar 00000000000000[package] name = "libcramjam" version = "0.8.0" edition = "2021" license = "MIT" description = "Compression library combining a plethora of algorithms in a similar as possible API" readme = "README.md" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] default = ["snappy", "lz4", "bzip2", "brotli", "xz", "zstd", "gzip", "deflate", "zlib"] capi = ["dep:libc"] snappy = ["dep:snap"] lz4 = ["dep:lz4"] bzip2 = ["dep:bzip2"] brotli = ["dep:brotli"] zstd = ["dep:zstd"] igzip = ["isal-static"] igzip-static = ["isal-static"] igzip-shared = ["isal-shared"] ideflate = ["isal-static"] ideflate-static = ["isal-static"] ideflate-shared = ["isal-shared"] izlib = ["isal-static"] izlib-static = ["isal-static"] izlib-shared = ["isal-shared"] isal-static = ["dep:isal-rs", "isal-rs/static"] isal-shared = ["dep:isal-rs", "isal-rs/shared"] use-system-isal = ["dep:isal-rs", "isal-rs/use-system-isal"] gzip = ["gzip-static"] gzip-static = ["dep:libdeflate-sys", "dep:flate2", "deflate-static"] gzip-shared = ["dep:libdeflate-sys", "dep:flate2", "libdeflate-sys/dynamic", "deflate-shared"] zlib = ["zlib-static"] zlib-static = ["dep:libdeflate-sys", "dep:flate2", "deflate-static"] zlib-shared = ["dep:libdeflate-sys", "dep:flate2", "libdeflate-sys/dynamic", "deflate-shared"] deflate = ["deflate-static"] deflate-static = ["dep:libdeflate-sys", "dep:flate2"] deflate-shared = ["dep:libdeflate-sys", "dep:flate2", "libdeflate-sys/dynamic"] xz = ["xz-static"] xz-static = ["dep:xz2", "xz2/static"] # builds from vendored source of xz xz-shared = ["dep:xz2"] # _maybe_ uses system xz; probes for it but falls back to building static lib blosc2 = ["blosc2-static"] blosc2-static = ["dep:blosc2-rs", "blosc2-rs/static"] blosc2-shared = ["dep:blosc2-rs", "blosc2-rs/shared"] use-system-blosc2 = ["dep:blosc2-rs", "blosc2-rs/use-system-blosc2"] wasm32-compat = ["blosc2-rs/deactivate-zlib-optim"] [dependencies] libc = { version = "0.2", optional = true } snap = { version = "^1", optional = true } brotli = { version = "^7", default-features = false, features = ["std", "ffi-api"], optional = true } bzip2 = { version = ">=0.4,<0.6", optional = true } lz4 = { version = "^1", optional = true } flate2 = { version = "^1", optional = true } libdeflate-sys = { version = "<1.20.0", optional = true } # TODO: requires gcc>=4.9 not available on Python's CI wheel builds blosc2-rs = { version = "0.4.0+2.15.2", optional = true, default-features = false } zstd = { version = "^0.13", optional = true } xz2 = { version = "0.1.7", optional = true } # ISA-L not supported on 32-bit; Cargo doesn't offer a way to disable a single feature [target.'cfg(target_pointer_width = "64")'.dependencies] isal-rs = { version = "^0.5", optional = true, default-features = false } [package.metadata.capi.pkg_config] strip_include_path_components = 1 [package.metadata.capi.library] rustflags = "-Cpanic=abort" name = "cramjam" [package.metadata.capi.header] name = "cramjam" subdirectory = "cramjam" [profile.release] strip = true lto = "fat" codegen-units = 1 opt-level = 3 libcramjam-0.8.0/LICENSE000064400000000000000000000020561046102023000127420ustar 00000000000000MIT License Copyright (c) 2020 Miles Granger Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. libcramjam-0.8.0/README.md000064400000000000000000000021551046102023000132140ustar 00000000000000 # cramjam library A Rust library combining different compression algorithms/libraries in a common (as possible) API. [![CI](https://github.com/cramjam/libcramjam/actions/workflows/CI.yml/badge.svg?branch=main)](https://github.com/cramjam/libcramjam/actions/workflows/CI.yml) [![Latest version](https://img.shields.io/crates/v/libcramjam.svg)](https://crates.io/crates/libcramjam) [![Documentation](https://docs.rs/libcramjam/badge.svg)](https://docs.rs/libcramjam) ![License](https://img.shields.io/crates/l/libcramjam.svg) --- #### Features (dynamic/static build features available on some variants, check [Cargo.toml](./Cargo.toml)): - `snappy` - `lz4` - `bzip2` - `brotli` - `zstd` - `zlib` - `xz` - `gzip` - `deflate` - `blosc2` - `igzip` (GZIP using ISA-L backend) - `ideflate` (DEFLATE using ISA-L backend) - `izlib` (ZLIB using ISA-L backend) - `capi`: Build a C-ABI library. Compatible with [`cargo-c`](https://github.com/lu-zero/cargo-c) Pre-compiled libraries available on [![Anaconda-Server Badge](https://anaconda.org/conda-forge/libcramjam/badges/version.svg)](https://anaconda.org/conda-forge/libcramjam) libcramjam-0.8.0/build.rs000064400000000000000000000011501046102023000133740ustar 00000000000000fn main() { // See note in Cargo.toml if cfg!(target_pointer_width = "32") { #[cfg(any( feature = "use-system-isal", feature = "isal-static", feature = "isal-shared" ))] let msg = "feature set but ISA-L not supported on 32 bit systems."; #[cfg(feature = "use-system-isal")] println!("cargo:warning='use-system-isal' {}", msg); #[cfg(feature = "isal-static")] println!("cargo:warning='isal-static' {}", msg); #[cfg(feature = "isal-shared")] println!("cargo:warning='isal-shared' {}", msg); } } libcramjam-0.8.0/cbindgen.toml000064400000000000000000000001161046102023000143760ustar 00000000000000language = "C" cpp_compat = true include_version = true namespace = "cramjam" libcramjam-0.8.0/src/blosc2.rs000064400000000000000000000042771046102023000142650ustar 00000000000000//! blosc2 de/compression interface use ::blosc2::CParams; pub use blosc2; use std::io::{self, BufReader, Read, Write}; // TODO: Could downcast to check for file, then use file-backed SChunk /// Compress using Blosc2 SChunk pub fn compress(rdr: R, wtr: &mut W) -> io::Result where R: Read, W: Write + ?Sized, { let mut schunk = blosc2::schunk::SChunk::new( blosc2::schunk::Storage::default() .set_contiguous(true) .set_cparams(CParams::default().set_typesize::()) .set_dparams(Default::default()), ); let mut rdr = BufReader::new(rdr); // stream compress into schunk io::copy(&mut rdr, &mut schunk)?; let buf = schunk.into_vec()?; wtr.write_all(&buf)?; Ok(buf.len()) } /// Decompress, assumed reader will be giving a SChunk compatible input pub fn decompress(input: R, output: &mut W) -> io::Result where R: Read, W: Write + ?Sized, { // TODO: Avoid the double copy somehow let mut buf = vec![]; io::copy(&mut BufReader::new(input), &mut buf)?; let mut schunk = blosc2::schunk::SChunk::from_vec(buf)?; let mut decoder = blosc2::schunk::SChunkDecoder::new(&mut schunk); io::copy(&mut decoder, output).map(|v| v as usize) } #[inline(always)] pub fn compress_chunk(input: &[T]) -> io::Result> { let buf = blosc2::compress(input, None, None, None, None)?; Ok(buf) } pub fn compress_chunk_into(input: &[T], output: &mut [u8]) -> io::Result { let nbytes = blosc2::compress_into(input, output, None, None, None, None)?; Ok(nbytes) } #[inline(always)] pub fn decompress_chunk(input: &[u8]) -> io::Result> { let buf = blosc2::decompress(input)?; Ok(buf) } pub fn decompress_chunk_into(input: &[u8], output: &mut [T]) -> io::Result { let nbytes = blosc2::decompress_into(input, output)?; Ok(nbytes) } #[cfg(test)] mod tests { use std::io::Cursor; use super::*; #[test] fn test_compress() { let _guard = blosc2::Blosc2Guard::get_or_init(); let mut compressed = vec![]; let data = b"bytes"; compress(Cursor::new(data), &mut compressed).unwrap(); } } libcramjam-0.8.0/src/brotli.rs000064400000000000000000000024431046102023000143650ustar 00000000000000//! brotli de/compression interface use std::io::Write; const DEFAULT_COMPRESSION_LEVEL: u32 = 11; const BUF_SIZE: usize = 1 << 17; // Taken from brotli kCompressFragementTwoPassBlockSize const LGWIN: u32 = 22; pub use brotli; use std::io::prelude::*; use std::io::Error; /// Decompress via Brotli #[inline(always)] pub fn decompress(input: R, output: &mut W) -> Result { let mut decoder = brotli::Decompressor::new(input, BUF_SIZE); let n_bytes = std::io::copy(&mut decoder, output)?; Ok(n_bytes as usize) } /// Compress via Brotli #[inline(always)] pub fn compress(input: R, output: &mut W, level: Option) -> Result { let level = level.unwrap_or_else(|| DEFAULT_COMPRESSION_LEVEL); let mut encoder = brotli::CompressorReader::new(input, BUF_SIZE, level, LGWIN); let n_bytes = std::io::copy(&mut encoder, output)?; Ok(n_bytes as usize) } pub fn make_write_compressor(w: W, level: Option) -> brotli::CompressorWriter { let level = level.unwrap_or_else(|| DEFAULT_COMPRESSION_LEVEL); brotli::CompressorWriter::new(w, BUF_SIZE, level, LGWIN) } pub fn compress_bound(input_len: usize) -> usize { brotli::ffi::compressor::BrotliEncoderMaxCompressedSize(input_len) } libcramjam-0.8.0/src/bzip2.rs000064400000000000000000000015161046102023000141200ustar 00000000000000//! bzip2 de/compression interface use std::io::prelude::*; use std::io::Error; const DEFAULT_COMPRESSION_LEVEL: u32 = 6; pub use bzip2; use bzip2::read::{BzEncoder, MultiBzDecoder}; /// Decompress via bzip2 #[inline(always)] pub fn decompress(input: R, output: &mut W) -> Result { let mut decoder = MultiBzDecoder::new(input); let n_bytes = std::io::copy(&mut decoder, output)?; Ok(n_bytes as usize) } /// Compress via bzip2 #[inline(always)] pub fn compress(input: R, output: &mut W, level: Option) -> Result { let level = level.unwrap_or_else(|| DEFAULT_COMPRESSION_LEVEL); let mut encoder = BzEncoder::new(input, bzip2::Compression::new(level)); let n_bytes = std::io::copy(&mut encoder, output)?; Ok(n_bytes as usize) } libcramjam-0.8.0/src/capi.rs000064400000000000000000001123171046102023000140100ustar 00000000000000use libc::c_void; use std::ffi::{c_char, CString}; use std::io::Cursor; use std::io::Write; use std::slice; #[cfg(feature = "brotli")] use crate::brotli; #[cfg(feature = "bzip2")] use crate::bzip2; #[cfg(feature = "deflate")] use crate::deflate; #[cfg(feature = "gzip")] use crate::gzip; #[cfg(feature = "lz4")] use crate::lz4; #[cfg(feature = "snappy")] use crate::snappy; #[cfg(feature = "zstd")] use crate::zstd; #[repr(C)] pub struct Buffer { data: *const u8, len: usize, owned: bool, } impl Buffer { pub fn empty() -> Self { Buffer { data: std::ptr::null(), len: 0, owned: false, } } } impl From<&Vec> for Buffer { fn from(v: &Vec) -> Self { Buffer { data: v.as_ptr(), len: v.len(), owned: false, } } } impl From> for Buffer { fn from(mut v: Vec) -> Self { v.shrink_to_fit(); let buffer = Buffer { data: v.as_ptr(), len: v.len(), owned: true, }; std::mem::forget(v); buffer } } /// All codecs supported by the de/compress and de/compress_into APIs #[derive(Debug, Copy, Clone)] #[repr(C)] pub enum Codec { #[cfg(feature = "snappy")] #[allow(dead_code)] Snappy, #[cfg(feature = "snappy")] #[allow(dead_code)] SnappyRaw, #[cfg(feature = "bzip2")] #[allow(dead_code)] Bzip2, #[cfg(feature = "lz4")] #[allow(dead_code)] Lz4, #[cfg(feature = "lz4")] #[allow(dead_code)] Lz4Block, #[cfg(feature = "zstd")] #[allow(dead_code)] Zstd, #[cfg(feature = "gzip")] #[allow(dead_code)] Gzip, #[cfg(feature = "brotli")] #[allow(dead_code)] Brotli, } /// Streaming only codecs, which can create De/Compressors using the de/compressor APIs #[derive(Debug)] #[repr(C)] pub enum StreamingCodec { #[cfg(feature = "bzip2")] #[allow(dead_code)] StreamingBzip2, #[cfg(feature = "snappy")] #[allow(dead_code)] StreamingSnappy, #[cfg(feature = "lz4")] #[allow(dead_code)] StreamingLz4, #[cfg(feature = "zstd")] #[allow(dead_code)] StreamingZstd, #[cfg(feature = "gzip")] #[allow(dead_code)] StreamingGzip, #[cfg(feature = "brotli")] #[allow(dead_code)] StreamingBrotli, } #[cfg(feature = "snappy")] type SnappyFrameCompressor = snappy::snap::write::FrameEncoder>; #[cfg(feature = "bzip2")] type Bzip2Compressor = bzip2::bzip2::write::BzEncoder>; #[cfg(feature = "lz4")] type Lz4Compressor = crate::lz4::lz4::Encoder>; #[cfg(feature = "gzip")] type GzipCompressor = crate::gzip::flate2::write::GzEncoder>; #[cfg(feature = "brotli")] type BrotliCompressor = brotli::brotli::CompressorWriter>; #[cfg(feature = "zstd")] type ZstdCompressor<'a> = crate::zstd::zstd::Encoder<'a, Vec>; type Decompressor = Cursor>; // Set the error string to a error message pointer #[inline(always)] fn error_to_ptr(err: impl ToString, ptr: &mut *mut c_char) { let err_msg = CString::new(err.to_string()).unwrap(); *ptr = err_msg.into_raw(); } /// Safe to call on a nullptr #[no_mangle] pub extern "C" fn free_string(ptr: *mut c_char) { if !ptr.is_null() { let _ = unsafe { CString::from_raw(ptr) }; } } #[no_mangle] pub extern "C" fn free_buffer(buf: Buffer) { if !buf.data.is_null() && buf.owned { let _ = unsafe { Vec::from_raw_parts(buf.data as *mut u8, buf.len, buf.len) }; } } #[no_mangle] pub extern "C" fn decompress( codec: Codec, input: *const u8, input_len: usize, nbytes_read: &mut usize, nbytes_written: &mut usize, error: &mut *mut c_char, ) -> Buffer { let mut decompressed = Cursor::new(vec![]); let mut compressed = Cursor::new(unsafe { std::slice::from_raw_parts(input, input_len) }); let ret: Result = match codec { #[cfg(feature = "snappy")] Codec::Snappy => snappy::decompress(&mut compressed, &mut decompressed), #[cfg(feature = "snappy")] Codec::SnappyRaw => snappy::raw::decompress_vec(compressed.get_ref()).map(|v| { let len = v.len(); *decompressed.get_mut() = v; decompressed.set_position(len as _); compressed.set_position(input_len as _); // todo, assuming it read the whole thing len }), #[cfg(feature = "bzip2")] Codec::Bzip2 => bzip2::decompress(&mut compressed, &mut decompressed), #[cfg(feature = "brotli")] Codec::Brotli => brotli::decompress(&mut compressed, &mut decompressed), #[cfg(feature = "gzip")] Codec::Gzip => gzip::decompress(&mut compressed, &mut decompressed), #[cfg(feature = "zstd")] Codec::Zstd => zstd::decompress(&mut compressed, &mut decompressed), #[cfg(feature = "lz4")] Codec::Lz4 => lz4::decompress(&mut compressed, &mut decompressed), #[cfg(feature = "lz4")] Codec::Lz4Block => lz4::block::decompress_vec(compressed.get_ref()).map(|v| { let len = v.len(); *decompressed.get_mut() = v; decompressed.set_position(len as _); compressed.set_position(input_len as _); // todo, assuming it read the whole thing len }), }; match ret { Ok(n) => { *nbytes_read = compressed.position() as usize; *nbytes_written = n; match decompressed.flush() { Ok(_) => Buffer::from(decompressed.into_inner()), Err(err) => { error_to_ptr(err, error); Buffer::empty() } } } Err(err) => { error_to_ptr(err, error); Buffer::empty() } } } #[no_mangle] pub extern "C" fn compress( codec: Codec, level: i32, input: *const u8, input_len: usize, nbytes_read: &mut usize, nbytes_written: &mut usize, error: &mut *mut c_char, ) -> Buffer { if level < 0 { error_to_ptr("Requires compression >= 0", error); return Buffer::empty(); } let level = Some(level); let mut compressed = Cursor::new(vec![]); let mut decompressed = Cursor::new(unsafe { std::slice::from_raw_parts(input, input_len) }); let ret: Result = match codec { #[cfg(feature = "snappy")] Codec::Snappy => snappy::compress(&mut decompressed, &mut compressed), #[cfg(feature = "snappy")] Codec::SnappyRaw => snappy::raw::compress_vec(decompressed.get_ref()).map(|v| { let len = v.len(); *compressed.get_mut() = v; compressed.set_position(len as _); decompressed.set_position(input_len as _); len }), #[cfg(feature = "bzip2")] Codec::Bzip2 => bzip2::compress(&mut decompressed, &mut compressed, level.map(|v| v as _)), #[cfg(feature = "brotli")] Codec::Brotli => { brotli::compress(&mut decompressed, &mut compressed, level.map(|v| v as _)) } #[cfg(feature = "gzip")] Codec::Gzip => gzip::compress(&mut decompressed, &mut compressed, level.map(|v| v as _)), #[cfg(feature = "zstd")] Codec::Zstd => zstd::compress( &mut decompressed, &mut compressed, level.map(|v: i32| v as i32), Some(input_len), ), #[cfg(feature = "lz4")] Codec::Lz4 => lz4::compress(&mut decompressed, &mut compressed, level.map(|v| v as _)), // TODO: Support passing acceleration #[cfg(feature = "lz4")] Codec::Lz4Block => lz4::block::compress_vec( decompressed.get_ref(), level.map(|v| v as _), None, Some(true), ) .map(|v| { let len = v.len(); *compressed.get_mut() = v; compressed.set_position(len as _); decompressed.set_position(input_len as _); len }), // TODO }; match ret { Ok(n) => { *nbytes_read = decompressed.get_ref().len(); *nbytes_written = n; match compressed.flush() { Ok(_) => Buffer::from(compressed.into_inner()), Err(err) => { error_to_ptr(err, error); Buffer::empty() } } } Err(err) => { error_to_ptr(err, error); Buffer::empty() } } } #[no_mangle] pub extern "C" fn decompress_into( codec: Codec, input: *const u8, input_len: usize, output: *mut u8, output_len: usize, nbytes_read: &mut usize, nbytes_written: &mut usize, error: &mut *mut c_char, ) { let mut compressed = Cursor::new(unsafe { std::slice::from_raw_parts(input, input_len) }); let mut decompressed = Cursor::new(unsafe { std::slice::from_raw_parts_mut(output, output_len) }); let ret: Result = match codec { #[cfg(feature = "snappy")] Codec::Snappy => snappy::decompress(&mut compressed, &mut decompressed), #[cfg(feature = "snappy")] Codec::SnappyRaw => snappy::raw::decompress(compressed.get_ref(), decompressed.get_mut()), #[cfg(feature = "bzip2")] Codec::Bzip2 => bzip2::decompress(&mut compressed, &mut decompressed), #[cfg(feature = "brotli")] Codec::Brotli => brotli::decompress(&mut compressed, &mut decompressed), #[cfg(feature = "gzip")] Codec::Gzip => gzip::decompress(&mut compressed, &mut decompressed), #[cfg(feature = "zstd")] Codec::Zstd => zstd::decompress(&mut compressed, &mut decompressed), #[cfg(feature = "lz4")] Codec::Lz4 => lz4::decompress(&mut compressed, &mut decompressed), #[cfg(feature = "lz4")] Codec::Lz4Block => { lz4::block::decompress_into(&compressed.get_ref(), decompressed.get_mut(), None) } }; match ret { Ok(n) => { *nbytes_written = n; *nbytes_read = compressed.get_ref().len(); } Err(err) => { error_to_ptr(err, error); *nbytes_written = 0; *nbytes_read = 0; } } } #[no_mangle] pub extern "C" fn compress_into( codec: Codec, level: i32, input: *const u8, input_len: usize, output: *mut u8, output_len: usize, nbytes_read: &mut usize, nbytes_written: &mut usize, error: &mut *mut c_char, ) { let mut decompressed = unsafe { std::slice::from_raw_parts(input, input_len) }; let mut compressed = unsafe { std::slice::from_raw_parts_mut(output, output_len) }; if level < 0 { error_to_ptr("Requires compression >= 0", error); return; } let level = Some(level); let ret: Result = match codec { #[cfg(feature = "snappy")] Codec::Snappy => snappy::compress(&mut decompressed, &mut compressed), #[cfg(feature = "snappy")] Codec::SnappyRaw => snappy::raw::compress(decompressed, &mut compressed), #[cfg(feature = "bzip2")] Codec::Bzip2 => bzip2::compress(&mut decompressed, &mut compressed, level.map(|v| v as _)), #[cfg(feature = "brotli")] Codec::Brotli => { brotli::compress(&mut decompressed, &mut compressed, level.map(|v| v as _)) } #[cfg(feature = "gzip")] Codec::Gzip => gzip::compress(&mut decompressed, &mut compressed, level.map(|v| v as _)), #[cfg(feature = "zstd")] Codec::Zstd => zstd::compress( &mut decompressed, &mut compressed, level.map(|v: i32| v as i32), Some(input_len), ), #[cfg(feature = "lz4")] Codec::Lz4 => lz4::compress(&mut decompressed, &mut compressed, level.map(|v| v as _)), // TODO: Support passing acceleration #[cfg(feature = "lz4")] Codec::Lz4Block => lz4::block::compress_into( decompressed, compressed, level.map(|v| v as _), None, Some(true), ), }; match ret { Ok(n) => { *nbytes_written = n; *nbytes_read = decompressed.len(); } Err(err) => { error_to_ptr(err, error); *nbytes_written = 0; *nbytes_read = 0; } } } /* ---------- Streaming Compressor --------------- */ #[no_mangle] #[allow(unused_variables)] pub extern "C" fn compressor_init( codec: StreamingCodec, level: i32, error: &mut *mut c_char, ) -> *mut c_void { match codec { #[cfg(feature = "bzip2")] StreamingCodec::StreamingBzip2 => { if level < 0 { error_to_ptr("Bzip2 requires compression level >= 0", error); return std::ptr::null_mut(); } let compressor = bzip2::bzip2::write::BzEncoder::new( vec![], bzip2::bzip2::Compression::new(level as _), ); Box::into_raw(Box::new(compressor)) as _ } #[cfg(feature = "brotli")] StreamingCodec::StreamingBrotli => { if level < 0 { error_to_ptr("Brotli requires compression level >= 0", error); return std::ptr::null_mut(); } let compressor = brotli::make_write_compressor(vec![], Some(level as _)); Box::into_raw(Box::new(compressor)) as _ } #[cfg(feature = "gzip")] StreamingCodec::StreamingGzip => { if level < 1 { error_to_ptr("Gzip requires compression level >= 1", error); return std::ptr::null_mut(); } let compressor = gzip::flate2::write::GzEncoder::new( vec![], gzip::flate2::Compression::new(level as _), ); Box::into_raw(Box::new(compressor)) as _ } #[cfg(feature = "zstd")] StreamingCodec::StreamingZstd => { let compressor = zstd::zstd::Encoder::new(vec![], level); Box::into_raw(Box::new(compressor)) as _ } #[cfg(feature = "snappy")] StreamingCodec::StreamingSnappy => { let compressor = snappy::snap::write::FrameEncoder::new(vec![]); Box::into_raw(Box::new(compressor)) as _ } #[cfg(feature = "lz4")] StreamingCodec::StreamingLz4 => { if level < 0 { error_to_ptr("Lz4 requires compression level >= 0", error); return std::ptr::null_mut(); } let compressor = lz4::make_write_compressor(vec![], Some(level as _)); Box::into_raw(Box::new(compressor)) as _ } } } #[no_mangle] pub extern "C" fn free_compressor(codec: StreamingCodec, compressor_ptr: &mut *mut c_void) { if !(*compressor_ptr).is_null() { { match codec { #[cfg(feature = "bzip2")] StreamingCodec::StreamingBzip2 => { let _ = unsafe { Box::from_raw(*compressor_ptr as *mut Bzip2Compressor) }; } #[cfg(feature = "brotli")] StreamingCodec::StreamingBrotli => { let _ = unsafe { Box::from_raw(*compressor_ptr as *mut BrotliCompressor) }; } #[cfg(feature = "gzip")] StreamingCodec::StreamingGzip => { let _ = unsafe { Box::from_raw(*compressor_ptr as *mut GzipCompressor) }; } #[cfg(feature = "zstd")] StreamingCodec::StreamingZstd => { let _ = unsafe { Box::from_raw(*compressor_ptr as *mut ZstdCompressor) }; } #[cfg(feature = "snappy")] StreamingCodec::StreamingSnappy => { let _ = unsafe { Box::from_raw(*compressor_ptr as *mut SnappyFrameCompressor) }; } #[cfg(feature = "lz4")] StreamingCodec::StreamingLz4 => { let _ = unsafe { Box::from_raw(*compressor_ptr as *mut Lz4Compressor) }; } } } *compressor_ptr = std::ptr::null_mut(); } } #[no_mangle] pub extern "C" fn compressor_inner( codec: StreamingCodec, compressor_ptr: &mut *mut c_void, ) -> Buffer { match codec { #[cfg(feature = "bzip2")] StreamingCodec::StreamingBzip2 => { let compressor = unsafe { Box::from_raw(*compressor_ptr as *mut Bzip2Compressor) }; let buffer = Buffer::from(compressor.get_ref()); *compressor_ptr = Box::into_raw(compressor) as _; buffer } #[cfg(feature = "brotli")] StreamingCodec::StreamingBrotli => { let compressor = unsafe { Box::from_raw(*compressor_ptr as *mut BrotliCompressor) }; let buffer = Buffer::from(compressor.get_ref()); *compressor_ptr = Box::into_raw(compressor) as _; buffer } #[cfg(feature = "gzip")] StreamingCodec::StreamingGzip => { let compressor = unsafe { Box::from_raw(*compressor_ptr as *mut GzipCompressor) }; let buffer = Buffer::from(compressor.get_ref()); *compressor_ptr = Box::into_raw(compressor) as _; buffer } #[cfg(feature = "zstd")] StreamingCodec::StreamingZstd => { let compressor = unsafe { Box::from_raw(*compressor_ptr as *mut ZstdCompressor) }; let buffer = Buffer::from(compressor.get_ref()); *compressor_ptr = Box::into_raw(compressor) as _; buffer } #[cfg(feature = "snappy")] StreamingCodec::StreamingSnappy => { let compressor = unsafe { Box::from_raw(*compressor_ptr as *mut SnappyFrameCompressor) }; let buffer = Buffer::from(compressor.get_ref()); *compressor_ptr = Box::into_raw(compressor) as _; buffer } #[cfg(feature = "lz4")] StreamingCodec::StreamingLz4 => { let compressor = unsafe { Box::from_raw(*compressor_ptr as *mut Lz4Compressor) }; let buffer = Buffer::from(compressor.writer()); *compressor_ptr = Box::into_raw(compressor) as _; buffer } } } /// Finish the decompression stream and return the underlying buffer, transfering ownership to caller #[no_mangle] pub extern "C" fn compressor_finish( codec: StreamingCodec, compressor_ptr: &mut *mut c_void, error: &mut *mut c_char, ) -> Buffer { let buf = match codec { #[cfg(feature = "bzip2")] StreamingCodec::StreamingBzip2 => { let compressor = unsafe { Box::from_raw(*compressor_ptr as *mut Bzip2Compressor) }; match compressor.finish() { Ok(buf) => Buffer::from(buf), Err(err) => { error_to_ptr(err, error); Buffer::empty() } } } #[cfg(feature = "brotli")] StreamingCodec::StreamingBrotli => { let mut compressor = unsafe { Box::from_raw(*compressor_ptr as *mut BrotliCompressor) }; if let Err(err) = compressor.flush() { error_to_ptr(err, error); return Buffer::empty(); } Buffer::from(compressor.into_inner()) } #[cfg(feature = "gzip")] StreamingCodec::StreamingGzip => { let compressor = unsafe { Box::from_raw(*compressor_ptr as *mut GzipCompressor) }; match compressor.finish() { Ok(buf) => Buffer::from(buf), Err(err) => { error_to_ptr(err, error); Buffer::empty() } } } #[cfg(feature = "zstd")] StreamingCodec::StreamingZstd => { let compressor = unsafe { Box::from_raw(*compressor_ptr as *mut ZstdCompressor) }; match compressor.finish() { Ok(buf) => Buffer::from(buf), Err(err) => { error_to_ptr(err, error); Buffer::empty() } } } #[cfg(feature = "snappy")] StreamingCodec::StreamingSnappy => { let compressor = unsafe { Box::from_raw(*compressor_ptr as *mut SnappyFrameCompressor) }; match compressor.into_inner() { Ok(buf) => Buffer::from(buf), Err(err) => { error_to_ptr(err, error); Buffer::empty() } } } #[cfg(feature = "lz4")] StreamingCodec::StreamingLz4 => { let compressor = unsafe { Box::from_raw(*compressor_ptr as *mut Lz4Compressor) }; let (w, ret) = compressor.finish(); match ret { Ok(_) => Buffer::from(w), Err(err) => { error_to_ptr(err, error); Buffer::empty() } } } }; *compressor_ptr = std::ptr::null_mut(); buf } #[no_mangle] pub extern "C" fn compressor_flush( codec: StreamingCodec, compressor_ptr: &mut *mut c_void, error: &mut *mut c_char, ) { match codec { #[cfg(feature = "bzip2")] StreamingCodec::StreamingBzip2 => { let mut compressor = unsafe { Box::from_raw(*compressor_ptr as *mut Bzip2Compressor) }; if let Err(err) = compressor.flush() { error_to_ptr(err, error); } *compressor_ptr = Box::into_raw(compressor) as _; } #[cfg(feature = "brotli")] StreamingCodec::StreamingBrotli => { let mut compressor = unsafe { Box::from_raw(*compressor_ptr as *mut BrotliCompressor) }; if let Err(err) = compressor.flush() { error_to_ptr(err, error); } *compressor_ptr = Box::into_raw(compressor) as _; } #[cfg(feature = "gzip")] StreamingCodec::StreamingGzip => { let mut compressor = unsafe { Box::from_raw(*compressor_ptr as *mut GzipCompressor) }; if let Err(err) = compressor.flush() { error_to_ptr(err, error); } *compressor_ptr = Box::into_raw(compressor) as _; } #[cfg(feature = "zstd")] StreamingCodec::StreamingZstd => { let mut compressor = unsafe { Box::from_raw(*compressor_ptr as *mut ZstdCompressor) }; if let Err(err) = compressor.flush() { error_to_ptr(err, error); } *compressor_ptr = Box::into_raw(compressor) as _; } #[cfg(feature = "snappy")] StreamingCodec::StreamingSnappy => { let mut compressor = unsafe { Box::from_raw(*compressor_ptr as *mut SnappyFrameCompressor) }; if let Err(err) = compressor.flush() { error_to_ptr(err, error); } *compressor_ptr = Box::into_raw(compressor) as _; } #[cfg(feature = "lz4")] StreamingCodec::StreamingLz4 => { let mut compressor = unsafe { Box::from_raw(*compressor_ptr as *mut Lz4Compressor) }; if let Err(err) = compressor.flush() { error_to_ptr(err, error); } *compressor_ptr = Box::into_raw(compressor) as _; } } } #[no_mangle] pub extern "C" fn compressor_compress( codec: StreamingCodec, compressor_ptr: &mut *mut c_void, input: *const u8, input_len: usize, nbytes_read: &mut usize, nbytes_written: &mut usize, error: &mut *mut c_char, ) { let mut decompressed = Cursor::new(unsafe { slice::from_raw_parts(input, input_len) }); match codec { #[cfg(feature = "bzip2")] StreamingCodec::StreamingBzip2 => { let mut compressor = unsafe { Box::from_raw(*compressor_ptr as *mut Bzip2Compressor) }; match std::io::copy(&mut decompressed, &mut compressor) { Ok(n) => { *nbytes_written = n as _; *nbytes_read = decompressed.position() as _; } Err(err) => { error_to_ptr(err, error); } } *compressor_ptr = Box::into_raw(compressor) as _; } #[cfg(feature = "brotli")] StreamingCodec::StreamingBrotli => { let mut compressor = unsafe { Box::from_raw(*compressor_ptr as *mut BrotliCompressor) }; match std::io::copy(&mut decompressed, &mut compressor) { Ok(n) => { *nbytes_written = n as _; *nbytes_read = decompressed.position() as _; } Err(err) => { error_to_ptr(err, error); } } *compressor_ptr = Box::into_raw(compressor) as _; } #[cfg(feature = "gzip")] StreamingCodec::StreamingGzip => { let mut compressor = unsafe { Box::from_raw(*compressor_ptr as *mut GzipCompressor) }; match std::io::copy(&mut decompressed, &mut compressor) { Ok(n) => { *nbytes_written = n as _; *nbytes_read = decompressed.position() as _; } Err(err) => { error_to_ptr(err, error); } } *compressor_ptr = Box::into_raw(compressor) as _; } #[cfg(feature = "zstd")] StreamingCodec::StreamingZstd => { let mut compressor = unsafe { Box::from_raw(*compressor_ptr as *mut ZstdCompressor) }; match std::io::copy(&mut decompressed, &mut compressor) { Ok(n) => { *nbytes_written = n as _; *nbytes_read = decompressed.position() as _; } Err(err) => { error_to_ptr(err, error); } } *compressor_ptr = Box::into_raw(compressor) as _; } #[cfg(feature = "snappy")] StreamingCodec::StreamingSnappy => { let mut compressor = unsafe { Box::from_raw(*compressor_ptr as *mut SnappyFrameCompressor) }; match std::io::copy(&mut decompressed, &mut compressor) { Ok(n) => { *nbytes_written = n as _; *nbytes_read = decompressed.position() as _; } Err(err) => { error_to_ptr(err, error); } } *compressor_ptr = Box::into_raw(compressor) as _; } #[cfg(feature = "lz4")] StreamingCodec::StreamingLz4 => { let mut compressor = unsafe { Box::from_raw(*compressor_ptr as *mut Lz4Compressor) }; match std::io::copy(&mut decompressed, &mut compressor) { Ok(n) => { *nbytes_written = n as _; *nbytes_read = decompressed.position() as _; } Err(err) => { error_to_ptr(err, error); } } *compressor_ptr = Box::into_raw(compressor) as _; } } } #[no_mangle] #[allow(unused_variables)] pub extern "C" fn decompressor_init(codec: StreamingCodec) -> *mut c_void { // for decompression, we really only need a buffer for storing output // some streaming codecs, like snappy, don't have a write impl and only a // read impl for decompressors let buf: Vec = vec![]; Box::into_raw(Box::new(Cursor::new(buf))) as _ } #[no_mangle] #[allow(unused_variables)] pub extern "C" fn free_decompressor(codec: StreamingCodec, decompressor_ptr: &mut *mut c_void) { if !(*decompressor_ptr).is_null() { { let _ = unsafe { Box::from_raw(*decompressor_ptr as *mut Decompressor) }; } *decompressor_ptr = std::ptr::null_mut(); } } #[no_mangle] #[allow(unused_variables)] pub extern "C" fn decompressor_inner( codec: StreamingCodec, decompressor_ptr: &mut *mut c_void, ) -> Buffer { let decompressor = unsafe { Box::from_raw(*decompressor_ptr as *mut Decompressor) }; let buf = Buffer::from(decompressor.get_ref()); *decompressor_ptr = Box::into_raw(decompressor) as _; buf } /// Finish the decompression stream and return the underlying buffer, transfering ownership to caller #[no_mangle] #[allow(unused_variables)] pub extern "C" fn decompressor_finish( codec: StreamingCodec, decompressor_ptr: &mut *mut c_void, error: &mut *mut c_char, ) -> Buffer { let mut cursor = unsafe { Box::from_raw(*decompressor_ptr as *mut Decompressor) }; if let Err(err) = cursor.flush() { error_to_ptr(err, error); return Buffer::empty(); }; *decompressor_ptr = std::ptr::null_mut(); Buffer::from(cursor.into_inner()) } #[no_mangle] #[allow(unused_variables)] pub extern "C" fn decompressor_flush( codec: StreamingCodec, decompressor_ptr: &mut *mut c_void, error: &mut *mut c_char, ) { let mut cursor = unsafe { Box::from_raw(*decompressor_ptr as *mut Decompressor) }; if let Err(err) = cursor.flush() { error_to_ptr(err, error); } *decompressor_ptr = Box::into_raw(cursor) as _; } #[no_mangle] pub extern "C" fn decompressor_decompress( codec: StreamingCodec, decompressor_ptr: &mut *mut c_void, input: *const u8, input_len: usize, nbytes_read: &mut usize, nbytes_written: &mut usize, error: &mut *mut c_char, ) { let mut decompressed = unsafe { Box::from_raw(*decompressor_ptr as *mut Decompressor) }; let start_pos = decompressed.position(); let mut compressed = Cursor::new(unsafe { std::slice::from_raw_parts(input, input_len) }); let ret: Result = match codec { #[cfg(feature = "bzip2")] StreamingCodec::StreamingBzip2 => bzip2::decompress(&mut compressed, &mut decompressed), #[cfg(feature = "gzip")] StreamingCodec::StreamingGzip => gzip::decompress(&mut compressed, &mut decompressed), #[cfg(feature = "brotli")] StreamingCodec::StreamingBrotli => brotli::decompress(&mut compressed, &mut decompressed), #[cfg(feature = "zstd")] StreamingCodec::StreamingZstd => zstd::decompress(&mut compressed, &mut decompressed), #[cfg(feature = "snappy")] StreamingCodec::StreamingSnappy => snappy::decompress(&mut compressed, &mut decompressed), #[cfg(feature = "lz4")] StreamingCodec::StreamingLz4 => lz4::decompress(&mut compressed, &mut decompressed), }; match ret { Ok(_) => { *nbytes_read = compressed.position() as _; *nbytes_written = (decompressed.position() - start_pos) as _; } Err(err) => { error_to_ptr(err, error); } }; *decompressor_ptr = Box::into_raw(decompressed) as _; } /* -------- Codec specific functions ----------*/ #[cfg(feature = "lz4")] #[no_mangle] pub extern "C" fn lz4_frame_max_compression_level() -> usize { lz4::LZ4_ACCELERATION_MAX as _ } #[cfg(feature = "lz4")] #[no_mangle] pub extern "C" fn lz4_frame_max_compressed_len(input_len: usize, compression_level: i32) -> usize { lz4::compress_bound(input_len, Some(compression_level as _)) } #[cfg(feature = "lz4")] #[no_mangle] #[allow(unused_variables)] pub extern "C" fn lz4_block_max_compressed_len(input_len: usize, error: &mut *mut c_char) -> usize { lz4::block::compress_bound(input_len, Some(true)) } #[cfg(feature = "deflate")] #[no_mangle] pub extern "C" fn deflate_max_compressed_len(input_len: usize) -> usize { deflate::compress_bound(input_len) } #[cfg(feature = "gzip")] #[no_mangle] pub extern "C" fn gzip_max_compressed_len(input_len: usize) -> usize { gzip::compress_bound(input_len) } #[cfg(feature = "zstd")] #[no_mangle] pub extern "C" fn zstd_max_compressed_len(input_len: usize) -> usize { zstd::compress_bound(input_len) } #[cfg(feature = "snappy")] #[no_mangle] pub extern "C" fn snappy_raw_max_compressed_len(input_len: usize) -> usize { snap::raw::max_compress_len(input_len) } #[cfg(feature = "brotli")] #[no_mangle] pub extern "C" fn brotli_max_compressed_len(input_len: usize) -> usize { brotli::compress_bound(input_len) } #[cfg(feature = "snappy")] #[no_mangle] pub extern "C" fn snappy_raw_decompressed_len( input: *const u8, input_len: usize, error: &mut *mut c_char, ) -> isize { let input = unsafe { slice::from_raw_parts(input, input_len) }; match snap::raw::decompress_len(input) { Ok(n) => n as _, Err(err) => { error_to_ptr(err, error); -1 } } } #[cfg(test)] mod tests { use super::*; const DATA: &[u8; 5] = b"bytes"; #[cfg(feature = "lz4")] #[test] fn test_lz4_frame_max_compressed_len() { // A known simple test case, expected len taken from lz4/lz4 repo let len = lz4_frame_max_compressed_len(25, 4); assert_eq!(len, 65544); } #[cfg(feature = "lz4")] #[test] fn test_lz4_block_max_compressed_len() { let mut error: *mut c_char = std::ptr::null_mut(); let len = lz4_block_max_compressed_len(10, &mut error); assert!(error.is_null()); assert_eq!(len, 30); } #[cfg(feature = "snappy")] #[test] fn test_snappy_raw_max_compressed_len() { let len = snappy_raw_max_compressed_len(10); assert_eq!(len, 43); } #[cfg(feature = "snappy")] #[test] fn test_snappy_raw_decompressed_len() { let uncompressed = b"bytes"; let mut compressed = vec![0; snappy_raw_max_compressed_len(uncompressed.len())]; let nbytes_written = snappy::raw::compress(uncompressed, &mut compressed).unwrap(); let mut error: *mut c_char = std::ptr::null_mut(); let len = snappy_raw_decompressed_len(compressed.as_ptr(), nbytes_written, &mut error); assert!(error.is_null()); assert_eq!(len as usize, uncompressed.len()); } #[cfg(feature = "snappy")] #[test] fn test_snappy_roundtrip() { let mut expected = vec![]; snappy::compress(Cursor::new(DATA), &mut expected).unwrap(); roundtrip(Codec::Snappy, &expected, 0); } #[cfg(feature = "snappy")] #[test] fn test_snappy_raw_roundtrip() { let expected = snappy::raw::compress_vec(DATA).unwrap(); roundtrip(Codec::SnappyRaw, &expected, 0); } #[cfg(feature = "lz4")] #[test] fn test_lz4_roundtrip() { let mut expected = Cursor::new(vec![]); lz4::compress(Cursor::new(DATA), &mut expected, Some(6)).unwrap(); let expected = expected.into_inner(); roundtrip(Codec::Lz4, &expected, 6); } #[cfg(feature = "lz4")] #[test] fn test_lz4_block_roundtrip() { let expected = lz4::block::compress_vec(DATA, Some(6), Some(1), Some(true)).unwrap(); roundtrip(Codec::Lz4Block, &expected, 6); } #[cfg(feature = "bzip2")] #[test] fn test_bzip2_roundtrip() { let mut expected = Cursor::new(vec![]); bzip2::compress(Cursor::new(DATA), &mut expected, Some(6)).unwrap(); let expected = expected.into_inner(); roundtrip(Codec::Bzip2, &expected, 6); } #[cfg(feature = "brotli")] #[test] fn test_brotli_roundtrip() { let mut expected = Cursor::new(vec![]); brotli::compress(Cursor::new(DATA), &mut expected, Some(6)).unwrap(); let expected = expected.into_inner(); roundtrip(Codec::Brotli, &expected, 6); } #[cfg(feature = "zstd")] #[test] fn test_zstd_roundtrip() { let mut expected = Cursor::new(vec![]); zstd::compress(Cursor::new(DATA), &mut expected, Some(6), Some(DATA.len())).unwrap(); let expected = expected.into_inner(); roundtrip(Codec::Zstd, &expected, 6); } fn roundtrip(codec: Codec, expected: &[u8], level: i32) { let mut nbytes_read = 0; let mut nbytes_written = 0; let mut error = std::ptr::null_mut(); let buffer = compress( codec, level, DATA.as_ptr(), DATA.len(), &mut nbytes_read, &mut nbytes_written, &mut error, ); if !error.is_null() { let error = unsafe { CString::from_raw(error) }; panic!("Failed: {}", error.to_str().unwrap()); } assert_eq!(nbytes_read, DATA.len()); assert_eq!(nbytes_written, buffer.len); assert!(buffer.owned); // retrieve compressed data and compare to actual rust impl let compressed = unsafe { Vec::from_raw_parts(buffer.data as *mut u8, buffer.len, buffer.len) }; assert_eq!(&compressed, expected); // And decompress nbytes_read = 0; nbytes_written = 0; let buffer = decompress( codec, compressed.as_ptr(), compressed.len(), &mut nbytes_read, &mut nbytes_written, &mut error, ); if !error.is_null() { let error = unsafe { CString::from_raw(error) }; panic!("Failed: {}", error.to_str().unwrap()); } assert_eq!(nbytes_read, compressed.len()); assert_eq!(nbytes_written, buffer.len); assert_eq!(nbytes_written, DATA.len()); assert!(buffer.owned); let decompressed = unsafe { Vec::from_raw_parts(buffer.data as *mut u8, buffer.len, buffer.len) }; assert_eq!(DATA.as_slice(), &decompressed); } } libcramjam-0.8.0/src/deflate.rs000064400000000000000000000023701046102023000144750ustar 00000000000000//! deflate de/compression interface pub use flate2; use flate2::read::{DeflateDecoder, DeflateEncoder}; use flate2::Compression; use std::io::prelude::*; use std::io::Error; pub const DEFAULT_COMPRESSION_LEVEL: u32 = 6; pub const MIN_BLOCK_LENGTH: usize = 5_000; /// Compression upper bound // xref: https://github.com/ebiggers/libdeflate/blob/6bb493615b0ef35c98fc4aa4ec04f448788db6a5/lib/deflate_compress.c#L4081 pub fn compress_bound(input_len: usize) -> usize { let max_blocks = std::cmp::max((input_len + MIN_BLOCK_LENGTH - 1) / MIN_BLOCK_LENGTH, 1); (5 * max_blocks) + input_len } /// Decompress gzip data #[inline(always)] pub fn decompress(input: R, output: &mut W) -> Result { let mut decoder = DeflateDecoder::new(input); let n_bytes = std::io::copy(&mut decoder, output)?; Ok(n_bytes as usize) } /// Compress gzip data #[inline(always)] pub fn compress( input: R, output: &mut W, level: Option, ) -> Result { let level = level.unwrap_or_else(|| DEFAULT_COMPRESSION_LEVEL); let mut encoder = DeflateEncoder::new(input, Compression::new(level)); let n_bytes = std::io::copy(&mut encoder, output)?; Ok(n_bytes as usize) } libcramjam-0.8.0/src/gzip.rs000064400000000000000000000035401046102023000140420ustar 00000000000000//! gzip de/compression interface pub use flate2; use flate2::read::{GzEncoder, MultiGzDecoder}; use flate2::Compression; use std::io::prelude::*; use std::io::{Cursor, Error}; pub const DEFAULT_COMPRESSION_LEVEL: u32 = 6; pub const GZIP_FOOTER_SIZE: usize = 8; pub const GZIP_MIN_HEADER_SIZE: usize = 10; pub const GZIP_MIN_OVERHEAD: usize = GZIP_MIN_HEADER_SIZE + GZIP_FOOTER_SIZE; /// Compression upper bound // xref: https://github.com/ebiggers/libdeflate/blob/6bb493615b0ef35c98fc4aa4ec04f448788db6a5/lib/gzip_compress.c#L85 pub fn compress_bound(input_len: usize) -> usize { GZIP_MIN_OVERHEAD + crate::deflate::compress_bound(input_len) } /// Decompress gzip data #[inline(always)] pub fn decompress(input: R, output: &mut W) -> Result { let mut decoder = MultiGzDecoder::new(input); let mut out = vec![]; let n_bytes = decoder.read_to_end(&mut out)?; std::io::copy(&mut Cursor::new(out.as_slice()), output)?; Ok(n_bytes as usize) } /// Compress gzip data #[inline(always)] pub fn compress( input: R, output: &mut W, level: Option, ) -> Result { let level = level.unwrap_or_else(|| DEFAULT_COMPRESSION_LEVEL); let mut encoder = GzEncoder::new(input, Compression::new(level)); let n_bytes = std::io::copy(&mut encoder, output)?; Ok(n_bytes as usize) } #[cfg(test)] mod tests { #[test] fn test_gzip_multiple_streams() { let mut out1 = vec![]; let mut out2 = vec![]; super::compress(b"foo".to_vec().as_slice(), &mut out1, None).unwrap(); super::compress(b"bar".to_vec().as_slice(), &mut out2, None).unwrap(); let mut out3 = vec![]; out1.extend_from_slice(&out2); super::decompress(out1.as_slice(), &mut out3).unwrap(); assert_eq!(out3, b"foobar".to_vec()); } } libcramjam-0.8.0/src/ideflate.rs000064400000000000000000000020461046102023000146460ustar 00000000000000//! DEFLATE w/ ISA-L de/compression interface #[cfg(all( any( feature = "ideflate", feature = "ideflate-static", feature = "ideflate-shared" ), target_pointer_width = "64", ))] pub use isal; use std::io::prelude::*; use std::io::Error; const DEFAULT_COMPRESSION_LEVEL: u32 = 3; /// Decompress igzip data #[inline(always)] pub fn decompress(input: R, output: &mut W) -> Result { let mut decoder = isal::read::DeflateDecoder::new(input); let nbytes = std::io::copy(&mut decoder, output)?; Ok(nbytes as usize) } /// Compress igzip data #[inline(always)] pub fn compress( input: R, output: &mut W, level: Option, ) -> Result { let level = level.unwrap_or_else(|| DEFAULT_COMPRESSION_LEVEL); let level = isal::CompressionLevel::try_from(level as isize)?; let mut encoder = isal::read::DeflateEncoder::new(input, level); let n_bytes = std::io::copy(&mut encoder, output)?; Ok(n_bytes as usize) } libcramjam-0.8.0/src/igzip.rs000064400000000000000000000027341046102023000142170ustar 00000000000000//! GZIP w/ ISA-L de/compression interface #[cfg(all( any(feature = "igzip", feature = "igzip-static", feature = "igzip-shared"), target_pointer_width = "64", ))] pub use isal; use std::io::prelude::*; use std::io::Error; const DEFAULT_COMPRESSION_LEVEL: u32 = 3; /// Decompress igzip data #[inline(always)] pub fn decompress(input: R, output: &mut W) -> Result { let mut decoder = isal::read::GzipDecoder::new(input); let nbytes = std::io::copy(&mut decoder, output)?; Ok(nbytes as usize) } /// Compress igzip data #[inline(always)] pub fn compress( input: R, output: &mut W, level: Option, ) -> Result { let level = level.unwrap_or_else(|| DEFAULT_COMPRESSION_LEVEL); let level = isal::CompressionLevel::try_from(level as isize)?; let mut encoder = isal::read::GzipEncoder::new(input, level); let n_bytes = std::io::copy(&mut encoder, output)?; Ok(n_bytes as usize) } #[cfg(test)] mod tests { #[test] fn test_gzip_multiple_streams() { let mut out1 = vec![]; let mut out2 = vec![]; super::compress(b"foo".to_vec().as_slice(), &mut out1, None).unwrap(); super::compress(b"bar".to_vec().as_slice(), &mut out2, None).unwrap(); let mut out3 = vec![]; out1.extend_from_slice(&out2); super::decompress(out1.as_slice(), &mut out3).unwrap(); assert_eq!(out3, b"foobar".to_vec()); } } libcramjam-0.8.0/src/izlib.rs000064400000000000000000000017661046102023000142120ustar 00000000000000//! Zlib w/ ISA-L de/compression interface #[cfg(all( any(feature = "izlib", feature = "izlib-static", feature = "izlib-shared"), target_pointer_width = "64", ))] pub use isal; use std::io::prelude::*; use std::io::Error; const DEFAULT_COMPRESSION_LEVEL: u32 = 3; /// Decompress igzip data #[inline(always)] pub fn decompress(input: R, output: &mut W) -> Result { let mut decoder = isal::read::ZlibDecoder::new(input); let nbytes = std::io::copy(&mut decoder, output)?; Ok(nbytes as usize) } /// Compress igzip data #[inline(always)] pub fn compress( input: R, output: &mut W, level: Option, ) -> Result { let level = level.unwrap_or_else(|| DEFAULT_COMPRESSION_LEVEL); let level = isal::CompressionLevel::try_from(level as isize)?; let mut encoder = isal::read::ZlibEncoder::new(input, level); let n_bytes = std::io::copy(&mut encoder, output)?; Ok(n_bytes as usize) } libcramjam-0.8.0/src/lib.rs000064400000000000000000000151111046102023000136340ustar 00000000000000#[cfg(any( feature = "blosc2", feature = "blosc2-static", feature = "blosc2-shared" ))] pub mod blosc2; #[cfg(feature = "brotli")] pub mod brotli; #[cfg(feature = "bzip2")] pub mod bzip2; #[cfg(feature = "capi")] mod capi; #[cfg(any( feature = "deflate", feature = "deflate-static", feature = "deflate-shared" ))] pub mod deflate; #[cfg(any(feature = "gzip", feature = "gzip-static", feature = "gzip-shared"))] pub mod gzip; #[cfg(all( any( feature = "ideflate", feature = "ideflate-static", feature = "ideflate-shared" ), target_pointer_width = "64" ))] pub mod ideflate; #[cfg(all( any(feature = "igzip", feature = "igzip-static", feature = "igzip-shared"), target_pointer_width = "64" ))] pub mod igzip; #[cfg(all( any(feature = "izlib", feature = "izlib-static", feature = "izlib-shared"), target_pointer_width = "64" ))] pub mod izlib; #[cfg(feature = "lz4")] pub mod lz4; #[cfg(feature = "snappy")] pub mod snappy; #[cfg(any(feature = "xz", feature = "xz-static", feature = "xz-shared"))] pub mod xz; #[cfg(any(feature = "zlib", feature = "zlib-static", feature = "zlib-shared"))] pub mod zlib; #[cfg(feature = "zstd")] pub mod zstd; #[cfg(test)] mod tests { use std::io::Cursor; use std::str::FromStr; // Generate some 'real-world' data by reading src code and duplicating until well over buf size static LARGE_DATA: std::sync::LazyLock> = std::sync::LazyLock::new(|| { // use src code as base, and we have at least 2mb of data let mut bytes = read_dir_files(std::path::PathBuf::from_str("./src").unwrap()); while bytes.len() < 5e6 as usize { bytes.extend(bytes.clone()); } bytes }); fn read_dir_files(dir: std::path::PathBuf) -> Vec { let mut all_bytes = vec![]; for entry in std::fs::read_dir(dir).unwrap().into_iter() { let entry = entry.unwrap(); if entry.file_type().unwrap().is_file() { all_bytes.extend(std::fs::read(entry.path()).unwrap()); } else if entry.file_type().unwrap().is_dir() { all_bytes.extend(read_dir_files(entry.path())); } } all_bytes } // Default testing data fn gen_data() -> Vec { (&*LARGE_DATA).clone() } // Single test generation macro_rules! round_trip { ($name:ident($compress_output:ident -> $decompress_output:ident), variant=$variant:ident, $(, $args:ident)*) => { #[test] fn $name() { let data = gen_data(); let mut compressed = Vec::new(); let compressed_size = if stringify!($decompress_output) == "Slice" { compressed = (0..data.len()).map(|_| 0).collect::>(); let mut cursor = Cursor::new(compressed.as_mut_slice()); crate::$variant::compress(&mut Cursor::new(data.as_slice()), &mut cursor $(, $args)*).unwrap() } else { crate::$variant::compress(&mut Cursor::new(data.as_slice()), &mut Cursor::new(&mut compressed) $(, $args)*).unwrap() }; println!("Compressed size: {}", compressed_size); compressed.truncate(compressed_size); let mut decompressed = Vec::new(); let decompressed_size = if stringify!($decompress_output) == "Slice" { decompressed = (0..data.len()).map(|_| 0).collect::>(); let mut cursor = Cursor::new(decompressed.as_mut_slice()); crate::$variant::decompress(&mut Cursor::new(&compressed), &mut cursor).unwrap() } else { crate::$variant::decompress(&mut Cursor::new(&compressed), &mut decompressed).unwrap() }; assert_eq!(decompressed_size, data.len()); if &decompressed[..decompressed_size] != &data { panic!("Decompressed and original data do not match! :-(") } } } } // macro to generate each variation of Output::* roundtrip. macro_rules! test_variant { ($variant:ident $(, $args:tt)*) => { #[cfg(test)] mod $variant { use super::*; round_trip!(roundtrip_compress_via_slice_decompress_via_slice(Slice -> Slice), variant=$variant, $(, $args)* ); round_trip!(roundtrip_compress_via_slice_decompress_via_vector(Slice -> Vector), variant=$variant, $(, $args)* ); round_trip!(roundtrip_compress_via_vector_decompress_via_slice(Vector -> Slice), variant=$variant, $(, $args)* ); round_trip!(roundtrip_compress_via_vector_decompress_via_vector(Vector -> Vector), variant=$variant, $(, $args)* ); } } } // Expected compressed_len, subsequent args are supplied to the variant's `compress` call. #[cfg(feature = "snappy")] test_variant!(snappy); #[cfg(feature = "gzip")] test_variant!(gzip, None); #[cfg(all( any(feature = "igzip", feature = "igzip-static", feature = "igzip-shared"), target_pointer_width = "64" ))] test_variant!(igzip, None); #[cfg(all( any( feature = "ideflate", feature = "ideflate-static", feature = "ideflate-shared" ), target_pointer_width = "64" ))] test_variant!(ideflate, None); #[cfg(all( any(feature = "izlib", feature = "izlib-static", feature = "izlib-shared"), target_pointer_width = "64" ))] test_variant!(izlib, None); #[cfg(feature = "brotli")] test_variant!(brotli, None); #[cfg(feature = "bzip2")] test_variant!(bzip2, None); #[cfg(feature = "deflate")] test_variant!(deflate, None); #[cfg(feature = "zstd")] test_variant!(zstd, None, None); #[cfg(feature = "zlib")] test_variant!(zlib, None); #[cfg(feature = "lz4")] test_variant!(lz4, None); #[cfg(feature = "blosc2")] test_variant!(blosc2); #[cfg(feature = "xz")] #[allow(non_upper_case_globals)] const format: Option = None; #[allow(non_upper_case_globals)] #[cfg(feature = "xz")] const check: Option = None; #[allow(non_upper_case_globals)] #[cfg(feature = "xz")] const filters: Option = None; #[allow(non_upper_case_globals)] #[cfg(feature = "xz")] const opts: Option = None; #[cfg(feature = "xz")] test_variant!(xz, None, format, check, filters, opts); } libcramjam-0.8.0/src/lz4.rs000064400000000000000000000160451046102023000136060ustar 00000000000000//! lz4 de/compression interface pub use lz4; use std::io::{BufReader, Cursor, Error, Read, Write}; pub const DEFAULT_COMPRESSION_LEVEL: u32 = 4; pub const LZ4_ACCELERATION_MAX: u32 = 65537; #[inline(always)] pub fn make_write_compressor(output: W, level: Option) -> Result, Error> { let comp = lz4::EncoderBuilder::new() .level(level.unwrap_or(DEFAULT_COMPRESSION_LEVEL)) .auto_flush(true) .favor_dec_speed(true) .build(output)?; Ok(comp) } /// Decompress lz4 data #[inline(always)] pub fn decompress(input: R, output: &mut W) -> Result { let mut decoder = lz4::Decoder::new(input)?; let n_bytes = std::io::copy(&mut decoder, output)?; decoder.finish().1?; Ok(n_bytes as usize) } #[inline(always)] pub fn compress_bound(input_len: usize, level: Option) -> usize { let mut prefs: std::mem::MaybeUninit = std::mem::MaybeUninit::zeroed(); let prefs_ptr = prefs.as_mut_ptr(); unsafe { std::ptr::write( std::ptr::addr_of_mut!((*prefs_ptr).compression_level), level.unwrap_or(DEFAULT_COMPRESSION_LEVEL), ) }; let n = unsafe { lz4::liblz4::LZ4F_compressBound(input_len, prefs.as_ptr()) }; unsafe { std::ptr::drop_in_place(std::ptr::addr_of_mut!((*prefs_ptr).compression_level)) }; n } /// Compress lz4 data #[inline(always)] pub fn compress(input: R, output: &mut W, level: Option) -> Result { // Can add an additional constraint to `Seek` for output but that is not great for API // so very unfortunately, we have an intermediate buffer to get bytes written to output // as lz4::Encoder is Write only let out_buffer = vec![]; let mut encoder = make_write_compressor(out_buffer, level)?; // this returns, bytes read from uncompressed, input; we want bytes written // but lz4 only implements Read for Encoder let mut buf = BufReader::new(input); std::io::copy(&mut buf, &mut encoder)?; let (w, r) = encoder.finish(); r?; // Now copy bytes from temp output buffer to actual output, returning number of bytes written to 'output'. let nbytes = std::io::copy(&mut Cursor::new(w), output)?; Ok(nbytes as _) } pub mod block { use lz4::block::CompressionMode; use std::io::Error; const PREPEND_SIZE: bool = true; #[inline(always)] pub fn compress_bound(input_len: usize, prepend_size: Option) -> usize { match lz4::block::compress_bound(input_len) { Ok(len) => { if prepend_size.unwrap_or(true) { len + 4 } else { len } } Err(_) => 0, } } /// Decompress into Vec. Must have been compressed with prepended uncompressed size. /// will panic otherwise. #[inline(always)] pub fn decompress_vec(input: &[u8]) -> Result, Error> { if input.len() < 4 { return Err(std::io::Error::new( std::io::ErrorKind::InvalidInput, "Input not long enough", )); } let bytes: [u8; 4] = input[..4].try_into().unwrap(); let len = u32::from_le_bytes(bytes); let mut buf = vec![0u8; len as usize]; let nbytes = decompress_into(&input[4..], &mut buf, Some(false))?; buf.truncate(nbytes); Ok(buf) } /// NOTE: input is expected to **not** have the size prepended. Calling decompress_into is /// saying you already know the output buffer min size. `output` can be larger, but it cannot /// be smaller than what's required. #[inline(always)] pub fn decompress_into(input: &[u8], output: &mut [u8], size_prepended: Option) -> Result { let uncompressed_size = if size_prepended.is_some_and(|v| v) { None // decompress_to_buffer will read from prepended size } else { Some(output.len() as _) }; let nbytes = lz4::block::decompress_to_buffer(input, uncompressed_size, output)?; Ok(nbytes) } #[inline(always)] pub fn compress_vec( input: &[u8], level: Option, acceleration: Option, prepend_size: Option, ) -> Result, Error> { let len = compress_bound(input.len(), prepend_size); let mut buffer = vec![0u8; len]; let nbytes = compress_into(input, &mut buffer, level, acceleration, prepend_size)?; buffer.truncate(nbytes); Ok(buffer) } #[inline(always)] pub fn compress_into( input: &[u8], output: &mut [u8], level: Option, acceleration: Option, prepend_size: Option, ) -> Result { let prepend_size = prepend_size.unwrap_or(PREPEND_SIZE); let mode = compression_mode(None, level.map(|v| v as _), acceleration)?; let nbytes = lz4::block::compress_to_buffer(input, Some(mode), prepend_size, output)?; Ok(nbytes) } #[inline] fn compression_mode( mode: Option<&str>, compression: Option, acceleration: Option, ) -> Result { let m = match mode { Some(m) => match m { "default" => CompressionMode::DEFAULT, "fast" => CompressionMode::FAST(acceleration.unwrap_or(1)), "high_compression" => CompressionMode::HIGHCOMPRESSION(compression.unwrap_or(9)), _ => { return Err(std::io::Error::new( std::io::ErrorKind::InvalidInput, "Invalid compression string, needed one of 'default', 'fast', or 'high_compression'", )) } }, None => CompressionMode::DEFAULT, }; Ok(m) } #[cfg(test)] mod tests { use super::{compress_vec, decompress_into, decompress_vec}; const DATA: &[u8; 14] = b"howdy neighbor"; #[test] fn round_trip_store_size() { let compressed = compress_vec(DATA, None, None, Some(true)).unwrap(); let decompressed = decompress_vec(&compressed).unwrap(); assert_eq!(&decompressed, DATA); } #[test] fn round_trip_no_store_size() { let compressed = compress_vec(DATA, None, None, Some(false)).unwrap(); // decompressed_vec depends on prepended_size, so we can't use that. assert!(decompress_vec(&compressed).is_err()); let mut decompressed = vec![0u8; DATA.len()]; decompress_into(&compressed, &mut decompressed, Some(false)).unwrap(); assert_eq!(&decompressed, DATA); // decompressed_into will allow a larger output buffer than what's needed let mut decompressed = vec![0u8; DATA.len() + 5_000]; let n = decompress_into(&compressed, &mut decompressed, Some(false)).unwrap(); assert_eq!(&decompressed[..n], DATA); } } } libcramjam-0.8.0/src/snappy.rs000064400000000000000000000032321046102023000144010ustar 00000000000000//! snappy de/compression interface pub use snap; use snap::read::{FrameDecoder, FrameEncoder}; use std::io; use std::io::{Read, Result, Write}; /// Decompress snappy data framed #[inline(always)] pub fn decompress(input: R, output: &mut W) -> Result { let mut decoder = FrameDecoder::new(input); let n_bytes = io::copy(&mut decoder, output)?; Ok(n_bytes as usize) } /// Decompress snappy data framed #[inline(always)] pub fn compress(data: R, output: &mut W) -> Result { let mut encoder = FrameEncoder::new(data); let n_bytes = io::copy(&mut encoder, output)?; Ok(n_bytes as usize) } pub mod raw { use super::*; #[inline(always)] pub fn compress_vec(input: &[u8]) -> Result> { snap::raw::Encoder::new() .compress_vec(input) .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string())) } #[inline(always)] pub fn compress(input: &[u8], output: &mut [u8]) -> Result { snap::raw::Encoder::new() .compress(input, output) .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string())) } #[inline(always)] pub fn decompress_vec(input: &[u8]) -> Result> { snap::raw::Decoder::new() .decompress_vec(input) .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string())) } #[inline(always)] pub fn decompress(input: &[u8], output: &mut [u8]) -> Result { snap::raw::Decoder::new() .decompress(input, output) .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string())) } } libcramjam-0.8.0/src/xz.rs000064400000000000000000000064131046102023000135340ustar 00000000000000//! LZMA / XZ de/compression interface //! Note this is still a bit of a work in progress, especially when it comes //! to filter chain support. use std::io::{self, BufRead, BufReader}; use std::io::{Read, Result, Write}; pub use xz2; use xz2::read::{XzDecoder, XzEncoder}; use xz2::stream::{Check as xz2Check, Stream, TELL_ANY_CHECK}; pub use xz2::stream::{Filters, LzmaOptions, MatchFinder, Mode}; /// Possible formats #[derive(Clone, Debug, Copy)] pub enum Format { /// Auto select the format, for compression this is XZ, /// for decompression it will be determined by the compressed input. AUTO, /// The `.xz` format (default) XZ, /// Legacy `.lzma` format. ALONE, /// Raw data stream RAW, } impl Default for Format { fn default() -> Self { Format::XZ } } /// Possible Check configurations #[derive(Debug, Clone, Copy)] pub enum Check { Crc64, Crc32, Sha256, None, } impl Into for Check { fn into(self) -> xz2Check { match self { Self::Crc64 => xz2Check::Crc64, Self::Crc32 => xz2Check::Crc32, Self::Sha256 => xz2Check::Sha256, Self::None => xz2Check::None, } } } /// Decompress snappy data framed #[inline(always)] pub fn decompress(input: R, output: &mut W) -> Result { let xz_magicbytes = b"\xfd7zXZ\x00"; let mut input = BufReader::new(input); let stream = { let innerbuf = input.fill_buf()?; if innerbuf.len() >= xz_magicbytes.len() && &innerbuf[..xz_magicbytes.len()] == xz_magicbytes { Stream::new_auto_decoder(u64::MAX, TELL_ANY_CHECK)? } else { Stream::new_lzma_decoder(u64::MAX)? } }; let mut decoder = XzDecoder::new_stream(input, stream); let n_bytes = io::copy(&mut decoder, output)?; Ok(n_bytes as usize) } /// Decompress snappy data framed #[inline(always)] pub fn compress( data: R, output: &mut W, preset: Option, format: Option>, check: Option>, filters: Option>, options: Option>, ) -> Result { let preset = preset.unwrap_or(6); // same as python default let stream = match format.map(Into::into).unwrap_or_default() { Format::AUTO | Format::XZ => { let check = check.map(Into::into).unwrap_or(Check::Crc64); // default for xz let stream = Stream::new_easy_encoder(preset, check.into())?; stream } Format::ALONE => { let opts = match options { Some(opts) => opts.into(), None => LzmaOptions::new_preset(preset)?, }; let stream = Stream::new_lzma_encoder(&opts)?; stream } Format::RAW => { let check = check.map(Into::into).unwrap_or(Check::None); // default for Alone and Raw formats let filters = filters.map(Into::into).unwrap_or_else(|| Filters::new()); let stream = Stream::new_stream_encoder(&filters, check.into())?; stream } }; let mut encoder = XzEncoder::new_stream(data, stream); let n_bytes = io::copy(&mut encoder, output)?; Ok(n_bytes as usize) } libcramjam-0.8.0/src/zlib.rs000064400000000000000000000025501046102023000140310ustar 00000000000000//! zlib de/compression interface pub use flate2; use flate2::read::{ZlibDecoder, ZlibEncoder}; use flate2::Compression; use std::io::prelude::*; use std::io::{Cursor, Error}; const DEFAULT_COMPRESSION_LEVEL: u32 = 6; pub const ZLIB_MIN_HEADER_SIZE: usize = 2; pub const ZLIB_FOOTER_SIZE: usize = 4; pub const ZLIB_MIN_OVERHEAD: usize = ZLIB_MIN_HEADER_SIZE + ZLIB_FOOTER_SIZE; /// Compression upper bound // xref: https://github.com/ebiggers/libdeflate/blob/6bb493615b0ef35c98fc4aa4ec04f448788db6a5/lib/zlib_compress.c#L77 pub fn compress_bound(len: usize) -> usize { ZLIB_MIN_OVERHEAD + crate::deflate::compress_bound(len) } /// Decompress zlib data #[inline(always)] pub fn decompress(input: R, output: &mut W) -> Result { let mut decoder = ZlibDecoder::new(input); let mut out = vec![]; let n_bytes = decoder.read_to_end(&mut out)?; std::io::copy(&mut Cursor::new(out.as_slice()), output)?; Ok(n_bytes as usize) } /// Compress zlib data #[inline(always)] pub fn compress( input: R, output: &mut W, level: Option, ) -> Result { let level = level.unwrap_or_else(|| DEFAULT_COMPRESSION_LEVEL); let mut encoder = ZlibEncoder::new(input, Compression::new(level)); let n_bytes = std::io::copy(&mut encoder, output)?; Ok(n_bytes as usize) } libcramjam-0.8.0/src/zstd.rs000064400000000000000000000021031046102023000140470ustar 00000000000000//! zstd de/compression interface use std::io::{Error, Read, Write}; pub use zstd; const DEFAULT_COMPRESSION_LEVEL: i32 = 0; /// Get the max compressed length for a single pass pub fn compress_bound(len: usize) -> usize { zstd::zstd_safe::compress_bound(len) } /// Decompress gzip data #[inline(always)] pub fn decompress(input: R, output: &mut W) -> Result { let mut decoder = zstd::stream::read::Decoder::new(input)?; let n_bytes = std::io::copy(&mut decoder, output)?; Ok(n_bytes as usize) } /// Compress gzip data #[inline(always)] pub fn compress( input: R, output: &mut W, level: Option, input_size: Option, ) -> Result { let level = level.unwrap_or_else(|| DEFAULT_COMPRESSION_LEVEL); // 0 will use zstd's default, currently 3 let mut encoder = zstd::stream::read::Encoder::new(input, level)?; encoder.set_pledged_src_size(input_size.map(|v| v as u64))?; let n_bytes = std::io::copy(&mut encoder, output)?; Ok(n_bytes as usize) }